diff --git a/Cargo.lock b/Cargo.lock index 0cf2039bd2..a56f39bb0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3215,6 +3215,7 @@ dependencies = [ name = "consensus" version = "0.1.0" dependencies = [ + "aligned-vec", "bit-set 0.11.1", "bytemuck", "bytes", @@ -6842,6 +6843,7 @@ dependencies = [ "enumset", "secrecy", "thiserror 2.0.19", + "twox-hash", ] [[package]] diff --git a/core/binary_protocol/Cargo.toml b/core/binary_protocol/Cargo.toml index ec9b25e751..4fcdab1c5d 100644 --- a/core/binary_protocol/Cargo.toml +++ b/core/binary_protocol/Cargo.toml @@ -35,6 +35,7 @@ bytes = { workspace = true } enumset = { workspace = true } secrecy = { workspace = true } thiserror = { workspace = true } +twox-hash = { workspace = true } [dev-dependencies] aligned-vec = { workspace = true } diff --git a/core/binary_protocol/src/consensus/error.rs b/core/binary_protocol/src/consensus/error.rs index 13af4a92e6..805ffab731 100644 --- a/core/binary_protocol/src/consensus/error.rs +++ b/core/binary_protocol/src/consensus/error.rs @@ -29,6 +29,15 @@ pub enum ConsensusError { #[error("invalid checksum")] InvalidChecksum, + #[error( + "{command:?}: header checksum {found:#034x} does not cover the frame (expected {expected:#034x})" + )] + FrameChecksumMismatch { + command: Command2, + expected: u128, + found: u128, + }, + #[error("invalid cluster ID")] InvalidCluster, diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index a4661cd27a..c7d5116681 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs @@ -18,6 +18,23 @@ //! All consensus headers are exactly 256 bytes with `#[repr(C)]` layout. //! Size and field offsets are enforced at compile time. Deserialization //! is a pointer cast (zero-copy) via `bytemuck::try_from_bytes`. +//! +//! # Wire compatibility +//! +//! The replica-to-replica control headers are a BREAKING, non-negotiable change +//! against any build predating [`ConsensusHeader::FRAME_SEALED`]: `checksum` went +//! from a field nobody wrote to one every receiver verifies, so each side reads the +//! other's frames as corrupt. `release` must be zero on every header, so there is no +//! version channel to gate on and no way for the two to detect each other. +//! +//! Replicas must therefore be upgraded together, with the cluster down. A rolling +//! upgrade does not degrade, it stops the cluster: every control frame between a +//! mixed pair is dropped, so no view change reaches a quorum. Nothing enforces this, +//! because there is nothing left to enforce it with; this note is the declaration. +//! +//! `Prepare`, `Request`, `Reply`, and `Eviction` are unaffected. Prepares keep +//! `checksum` as their view-independent identity, and the three client-facing +//! headers are sealed on neither side, so SDKs are untouched. use super::{Command2, ConsensusError, Operation}; use bytemuck::{CheckedBitPattern, NoUninit}; @@ -54,6 +71,16 @@ pub fn read_size_field(header: &[u8]) -> Option { .map(u32::from_le_bytes) } +/// Frame checksum over a raw header: every byte past `checksum` itself. +/// +/// Byte-level twin of [`ConsensusHeader::frame_checksum`], which delegates here so +/// the typed and raw seals cannot disagree. For callers that do not know the +/// concrete header type statically, such as a wire-level test fixture. +#[must_use] +pub fn frame_checksum_bytes(header: &[u8; HEADER_SIZE]) -> u128 { + u128::from(twox_hash::XxHash3_64::oneshot(&header[size_of::()..])) +} + /// Trait implemented by all consensus header types. /// /// Every header is exactly [`HEADER_SIZE`] bytes, `#[repr(C)]`, and supports @@ -78,12 +105,82 @@ pub trait ConsensusHeader: Sized + CheckedBitPattern + NoUninit { command == Self::COMMAND } + /// Whether this header's `checksum` field seals the frame. + /// + /// True for replica-to-replica control frames, whose header carries every + /// decision field: view number, commit point, and the nack bitset that + /// authorises truncation. TCP's 16-bit checksum does not reliably catch a + /// flipped bit on a plaintext replica link. + /// + /// False for three groups: [`PrepareHeader`] / [`RepairPrepareHeader`] spend + /// `checksum` on [`PrepareHeader::identity_checksum`], which excludes `view` so + /// a re-stamped prepare keeps one identity, and a seal cannot share the field; + /// [`RequestHeader`] / [`ReplyHeader`] / [`EvictionHeader`] cross the client + /// boundary, so sealing them is an SDK change on both ends; [`GenericHeader`] is + /// the type-erased pre-dispatch view and defers to the typed parse, where + /// [`Self::verify_frame`] runs. + const FRAME_SEALED: bool = true; + /// # Errors /// Returns `ConsensusError` if the header fields are inconsistent. fn validate(&self) -> Result<(), ConsensusError>; fn operation(&self) -> Operation; fn command(&self) -> Command2; fn size(&self) -> u32; + + /// The `checksum` field, whatever this header spends it on. + fn checksum(&self) -> u128; + + /// Overwrite the `checksum` field. + fn set_checksum(&mut self, checksum: u128); + + /// Checksum over every byte of the header past `checksum` itself. + /// + /// `checksum_body` sits inside that range, so sealing the header also + /// pins the body seal, and the two together cover the whole frame. + #[must_use] + fn frame_checksum(&self) -> u128 { + let bytes: &[u8; HEADER_SIZE] = bytemuck::bytes_of(self) + .try_into() + .expect("every consensus header is HEADER_SIZE bytes"); + frame_checksum_bytes(bytes) + } + + /// Stamp [`Self::frame_checksum`]. Call last when building a frame: it covers + /// every other field, `checksum_body` included, so later writes are uncovered. + fn seal(&mut self) { + debug_assert!( + Self::FRAME_SEALED, + "sealing a header whose checksum field means something else", + ); + let checksum = self.frame_checksum(); + self.set_checksum(checksum); + } + + /// Reject a frame whose header does not match its own checksum. + /// + /// Runs before [`Self::validate`] on every typed parse: a header that did not + /// arrive intact cannot have any field believed, `validate`'s included. + /// + /// # Errors + /// [`ConsensusError::FrameChecksumMismatch`] on a bad seal. Unsealed header + /// types return `Ok` unconditionally. + fn verify_frame(&self) -> Result<(), ConsensusError> { + if !Self::FRAME_SEALED { + return Ok(()); + } + let expected = self.frame_checksum(); + let found = self.checksum(); + if found == expected { + Ok(()) + } else { + Err(ConsensusError::FrameChecksumMismatch { + command: self.command(), + expected, + found, + }) + } + } } // GenericHeader - type-erased dispatch @@ -121,6 +218,15 @@ const _: () = { impl ConsensusHeader for GenericHeader { const COMMAND: Command2 = Command2::Reserved; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -225,6 +331,15 @@ impl Default for RequestHeader { impl ConsensusHeader for RequestHeader { const COMMAND: Command2 = Command2::Request; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { self.operation } @@ -376,6 +491,15 @@ impl Default for ReplyHeader { impl ConsensusHeader for ReplyHeader { const COMMAND: Command2 = Command2::Reply; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { self.operation } @@ -558,6 +682,15 @@ impl EvictionHeader { impl ConsensusHeader for EvictionHeader { const COMMAND: Command2 = Command2::Eviction; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } /// Session-level (not per-op): always `Reserved`. fn operation(&self) -> Operation { Operation::Reserved @@ -633,7 +766,7 @@ impl ConsensusHeader for EvictionHeader { /// Primary -> replicas: replicate this operation. #[repr(C)] -#[derive(Debug, Clone, Copy, CheckedBitPattern, NoUninit)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] pub struct PrepareHeader { pub checksum: u128, pub checksum_body: u128, @@ -704,6 +837,15 @@ impl Default for PrepareHeader { impl ConsensusHeader for PrepareHeader { const COMMAND: Command2 = Command2::Prepare; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { self.operation } @@ -721,10 +863,63 @@ impl ConsensusHeader for PrepareHeader { found: self.command, }); } + // Both reserved regions must be zero. They sit inside + // [`Self::identity_checksum`], so a peer that fills them changes the op's + // identity while changing nothing the merge can see; and `dvc_blank` + // classifies a slot by exact struct equality, so a non-zero reserved byte + // turns a blank into a `Valid` header the merge then indexes. + if self.reserved_frame.iter().any(|&byte| byte != 0) { + return Err(ConsensusError::InvalidField( + "prepare: reserved_frame bytes must be zero".to_string(), + )); + } + if self.reserved.iter().any(|&byte| byte != 0) { + return Err(ConsensusError::InvalidField( + "prepare: reserved bytes must be zero".to_string(), + )); + } Ok(()) } } +/// `checksum` of a prepare no producer sealed. +/// +/// Written by a build predating the identity seal, or by the partition plane. +/// Verification skips such entries so an older build's WAL still replays. +pub const CHECKSUM_UNSEALED: u128 = 0; + +/// The frame's body, bounded by `size`. What `checksum_body` covers. +/// +/// Not `&frame[HEADER_SIZE..]`: `Message::try_from` accepts a buffer longer than +/// `size` without trimming, while the WAL scan reads exactly `size`, so slicing to +/// the end makes the two disagree. Empty when `size` overruns the buffer. +#[must_use] +pub fn frame_body(frame: &[u8], size: u32) -> &[u8] { + let end = size as usize; + if end <= HEADER_SIZE || end > frame.len() { + return &[]; + } + &frame[HEADER_SIZE..end] +} + +impl PrepareHeader { + /// Which prepare this is, independent of which view re-sent it. + /// + /// Covers the whole 256-byte header except `checksum` (a field cannot hash + /// itself) and `view`, so a retransmission that re-stamps `view` stays valid. + /// The body reaches the value through the covered `checksum_body`. + /// + /// Lives here, not in the consensus crate, because the WAL scan verifies it too + /// and the two must agree byte for byte: it hashes this struct's layout. + #[must_use] + pub fn identity_checksum(&self) -> u128 { + let mut covered = *self; + covered.checksum = 0; + covered.view = 0; + u128::from(twox_hash::XxHash3_64::oneshot(bytemuck::bytes_of(&covered))) + } +} + // RepairPrepareHeader - repair peer -> recovering replica (journal repair) /// A stored prepare served for journal repair. @@ -740,6 +935,15 @@ pub struct RepairPrepareHeader(pub PrepareHeader); impl ConsensusHeader for RepairPrepareHeader { const COMMAND: Command2 = Command2::RepairPrepare; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.0.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.0.checksum = checksum; + } fn operation(&self) -> Operation { self.0.operation } @@ -825,6 +1029,14 @@ impl Default for PrepareOkHeader { impl ConsensusHeader for PrepareOkHeader { const COMMAND: Command2 = Command2::PrepareOk; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { self.operation } @@ -880,6 +1092,14 @@ const _: () = { impl ConsensusHeader for CommitHeader { const COMMAND: Command2 = Command2::Commit; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -931,6 +1151,14 @@ const _: () = { impl ConsensusHeader for StartViewChangeHeader { const COMMAND: Command2 = Command2::StartViewChange; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -978,7 +1206,28 @@ pub struct DoViewChangeHeader { pub namespace: u64, /// View when status was last normal (key for log selection). pub log_view: u32, - pub reserved: [u8; 100], + pub reserved: [u8; 68], + /// Bit `i` set means the sender proves it never prepared suffix entry `i`, so + /// that entry never reached a replication quorum through this replica. A new + /// primary may truncate only once `quorum_nack_prepare` senders nack an entry; + /// short of that it might be committed and must be preserved. + /// + /// A corrupt local entry is deliberately NOT nacked: the sender cannot tell a + /// prepare it never saw from one it saw and lost, and only the former is proof. + /// Silence costs availability; a false nack costs data. + /// + /// Carved from the tail of the former `reserved` region, with `present_bitset` + /// LAST so both land 16-aligned with no padding and `op`/`commit`/`namespace`/ + /// `log_view` keep their offsets. A sender with nothing to nack sends zeros, + /// decoding as "nacks nothing": safe, since that can only slow a view change. + pub nack_bitset: u128, + /// Bit `i` set means the sender can serve the BODY of suffix entry `i`, not just + /// its header. The new primary needs one such sender per surviving entry, since + /// a header whose body it cannot fetch is an entry it can never commit. + /// + /// Zero from a sender offering nothing, reading as "offers no bodies": safe, + /// since the new primary waits rather than adopting an entry it cannot complete. + pub present_bitset: u128, } const _: () = { assert!(size_of::() == HEADER_SIZE); @@ -986,11 +1235,36 @@ const _: () = { offset_of!(DoViewChangeHeader, op) == offset_of!(DoViewChangeHeader, reserved_frame) + size_of::<[u8; 66]>() ); - assert!(offset_of!(DoViewChangeHeader, reserved) + size_of::<[u8; 100]>() == HEADER_SIZE); + // op/commit/namespace/log_view keep their pre-bitset offsets. + assert!(offset_of!(DoViewChangeHeader, reserved) == 156); + // Both bitsets are last and 16-aligned, so the struct has no padding + // (`NoUninit` would reject any). + assert!(offset_of!(DoViewChangeHeader, nack_bitset) % 16 == 0); + assert!(offset_of!(DoViewChangeHeader, present_bitset) % 16 == 0); + assert!( + offset_of!(DoViewChangeHeader, nack_bitset) + == offset_of!(DoViewChangeHeader, reserved) + size_of::<[u8; 68]>() + ); + assert!(offset_of!(DoViewChangeHeader, present_bitset) + size_of::() == HEADER_SIZE); }; +/// Suffix headers a `DoViewChange` may carry: one bit per entry in each of the two +/// `u128` bitsets. +/// +/// Mirrors `consensus::DVC_HEADERS_MAX` as a literal so this crate need not depend +/// on the consensus crate, as with `REPLICAS_MAX` in [`EvictionHeader::new`]. +pub const DVC_HEADERS_MAX: usize = 128; + impl ConsensusHeader for DoViewChangeHeader { const COMMAND: Command2 = Command2::DoViewChange; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1023,10 +1297,52 @@ impl ConsensusHeader for DoViewChangeHeader { "commit cannot exceed op".to_string(), )); } + let suffix_len = self.suffix_len()?; + // Bits past the suffix describe entries never sent: unchecked, a peer could + // smuggle a nack for an op the new primary would then truncate. + if suffix_len < DVC_HEADERS_MAX { + let beyond = !((1u128 << suffix_len) - 1); + if self.nack_bitset & beyond != 0 || self.present_bitset & beyond != 0 { + return Err(ConsensusError::InvalidField(format!( + "do_view_change: bitset bits set past the {suffix_len}-entry suffix" + ))); + } + } Ok(()) } } +impl DoViewChangeHeader { + /// Number of `PrepareHeader`s in the body. + /// + /// Zero is valid and means "no suffix": a replica with nothing uncommitted, or a + /// peer predating the suffix. Both contribute numbers only. + /// + /// # Errors + /// [`ConsensusError::InvalidField`] when `size` is short of the header, is not a + /// whole number of headers, or exceeds what the bitsets can address. + pub fn suffix_len(&self) -> Result { + let size = self.size as usize; + let Some(body_len) = size.checked_sub(HEADER_SIZE) else { + return Err(ConsensusError::InvalidField(format!( + "do_view_change: size {size} is shorter than the {HEADER_SIZE}-byte header" + ))); + }; + if body_len % HEADER_SIZE != 0 { + return Err(ConsensusError::InvalidField(format!( + "do_view_change: body of {body_len} bytes is not a whole number of headers" + ))); + } + let suffix_len = body_len / HEADER_SIZE; + if suffix_len > DVC_HEADERS_MAX { + return Err(ConsensusError::InvalidField(format!( + "do_view_change: {suffix_len} suffix entries exceeds the maximum {DVC_HEADERS_MAX}" + ))); + } + Ok(suffix_len) + } +} + // StartViewHeader - new view announcement (header-only) /// New primary -> all replicas: start new view. Header-only. @@ -1076,6 +1392,14 @@ const _: () = { impl ConsensusHeader for StartViewHeader { const COMMAND: Command2 = Command2::StartView; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1103,10 +1427,43 @@ impl ConsensusHeader for StartViewHeader { "commit cannot exceed op".to_string(), )); } + self.suffix_len()?; Ok(()) } } +impl StartViewHeader { + /// Number of `PrepareHeader`s in the body: the view's suffix, high-to-low op + /// from `op` down toward `commit`. + /// + /// Zero means numbers only, which is what a peer predating the suffix sends and + /// what the probe-answer path sends. A backup then falls back to trusting `op`. + /// + /// # Errors + /// [`ConsensusError::InvalidField`] when `size` is short of the header, is not a + /// whole number of headers, or exceeds what a view change can address. + pub fn suffix_len(&self) -> Result { + let size = self.size as usize; + let Some(body_len) = size.checked_sub(HEADER_SIZE) else { + return Err(ConsensusError::InvalidField(format!( + "start_view: size {size} is shorter than the {HEADER_SIZE}-byte header" + ))); + }; + if body_len % HEADER_SIZE != 0 { + return Err(ConsensusError::InvalidField(format!( + "start_view: body of {body_len} bytes is not a whole number of headers" + ))); + } + let suffix_len = body_len / HEADER_SIZE; + if suffix_len > DVC_HEADERS_MAX { + return Err(ConsensusError::InvalidField(format!( + "start_view: {suffix_len} suffix entries exceeds the maximum {DVC_HEADERS_MAX}" + ))); + } + Ok(suffix_len) + } +} + // RequestStartViewHeader - restarted replica asking for the current view /// Recovering replica -> all replicas: resend me the current `StartView`. @@ -1155,6 +1512,14 @@ const _: () = { impl ConsensusHeader for RequestStartViewHeader { const COMMAND: Command2 = Command2::RequestStartView; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1219,6 +1584,14 @@ const _: () = { impl ConsensusHeader for RequestPreparesHeader { const COMMAND: Command2 = Command2::RequestPrepares; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1282,6 +1655,14 @@ const _: () = { impl ConsensusHeader for RepairRangeReplyHeader { const COMMAND: Command2 = Command2::RepairDone; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } // One layout, two commands: `RepairDone` terminates a stream, // `RangeEvicted` prefixes it. Without this widening, `try_into_typed` // rejects `RangeEvicted` frames before `validate` ever sees them. @@ -1354,6 +1735,14 @@ const _: () = { impl ConsensusHeader for RequestStateTransferHeader { const COMMAND: Command2 = Command2::RequestStateTransfer; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1468,6 +1857,14 @@ const _: () = { impl ConsensusHeader for StateTransferTargetHeader { const COMMAND: Command2 = Command2::StateTransferTarget; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1559,6 +1956,14 @@ const _: () = { impl ConsensusHeader for RequestStateChunkHeader { const COMMAND: Command2 = Command2::RequestStateChunk; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1633,6 +2038,14 @@ const _: () = { impl ConsensusHeader for StateChunkHeader { const COMMAND: Command2 = Command2::StateChunk; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1664,9 +2077,12 @@ impl ConsensusHeader for StateChunkHeader { #[cfg(test)] mod tests { use super::{ - Command2, CommitHeader, ConsensusHeader, DoViewChangeHeader, EvictionHeader, - EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, PrepareOkHeader, - ReplyHeader, RequestHeader, StartViewChangeHeader, StartViewHeader, + Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, + EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, + PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, + RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, + StateTransferTargetHeader, }; use aligned_vec::{AVec, ConstAlign}; @@ -1678,6 +2094,128 @@ mod tests { v } + /// A header-sized frame that satisfies `bytemuck`'s 16-byte alignment. + #[repr(C, align(16))] + struct AlignedFrame([u8; HEADER_SIZE]); + + /// A minimal well-formed header of type `H`: own command and size, everything + /// else zero. Enough for the seal, which reads bytes rather than fields. + fn control_header() -> H { + const COMMAND_OFF: usize = std::mem::offset_of!(GenericHeader, command); + const SIZE_OFF: usize = std::mem::offset_of!(GenericHeader, size); + + let frame_len = u32::try_from(HEADER_SIZE).expect("HEADER_SIZE fits u32"); + let mut frame = AlignedFrame([0u8; HEADER_SIZE]); + frame.0[COMMAND_OFF] = H::COMMAND as u8; + frame.0[SIZE_OFF..SIZE_OFF + 4].copy_from_slice(&frame_len.to_le_bytes()); + *bytemuck::checked::try_from_bytes::(&frame.0).expect("a zeroed frame is a valid header") + } + + /// Seal a header, flip one bit at `offset`, and report `verify_frame`'s verdict. + fn tamper(mut header: H, offset: usize) -> Result<(), ConsensusError> { + header.seal(); + let mut frame = AlignedFrame([0u8; HEADER_SIZE]); + frame.0.copy_from_slice(bytemuck::bytes_of(&header)); + frame.0[offset] ^= 0x01; + let tampered = bytemuck::checked::try_from_bytes::(&frame.0) + .expect("a single flipped bit stays a valid bit pattern here"); + tampered.verify_frame() + } + + #[test] + fn given_a_sealed_control_header_when_verifying_should_accept() { + macro_rules! seals { + ($($header:ty),+ $(,)?) => {$({ + assert!( + <$header>::FRAME_SEALED, + "{} is a replica-to-replica control header and must seal", + stringify!($header), + ); + let mut header = control_header::<$header>(); + header.seal(); + assert_eq!( + header.verify_frame(), + Ok(()), + "{} must accept its own seal", + stringify!($header), + ); + })+}; + } + seals!( + PrepareOkHeader, + CommitHeader, + StartViewChangeHeader, + DoViewChangeHeader, + StartViewHeader, + RequestStartViewHeader, + RequestPreparesHeader, + RepairRangeReplyHeader, + RequestStateTransferHeader, + StateTransferTargetHeader, + RequestStateChunkHeader, + StateChunkHeader, + ); + } + + #[test] + fn given_any_covered_byte_when_flipped_should_reject() { + // Why this seal exists. A `DoViewChange` nack bitset is a new primary's + // authority to truncate: two nacks on three replicas reach + // `quorum_nack_prepare` and discard a committed, client-acked op. The bitsets + // ride the header, and TCP's checksum will not reliably catch one bit. + // Every byte past `checksum` is covered, `checksum_body` included. + for offset in size_of::()..HEADER_SIZE { + let header = control_header::(); + // Skip offsets where the flipped bit is an invalid bit pattern, which + // `try_from_bytes` rejects one layer earlier. + if offset == std::mem::offset_of!(DoViewChangeHeader, command) { + continue; + } + assert!( + matches!( + tamper(header, offset), + Err(ConsensusError::FrameChecksumMismatch { .. }) + ), + "byte {offset} is inside the seal and must be covered" + ); + } + } + + #[test] + fn given_an_unsealed_control_header_when_verifying_should_reject() { + // No presence-keying: a zero checksum is a corrupt frame, not an old one. + // Keying on "does this look sealed" leaves the layer bypassable by zeroing + // the one field that decides whether anything is checked. + let header = control_header::(); + assert_eq!(header.checksum, 0); + assert!(matches!( + header.verify_frame(), + Err(ConsensusError::FrameChecksumMismatch { found: 0, .. }) + )); + } + + #[test] + fn given_an_identity_or_client_header_when_verifying_should_abstain() { + // `PrepareHeader` spends `checksum` on `identity_checksum`, which excludes + // `view` so a re-stamped prepare keeps one identity; the client-facing three + // are sealed on neither side yet. All must parse unchanged. + const { + assert!(!PrepareHeader::FRAME_SEALED); + assert!(!RepairPrepareHeader::FRAME_SEALED); + assert!(!RequestHeader::FRAME_SEALED); + assert!(!ReplyHeader::FRAME_SEALED); + assert!(!EvictionHeader::FRAME_SEALED); + assert!(!GenericHeader::FRAME_SEALED); + } + + let prepare = PrepareHeader { + command: Command2::Prepare, + checksum: 0xdead_beef, + ..Default::default() + }; + assert_eq!(prepare.verify_frame(), Ok(())); + } + #[test] fn all_headers_are_256_bytes() { assert_eq!(size_of::(), 256); diff --git a/core/binary_protocol/src/consensus/mod.rs b/core/binary_protocol/src/consensus/mod.rs index 83bf5ae616..0cf8eefa3a 100644 --- a/core/binary_protocol/src/consensus/mod.rs +++ b/core/binary_protocol/src/consensus/mod.rs @@ -45,12 +45,12 @@ mod reply_result; pub use command::Command2; pub use error::ConsensusError; pub use header::{ - CommitHeader, ConsensusHeader, DoViewChangeHeader, EvictionHeader, EvictionReason, - GenericHeader, HEADER_SIZE, PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, - RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, - RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, SIZE_FIELD_OFFSET, - StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, - read_size_field, + CHECKSUM_UNSEALED, CommitHeader, ConsensusHeader, DVC_HEADERS_MAX, DoViewChangeHeader, + EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, PrepareHeader, PrepareOkHeader, + RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, + RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, + StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, read_size_field, }; pub use operation::Operation; pub use reply_result::{RESULT_COUNT_LEN, RESULT_ENTRY_LEN, result_code, result_section_len}; diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index 1f4baf3fb6..4d66bbc7d2 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs @@ -71,12 +71,13 @@ pub mod version; pub use codec::{WireDecode, WireEncode}; pub use consensus::{ - Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, EvictionHeader, - EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, PrepareOkHeader, - RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, - StateChunkHeader, StateTransferTargetHeader, read_size_field, result_code, result_section_len, + CHECKSUM_UNSEALED, Command2, CommitHeader, ConsensusError, ConsensusHeader, DVC_HEADERS_MAX, + DoViewChangeHeader, EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, Operation, + PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, RepairPrepareHeader, + RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, SIZE_FIELD_OFFSET, + StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, + frame_body, frame_checksum_bytes, read_size_field, result_code, result_section_len, }; pub use dispatch::{COMMAND_TABLE, CommandMeta, lookup_by_operation, lookup_command}; pub use error::WireError; diff --git a/core/configs/src/server_ng_config/metadata.rs b/core/configs/src/server_ng_config/metadata.rs index f38840f893..fa822bbf72 100644 --- a/core/configs/src/server_ng_config/metadata.rs +++ b/core/configs/src/server_ng_config/metadata.rs @@ -60,10 +60,15 @@ pub const DEFAULT_METADATA_JOURNAL_SLOTS: usize = 1024; /// margin is `max(this, prepare_queue_depth)`. pub const METADATA_CHECKPOINT_MARGIN_FLOOR: usize = 64; -/// Upper bound on `prepare_queue_depth`. Every queued prepare pins a -/// full message buffer; four thousand in-flight metadata ops is far past -/// any sane deployment and a likely unit typo. -pub const MAX_METADATA_PREPARE_QUEUE_DEPTH: usize = 4096; +/// Upper bound on `prepare_queue_depth`. +/// +/// Pinned by the view-change wire format, not by memory: a `DoViewChange` carries +/// the sender's uncommitted suffix plus one nack bit and one present bit per entry, +/// each bitset a single `u128` (`consensus::DVC_HEADERS_MAX` = 128). The suffix +/// spans `commit_max..=op`, which this depth bounds, so a deeper queue produces +/// entries the new primary can neither adopt nor prove dead. The reserved head slot +/// leaves room for the head op. +pub const MAX_METADATA_PREPARE_QUEUE_DEPTH: usize = 127; /// Upper bound on `journal_slots`. Each slot costs index memory and every /// checkpoint rewrites the live WAL suffix; a million slots is the sanity @@ -187,33 +192,52 @@ mod tests { #[test] fn margin_tracks_deep_prepare_queue() { let config = MetadataConfig { - prepare_queue_depth: 256, + prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH, journal_slots: 4096, clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(config.validate().is_ok()); - assert_eq!(config.checkpoint_margin(), 256); + assert_eq!(config.checkpoint_margin(), MAX_METADATA_PREPARE_QUEUE_DEPTH); } #[test] fn journal_must_outsize_margin() { - // Deep queue, journal kept at the old default: margin becomes 256, - // 4 * 256 = 1024 == journal_slots, boundary accepted... + // Deepest permitted queue: margin becomes the depth, and the journal + // must hold 4x that. At exactly 4x the boundary is accepted... + let min_slots = 4 * MAX_METADATA_PREPARE_QUEUE_DEPTH; let boundary = MetadataConfig { - prepare_queue_depth: 256, - journal_slots: 1024, + prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH, + journal_slots: min_slots, clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(boundary.validate().is_ok()); // ...one slot fewer is refused. let starved = MetadataConfig { - prepare_queue_depth: 256, - journal_slots: 1023, + prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH, + journal_slots: min_slots - 1, clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(starved.validate().is_err()); } + #[test] + fn prepare_queue_depth_capped_by_view_change_bitset_width() { + // Not a memory guard: it keeps every uncommitted suffix entry addressable by + // a `u128` bitset in a `DoViewChange`. One past it must be refused, or a view + // change meets an entry it can neither adopt nor prove dead. + let over = MetadataConfig { + prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH + 1, + journal_slots: MAX_METADATA_JOURNAL_SLOTS, + clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, + }; + assert!(over.validate().is_err()); + assert_eq!( + MAX_METADATA_PREPARE_QUEUE_DEPTH + 1, + 128, + "cap must leave the head op a slot inside the 128-bit bitset" + ); + } + #[test] fn zero_depth_is_refused() { let config = MetadataConfig { diff --git a/core/configs/src/server_ng_config/partition.rs b/core/configs/src/server_ng_config/partition.rs index 011e82a9b8..87d5a4f7ad 100644 --- a/core/configs/src/server_ng_config/partition.rs +++ b/core/configs/src/server_ng_config/partition.rs @@ -27,11 +27,11 @@ //! (the per-partition journal-repair retention ring's dual ceilings) //! //! Distinct from `[metadata]` (a single, shard-0-global VSR plane) because -//! partition pipelines exist PER PARTITION. The default mirrors the runtime -//! constant so a default deployment is byte-identical; the ceiling is far -//! below metadata's because the request queue (`depth * 2` slots) pins full -//! inbound produce batches, so pinned memory scales with the partition count -//! (see [`MAX_PARTITION_PREPARE_QUEUE_DEPTH`]). +//! partition pipelines exist PER PARTITION: the request queue (`depth * 2` slots) +//! pins full inbound produce batches, so pinned memory scales with the partition +//! count. The default mirrors the runtime constant; the ceiling matches metadata's, +//! since both planes ship the same `DoViewChange` suffix over the same bitsets (see +//! [`MAX_PARTITION_PREPARE_QUEUE_DEPTH`]). //! //! The default is a duplicated literal rather than an import so //! `core/configs` does not grow a build-time edge onto `core/consensus` @@ -47,13 +47,19 @@ use serde::{Deserialize, Serialize}; /// Mirrors `consensus::PIPELINE_PREPARE_QUEUE_MAX`. pub const DEFAULT_PARTITION_PREPARE_QUEUE_DEPTH: usize = 32; -/// Upper bound on `prepare_queue_depth`. Unlike the single metadata pipeline, -/// a pipeline exists per partition, and each queued request pins a full -/// inbound produce batch (a 4 KiB floor up to megabytes). Worst-case pinned -/// memory therefore scales as `depth * 2 * partition_count * batch_size`, so -/// this ceiling sits far below metadata's 4096: it is a typo guard, not a -/// sizing endorsement. -pub const MAX_PARTITION_PREPARE_QUEUE_DEPTH: usize = 256; +/// Upper bound on `prepare_queue_depth`. +/// +/// Pinned by the view-change wire format, and equal to +/// [`super::metadata::MAX_METADATA_PREPARE_QUEUE_DEPTH`] for that reason: a +/// `DoViewChange` carries the sender's uncommitted suffix spanning `commit..=op` +/// with one nack bit and one present bit per entry, each bitset a single `u128` +/// (`consensus::DVC_HEADERS_MAX` = 128). This depth bounds `op - commit`, so a +/// deeper queue produces entries the new primary can neither adopt nor prove dead. +/// The reserved head slot leaves room for the head op. +/// +/// The memory bound (`depth * 2 * partition_count * batch_size` of pinned produce +/// batches) still holds and is looser, so the wire is what decides. +pub const MAX_PARTITION_PREPARE_QUEUE_DEPTH: usize = 127; /// Mirrors the free const `shard::PARTITION_ARTIFACT_LEN_DEFAULT` (segment /// ceiling plus the one whole batch a segment may close past it). diff --git a/core/configs/src/server_ng_config/validators.rs b/core/configs/src/server_ng_config/validators.rs index f7408b7d35..cf47cce379 100644 --- a/core/configs/src/server_ng_config/validators.rs +++ b/core/configs/src/server_ng_config/validators.rs @@ -400,11 +400,6 @@ fn reject_unsupported_and_warn_inert(config: &ServerNgConfig) -> Result<(), Conf if config.tcp.socket_migration != defaults.tcp.socket_migration { warn!("tcp.socket_migration is not implemented in server-ng"); } - if config.system.partition.validate_checksum != defaults.system.partition.validate_checksum { - warn!( - "system.partition.validate_checksum is not applied in server-ng; nothing verifies checksums on load" - ); - } if config.system.segment.cache_indexes != defaults.system.segment.cache_indexes { warn!("system.segment.cache_indexes is not applied in server-ng"); } @@ -698,10 +693,6 @@ mod tests { let defaults = ServerNgConfig::default(); assert_eq!(shipped.tcp.socket_migration, defaults.tcp.socket_migration); - assert_eq!( - shipped.system.partition.validate_checksum, - defaults.system.partition.validate_checksum - ); assert_eq!( shipped.system.segment.cache_indexes, defaults.system.segment.cache_indexes diff --git a/core/consensus/Cargo.toml b/core/consensus/Cargo.toml index 1ba5610c24..342ce604d8 100644 --- a/core/consensus/Cargo.toml +++ b/core/consensus/Cargo.toml @@ -45,6 +45,7 @@ tracing = { workspace = true } twox-hash = { workspace = true } [dev-dependencies] +aligned-vec = { workspace = true } futures = { workspace = true } [lints.clippy] diff --git a/core/consensus/src/dvc_merge.rs b/core/consensus/src/dvc_merge.rs new file mode 100644 index 0000000000..bbf265248b --- /dev/null +++ b/core/consensus/src/dvc_merge.rs @@ -0,0 +1,911 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Merging a `DoViewChange` quorum into the new view's log: for every op that +//! might be uncommitted, does the new view keep it or discard it? +//! +//! Keeping an op that was never committed costs a wasted slot. Discarding one +//! that WAS committed loses acknowledged client data, so the only proof accepted +//! for discarding is a nack quorum: enough replicas stating they never prepared +//! it that a replication quorum provably never formed. Absent that the op is +//! kept, and if no replica offers its body the view does not start. Stalling is +//! visible and recoverable; losing the op is neither. + +#[cfg(test)] +use crate::view_change_quorum::DvcSuffix; +use crate::view_change_quorum::{DvcQuorumArray, StoredDvc, dvc_count, dvc_iter}; +use iggy_binary_protocol::PrepareHeader; + +/// Sizes the merge needs from the replica. +#[derive(Debug, Clone, Copy)] +pub struct MergeQuorums { + /// `DoViewChange` messages needed before a view may start. + pub view_change: usize, + /// Nacks needed to prove an op uncommitted, so it may be discarded. + pub nack_prepare: usize, + /// Cluster size, which bounds how many more DVCs could still arrive. + pub replica_count: usize, + /// Cluster-wide pipeline ceiling: an op further than this below a sender's head + /// cannot still be uncommitted, since no node could have kept it in flight. + /// + /// NOT this node's configured depth. The bound applies to a *peer's* head op, + /// and a local depth larger than that peer's manufactures a commit the peer + /// never made. Every node's depth is pinned below `DVC_HEADERS_MAX`, so the + /// ceiling holds for all of them. + pub prepare_queue_ceiling: u64, +} + +/// What the collected DVCs say about starting the view. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MergeOutcome { + /// Fewer than `view_change` DVCs so far. + AwaitingQuorum, + /// Quorum is in, but some op is neither provably dead nor recoverable and an + /// unreported replica could still settle it. Wait for them. + AwaitingRepair { + /// The op that cannot yet be decided. + undecided_op: u64, + }, + /// Every replica reported and an op is still neither provably dead nor + /// recoverable. No further message changes that: data loss already happened, + /// and truncating here would turn it from detected into silent. + Deadlocked { + /// The op that cannot be decided. + undecided_op: u64, + }, + /// The view can start. + Ready(MergedLog), +} + +/// The log the new primary adopts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MergedLog { + /// New head op. Below the highest op any canonical sender reported when a + /// nack quorum proved the ops above it dead. + pub op_head: u64, + /// Highest op the quorum proves committed. The merge never discards at or + /// below this. + pub commit_max: u64, + /// Canonical headers for `commit_max..=op_head`, ordered high-to-low op. + /// The new primary installs these over its own log. + pub headers: Vec, + /// Headers non-canonical senders report committed. Installed unconditionally, + /// because header repair will not cross a gap to reach them later. + pub committed_elsewhere: Vec, +} + +/// Highest op the quorum proves committed. +/// +/// Three independent lower bounds, because a single sender's view of the commit +/// point can lag arbitrarily while the cluster's cannot: +/// * each sender's own reported commit, +/// * the `commit` its head prepare carries, stamped by that op's primary, +/// * its head minus the cluster-wide pipeline ceiling, since nothing further back +/// than one pipeline can still be in flight. +/// +/// The lowest op in a sender's suffix is deliberately NOT a fourth bound, and +/// re-adding one is a data-loss bug. It would be sound only if the suffix floor +/// were the commit point by construction; here it is computed, and two paths in +/// `build_dvc_suffix` raise it above the sender's commit (ops are 1-based, so +/// commit 0 floors at op 1; the `DVC_HEADERS_MAX` clamp drops the bottom of an +/// over-wide window). Nothing on the wire distinguishes the two. +/// +/// Nothing is lost by omitting it: the snapshot is tagged with the `(op, commit)` +/// the header is stamped with and dropped on a mismatch, so the floor either +/// equals `dvc.commit`, already the first bound, or exceeds it, the unsound case. +#[must_use] +pub fn merge_commit_max(quorum: &DvcQuorumArray, prepare_queue_ceiling: u64) -> u64 { + let mut commit_max = 0; + for dvc in dvc_iter(quorum) { + commit_max = commit_max.max(dvc.commit); + commit_max = commit_max.max(dvc.op.saturating_sub(prepare_queue_ceiling)); + if let Some(head) = dvc.suffix.headers().first() { + commit_max = commit_max.max(head.commit); + } + } + commit_max +} + +/// Highest `log_view` any sender reported. +/// +/// Senders at this `log_view` were in every earlier view change, so their headers +/// already reflect the truncations those views decided. That makes them canonical, +/// and a lower-`log_view` sender disagreeing is evidence against its own header. +fn log_view_canonical(quorum: &DvcQuorumArray) -> Option { + dvc_iter(quorum).map(|dvc| dvc.log_view).max() +} + +/// Per-op tally over the whole quorum. +struct OpVerdict<'a> { + canonical: Option<&'a PrepareHeader>, + /// Senders holding the canonical header AND able to serve its body. + copies: usize, + nacks: usize, + /// Canonical senders disagree about this op, so no header here is trustworthy. + conflict: bool, +} + +/// What the canonical senders say about one op. +struct CanonicalAt<'a> { + header: Option<&'a PrepareHeader>, + /// Two senders at the canonical `log_view` disagree here. + conflict: bool, +} + +/// The canonical header at `op`, and whether the canonical senders agree. +/// +/// Every canonical sender is consulted, not just the first: they were all in +/// normal status in that view and a primary prepares one thing per op, so a +/// disagreement is not a vote but proof that one header is wrong with no way to +/// tell which. The caller treats the op as undecidable. +fn canonical_header_at<'a>(canonical_senders: &[&'a StoredDvc], op: u64) -> CanonicalAt<'a> { + let mut header: Option<&'a PrepareHeader> = None; + let mut conflict = false; + for dvc in canonical_senders { + let Some(index) = dvc.suffix.index_of(dvc.op, op) else { + continue; + }; + let Some(candidate) = dvc.suffix.valid_header_at(index) else { + continue; + }; + match header { + Some(existing) if existing.checksum != candidate.checksum => conflict = true, + Some(_) => {} + None => header = Some(candidate), + } + } + CanonicalAt { header, conflict } +} + +/// Tally every sender's position on `op`. +fn tally_op<'a>( + quorum: &'a DvcQuorumArray, + canonical_senders: &[&'a StoredDvc], + canonical_log_view: u32, + op: u64, +) -> OpVerdict<'a> { + let CanonicalAt { + header: canonical, + conflict, + } = canonical_header_at(canonical_senders, op); + let mut copies = 0; + let mut nacks = 0; + + for dvc in dvc_iter(quorum) { + // The sender's log stops below this op, so it never prepared it. + if dvc.op < op { + nacks += 1; + continue; + } + let Some(index) = dvc.suffix.index_of(dvc.op, op) else { + // The sender said nothing about this op, so it abstains: no nack, no + // copy. Counting silence as a nack is sound only when every DVC carries + // headers, since then falling outside a window means being above it. + // Two cases here are not, and abstaining costs a slower view change + // where nacking costs data. + // + // An empty suffix is not a vote. It comes from a replica with + // nothing uncommitted, or one whose snapshot no longer matches its + // log; reading it as + // agreement with someone else's nack discards a committed op on one real + // nack plus one silence. And with `dvc.op >= op` established above, a + // non-empty suffix not covering `op` puts `op` below the sender's window + // floor, at or below its own commit point, so nacking it is backwards. + // Only the defensive `DVC_HEADERS_MAX` clamp reaches that. + continue; + }; + + let held = dvc.suffix.valid_header_at(index); + if let (Some(held), Some(canonical)) = (held, canonical) + && dvc.suffix.offers_body(index) + && held.checksum == canonical.checksum + { + copies += 1; + } + + if dvc.suffix.nacks(index) { + // Explicit: the sender proves it never prepared this op. + nacks += 1; + } else if let Some(held) = held { + // Only a sender BEHIND the canonical log_view can implicitly nack. + // Without this, corrupting one canonical header in transit turns every + // honest sender's correct header into an implicit nack against the + // garbage: a nack quorum on three replicas. A same-log_view + // disagreement is evidence, not a vote, and goes through `conflict`. + let may_nack_implicitly = dvc.log_view < canonical_log_view; + match canonical { + // Implicit: the sender holds a DIFFERENT prepare, so not this one. + Some(canonical) if may_nack_implicitly && held.checksum != canonical.checksum => { + nacks += 1; + } + // Implicit: no canonical sender holds anything here, so a newer + // view already truncated this op and the sender holds a corpse. + None if may_nack_implicitly => nacks += 1, + _ => {} + } + } + } + + OpVerdict { + canonical, + copies, + nacks, + conflict, + } +} + +/// Collect the canonical headers for `commit_max..=op_head`, high-to-low. +/// +/// Checks the hash chain as it walks: a break means the canonical senders agreed +/// on individual ops but not on one history, which no later step would notice. +fn canonical_headers( + canonical_senders: &[&StoredDvc], + op_head: u64, + commit_max: u64, +) -> Option> { + if op_head == 0 { + return Some(Vec::new()); + } + let floor = commit_max.max(1); + let mut headers = Vec::new(); + let mut child: Option = None; + let mut op = op_head; + loop { + let at = canonical_header_at(canonical_senders, op); + if at.conflict { + return None; + } + let header = *at.header?; + if let Some(child) = child + && child.parent != header.checksum + { + tracing::error!( + op, + child_op = child.op, + "view-change headers do not hash-chain; refusing to install" + ); + return None; + } + child = Some(header); + headers.push(header); + if op <= floor { + break; + } + op -= 1; + } + Some(headers) +} + +/// Headers that a non-canonical sender reports committed. +/// +/// Trusted on that sender's word alone, unlike anything above its commit point: +/// header repair walks the chain backwards and stops at a gap, so an op missing +/// below the new primary's commit point can never be repaired into place and +/// refusing it here strands the log permanently. The claim is already +/// quorum-backed, since a sender cannot report an op committed unless a +/// replication quorum held it. +/// +/// `None` when two senders report *different* prepares committed at one op. These +/// install unconditionally, so guessing is least affordable here; as in +/// [`canonical_header_at`], the caller treats it as undecidable. +fn committed_elsewhere( + quorum: &DvcQuorumArray, + canonical_log_view: u32, + already_installed: &[PrepareHeader], +) -> Option> { + let mut extra: Vec = Vec::new(); + for dvc in dvc_iter(quorum).filter(|dvc| dvc.log_view < canonical_log_view) { + for (index, header) in dvc.suffix.headers().iter().enumerate() { + if header.op > dvc.commit { + continue; + } + if dvc.suffix.valid_header_at(index).is_none() { + continue; + } + if already_installed + .iter() + .any(|installed| installed.op == header.op) + { + continue; + } + if let Some(queued) = extra.iter().find(|queued| queued.op == header.op) { + if queued.checksum != header.checksum { + tracing::error!( + op = header.op, + "replicas disagree about the prepare committed at op {}; refusing to \ + choose one to install", + header.op + ); + return None; + } + continue; + } + extra.push(*header); + } + } + Some(extra) +} + +/// Decide whether the new view can start, and with what log. +/// +/// Walks every op that might be uncommitted, from the proven commit point to the +/// highest a canonical sender reported, stopping at the first proved dead. +#[must_use] +pub fn merge_dvc_quorum(quorum: &DvcQuorumArray, quorums: MergeQuorums) -> MergeOutcome { + let received = dvc_count(quorum); + if received < quorums.view_change { + return MergeOutcome::AwaitingQuorum; + } + + let Some(canonical_log_view) = log_view_canonical(quorum) else { + return MergeOutcome::AwaitingQuorum; + }; + let canonical_senders: Vec<&StoredDvc> = dvc_iter(quorum) + .filter(|dvc| dvc.log_view == canonical_log_view) + .collect(); + debug_assert!( + !canonical_senders.is_empty(), + "the max log_view must be held by at least one sender" + ); + + let commit_max = merge_commit_max(quorum, quorums.prepare_queue_ceiling); + let op_head_max = canonical_senders + .iter() + .map(|dvc| dvc.op) + .max() + .unwrap_or(commit_max) + .max(commit_max); + + if op_head_max == 0 { + // Nothing was ever prepared, so nothing to decide. Ops are 1-based, and + // scanning op 0 reads every sender's absent entry as a nack, manufacturing + // a nack quorum for an op that does not exist. + return MergeOutcome::Ready(MergedLog { + op_head: 0, + commit_max: 0, + headers: Vec::new(), + committed_elsewhere: Vec::new(), + }); + } + + let mut op_head = op_head_max; + // Start at the proven commit point, or op 1 when nothing is committed. The + // commit point is scanned so the adopted log is anchored on a servable header. + let mut op = commit_max.max(1); + while op <= op_head_max { + let verdict = tally_op(quorum, &canonical_senders, canonical_log_view, op); + + if verdict.nacks >= quorums.nack_prepare { + if op <= commit_max { + // A nack quorum for a committed op is impossible under quorum + // intersection, so a peer lied or a bitset is wrong. Refuse the + // view rather than assert, so one bad peer stalls the group instead + // of panicking a node into a restart loop. + tracing::error!( + op, + commit_max, + nacks = verdict.nacks, + nack_quorum = quorums.nack_prepare, + "nack quorum for an op the quorum proves committed; refusing the view" + ); + return MergeOutcome::Deadlocked { undecided_op: op }; + } + op_head = op - 1; + break; + } + + if verdict.conflict { + // Senders all in normal status in the same view disagree about what it + // prepared here. One header is wrong with no way to tell which, so + // refuse rather than pick. + tracing::error!( + op, + log_view = canonical_log_view, + "replicas at the same log_view disagree about op {op}; refusing to choose a \ + canonical header for it" + ); + return if received < quorums.replica_count { + MergeOutcome::AwaitingRepair { undecided_op: op } + } else { + MergeOutcome::Deadlocked { undecided_op: op } + }; + } + + if verdict.canonical.is_none() || verdict.copies == 0 { + // Neither provably dead nor recoverable. An outstanding replica may + // hold the body or supply the deciding nack. + return if received < quorums.replica_count { + MergeOutcome::AwaitingRepair { undecided_op: op } + } else { + tracing::error!( + op, + canonical = verdict.canonical.is_some(), + copies = verdict.copies, + nacks = verdict.nacks, + nack_quorum = quorums.nack_prepare, + "every replica reported and op {op} is neither recoverable nor provably \ + uncommitted; the view cannot start" + ); + MergeOutcome::Deadlocked { undecided_op: op } + }; + } + + op += 1; + } + + debug_assert!(op_head >= commit_max); + let Some(headers) = canonical_headers(&canonical_senders, op_head, commit_max) else { + return MergeOutcome::Deadlocked { + undecided_op: op_head, + }; + }; + let Some(committed_elsewhere) = committed_elsewhere(quorum, canonical_log_view, &headers) + else { + // Two senders disagree about a committed op. Not a repair problem: these + // install without a nack quorum, and no message resolves which is true. + return MergeOutcome::Deadlocked { + undecided_op: op_head, + }; + }; + + MergeOutcome::Ready(MergedLog { + op_head, + commit_max, + headers, + committed_elsewhere, + }) +} + +/// Build a suffix for a sender that holds every op in `commit..=op` with a +/// servable body. Test helper. +#[cfg(test)] +#[must_use] +pub fn suffix_all_present(headers: Vec) -> DvcSuffix { + // `1 << 128` overflows, and a full-width suffix is exactly what the clamp + // produces, so the widest case cannot use the shift. Past the width is left to + // `DvcSuffix::new`, which rejects it by name rather than as an overflow. + let count = u32::try_from(headers.len()).unwrap_or(u32::MAX).min(128); + let mask = u128::MAX.checked_shr(128 - count).unwrap_or(0); + DvcSuffix::new(headers, 0, mask) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::DVC_HEADERS_MAX; + use crate::view_change_quorum::{dvc_blank, dvc_quorum_array_empty, dvc_record}; + use iggy_binary_protocol::{Command2, Operation}; + + /// Three replicas: replication 2, view-change 2, nack 2. + fn quorums_r3() -> MergeQuorums { + MergeQuorums { + view_change: 2, + nack_prepare: 2, + replica_count: 3, + prepare_queue_ceiling: 32, + } + } + + /// A prepare whose checksum derives from its op, so the hash chain connects. + fn prepare(op: u64, view: u32) -> PrepareHeader { + PrepareHeader { + command: Command2::Prepare, + operation: Operation::CreateStream, + op, + view, + checksum: u128::from(op) | (u128::from(view) << 64), + parent: if op <= 1 { + 0 + } else { + u128::from(op - 1) | (u128::from(view) << 64) + }, + // Left at zero so each test drives `commit_max` through the bound it is + // about; `merge_commit_max` honours this field, covered separately below. + commit: 0, + ..Default::default() + } + } + + /// Headers for `low..=high`, ordered high-to-low as a suffix requires. + fn suffix_headers(low: u64, high: u64, view: u32) -> Vec { + (low..=high).rev().map(|op| prepare(op, view)).collect() + } + + fn dvc(replica: u8, log_view: u32, op: u64, commit: u64, suffix: DvcSuffix) -> StoredDvc { + StoredDvc { + replica, + log_view, + op, + commit, + suffix, + } + } + + #[test] + fn given_agreeing_quorum_when_merging_should_adopt_the_shared_head() { + let mut quorum = dvc_quorum_array_empty(); + dvc_record( + &mut quorum, + dvc(0, 1, 5, 3, suffix_all_present(suffix_headers(3, 5, 1))), + ); + dvc_record( + &mut quorum, + dvc(1, 1, 5, 3, suffix_all_present(suffix_headers(3, 5, 1))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("an agreeing quorum must be ready"); + }; + assert_eq!(log.op_head, 5); + assert_eq!(log.commit_max, 3); + assert_eq!( + log.headers.iter().map(|h| h.op).collect::>(), + vec![5, 4, 3], + "headers run high-to-low from the head down to commit_max" + ); + } + + #[test] + fn given_nack_quorum_above_commit_when_merging_should_truncate_to_the_nacked_op() { + // Both survivors hold 1..=3 and never prepared 4, so the head drops to 3. + let mut quorum = dvc_quorum_array_empty(); + let mut headers = suffix_headers(2, 4, 1); + headers[0] = dvc_blank(4); + let nack_op_four = DvcSuffix::new(headers.clone(), 0b001, 0b110); + dvc_record(&mut quorum, dvc(0, 1, 4, 2, nack_op_four.clone())); + dvc_record(&mut quorum, dvc(1, 1, 4, 2, nack_op_four)); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("a nack quorum must decide the view"); + }; + assert_eq!(log.op_head, 3, "op 4 is provably uncommitted"); + assert!(log.headers.iter().all(|header| header.op <= 3)); + } + + #[test] + fn given_one_nack_short_of_quorum_when_merging_should_keep_the_op() { + // Replica 0 never saw op 4; replica 1 holds it and can serve it. One nack + // is short of the quorum of 2, so op 4 survives. + let mut quorum = dvc_quorum_array_empty(); + let mut holed = suffix_headers(2, 4, 1); + holed[0] = dvc_blank(4); + dvc_record( + &mut quorum, + dvc(0, 1, 4, 2, DvcSuffix::new(holed, 0b001, 0b110)), + ); + dvc_record( + &mut quorum, + dvc(1, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("one nack must not decide the view"); + }; + assert_eq!(log.op_head, 4, "a single nack cannot discard op 4"); + assert!( + log.headers.iter().any(|header| header.op == 4), + "the surviving op must be installed" + ); + } + + #[test] + fn given_committed_op_when_nacked_by_quorum_should_refuse_rather_than_truncate() { + // A nack quorum at or below the proven commit point is impossible under + // quorum intersection. If it appears anyway, refuse; never truncate. + let mut quorum = dvc_quorum_array_empty(); + let blanks = vec![dvc_blank(3)]; + let all_nacked = DvcSuffix::new(blanks, 0b1, 0b0); + dvc_record(&mut quorum, dvc(0, 1, 3, 3, all_nacked.clone())); + dvc_record(&mut quorum, dvc(1, 1, 3, 3, all_nacked)); + + assert_eq!( + merge_dvc_quorum(&quorum, quorums_r3()), + MergeOutcome::Deadlocked { undecided_op: 3 }, + "a committed op must never be truncated" + ); + } + + #[test] + fn given_blank_commit_point_from_every_sender_should_deadlock() { + // The commit point is scanned and may not be discarded, so a sender that + // reports it blank is deferring to a peer. When every sender defers there + // is no peer left and the view cannot start. + // + // Nothing in the merge can rescue this, which is why the senders must not + // produce it: a replica keeps the header at its own commit point through + // compaction (the metadata checkpoint drain stops one op short, a + // partition answers from its evicted ring). + let mut quorum = dvc_quorum_array_empty(); + let blank_at_commit = DvcSuffix::new(vec![dvc_blank(5)], 0, 0); + for replica in 0..3 { + dvc_record(&mut quorum, dvc(replica, 1, 5, 5, blank_at_commit.clone())); + } + + assert_eq!( + merge_dvc_quorum(&quorum, quorums_r3()), + MergeOutcome::Deadlocked { undecided_op: 5 }, + "a blank commit point is neither adoptable nor discardable" + ); + } + + #[test] + fn given_blank_commit_point_from_one_sender_should_adopt_the_peer_header() { + // The same suffix stops being fatal the moment one sender still holds the + // header: that one is canonical and serves the body, and the deferring + // sender neither nacks it nor conflicts with it. + let mut quorum = dvc_quorum_array_empty(); + dvc_record( + &mut quorum, + dvc(0, 1, 5, 5, DvcSuffix::new(vec![dvc_blank(5)], 0, 0)), + ); + dvc_record( + &mut quorum, + dvc(1, 1, 5, 5, suffix_all_present(suffix_headers(5, 5, 1))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("one surviving copy of the commit point is enough to start the view"); + }; + assert_eq!(log.op_head, 5); + assert_eq!(log.commit_max, 5); + } + + #[test] + fn given_header_without_a_servable_body_when_replicas_outstanding_should_await_repair() { + // Both senders have op 4's header, neither can serve its body, and replica 2 + // has not reported. A head whose body nobody holds would wedge the view. + let mut quorum = dvc_quorum_array_empty(); + let headers = suffix_headers(2, 4, 1); + let header_only = DvcSuffix::new(headers, 0, 0b110); + dvc_record(&mut quorum, dvc(0, 1, 4, 2, header_only.clone())); + dvc_record(&mut quorum, dvc(1, 1, 4, 2, header_only)); + + assert_eq!( + merge_dvc_quorum(&quorum, quorums_r3()), + MergeOutcome::AwaitingRepair { undecided_op: 4 } + ); + } + + #[test] + fn given_all_replicas_reported_and_op_undecidable_should_deadlock() { + let mut quorum = dvc_quorum_array_empty(); + let headers = suffix_headers(2, 4, 1); + let header_only = DvcSuffix::new(headers, 0, 0b110); + dvc_record(&mut quorum, dvc(0, 1, 4, 2, header_only.clone())); + dvc_record(&mut quorum, dvc(1, 1, 4, 2, header_only.clone())); + dvc_record(&mut quorum, dvc(2, 1, 4, 2, header_only)); + + assert_eq!( + merge_dvc_quorum(&quorum, quorums_r3()), + MergeOutcome::Deadlocked { undecided_op: 4 }, + "with every replica in, an unrecoverable op stalls the view forever" + ); + } + + #[test] + fn given_lower_log_view_sender_when_merging_should_prefer_the_canonical_log() { + // Replica 1 is at the newer log_view, so its op 4 is canonical and + // replica 0's stale op 4 counts as an implicit nack against itself. + let mut quorum = dvc_quorum_array_empty(); + dvc_record( + &mut quorum, + dvc(0, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), + ); + dvc_record( + &mut quorum, + dvc(1, 2, 4, 2, suffix_all_present(suffix_headers(2, 4, 2))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("the canonical log must win"); + }; + assert_eq!(log.op_head, 4); + assert!( + log.headers.iter().all(|header| header.view == 2), + "installed headers must come from the canonical log_view" + ); + } + + #[test] + fn given_committed_header_on_a_stale_sender_should_be_installed_anyway() { + // Replica 0 is behind on log_view but reports op 2 committed. The canonical + // window starts at 3, so op 2 is otherwise unreachable across the gap. + let mut quorum = dvc_quorum_array_empty(); + dvc_record( + &mut quorum, + dvc(0, 1, 2, 2, suffix_all_present(suffix_headers(2, 2, 1))), + ); + dvc_record( + &mut quorum, + dvc(1, 2, 4, 3, suffix_all_present(suffix_headers(3, 4, 2))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("the view must start"); + }; + assert!( + log.committed_elsewhere.iter().any(|header| header.op == 2), + "a committed header from a stale sender must still be installed" + ); + } + + #[test] + fn given_a_corrupted_canonical_header_should_not_let_honest_senders_nack_it() { + // Transit corruption of one canonical sender's suffix entry must not turn + // every other sender's CORRECT header at that op into an implicit nack + // against the garbage, reaching a nack quorum through the one path that does + // not verify. Senders at the canonical log_view cannot legitimately disagree, + // since a primary prepares one thing per op, so it is evidence, not a vote. + let mut quorum = dvc_quorum_array_empty(); + + let mut corrupted = suffix_headers(2, 4, 1); + corrupted[0].checksum ^= 0xFFFF; + dvc_record(&mut quorum, dvc(0, 1, 4, 2, suffix_all_present(corrupted))); + // Two honest senders at the same log_view holding the real op 4. + dvc_record( + &mut quorum, + dvc(1, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), + ); + dvc_record( + &mut quorum, + dvc(2, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), + ); + + let outcome = merge_dvc_quorum(&quorum, quorums_r3()); + if let MergeOutcome::Ready(log) = &outcome { + assert_eq!( + log.op_head, 4, + "op 4 is held by two honest senders and must not be discarded" + ); + } + assert!( + !matches!(&outcome, MergeOutcome::Ready(log) if log.op_head < 4), + "a corrupted canonical header must never authorise truncating op 4, got {outcome:?}" + ); + } + + #[test] + fn given_a_mixed_version_quorum_when_one_upgraded_sender_nacks_should_not_truncate() { + // A silent sender must never count as agreement with someone else's nack. + // Only replica 1 proves it never held op 4; replica 0 sends no suffix and so + // says nothing. One real nack is short of the quorum of two, so op 4 has to + // survive -- reading the empty suffix as a second nack discards it. + let mut quorum = dvc_quorum_array_empty(); + dvc_record(&mut quorum, dvc(0, 1, 4, 2, DvcSuffix::empty())); + let mut holed = suffix_headers(2, 4, 1); + holed[0] = dvc_blank(4); + dvc_record( + &mut quorum, + dvc(1, 1, 4, 2, DvcSuffix::new(holed, 0b001, 0b110)), + ); + dvc_record( + &mut quorum, + dvc(2, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("op 4 is recoverable from replica 2, so the view must start"); + }; + assert_eq!( + log.op_head, 4, + "an empty suffix must not stand in for the second nack" + ); + } + + #[test] + fn given_a_clamped_suffix_floor_when_merging_should_not_raise_commit_max() { + // `build_dvc_suffix` clamps a window wider than `DVC_HEADERS_MAX` from below, + // so its floor stops being the sender's commit point with nothing on the wire + // saying so. Reading that floor as a commit point marks every op between the + // real commit and the clamp committed: applied and replied to without a + // replication quorum, and unreachable by later truncation via `Deadlocked`. + let mut quorum = dvc_quorum_array_empty(); + // Sender at op 400, commit 200, whose window clamped to 273..=400. + let clamped = suffix_headers(273, 400, 1); + assert_eq!(clamped.len(), DVC_HEADERS_MAX); + dvc_record( + &mut quorum, + dvc(0, 1, 400, 200, suffix_all_present(clamped.clone())), + ); + dvc_record( + &mut quorum, + dvc(1, 1, 400, 200, suffix_all_present(clamped)), + ); + + // A ceiling wide enough not to bind, so this asserts the floor rule alone. In + // production the ceiling is `PREPARE_QUEUE_CEILING`, which already puts + // `commit_max` at or above any clamped floor. + assert_eq!( + merge_commit_max(&quorum, 1000), + 200, + "a clamped window floor is a scan bound, not a proven commit point" + ); + } + + #[test] + fn given_a_sender_at_commit_zero_when_merging_should_not_commit_op_one() { + // The other floor-raising path: ops are 1-based, so a sender with nothing + // committed still floors its window at op 1. Every sender says commit 0, so + // treating op 1 as committed would ack a client for an unheld op. + let mut quorum = dvc_quorum_array_empty(); + let floored = suffix_headers(1, 1, 1); + dvc_record( + &mut quorum, + dvc(0, 1, 1, 0, suffix_all_present(floored.clone())), + ); + dvc_record(&mut quorum, dvc(1, 1, 1, 0, suffix_all_present(floored))); + + assert_eq!( + merge_commit_max(&quorum, 32), + 0, + "nothing is committed, so the merge must prove nothing committed" + ); + } + + #[test] + fn given_disagreeing_committed_elsewhere_headers_should_refuse_the_view() { + // These headers install unconditionally, with no nack quorum behind them, so + // first-wins is the defect `canonical_header_at` refuses for the canonical + // range: only one of two committed claims can be true. + let mut quorum = dvc_quorum_array_empty(); + // Canonical sender at the higher log_view. Its window starts at the proven + // commit point, so ops below it come only from a stale sender. + dvc_record( + &mut quorum, + dvc(0, 2, 6, 5, suffix_all_present(suffix_headers(5, 6, 2))), + ); + // Two senders behind on log_view but level on op, so no nack and no conflict + // inside the canonical range. They disagree only at op 3, which both report + // committed; `prepare` derives the checksum from `(op, view)`. + dvc_record( + &mut quorum, + dvc(1, 1, 6, 5, suffix_all_present(suffix_headers(3, 6, 2))), + ); + let mut divergent = suffix_headers(3, 6, 2); + *divergent.last_mut().expect("suffix is non-empty") = prepare(3, 5); + dvc_record(&mut quorum, dvc(2, 1, 6, 5, suffix_all_present(divergent))); + + assert!( + matches!( + merge_dvc_quorum(&quorum, quorums_r3()), + MergeOutcome::Deadlocked { .. } + ), + "two committed claims at one op must refuse the view, not pick one" + ); + } + + #[test] + fn given_head_header_claiming_a_higher_commit_should_raise_commit_max() { + // The head prepare's `commit` was stamped by the primary that prepared it, so + // it proves a commit point even when every sender's own tracking lags. + // Without it the merge rescans committed ops and could accept nacks for them. + let mut quorum = dvc_quorum_array_empty(); + let mut headers = suffix_headers(2, 4, 1); + headers[0].commit = 3; + dvc_record( + &mut quorum, + dvc(0, 1, 4, 2, suffix_all_present(headers.clone())), + ); + dvc_record(&mut quorum, dvc(1, 1, 4, 2, suffix_all_present(headers))); + + assert_eq!( + merge_commit_max(&quorum, 32), + 3, + "the head header's commit field is a commit_max lower bound" + ); + } +} diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs index d23378f154..4c3f8c1ad3 100644 --- a/core/consensus/src/impls.rs +++ b/core/consensus/src/impls.rs @@ -18,17 +18,18 @@ use crate::oneshot::{self, Receiver, Sender}; use crate::vsr_timeout::{TimeoutKind, TimeoutManager}; use crate::{ - AckLogEvent, Consensus, ControlActionLogEvent, DvcQuorumArray, IgnoreReason, Pipeline, - PlaneKind, PrepareLogEvent, Project, ReplicaLogContext, SimEventKind, StoredDvc, - ViewChangeLogEvent, ViewChangeReason, VsrState, dvc_count, dvc_max_commit, - dvc_quorum_array_empty, dvc_record, dvc_reset, dvc_select_winner, emit_replica_event, - emit_sim_event, + AckLogEvent, Consensus, ControlActionLogEvent, DvcQuorumArray, DvcSuffix, IgnoreReason, + MergeOutcome, MergeQuorums, MergedLog, Pipeline, PlaneKind, PrepareLogEvent, Project, + ReplicaLogContext, SimEventKind, StoredDvc, ViewChangeLogEvent, ViewChangeReason, VsrState, + dvc_count, dvc_iter, dvc_quorum_array_empty, dvc_record, dvc_reset, dvc_suffix_decode, + emit_replica_event, emit_sim_event, merge_dvc_quorum, seal_prepare_checksum, }; use bit_set::BitSet; use clock::{Clock, IggySystemClock}; use iggy_binary_protocol::{ Command2, ConsensusHeader, DoViewChangeHeader, GenericHeader, PrepareHeader, PrepareOkHeader, ReplyHeader, RequestHeader, RequestStartViewHeader, StartViewChangeHeader, StartViewHeader, + frame_body, }; use iggy_common::IggyTimestamp; use iggy_common::calculate_checksum; @@ -146,6 +147,27 @@ pub const PIPELINE_REQUEST_QUEUE_MAX: usize = 64; /// Maximum number of replicas in a cluster. pub const REPLICAS_MAX: usize = 32; +/// Ceiling on [`VsrConsensus::quorum_replication`]. +/// +/// Past three acks marginal durability is small and every extra ack sits on the +/// commit path, so wide clusters spend the difference on the view-change quorum. +pub const QUORUM_REPLICATION_MAX: usize = 3; + +/// Headers a `DoViewChange` may carry, and so the widest uncommitted suffix a +/// view change can reason about. +/// +/// Pinned by the wire: `DoViewChangeHeader`'s nack and present bitsets are one +/// `u128` each, one bit per entry. The suffix spans `commit_max..=op`, bounded by +/// `prepare_queue_max`, so capping that depth here keeps every suffix addressable. +pub const DVC_HEADERS_MAX: usize = 128; + +/// Deepest prepare queue any node in the cluster may be configured with. +/// +/// One less than [`DVC_HEADERS_MAX`]: the suffix spans `commit..=op` and the head +/// needs the reserved slot. Config ceilings and [`LocalPipeline::with_capacities`] +/// both enforce it, so it holds for a peer as well as for this node. +pub const PREPARE_QUEUE_CEILING: usize = DVC_HEADERS_MAX - 1; + /// Unanswered `RequestStartView` probes tolerated before a recovering /// replica gives up waiting for a settled primary and falls back to an /// election (a full-cluster restart leaves nobody able to answer). @@ -325,7 +347,8 @@ impl LocalPipeline { /// `SnapshotCoordinator` in `core/metadata`). /// /// # Panics - /// If a depth is zero — a zero-depth pipeline can never admit an op. + /// If a depth is zero, or if the prepare depth would let the uncommitted + /// suffix outgrow what a `DoViewChange` can address. #[must_use] pub fn with_capacities(prepare_queue_max: usize, request_queue_max: usize) -> Self { assert!( @@ -333,6 +356,16 @@ impl LocalPipeline { "pipeline queue depths must be non-zero \ (prepare={prepare_queue_max}, request={request_queue_max})" ); + // Each `DoViewChange` bitset addresses one suffix entry with one bit of a + // `u128`, and the suffix spans `commit..=op`. Deeper, and the builder clamps + // its window from below, leaving undecidable ops and a stalled view change. + // Config ceilings also enforce this; a stall is worth a loud boot. + assert!( + prepare_queue_max < DVC_HEADERS_MAX, + "prepare queue depth {prepare_queue_max} would produce an uncommitted suffix wider \ + than a DoViewChange can address (max {})", + DVC_HEADERS_MAX - 1, + ); Self { prepare_queue: VecDeque::with_capacity(prepare_queue_max), request_queue: VecDeque::with_capacity(request_queue_max), @@ -732,6 +765,11 @@ pub enum VsrAction { op: u64, commit: u64, namespace: u64, + /// The sender's uncommitted suffix, snapshotted for this view. Carried on + /// the action rather than re-read by the dispatcher so the wire bytes match + /// this replica's own `StoredDvc`: a merge seeing two versions of one + /// sender's suffix could adopt a header no replica holds. + suffix: DvcSuffix, }, /// Broadcast a `RequestStartView` probe (recovering replica asking for /// the current view's `StartView`; only that view's primary answers). @@ -755,6 +793,13 @@ pub enum VsrAction { incarnation: u128, target: Option, namespace: u64, + /// The view's suffix, high-to-low op from `op` down toward `commit`. + /// + /// Lets a backup check the head it is told to adopt against real headers, + /// and gives it canonical checksums to verify repaired bodies against. + /// Empty on the probe-answer path, where the primary reports its own + /// frontier rather than concluding a view change; the backup trusts `op`. + suffix: Vec, }, /// Send `PrepareOK` for each op in `[from_op, to_op]` that is present in the WAL. /// @@ -923,6 +968,29 @@ where /// built-in default. probe_attempts_max: Cell, + /// This replica's own uncommitted suffix, with the `(op, commit)` the journal + /// was at when it was read. + /// + /// Installed by the shard via [`Self::set_local_dvc_suffix`] before any handler + /// that could enter a view change. Snapshotted rather than recomputed per send + /// so a retransmit is byte-identical: a nack is a durable claim about this + /// replica's log, and silently retracting one lets the new primary assemble a + /// quorum that never simultaneously existed. + /// + /// Tagged by `(op, commit)`, not by view, because that is what the suffix + /// describes: a view advance leaves the log alone so the snapshot survives, + /// while anything moving the head or commit point makes the tag mismatch, + /// which reads as no snapshot at all. + local_dvc_suffix: RefCell>, + + /// The log a DVC quorum settled on, parked until this replica's journal can + /// serve all of it. + /// + /// Non-`None` means "primary-elect, repairing": decided but not started, so + /// this replica prepares and announces nothing. Cleared by + /// [`VsrConsensus::start_pending_view`], or by `reset_view_change_state`. + pending_view_log: RefCell>, + /// Tracks DVC messages received (only used by primary candidate) /// Stores metadata; actual log comes from message do_view_change_from_all_replicas: RefCell, @@ -1030,6 +1098,8 @@ impl> VsrConsensus { start_view_change_from_all_replicas: RefCell::new(BitSet::with_capacity(REPLICAS_MAX)), probe_attempts: Cell::new(0), probe_attempts_max: Cell::new(PROBE_ATTEMPTS_MAX), + local_dvc_suffix: RefCell::new(None), + pending_view_log: RefCell::new(None), do_view_change_from_all_replicas: RefCell::new(dvc_quorum_array_empty()), do_view_change_quorum: Cell::new(false), sent_own_start_view_change: Cell::new(false), @@ -1249,10 +1319,45 @@ impl> VsrConsensus { (self.replica_count as usize - 1) / 2 } - /// Quorum size = f + 1 = `max_faulty` + 1 + /// Replicas that must ack before an op is committed. + /// + /// Capped at [`QUORUM_REPLICATION_MAX`] to keep a wide cluster's commit path + /// cheap; the view-change quorum grows so the two still sum above the count. + #[must_use] + pub const fn quorum_replication(&self) -> usize { + if self.replica_count == 2 { + // =1 would intersect, but =2 keeps a two-replica cluster durable. + return 2; + } + let half_rounded_up = (self.replica_count as usize).div_ceil(2); + if half_rounded_up < QUORUM_REPLICATION_MAX { + half_rounded_up + } else { + QUORUM_REPLICATION_MAX + } + } + + /// Replicas that must send a `DoViewChange` before a view can start. + /// + /// Pays for the cheaper replication quorum, which is the far hotter path. + #[must_use] + pub const fn quorum_view_change(&self) -> usize { + if self.replica_count == 2 { + // Avoids a single-replica view change special case. + return 2; + } + self.replica_count as usize - self.quorum_replication() + 1 + } + + /// Nacks required to prove an op was never committed, so the new primary may + /// truncate it. + /// + /// Sized so a nack quorum and a replication quorum cannot both exist for one + /// op. This is what makes truncation safe, so it is the one quorum that must + /// never be loosened. #[must_use] - pub const fn quorum(&self) -> usize { - self.max_faulty() + 1 + pub const fn quorum_nack_prepare(&self) -> usize { + self.replica_count as usize - self.quorum_replication() + 1 } /// Highest op locally executed (state machine applied, client table updated). @@ -1533,6 +1638,64 @@ impl> VsrConsensus { } } + /// Install this replica's uncommitted-suffix snapshot for the current view. + /// + /// Called by the shard, which owns the journal. Consensus keeps the snapshot + /// rather than deriving it so the copy in this replica's `StoredDvc` and the + /// copy on the wire are the same bytes: a merge seeing two versions of one + /// sender's suffix could adopt a header no replica holds. + /// + /// Installing twice for one view overwrites: the shard refreshes before each + /// handler, and a later suffix is at least as complete (repair only adds). + pub fn set_local_dvc_suffix(&self, suffix: DvcSuffix) { + let (op, commit) = self.local_dvc_suffix_tag(); + *self.local_dvc_suffix.borrow_mut() = Some((op, commit, suffix)); + } + + /// Drop the cached suffix snapshot. + /// + /// The `(op, commit)` tag tracks how far the log reaches, not what it still + /// contains, so a mutation that removes entries without moving either + /// (truncating a diverging uncommitted range) leaves a snapshot reading as + /// current while offering bodies this replica can no longer serve. A peer that + /// picks it as a body source then waits out the whole view change. + /// + /// Call from the mutation site. The next refresh re-reads the journal. + pub fn invalidate_local_dvc_suffix(&self) { + self.local_dvc_suffix.borrow_mut().take(); + } + + /// The `(op, commit)` a snapshot must match to still describe this log. + /// `commit` is clamped to `op` exactly as the outgoing DVC clamps it. + fn local_dvc_suffix_tag(&self) -> (u64, u64) { + let op = self.sequencer.current_sequence(); + (op, self.commit_max.get().min(op)) + } + + /// This replica's suffix snapshot, or an empty one when none matches the log's + /// current head and commit point. Empty is the safe direction: it nacks nothing + /// and offers no bodies, so it can only stall a view change, never authorise a + /// truncation. + #[must_use] + pub fn local_dvc_suffix(&self) -> DvcSuffix { + let tag = self.local_dvc_suffix_tag(); + match &*self.local_dvc_suffix.borrow() { + Some((op, commit, suffix)) if (*op, *commit) == tag => suffix.clone(), + _ => DvcSuffix::empty(), + } + } + + /// True when no snapshot matches the log's current head and commit point, + /// so the shard must read one from the journal before this replica votes. + #[must_use] + pub fn local_dvc_suffix_stale(&self) -> bool { + let tag = self.local_dvc_suffix_tag(); + !matches!( + &*self.local_dvc_suffix.borrow(), + Some((op, commit, _)) if (*op, *commit) == tag + ) + } + /// True when the current `(view, log_view)` is not yet in the superblock, so a /// view-scoped send would advertise a view a crash could lose. The split-brain /// gate: the dispatcher persists first when this holds. `commit_max` is @@ -1615,6 +1778,9 @@ impl> VsrConsensus { self.reset_dvc_quorum(); self.sent_own_start_view_change.set(false); self.sent_own_do_view_change.set(false); + // A merge parked for the superseded view may describe a different log, so + // drop it and let the new attempt re-derive from the DVCs it collects. + self.pending_view_log.borrow_mut().take(); self.loopback_queue.borrow_mut().clear(); let mut pipeline = self.pipeline.borrow_mut(); pipeline.cancel_all_subscribers(); @@ -1885,16 +2051,9 @@ impl> VsrConsensus { .borrow_mut() .reset(TimeoutKind::DoViewChangeMessage); - let current_op = self.sequencer.current_sequence(); - let action = VsrAction::SendDoViewChange { - view: self.view.get(), - target: self.primary_index(self.view.get()), - log_view: self.log_view.get(), - op: current_op, - // commit_max clamped to op: see `handle_start_view_change`. - commit: self.commit_max.get().min(current_op), - namespace: self.namespace, - }; + // The same snapshot the first send used. A retransmit that re-derived the + // suffix could retract a nack the candidate already counted. + let action = self.build_do_view_change(self.primary_index(self.view.get())); emit_sim_event( SimEventKind::ControlMessageScheduled, &ControlActionLogEvent::from_vsr_action( @@ -2166,36 +2325,14 @@ impl> VsrConsensus { let primary_candidate = self.primary_index(self.view.get()); let current_op = self.sequencer.current_sequence(); - // DVC carries commit_max (highest known-committed), not commit_min - // (locally applied). The new primary floors its pipeline rebuild at - // max(commit) across the quorum; only commit_max bounds that range - // to pipeline depth (every replica holds op - commit_max <= depth). - // commit_min can lag far behind and overflow the rebuild. The - // committed-but-unapplied tail (commit_min..commit_max] is replayed - // by the new primary's CommitJournal, not the pipeline. - // - // Clamp to op: a backup learns commit_max from a heartbeat before - // receiving the prepares, so commit_max can exceed its op. The wire - // contract `DoViewChangeHeader::validate` rejects commit > op and - // drops such a DVC (view-change liveness stall). The clamp is - // lossless for the rebuild floor: quorum intersection guarantees - // some sender whose op covers the true commit point carries it, so - // max(commit) across the quorum is unchanged. - let commit = self.commit_max.get().min(current_op); + let commit = self.dvc_commit(); // Start DVC timeout self.timeouts .borrow_mut() .start(TimeoutKind::DoViewChangeMessage); - let action = VsrAction::SendDoViewChange { - view: self.view.get(), - target: primary_candidate, - log_view: self.log_view.get(), - op: current_op, - commit, - namespace: self.namespace, - }; + let action = self.build_do_view_change(primary_candidate); emit_sim_event( SimEventKind::ControlMessageScheduled, &ControlActionLogEvent::from_vsr_action( @@ -2212,15 +2349,19 @@ impl> VsrConsensus { log_view: self.log_view.get(), op: current_op, commit, + suffix: self.local_dvc_suffix(), }; dvc_record( &mut self.do_view_change_from_all_replicas.borrow_mut(), own_dvc, ); - // Check if we now have quorum - if dvc_count(&self.do_view_change_from_all_replicas.borrow()) >= self.quorum() { - self.do_view_change_quorum.set(true); + // `complete_view_change_as_primary` latches only once the merge + // decides, so an undecidable quorum stays open to later DVCs. + if !self.do_view_change_quorum.get() + && dvc_count(&self.do_view_change_from_all_replicas.borrow()) + >= self.quorum_view_change() + { actions.extend(self.complete_view_change_as_primary(plane)); } } @@ -2235,12 +2376,75 @@ impl> VsrConsensus { /// replicas (including itself), it sets its view-number to that in the messages /// and selects as the new log the one contained in the message with the largest v'..." /// + /// The `commit` this replica advertises in a `DoViewChange`. + /// + /// `commit_max`, not `commit_min`: the new primary floors its pipeline rebuild + /// at `max(commit)` across the quorum, and only `commit_max` bounds that range + /// to the pipeline depth. `commit_min` can lag far enough to overflow the + /// rebuild; `CommitJournal` replays the committed-but-unapplied tail instead. + /// + /// Clamped to `op`, since a backup learns `commit_max` from a heartbeat before + /// the prepares and `DoViewChangeHeader::validate` rejects `commit > op`. + /// Lossless for the rebuild floor: quorum intersection guarantees some sender + /// whose head covers the true commit point carries it. + fn dvc_commit(&self) -> u64 { + let op = self.sequencer.current_sequence(); + self.commit_max.get().min(op) + } + + /// Build this replica's `DoViewChange` for the current view. + fn build_do_view_change(&self, target: u8) -> VsrAction { + VsrAction::SendDoViewChange { + view: self.view.get(), + target, + log_view: self.log_view.get(), + op: self.sequencer.current_sequence(), + commit: self.dvc_commit(), + namespace: self.namespace, + suffix: self.local_dvc_suffix(), + } + } + + /// Decode a peer's suffix, or `None` to drop the whole `DoViewChange`. + /// + /// A suffix that will not decode makes the numbers untrustworthy too. Dropping + /// the message lets the sender's retransmit try again, rather than seating a + /// vote whose nacks and offered bodies cannot be placed against an op. + fn decode_peer_suffix( + &self, + header: &DoViewChangeHeader, + suffix_body: &[u8], + ) -> Option { + match dvc_suffix_decode( + suffix_body, + header.op, + header.nack_bitset, + header.present_bitset, + ) { + Ok(suffix) => Some(suffix), + Err(error) => { + tracing::warn!( + replica = self.replica, + from_replica = header.replica, + view = header.view, + op = header.op, + "dropping do_view_change with an unreadable suffix: {error}" + ); + None + } + } + } + + /// `suffix_body` is the sender's uncommitted-suffix headers. Empty from a peer + /// unable to snapshot one, which then contributes numbers only. + /// /// # Panics /// If `header.namespace` does not match this replica's namespace. pub fn handle_do_view_change( &self, plane: PlaneKind, header: &DoViewChangeHeader, + suffix_body: &[u8], ) -> Vec { assert_eq!( header.namespace, self.namespace, @@ -2256,6 +2460,9 @@ impl> VsrConsensus { let msg_log_view = header.log_view; let msg_op = header.op; let msg_commit = header.commit; + let Some(msg_suffix) = self.decode_peer_suffix(header, suffix_body) else { + return Vec::new(); + }; // Ignore DVCs for old views if msg_view < self.view.get() { @@ -2332,6 +2539,7 @@ impl> VsrConsensus { log_view: self.log_view.get(), op: current_op, commit, + suffix: self.local_dvc_suffix(), }; dvc_record( &mut self.do_view_change_from_all_replicas.borrow_mut(), @@ -2345,14 +2553,16 @@ impl> VsrConsensus { log_view: msg_log_view, op: msg_op, commit: msg_commit, + suffix: msg_suffix, }; dvc_record(&mut self.do_view_change_from_all_replicas.borrow_mut(), dvc); - // Check if quorum achieved + // `complete_view_change_as_primary` latches only once the merge decides, + // so an undecidable quorum re-merges as each further DVC lands. if !self.do_view_change_quorum.get() - && dvc_count(&self.do_view_change_from_all_replicas.borrow()) >= self.quorum() + && dvc_count(&self.do_view_change_from_all_replicas.borrow()) + >= self.quorum_view_change() { - self.do_view_change_quorum.set(true); actions.extend(self.complete_view_change_as_primary(plane)); } @@ -2490,6 +2700,9 @@ impl> VsrConsensus { commit: self.commit_max.get(), incarnation: header.incarnation, target: Some(header.replica), + // A probe answer reports this primary's settled frontier, not a + // freshly merged log, so there is no canonical suffix to publish. + suffix: Vec::new(), namespace: self.namespace, }] } @@ -2519,6 +2732,44 @@ impl> VsrConsensus { .stop(TimeoutKind::RequestStartViewMessage); } + /// Decide which head to adopt from a `StartView`, and record the view's + /// canonical headers when it carried any. + /// + /// Headers go in `pending_view_log`, not the journal: a journal entry is a + /// header plus its body, and a backup adopting a view usually holds neither. + /// Keeping them lets the repair ingest reject a body that disagrees with what + /// the view decided, which is what makes fetching by op number safe. + /// + /// Falls back to the announced `op` on an empty body (probe answer, stale-view + /// correction). + fn adopt_start_view_suffix(&self, header: &StartViewHeader, suffix_body: &[u8]) -> u64 { + let suffix = match dvc_suffix_decode(suffix_body, header.op, 0, 0) { + Ok(suffix) => suffix, + Err(error) => { + tracing::warn!( + replica = self.replica, + from_replica = header.replica, + view = header.view, + op = header.op, + "start_view suffix did not decode, falling back to the announced op: {error}" + ); + return header.op; + } + }; + let headers = suffix.headers(); + if headers.is_empty() { + return header.op; + } + + *self.pending_view_log.borrow_mut() = Some(MergedLog { + op_head: header.op, + commit_max: header.commit, + headers: headers.to_vec(), + committed_elsewhere: Vec::new(), + }); + header.op + } + /// Handle a received `StartView` message (backups only). /// /// "When other replicas receive the STARTVIEW message, they replace their log @@ -2537,7 +2788,14 @@ impl> VsrConsensus { /// /// Gap: if a backup never received a prepare (lost message), /// `commit_journal` stops at the gap. Requires message repair. - pub fn handle_start_view(&self, plane: PlaneKind, header: &StartViewHeader) -> Vec { + /// `suffix_body` is the message body: the view's canonical headers, empty + /// when the announcement carries numbers only. + pub fn handle_start_view( + &self, + plane: PlaneKind, + header: &StartViewHeader, + suffix_body: &[u8], + ) -> Vec { assert_eq!(header.namespace, self.namespace, "SV routed to wrong group"); let from_replica = header.replica; let msg_view = header.view; @@ -2563,7 +2821,7 @@ impl> VsrConsensus { // incarnation is set (partition plane, tests). // // A zero `header.incarnation` makes no claim either way: it is what an - // unsolicited StartView carries, and what a peer predating the field sends. + // unsolicited StartView carries. // Classifying it stale would have this replica reject a current StartView // from a healthy primary purely because that primary is older, so it falls // through to the view checks that governed before the field existed. @@ -2640,11 +2898,11 @@ impl> VsrConsensus { // Stale pipeline entries from the old view must be discarded self.pipeline.borrow_mut().clear(); - // TODO: StartView should carry uncommitted headers so backup installs - // into WAL and sets op WAL-verified. Today we trust msg_op, correct - // for truncation (sequencer > msg_op) but wrong when behind - // (sequencer < msg_op): gap is unreachable without message repair. - self.sequencer.set_sequence(msg_op); + // Cross-check the announced head against the headers published with it: a + // suffix head disagreeing with `header.op` means an inconsistently built + // frame, and either value leaves this replica chasing an unservable head. + let announced = self.adopt_start_view_suffix(header, suffix_body); + self.sequencer.set_sequence(announced); // Update timeouts for normal backup operation { @@ -2783,22 +3041,83 @@ impl> VsrConsensus { /// contains entries for all committed ops it received. /// /// Gap: missing prepares (lost messages) require message repair. + /// + /// Re-entrant, called again for every `DoViewChange` landing while the merge is + /// undecided. Every non-`Ready` outcome leaves this replica untouched, so a + /// re-run costs only the merge. fn complete_view_change_as_primary(&self, plane: PlaneKind) -> Vec { - let dvc_array = self.do_view_change_from_all_replicas.borrow(); + let merged = { + let dvc_array = self.do_view_change_from_all_replicas.borrow(); + merge_dvc_quorum(&dvc_array, self.merge_quorums()) + }; - let Some(winner) = dvc_select_winner(&dvc_array) else { - return Vec::new(); + let merged = match merged { + MergeOutcome::Ready(merged) => merged, + // Every non-ready outcome keeps this replica in `ViewChange` with its + // log untouched. Picking a winner unconditionally and letting the + // pipeline rebuild truncate what it cannot find locally discards + // committed ops; an unavailable cluster that says so is the better + // failure. + // + // None of these latch `do_view_change_quorum`: an undecidable quorum is + // not a decision, and the replicas still to report are what would + // settle it. The flag belongs only where the quorum is decidable. + MergeOutcome::AwaitingQuorum => return Vec::new(), + MergeOutcome::AwaitingRepair { undecided_op } => { + tracing::debug!( + replica = self.replica, + view = self.view.get(), + undecided_op, + "view change waiting on more DoViewChange messages to decide an op" + ); + return Vec::new(); + } + MergeOutcome::Deadlocked { undecided_op } => { + tracing::error!( + replica = self.replica, + view = self.view.get(), + undecided_op, + "view change cannot start: op {undecided_op} is neither recoverable from any \ + replica nor provably uncommitted" + ); + return Vec::new(); + } }; - let new_op = winner.op; - let max_commit = dvc_max_commit(&dvc_array); + // The pipeline must hold the whole uncommitted range, and the merge decides + // that range against a cluster-wide ceiling, so a node configured shallower + // than its peers can be handed a range it cannot rebuild. + // + // Refuse rather than panic: a further DoViewChange can raise `commit_max` + // and shrink the range, and otherwise the status timeout escalates. A panic + // would restart into the same merge. + if merged.op_head.saturating_sub(merged.commit_max) > self.prepare_queue_max as u64 { + tracing::error!( + replica = self.replica, + view = self.view.get(), + commit_max = merged.commit_max, + op_head = merged.op_head, + prepare_queue_max = self.prepare_queue_max, + "view change cannot start: the merged log claims {} in-flight ops, more than this \ + replica's pipeline holds; refusing the view", + merged.op_head - merged.commit_max, + ); + return Vec::new(); + } - // Update state - self.log_view.set(self.view.get()); - self.status.set(Status::Normal); - self.ceded_primaryship.set(false); + // Quorum closed now the merge decided: re-merging after parking could + // produce a different log than the one already being repaired toward. + self.do_view_change_quorum.set(true); + + // The merged log is authoritative but this replica may not hold every body + // yet. Park it, let the shard repair up to it, and `start_pending_view` + // finishes once the journal covers the range. Until then this replica stays + // in `ViewChange` and prepares nothing, so no client op is stamped onto an + // unproven log. `log_view` does NOT advance here; see `start_pending_view`. + let max_commit = merged.commit_max; + let new_op = merged.op_head; self.advance_commit_max(max_commit); - self.sequencer.set_sequence(new_op); + *self.pending_view_log.borrow_mut() = Some(merged); // Stale pipeline entries are invalid in new view; reconciliation // replays from journal. @@ -2818,6 +3137,110 @@ impl> VsrConsensus { // the loopback queue directly. self.loopback_queue.borrow_mut().clear(); + tracing::info!( + replica = self.replica, + view = self.view.get(), + op_head = new_op, + commit_max = max_commit, + "view-change quorum merged; repairing up to the merged log before starting the view" + ); + emit_replica_event( + SimEventKind::ReplicaStateChanged, + &ReplicaLogContext::from_consensus(self, plane), + ); + + // No sends yet: `SendStartView` promises this replica can serve every op in + // the merged log, and a backup adopting the announced head asks it for the + // bodies behind it. + Vec::new() + } + + /// Sizes handed to the DVC merge. + const fn merge_quorums(&self) -> MergeQuorums { + MergeQuorums { + view_change: self.quorum_view_change(), + nack_prepare: self.quorum_nack_prepare(), + replica_count: self.replica_count as usize, + // The cluster-wide ceiling, not `self.prepare_queue_max`: this node's + // config says nothing about how deep a peer's pipeline is. + prepare_queue_ceiling: PREPARE_QUEUE_CEILING as u64, + } + } + + /// The merged log this replica is repairing toward, if a view change is + /// mid-transition. The shard reads it for the op range it must cover before the + /// view can start, and for which peers offered the bodies. + #[must_use] + pub fn pending_view_log(&self) -> Option { + self.pending_view_log.borrow().clone() + } + + /// Replicas that offered a body for `op`, most-recent-log_view first. + /// + /// Only meaningful while a merge is parked. These peers and nobody else: a + /// cleared present bit means the body was never held or cannot be read back, + /// and the view change is blocked on the round-trip. + #[must_use] + pub fn pending_view_body_sources(&self, op: u64) -> Vec { + let quorum = self.do_view_change_from_all_replicas.borrow(); + let mut sources: Vec<(u32, u8)> = dvc_iter(&quorum) + .filter(|dvc| dvc.replica != self.replica) + .filter_map(|dvc| { + let index = dvc.suffix.index_of(dvc.op, op)?; + dvc.suffix + .offers_body(index) + .then_some((dvc.log_view, dvc.replica)) + }) + .collect(); + sources.sort_unstable_by_key(|(log_view, _)| std::cmp::Reverse(*log_view)); + sources.into_iter().map(|(_, replica)| replica).collect() + } + + /// Finish the parked view change: this replica's journal now covers the merged + /// log, so it can serve any op it is about to announce. + /// + /// Called by the shard after repair progress. No-op when nothing is parked. + /// + /// # Panics + /// If the merged uncommitted range exceeds pipeline capacity, which needs a head + /// more than one pipeline depth above the proven commit point. + pub fn start_pending_view(&self, plane: PlaneKind) -> Vec { + // A backup's parked log (the `StartView` suffix) is only what its ingest + // verifies bodies against. It must never take this path: starting the view + // claims the primaryship of a view this replica did not win. + if !self.is_primary_for_view(self.view.get()) { + return Vec::new(); + } + let Some(merged) = self.pending_view_log.borrow_mut().take() else { + return Vec::new(); + }; + let new_op = merged.op_head; + let max_commit = merged.commit_max; + + self.status.set(Status::Normal); + self.ceded_primaryship.set(false); + self.sequencer.set_sequence(new_op); + if let Some(head) = merged.headers.first() { + // Keep the hash chain continuous: the next prepare must chain onto the + // head this view adopted, not onto whatever was appended last. + self.set_last_prepare_checksum(head.checksum); + } + for header in &merged.headers { + self.observe_prepare_timestamp(header.timestamp); + } + // Only now, with the merged head installed above. `log_view` claims "my log + // IS the log this view decided", and it selects the canonical senders of the + // next view change, whose headers outrank everyone else's. + // + // Raising it at merge time breaks that claim for the whole parked window, + // which can end in supersession or a crash (`log_view` is durable): the + // replica then votes as canonical carrying its own stale head, and ops the + // merge decided to keep fall outside the next scan range, discarded with no + // nack required. Merge-time assignment is only truthful where every merged + // header is installed there; parking installs nothing and the repair ingest + // only fills holes, so a parked replica still holds its old view's log. + self.log_view.set(self.view.get()); + // Update timeouts for normal primary operation { let mut timeouts = self.timeouts.borrow_mut(); @@ -2847,6 +3270,8 @@ impl> VsrConsensus { incarnation: 0, target: None, namespace: self.namespace, + // `merged` was taken out of the parked slot, so hand the headers over. + suffix: merged.headers, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2864,10 +3289,12 @@ impl> VsrConsensus { // The new primary must rebuild its pipeline from the journal so that // incoming PrepareOk messages can be matched and commits can proceed. if max_commit < new_op { - assert!( + // `complete_view_change_as_primary` already refused a non-fitting + // range. Asserted so the sites cannot drift; this one cannot decline. + debug_assert!( (new_op - max_commit) <= self.prepare_queue_max as u64, "view change: uncommitted range {}..={} ({} ops) exceeds pipeline capacity ({}); \ - DVC winner claims more in-flight ops than the pipeline can hold", + the merged log claims more in-flight ops than the pipeline can hold", max_commit + 1, new_op, new_op - max_commit, @@ -2948,7 +3375,7 @@ impl> VsrConsensus { // Record the ack from this replica let ack_count = entry.add_ack(header.replica); - let quorum = self.quorum(); + let quorum = self.quorum_replication(); let quorum_reached = ack_count >= quorum && !entry.ok_quorum_received; // Check if we've reached quorum @@ -3048,12 +3475,8 @@ where // stores the same value and the scan verifies it after a crash. The body is // never re-stamped (`restamp_prepare_view` patches only `view`), so this // survives view-change retransmits. The header `checksum` and its `parent` - // chain stay `0`: activating them needs the retransmit path to re-seal a - // re-stamped header, a separate change. Whoever activates it must also - // audit every `set_last_prepare_checksum` caller for cross-plane carry -- - // the repair router in `shard` drops metadata-plane frames it cannot - // journal precisely so one cannot stamp a PARTITION consensus, which is - // inert only while these values are structurally zero. + // chain are sealed too, for both planes, by `seal_prepare_checksum` below; + // they exclude `view`, which is what lets a restamp leave them valid. // // Metadata plane only. A partition produce prepare already carries a verified // `batch_checksum` over the same bytes, so a second full-payload pass is pure @@ -3062,15 +3485,25 @@ where // sealed region before the entry is journaled. Leaving those prepares at `0` // is the designed "nothing to verify" sentinel, so a future durable partition // journal skips verification instead of failing every entry as corrupt. + // + // So a partition prepare's `checksum` identifies its header alone, and two + // such prepares at one op with matching header fields are indistinguishable + // to the view-change merge. Closing that wants `checksum_body` here to BE + // the batch checksum, recomputed after stamping, so `checksum` covers the + // body for free by hashing this field. + // + // Bounded by `size`, the range every verifier re-reads; the prepare + // inherits it verbatim below. let checksum_body = if consensus.namespace == METADATA_CONSENSUS_NAMESPACE { - u128::from(calculate_checksum( - &self.as_slice()[size_of::()..], - )) + u128::from(calculate_checksum(frame_body( + self.as_slice(), + self.header().size, + ))) } else { 0 }; - self.transmute_header(|old, new| { + let prepared = self.transmute_header(|old, new| { *new = PrepareHeader { cluster: consensus.cluster, size: old.size, @@ -3099,7 +3532,11 @@ where user_id: old.user_id, ..Default::default() } - }) + }); + // Last, because the checksum covers every other field. Gives the op the + // stable identity the view-change merge compares across replicas; `parent` + // chains it, so the log is hash-linked rather than nominally so. + seal_prepare_checksum(prepared) } } @@ -3132,6 +3569,7 @@ where size: std::mem::size_of::() as u32, ..Default::default() }; + new.seal(); }) } } @@ -3600,7 +4038,7 @@ mod timestamp_clamp_tests { let stale = make_start_view(1, 4, 1, STALE); assert!( consensus - .handle_start_view(PlaneKind::Metadata, stale.header()) + .handle_start_view(PlaneKind::Metadata, stale.header(), &[]) .is_empty(), "a StartView echoing a previous incarnation must be ignored while recovering" ); @@ -3620,7 +4058,7 @@ mod timestamp_clamp_tests { let fresh = make_start_view(1, 4, 1, CURRENT); assert!( !consensus - .handle_start_view(PlaneKind::Metadata, fresh.header()) + .handle_start_view(PlaneKind::Metadata, fresh.header(), &[]) .is_empty(), "a StartView echoing our current incarnation must be adopted" ); @@ -3712,7 +4150,11 @@ mod timestamp_clamp_tests { // head covers every op it told us was committed. assert!( consensus - .handle_start_view(PlaneKind::Metadata, make_start_view(7, 104, 1, 0).header()) + .handle_start_view( + PlaneKind::Metadata, + make_start_view(7, 104, 1, 0).header(), + &[] + ) .is_empty(), "an equal-view StartView below the commit floor must be skipped" ); @@ -3726,7 +4168,11 @@ mod timestamp_clamp_tests { // Adopt it and drop the discarded suffix. assert!( !consensus - .handle_start_view(PlaneKind::Metadata, make_start_view(7, 105, 1, 0).header()) + .handle_start_view( + PlaneKind::Metadata, + make_start_view(7, 105, 1, 0).header(), + &[] + ) .is_empty(), "an equal-view StartView at or above the commit floor must be adopted, \ even when its head is behind a WAL suffix the view already discarded" @@ -3994,3 +4440,82 @@ mod state_transfer_stage_tests { assert_eq!(consensus.status(), Status::Recovering); } } + +#[cfg(test)] +mod quorum_tests { + //! Pin the three quorum sizes for replica counts 1 through 8. The + //! intersection asserts are the safety properties: replication and + //! view-change quorums must overlap, so a committed op is visible to the + //! next view, and replication and nack quorums must overlap, so an op that + //! may have committed can never gather a nack quorum. + + use super::*; + use crate::LocalPipeline; + use server_common::MESSAGE_ALIGN; + use server_common::iobuf::Frozen; + + struct NoopBus; + + impl MessageBus for NoopBus { + async fn send_to_client( + &self, + _client_id: u128, + _data: Frozen, + ) -> Result<(), message_bus::SendError> { + Ok(()) + } + + async fn send_to_replica( + &self, + _replica: u8, + _data: Frozen, + ) -> Result<(), message_bus::SendError> { + Ok(()) + } + + fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {} + fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {} + fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {} + fn track_background(&self, _handle: message_bus::JoinHandle<()>) {} + } + + fn consensus_with_replica_count(replica_count: u8) -> VsrConsensus { + VsrConsensus::new( + 1, + 0, + replica_count, + METADATA_CONSENSUS_NAMESPACE, + NoopBus, + LocalPipeline::new(), + ) + } + + #[test] + fn given_any_replica_count_when_sizing_quorums_should_intersect() { + for replica_count in 1u8..=REPLICAS_MAX_U8 { + let consensus = consensus_with_replica_count(replica_count); + let count = usize::from(replica_count); + + assert!( + consensus.quorum_replication() + consensus.quorum_view_change() > count, + "replication+view-change must intersect at replica_count={replica_count}" + ); + assert!( + consensus.quorum_nack_prepare() + consensus.quorum_replication() > count, + "nack+replication must intersect at replica_count={replica_count}" + ); + assert!(consensus.quorum_replication() <= count); + assert!(consensus.quorum_view_change() <= count); + assert!(consensus.quorum_nack_prepare() <= count); + } + } + + /// `REPLICAS_MAX` as a `u8` for loop bounds. + const REPLICAS_MAX_U8: u8 = { + assert!(REPLICAS_MAX <= u8::MAX as usize); + #[allow(clippy::cast_possible_truncation)] + { + REPLICAS_MAX as u8 + } + }; +} diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs index 4d7a53d53e..e135da9f33 100644 --- a/core/consensus/src/lib.rs +++ b/core/consensus/src/lib.rs @@ -180,6 +180,9 @@ pub use observability::*; mod view_change_quorum; pub use view_change_quorum::*; + +mod dvc_merge; +pub use dvc_merge::*; mod vsr_state; pub use vsr_state::{VsrState, VsrStateError}; mod vsr_timeout; diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index 4515f2ba7d..e8d52cd147 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -19,9 +19,13 @@ use crate::{ Consensus, IgnoreReason, Pipeline, PipelineEntry, PlaneKind, PrepareOkOutcome, Sequencer, Status, VsrConsensus, }; -use iggy_binary_protocol::{Command2, PrepareHeader, PrepareOkHeader, ReplyHeader, RequestHeader}; +use iggy_binary_protocol::{ + CHECKSUM_UNSEALED, Command2, ConsensusHeader, PrepareHeader, PrepareOkHeader, ReplyHeader, + RequestHeader, frame_body, +}; use message_bus::{MessageBus, SendError}; use server_common::{Message, iobuf::Owned}; +use std::mem::size_of; use std::ops::AsyncFnOnce; /// Shared pipeline-first request flow (metadata + partitions). @@ -114,6 +118,35 @@ where consensus.message_bus().send_to_replica(next, frozen).await } +/// Recompute a prepare's integrity fields and report the first that disagrees. +/// +/// Everywhere else `checksum` is an opaque token: the pipeline, the merge, and the +/// repair ingest compare it for equality without asking whether it describes the +/// bytes it arrived with, so a corrupted frame is admitted whenever its flipped +/// value satisfies those comparisons, then journaled and re-served to peers. +/// +/// `frame` is the whole message. The body range comes from [`frame_body`], not the +/// caller, so no ingress point can verify a different span than the producer sealed. +/// [`CHECKSUM_UNSEALED`] skips the partition plane, which carries `batch_checksum` +/// over the same bytes instead. +/// +/// # Errors +/// Returns a static description of which field failed. +pub fn verify_prepare_integrity(header: &PrepareHeader, frame: &[u8]) -> Result<(), &'static str> { + if header.checksum != CHECKSUM_UNSEALED && header.identity_checksum() != header.checksum { + return Err("prepare header does not match its own checksum"); + } + if header.checksum_body != 0 + && u128::from(iggy_common::calculate_checksum(frame_body( + frame, + header.size, + ))) != header.checksum_body + { + return Err("prepare body does not match its checksum"); + } + Ok(()) +} + /// Shared preflight checks for `on_replicate`. /// /// Returns current op on success. @@ -169,6 +202,41 @@ where Ok(current_op) } +/// Compute a prepare's identity checksum: which prepare this is, independent of +/// which view re-sent it. +/// +/// `view` is excluded from the covered bytes even though the frame checksum covers +/// it, because `restamp_prepare_view` rewrites `view` in place to clear the +/// receiver's `header.view < view` fence. Covering it would give one logical op a +/// different checksum per replica, and the merge would read those as competing +/// prepares nacking each other. Not merely a workaround either: two frames +/// differing only in `view` ARE the same op, and everything that makes them +/// genuinely different (client, request, timestamp, parent, operation, or the body +/// via `checksum_body`) stays covered. +/// +/// `checksum_body` must already be set, since it is how the body reaches this +/// value. A prepare left unsealed there gets an identity over its header alone. +#[must_use] +pub fn prepare_identity_checksum(header: &PrepareHeader) -> u128 { + header.identity_checksum() +} + +/// Stamp [`prepare_identity_checksum`] into a freshly built prepare. +/// +/// Call once, after every other field is final: the checksum covers them. +/// +/// # Panics +/// If the message is shorter than its own header. +#[must_use] +pub fn seal_prepare_checksum(mut message: Message) -> Message { + let checksum = prepare_identity_checksum(message.header()); + let bytes = &mut message.as_mut_slice()[..size_of::()]; + let header = bytemuck::checked::try_from_bytes_mut::(bytes) + .expect("a prepare header round-trips its own bit pattern"); + header.checksum = checksum; + message +} + /// Shared preflight checks for `on_ack`. /// /// # Errors @@ -625,9 +693,13 @@ pub async fn send_prepare_ok( ..Default::default() }; - let message: Message = - Message::::new(std::mem::size_of::()) - .transmute_header(|_, new| *new = prepare_ok_header); + let message: Message = Message::::new(std::mem::size_of::< + PrepareOkHeader, + >()) + .transmute_header(|_, new| { + *new = prepare_ok_header; + new.seal(); + }); let primary = consensus.primary_index(consensus.view()); consensus @@ -639,10 +711,18 @@ pub async fn send_prepare_ok( mod tests { use super::*; use crate::{Consensus, LocalPipeline, VsrAction}; + use aligned_vec::{AVec, ConstAlign}; use iggy_binary_protocol::{ConsensusHeader, Operation, StartViewChangeHeader}; + use iggy_common::calculate_checksum; use message_bus::SendError; use server_common::{MESSAGE_ALIGN, iobuf::Frozen}; + /// `PrepareHeader`'s alignment, which every suffix body has to satisfy. + const BODY_ALIGN: usize = align_of::(); + + /// A control-message body, aligned for the headers packed into it. + type Body = AVec>; + #[derive(Debug, Default)] struct NoopBus; @@ -816,7 +896,7 @@ mod tests { checksum: 0, checksum_body: 0, cluster: 0, - size: 0, + size: std::mem::size_of::() as u32, view: 1, release: 0, command: Command2::DoViewChange, @@ -826,7 +906,9 @@ mod tests { commit, namespace: 0, log_view: 0, - reserved: [0; 100], + reserved: [0; 68], + nack_bitset: 0, + present_bitset: 0, }; assert!(header(dvc_commit).validate().is_ok()); assert!( @@ -835,6 +917,61 @@ mod tests { ); } + #[test] + fn given_restamped_view_when_sealing_should_keep_the_same_identity() { + // `restamp_prepare_view` rewrites `view` on retransmission. If the identity + // moved with it, one op would carry different checksums per receiving view + // and the merge would read them as competing prepares nacking each other. + let base = PrepareHeader { + command: Command2::Prepare, + operation: iggy_binary_protocol::Operation::CreateStream, + op: 9, + view: 4, + client: 11, + request: 2, + timestamp: 1234, + checksum_body: 99, + ..Default::default() + }; + let restamped = PrepareHeader { view: 12, ..base }; + assert_eq!( + prepare_identity_checksum(&base), + prepare_identity_checksum(&restamped), + "view must not participate in a prepare's identity" + ); + } + + #[test] + fn given_different_prepares_at_one_op_when_sealing_should_differ() { + // The distinction the merge depends on: two prepares at one op number are + // told apart, so a canonical header is distinguishable from a stale one. + let first = PrepareHeader { + command: Command2::Prepare, + operation: iggy_binary_protocol::Operation::CreateStream, + op: 5, + client: 1, + request: 1, + timestamp: 100, + ..Default::default() + }; + let second = PrepareHeader { client: 2, ..first }; + assert_ne!( + prepare_identity_checksum(&first), + prepare_identity_checksum(&second), + "distinct prepares at the same op must not share an identity" + ); + + let body_differs = PrepareHeader { + checksum_body: 7, + ..first + }; + assert_ne!( + prepare_identity_checksum(&first), + prepare_identity_checksum(&body_differs), + "the body reaches the identity through checksum_body" + ); + } + #[test] fn loopback_push_and_drain() { let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); @@ -894,81 +1031,575 @@ mod tests { assert_eq!(typed.header().command, Command2::PrepareOk); } - #[test] - fn loopback_cleared_on_complete_view_change_as_primary() { - use iggy_binary_protocol::{DoViewChangeHeader, StartViewChangeHeader}; + /// A sender's suffix and matching body bytes for a replica that holds every op + /// in `commit..=op` and can serve each body. Checksums derive from `(op, view)` + /// so the hash chain connects, which the merge checks. + fn dvc_with_full_suffix( + replica: u8, + view: u32, + log_view: u32, + op: u64, + commit: u64, + ) -> (iggy_binary_protocol::DoViewChangeHeader, Body) { + dvc_with_suffix(replica, view, log_view, op, commit, None) + } - // 3 replicas, replica 0 is primary for view 0 (and view 3: 3 % 3 = 0). - let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); - consensus.init(); + /// As [`dvc_with_full_suffix`], but `withhold_body` names one op whose present + /// bit is cleared: header held, body unservable. A quorum where every sender + /// withholds the same op decides nothing yet. + fn dvc_with_suffix( + replica: u8, + view: u32, + log_view: u32, + op: u64, + commit: u64, + withhold_body: Option, + ) -> (iggy_binary_protocol::DoViewChangeHeader, Body) { + use iggy_binary_protocol::DoViewChangeHeader; + + let headers = suffix_headers(commit, op, log_view); + let body = encode_body(&headers); + let mut present = if headers.is_empty() { + 0 + } else { + (1u128 << headers.len()) - 1 + }; + if let Some(withheld) = withhold_body + && let Some(index) = headers.iter().position(|header| header.op == withheld) + { + present &= !(1u128 << index); + } + let header = DoViewChangeHeader { + checksum: 0, + checksum_body: 0, + cluster: 0, + size: u32::try_from(std::mem::size_of::() + body.len()) + .expect("synthetic DVC frame fits u32"), + view, + release: 0, + command: Command2::DoViewChange, + replica, + reserved_frame: [0; 66], + op, + commit, + namespace: 0, + log_view, + reserved: [0; 68], + nack_bitset: 0, + present_bitset: present, + }; + (header, body) + } - // SVC from replica 1, view 3. Replica 0 advances to view 3 - // (reset_view_change_state clears loopback), records own SVC+DVC and - // replica 1's SVC. DVC quorum needs 2; have 1. - let svc = StartViewChangeHeader { + /// Headers for `low..=high`, high-to-low as a suffix requires, sealed and + /// chained the way a real producer writes them. + /// + /// Built ascending so each `parent` is the previous entry's real identity, then + /// reversed. The decoder recomputes both, so fabricated checksums are rejected + /// before the code under test sees them. + fn suffix_headers(low: u64, high: u64, view: u32) -> Vec { + if high == 0 { + return Vec::new(); + } + let mut parent = 0u128; + let mut ascending = Vec::new(); + for op in low.max(1)..=high { + let mut header = PrepareHeader { + command: Command2::Prepare, + operation: iggy_binary_protocol::Operation::CreateStream, + op, + view, + parent, + // Strictly increasing with op, so the suffix reads decreasing. + timestamp: op, + // Zero so the DVC's own commit drives `commit_max`. + commit: 0, + ..Default::default() + }; + header.checksum = header.identity_checksum(); + parent = header.checksum; + ascending.push(header); + } + ascending.reverse(); + ascending + } + + fn svc_header(replica: u8, view: u32) -> iggy_binary_protocol::StartViewChangeHeader { + iggy_binary_protocol::StartViewChangeHeader { checksum: 0, checksum_body: 0, cluster: 0, - size: 0, - view: 3, + size: u32::try_from(std::mem::size_of::< + iggy_binary_protocol::StartViewChangeHeader, + >()) + .expect("header fits u32"), + view, release: 0, command: Command2::StartViewChange, - replica: 1, + replica, reserved_frame: [0; 66], namespace: 0, reserved: [0; 120], + } + } + + /// Install the suffix this replica would read from its own journal. + fn install_local_suffix( + consensus: &VsrConsensus, + op: u64, + commit: u64, + log_view: u32, + ) { + let headers = suffix_headers(commit, op, log_view); + consensus.set_local_dvc_suffix(crate::dvc_merge::suffix_all_present(headers)); + } + + #[test] + fn given_an_undecidable_quorum_when_a_later_dvc_decides_it_should_start_the_view() { + // Reaching a view-change quorum is not the same as deciding a log. Latching + // `do_view_change_quorum` at the quorum makes every non-Ready outcome + // terminal: later DoViewChanges are recorded, but the guard that calls the + // merge is already false, so the view burns its status timeout for nothing. + // + // 5 replicas, view_change quorum 3, replica 0 is primary for view 5. + let consensus = VsrConsensus::new(1, 0, 5, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.restore_commit_state(2, 2); + consensus.sequencer().set_sequence(4); + // This replica holds op 4's header but cannot serve its body. Suffix entries + // run high-to-low, so bit 0 is op 4: clearing it offers ops 3 and 2 only. + let local = suffix_headers(2, 4, 0); + consensus.set_local_dvc_suffix(crate::view_change_quorum::DvcSuffix::new(local, 0, 0b110)); + + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 5)); + + // Two peers report, reaching the quorum of 3. All three hold op 4's header, + // none can serve its body, and two replicas have yet to report. + for replica in [1u8, 2] { + let (dvc, body) = dvc_with_suffix(replica, 5, 0, 4, 2, Some(4)); + let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + assert!(actions.is_empty()); + } + assert!( + consensus.pending_view_log().is_none(), + "op 4 is neither servable nor provably dead, so nothing may be parked yet" + ); + + // Replica 3 arrives holding the body: the deciding message, still merged. + let (dvc, body) = dvc_with_full_suffix(3, 5, 0, 4, 2); + let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + + let pending = consensus + .pending_view_log() + .expect("the DVC that supplies the missing body must complete the merge"); + assert_eq!(pending.op_head, 4); + assert_eq!(pending.commit_max, 2); + } + + #[test] + fn given_a_sealed_prepare_when_verifying_integrity_should_accept() { + let message = Message::::new(size_of::()).transmute_header( + |_, header: &mut PrepareHeader| { + header.command = Command2::Prepare; + header.op = 7; + header.size = u32::try_from(size_of::()).expect("header fits u32"); + }, + ); + let sealed = seal_prepare_checksum(message); + assert_eq!(verify_prepare_integrity(sealed.header(), &[]), Ok(())); + } + + #[test] + fn given_a_prepare_whose_header_was_altered_when_verifying_should_reject() { + // Downstream compares `checksum` as an opaque token, so without this a frame + // corrupted in transit is journaled and then re-served to peers from the WAL. + let message = Message::::new(size_of::()).transmute_header( + |_, header: &mut PrepareHeader| { + header.command = Command2::Prepare; + header.op = 7; + header.size = u32::try_from(size_of::()).expect("header fits u32"); + }, + ); + let mut sealed = seal_prepare_checksum(message); + let bytes = &mut sealed.as_mut_slice()[..size_of::()]; + let header = bytemuck::checked::try_from_bytes_mut::(bytes) + .expect("a prepare header round-trips its own bit pattern"); + header.op = 8; + assert!(verify_prepare_integrity(&header.clone(), &[]).is_err()); + } + + #[test] + fn given_an_unsealed_prepare_when_verifying_should_abstain() { + // The partition plane leaves `checksum` at `CHECKSUM_UNSEALED` and carries a + // verified `batch_checksum` over the same bytes instead. + let header = PrepareHeader { + command: Command2::Prepare, + op: 7, + ..Default::default() }; - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc); + assert_eq!(header.checksum, CHECKSUM_UNSEALED); + assert_eq!(verify_prepare_integrity(&header, &[]), Ok(())); + } + + /// A frame carrying `body`, with `size` covering exactly header + body and + /// `checksum_body` sealed over it, as the metadata projection does. + /// `trailing` bytes of garbage past the sealed frame, which `size` does not + /// cover. The buffer is `MESSAGE_ALIGN`ed: `PrepareHeader` holds `u128`s, so a + /// `Vec` would only be 16-aligned by the allocator's good graces and miri + /// rejects the cast. + fn sealed_frame(body: &[u8], trailing: usize) -> Owned { + let size = size_of::() + body.len(); + let mut frame = Owned::::zeroed(size + trailing); + let bytes = frame.as_mut_slice(); + bytes[size_of::()..size].copy_from_slice(body); + bytes[size..].fill(0xAA); + let header = bytemuck::checked::from_bytes_mut::( + &mut bytes[..size_of::()], + ); + header.command = Command2::Prepare; + header.op = 7; + header.size = u32::try_from(size).expect("fits u32"); + header.checksum_body = u128::from(calculate_checksum(body)); + frame + } + + fn frame_header(frame: &Owned) -> PrepareHeader { + *bytemuck::checked::from_bytes::( + &frame.as_slice()[..size_of::()], + ) + } + + #[test] + fn given_a_prepare_whose_body_was_altered_when_verifying_should_reject() { + let mut frame = sealed_frame(b"body", 0); + let header = frame_header(&frame); + assert_eq!(verify_prepare_integrity(&header, frame.as_slice()), Ok(())); + + *frame + .as_mut_slice() + .last_mut() + .expect("the frame has a body") ^= 1; + assert!(verify_prepare_integrity(&header, frame.as_slice()).is_err()); + } + + #[test] + fn given_bytes_past_the_frame_size_when_verifying_should_ignore_them() { + // `try_from` accepts a buffer longer than `size` without trimming; hashing to + // the end would reject a correctly sealed prepare and disagree with the WAL scan. + let padded = sealed_frame(b"body", 16); + let header = frame_header(&padded); + assert_eq!( + verify_prepare_integrity(&header, padded.as_slice()), + Ok(()), + "only the bytes `size` covers are the body" + ); + } + + #[test] + fn given_a_size_that_overruns_the_buffer_when_verifying_should_reject() { + // Truncated frame, header still claims the full length: the body it names is + // not there to hash. + let frame = sealed_frame(b"body", 0); + let header = frame_header(&frame); + + let truncated = &frame.as_slice()[..frame.as_slice().len() - 1]; + assert!(verify_prepare_integrity(&header, truncated).is_err()); + } + + #[test] + fn given_a_parked_merge_when_not_yet_started_should_not_advance_log_view() { + // `log_view` claims "my log IS the log this view decided", which is what + // makes a sender canonical next time. Raising it when the merge parks, before + // the merged head is installed, lets a primary-elect that never finishes + // repair vote as canonical carrying its own stale head, and ops the merge + // kept then fall outside the next scan range, dropped with no nack. + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.restore_commit_state(2, 2); + consensus.sequencer().set_sequence(3); + install_local_suffix(&consensus, 3, 2, 0); + assert_eq!(consensus.log_view(), 0); + + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); + let (dvc, body) = dvc_with_full_suffix(2, 3, 0, 3, 2); + let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + + assert!(consensus.pending_view_log().is_some(), "the merge parks"); + assert_eq!( + consensus.log_view(), + 0, + "a parked merge has installed nothing, so log_view must still \ + describe the log this replica actually holds" + ); + assert_eq!(consensus.view(), 3, "the view itself did advance"); + + let _ = consensus.start_pending_view(PlaneKind::Metadata); + assert_eq!( + consensus.log_view(), + 3, + "installing the merged head is what earns the log_view claim" + ); + } + + #[test] + fn loopback_cleared_on_complete_view_change_as_primary() { + // 3 replicas, replica 0 is primary for view 0 (and view 3: 3 % 3 = 0). + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.restore_commit_state(2, 2); + consensus.sequencer().set_sequence(3); + install_local_suffix(&consensus, 3, 2, 0); + + // SVC from replica 1, view 3. Replica 0 advances, records own SVC+DVC and + // replica 1's SVC. DVC quorum needs 2; have 1. + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); // Stale loopback queued between SVC and DVC quorum. let stale_msg = Message::::new(std::mem::size_of::()); consensus.push_loopback(stale_msg.into_generic()); - // DVC from replica 2, view 3, quorum, complete_view_change_as_primary fires. - let dvc = DoViewChangeHeader { + // DVC from replica 2 forms the quorum and the merge settles the log. + let (dvc, body) = dvc_with_full_suffix(2, 3, 0, 3, 2); + let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + + // Parked: nothing is announced until the journal can serve it. + assert!( + actions.is_empty(), + "a merged view change must announce nothing until repair completes" + ); + let pending = consensus + .pending_view_log() + .expect("a decidable quorum must park a merged log"); + assert_eq!(pending.op_head, 3); + assert_eq!(pending.commit_max, 2); + assert_eq!(consensus.status(), Status::ViewChange); + + // Stale loopback must be cleared. + let mut buf = Vec::new(); + consensus.drain_loopback_into(&mut buf); + assert!( + buf.is_empty(), + "loopback queue must be empty after view change completion" + ); + + // Journal now covers the merged log, so the view starts and announces. + let actions = consensus.start_pending_view(PlaneKind::Metadata); + assert!( + actions + .iter() + .any(|a| matches!(a, crate::VsrAction::SendStartView { .. })), + "expected SendStartView once the view starts" + ); + assert_eq!(consensus.status(), Status::Normal); + assert!( + consensus.pending_view_log().is_none(), + "starting the view must consume the parked log" + ); + } + + /// Refusing to start a view must not be terminal. + /// + /// A parked merge leaves the replica in `ViewChange` announcing nothing if the + /// bodies never arrive, which is the intended trade against losing data but has + /// to stay recoverable: the status timeout fires, escalates, and drops the parked + /// log. Reusing a log merged for a superseded view would leak a truncation + /// decided there into a view that never voted for it. + /// A `StartView` from the view's primary, optionally carrying the view's + /// canonical headers. + fn start_view_with_suffix( + replica: u8, + view: u32, + op: u64, + commit: u64, + with_suffix: bool, + ) -> (iggy_binary_protocol::StartViewHeader, Body) { + use iggy_binary_protocol::StartViewHeader; + + let body = if with_suffix { + encode_body(&suffix_headers(commit, op, view)) + } else { + Body::new(BODY_ALIGN) + }; + let header = StartViewHeader { checksum: 0, checksum_body: 0, cluster: 0, - size: 0, - view: 3, + size: u32::try_from(std::mem::size_of::() + body.len()) + .expect("synthetic StartView fits u32"), + view, release: 0, - command: Command2::DoViewChange, - replica: 2, + command: Command2::StartView, + replica, reserved_frame: [0; 66], - op: 0, - commit: 0, + op, + commit, namespace: 0, - log_view: 0, - reserved: [0; 100], + reserved: [0; 88], + incarnation: 0, }; - let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc); + (header, body) + } + + /// Encode headers as a control-message body. + /// + /// Aligned, because `dvc_suffix_decode` uses a checked `bytemuck` cast per + /// 256-byte chunk: a `Vec` body reports `MalformedHeader` for entry 0 + /// instead of the failure under test. glibc over-aligns these; Miri does not. + fn encode_body(headers: &[PrepareHeader]) -> Body { + let mut body = Body::with_capacity(BODY_ALIGN, std::mem::size_of_val(headers)); + for header in headers { + body.extend_from_slice(bytemuck::bytes_of(header)); + } + body + } + + #[test] + fn given_a_corrupted_suffix_entry_when_decoding_should_reject_the_frame() { + // The worst failure mode: a flipped bit in a canonical sender's header makes + // it canonical for the view, so honest senders read as disagreeing and can + // reach a nack quorum against a committed op. Recomputing keeps that out. + let mut headers = suffix_headers(2, 4, 1); + headers[0].timestamp ^= 0xFF; + let body = encode_body(&headers); + + let error = crate::dvc_suffix_decode(&body, 4, 0, 0) + .expect_err("a header that does not match its own checksum must be rejected"); + assert_eq!(error, crate::DvcSuffixError::ChecksumMismatch { index: 0 }); + } + + #[test] + fn given_a_broken_suffix_chain_when_decoding_should_reject_the_frame() { + // Well-sealed entries that do not link: a log, not a bag of records. + let mut headers = suffix_headers(2, 4, 1); + headers[0].parent ^= 0xFF; + headers[0].checksum = headers[0].identity_checksum(); + let body = encode_body(&headers); + + let error = crate::dvc_suffix_decode(&body, 4, 0, 0) + .expect_err("a suffix whose entries do not chain must be rejected"); + assert_eq!(error, crate::DvcSuffixError::ChainBreak { index: 1 }); + } + + #[test] + fn given_an_unsealed_suffix_when_decoding_should_be_accepted() { + // A pre-sealing peer sends the unsealed sentinel. Rejecting those breaks + // rolling upgrades, so identity and chain checks skip them: no evidence. + let headers: Vec = suffix_headers(2, 4, 1) + .into_iter() + .map(|mut header| { + header.checksum = 0; + header.parent = 0; + header + }) + .collect(); + let body = encode_body(&headers); + + let suffix = + crate::dvc_suffix_decode(&body, 4, 0, 0).expect("an unsealed suffix must still decode"); + assert_eq!(suffix.len(), 3); + } + + #[test] + fn given_start_view_with_suffix_when_adopted_should_record_the_canonical_headers() { + // The backup keeps the view's headers so its repair ingest can reject a body + // that disagrees with the view's decision, and so a disagreeing local entry + // is reported rather than silently blocking its own repair forever. + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + + // Replica 1 is primary for view 1 (1 % 3). + let (header, body) = start_view_with_suffix(1, 1, 5, 3, true); + let actions = consensus.handle_start_view(PlaneKind::Metadata, &header, &body); + + assert!(!actions.is_empty(), "a valid StartView must be adopted"); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.sequencer().current_sequence(), 5); + + let recorded = consensus + .pending_view_log() + .expect("an adopted StartView carrying a suffix must record its headers"); + assert_eq!(recorded.op_head, 5); + assert_eq!(recorded.commit_max, 3); + assert_eq!( + recorded.headers.iter().map(|h| h.op).collect::>(), + vec![5, 4, 3], + "headers run high-to-low from the head down to the announced commit" + ); + } + + #[test] + fn given_start_view_without_suffix_when_adopted_should_trust_the_announced_op() { + // Probe answers and stale-view corrections carry numbers only: a backup must + // still adopt, and record nothing it could mistake for the view's decision. + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + + let (header, body) = start_view_with_suffix(1, 1, 5, 3, false); + assert!(body.is_empty()); + let actions = consensus.handle_start_view(PlaneKind::Metadata, &header, &body); - // View change complete → SendStartView action. assert!( - actions - .iter() - .any(|a| matches!(a, crate::VsrAction::SendStartView { .. })), - "expected SendStartView after DVC quorum" + !actions.is_empty(), + "a numbers-only StartView must still adopt" ); + assert_eq!(consensus.sequencer().current_sequence(), 5); + assert!( + consensus.pending_view_log().is_none(), + "no suffix means no canonical headers to verify against" + ); + } + + #[test] + fn given_parked_view_change_when_status_timeout_fires_should_escalate_and_drop_merged_log() { + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.restore_commit_state(2, 2); + consensus.sequencer().set_sequence(3); + install_local_suffix(&consensus, 3, 2, 0); + + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); + let (dvc, body) = dvc_with_full_suffix(2, 3, 0, 3, 2); + let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + + // Parked: no shard here reports coverage, so the view never starts. + assert!(consensus.pending_view_log().is_some()); + assert_eq!(consensus.status(), Status::ViewChange); + let parked_view = consensus.view(); + + // `VIEW_CHANGE_STATUS_TICKS` is 500; tick past it. Escalation shows as the + // view advancing, since the 50-tick SVC retransmit also emits a send. + let mut escalated = false; + for _ in 0..600 { + let _ = consensus.tick(PlaneKind::Metadata); + if consensus.view() > parked_view { + escalated = true; + break; + } + } - // Stale loopback must be cleared. - let mut buf = Vec::new(); - consensus.drain_loopback_into(&mut buf); assert!( - buf.is_empty(), - "loopback queue must be empty after view change completion" + escalated, + "a parked view change must still escalate on the status timeout" + ); + assert!( + consensus.view() > parked_view, + "escalation must advance the view past {parked_view}, got {}", + consensus.view() + ); + assert!( + consensus.pending_view_log().is_none(), + "the superseded merged log must be dropped, not carried into the next view" ); + assert_eq!(consensus.status(), Status::ViewChange); } - /// A DVC winner may claim an uncommitted range up to the *configured* + /// A merged log may claim an uncommitted range up to the *configured* /// prepare depth. With a pipeline deeper than the default const, the new /// primary schedules the rebuild rather than panicking on the old /// `PIPELINE_PREPARE_QUEUE_MAX` bound. #[test] #[allow(clippy::cast_possible_truncation)] - fn given_view_change_range_above_default_when_complete_as_primary_should_rebuild() { - use iggy_binary_protocol::{DoViewChangeHeader, StartViewChangeHeader}; - + fn given_view_change_range_above_default_when_starting_view_should_rebuild() { let depth = crate::PIPELINE_PREPARE_QUEUE_MAX * 2; // Strictly above the default const, still within the configured depth. let winner_op = (crate::PIPELINE_PREPARE_QUEUE_MAX + 8) as u64; @@ -983,48 +1614,23 @@ mod tests { LocalPipeline::with_capacities(depth, depth * 2), ); consensus.init(); + consensus.sequencer().set_sequence(winner_op); + install_local_suffix(&consensus, winner_op, 1, 0); // SVC from replica 1 moves replica 0 into view 3 and records its own DVC. - let svc = StartViewChangeHeader { - checksum: 0, - checksum_body: 0, - cluster: 0, - size: 0, - view: 3, - release: 0, - command: Command2::StartViewChange, - replica: 1, - reserved_frame: [0; 66], - namespace: 0, - reserved: [0; 120], - }; - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc); + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); - // DVC from replica 2 claims a log head far past commit, forming quorum. - let dvc = DoViewChangeHeader { - checksum: 0, - checksum_body: 0, - cluster: 0, - size: 0, - view: 3, - release: 0, - command: Command2::DoViewChange, - replica: 2, - reserved_frame: [0; 66], - op: winner_op, - commit: 0, - namespace: 0, - log_view: 0, - reserved: [0; 100], - }; - let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc); + // DVC from replica 2 claims the same deep log, forming quorum. + let (dvc, body) = dvc_with_full_suffix(2, 3, 0, winner_op, 1); + let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + let actions = consensus.start_pending_view(PlaneKind::Metadata); assert!( actions.iter().any(|action| matches!( action, - VsrAction::RebuildPipeline { from_op: 1, to_op } if *to_op == winner_op + VsrAction::RebuildPipeline { from_op: 2, to_op } if *to_op == winner_op )), - "expected RebuildPipeline over the full uncommitted range" + "expected RebuildPipeline over the uncommitted range, got {actions:?}" ); } diff --git a/core/consensus/src/view_change_quorum.rs b/core/consensus/src/view_change_quorum.rs index 2eaff15b36..5cded28eab 100644 --- a/core/consensus/src/view_change_quorum.rs +++ b/core/consensus/src/view_change_quorum.rs @@ -16,27 +16,201 @@ // under the License. use crate::REPLICAS_MAX; +use iggy_binary_protocol::{ + CHECKSUM_UNSEALED, Command2, DVC_HEADERS_MAX, Operation, PrepareHeader, +}; + +/// Write prepare headers into a control-message body, high-to-low op. +/// +/// Shared by `DoViewChange` and `StartView`, which both carry a suffix as a plain +/// run of 256-byte headers; stating the layout once keeps them from drifting. +/// +/// # Panics +/// When `dst` is not exactly `headers.len()` headers wide. +pub fn encode_prepare_headers(headers: &[PrepareHeader], dst: &mut [u8]) { + let stride = size_of::(); + assert_eq!( + dst.len(), + std::mem::size_of_val(headers), + "control-message body buffer must fit the headers exactly" + ); + for (index, header) in headers.iter().enumerate() { + dst[index * stride..(index + 1) * stride].copy_from_slice(bytemuck::bytes_of(header)); + } +} + +/// Placeholder standing in for a suffix entry the sender does not hold. +/// +/// The suffix stays consecutive so a `(head_op, op)` pair indexes it arithmetically +/// and one bitset bit lines up with one op. A gap is transmitted, not omitted. +/// +/// `Operation::Reserved` is the marker, since no real prepare carries it, and every +/// other field is zero. [`dvc_header_kind`] insists on exactly that, so arbitrary +/// bytes cannot pass as a blank the merge would index. +#[must_use] +pub fn dvc_blank(op: u64) -> PrepareHeader { + PrepareHeader { + command: Command2::Prepare, + operation: Operation::Reserved, + op, + ..Default::default() + } +} + +/// What a suffix slot says about the sender's log at that op. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DvcHeaderKind { + /// No header for this op, or one the sender cannot vouch for. The nack bit is + /// what distinguishes "never prepared" (proof) from "lost it" (no proof). + Blank, + /// A real prepare header the sender holds. + Valid, +} + +/// Classify a suffix slot, by exact comparison against the canonical blank. +#[must_use] +pub fn dvc_header_kind(header: &PrepareHeader) -> DvcHeaderKind { + if *header == dvc_blank(header.op) { + DvcHeaderKind::Blank + } else { + DvcHeaderKind::Valid + } +} + +/// A sender's uncommitted suffix: the headers spanning `commit..=op`, high-to-low, +/// plus one nack bit and one present bit per entry. +/// +/// Index 0 is the head (`StoredDvc::op`); index `i` is op `op - i`. Empty when the +/// sender has nothing uncommitted, or could not snapshot a suffix for this log. +#[derive(Debug, Clone, Default)] +pub struct DvcSuffix { + headers: Vec, + nack_bitset: u128, + present_bitset: u128, +} + +// These all read through the `Vec`, and neither `Vec::len` nor `Deref` is const on +// the pinned toolchain, so clippy's suggestion does not compile. +#[allow(clippy::missing_const_for_fn)] +impl DvcSuffix { + /// # Panics + /// When `headers` exceeds [`DVC_HEADERS_MAX`], or a bitset sets a bit past the + /// suffix. Sender-side programming errors; the same conditions off the wire go + /// through `DoViewChangeHeader::validate`. + #[must_use] + pub fn new(headers: Vec, nack_bitset: u128, present_bitset: u128) -> Self { + assert!( + headers.len() <= DVC_HEADERS_MAX, + "DVC suffix of {} entries exceeds the addressable maximum {DVC_HEADERS_MAX}", + headers.len() + ); + if headers.len() < DVC_HEADERS_MAX { + let beyond = !((1u128 << headers.len()) - 1); + assert_eq!( + nack_bitset & beyond, + 0, + "nack bit set past the {}-entry suffix", + headers.len() + ); + assert_eq!( + present_bitset & beyond, + 0, + "present bit set past the {}-entry suffix", + headers.len() + ); + } + Self { + headers, + nack_bitset, + present_bitset, + } + } + + /// A sender contributing numbers only: no headers, nacks, or offered bodies. + #[must_use] + pub fn empty() -> Self { + Self::default() + } + + #[must_use] + pub fn len(&self) -> usize { + self.headers.len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.headers.is_empty() + } + + #[must_use] + pub fn headers(&self) -> &[PrepareHeader] { + &self.headers + } + + #[must_use] + pub const fn nack_bitset(&self) -> u128 { + self.nack_bitset + } + + #[must_use] + pub const fn present_bitset(&self) -> u128 { + self.present_bitset + } + + /// Bytes the headers occupy on the wire. + #[must_use] + pub fn encoded_len(&self) -> usize { + self.headers.len() * size_of::() + } + + /// Write the headers into a `DoViewChange` body, high-to-low op. + /// + /// Paired with [`dvc_suffix_decode`], so the wire ordering is stated once. + /// + /// # Panics + /// When `dst` is not exactly [`Self::encoded_len`] bytes. + pub fn encode_into(&self, dst: &mut [u8]) { + encode_prepare_headers(&self.headers, dst); + } + + /// Slot index for `op`. `None` when `op` falls outside this sender's window. + #[must_use] + pub fn index_of(&self, head_op: u64, op: u64) -> Option { + let distance = usize::try_from(head_op.checked_sub(op)?).ok()?; + (distance < self.headers.len()).then_some(distance) + } + + /// The header at `index`, or `None` for a blank or out-of-range slot. + #[must_use] + pub fn valid_header_at(&self, index: usize) -> Option<&PrepareHeader> { + let header = self.headers.get(index)?; + matches!(dvc_header_kind(header), DvcHeaderKind::Valid).then_some(header) + } + + /// Whether the sender proves it never prepared the entry at `index`. + #[must_use] + pub fn nacks(&self, index: usize) -> bool { + index < self.headers.len() && self.nack_bitset & (1u128 << index) != 0 + } + + /// Whether the sender can serve the body of the entry at `index`. + #[must_use] + pub fn offers_body(&self, index: usize) -> bool { + index < self.headers.len() && self.present_bitset & (1u128 << index) != 0 + } +} /// Stored information from a `DoViewChange` message. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub struct StoredDvc { pub replica: u8, /// The view when the replica's status was last normal. pub log_view: u32, pub op: u64, pub commit: u64, -} - -impl StoredDvc { - /// Compare for log selection: highest `log_view`, then highest op. - #[must_use] - pub const fn is_better_than(&self, other: &Self) -> bool { - if self.log_view == other.log_view { - self.op > other.op - } else { - self.log_view > other.log_view - } - } + /// The sender's uncommitted suffix. Empty from a silent sender, which counts + /// toward the quorum and `max(commit)` but neither nacks nor offers bodies. + pub suffix: DvcSuffix, } /// Array type for storing DVC messages from all replicas. @@ -44,12 +218,13 @@ pub type DvcQuorumArray = [Option; REPLICAS_MAX]; /// Create an empty DVC quorum array. #[must_use] -pub const fn dvc_quorum_array_empty() -> DvcQuorumArray { - [None; REPLICAS_MAX] +pub fn dvc_quorum_array_empty() -> DvcQuorumArray { + // `[None; REPLICAS_MAX]` needs `StoredDvc: Copy`, ruled out by the `Vec`. + std::array::from_fn(|_| None) } /// Record a DVC in the array. Returns true if this is a new entry (not duplicate). -pub const fn dvc_record(array: &mut DvcQuorumArray, dvc: StoredDvc) -> bool { +pub fn dvc_record(array: &mut DvcQuorumArray, dvc: StoredDvc) -> bool { let slot = &mut array[dvc.replica as usize]; if slot.is_some() { return false; // Duplicate @@ -64,43 +239,196 @@ pub fn dvc_count(array: &DvcQuorumArray) -> usize { array.iter().filter(|m| m.is_some()).count() } -/// Check if a specific replica has sent a DVC. -#[must_use] -pub fn dvc_has_from(array: &DvcQuorumArray, replica: u8) -> bool { - array.get(replica as usize).is_some_and(Option::is_some) +/// Reset the DVC quorum array. +pub fn dvc_reset(array: &mut DvcQuorumArray) { + *array = dvc_quorum_array_empty(); } -/// Select the winning DVC (best log) from the quorum. -/// Returns the DVC with: highest `log_view`, then highest op. -#[must_use] -pub fn dvc_select_winner(array: &DvcQuorumArray) -> Option<&StoredDvc> { - array - .iter() - .filter_map(|m| m.as_ref()) - .max_by(|a, b| match a.log_view.cmp(&b.log_view) { - std::cmp::Ordering::Equal => a.op.cmp(&b.op), - other => other, - }) +/// Iterator over all stored DVCs. +pub fn dvc_iter(array: &DvcQuorumArray) -> impl Iterator { + array.iter().filter_map(|m| m.as_ref()) } -/// Get the maximum commit number across all DVCs. -#[must_use] -pub fn dvc_max_commit(array: &DvcQuorumArray) -> u64 { - array - .iter() - .filter_map(|m| m.as_ref()) - .map(|dvc| dvc.commit) - .max() - .unwrap_or(0) +/// Why a `DoViewChange` body could not be read as a suffix. +/// +/// Dropped whole rather than partially trusted: the merge indexes arithmetically +/// from the head op, so one bad offset misattributes a header, nack, or body. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DvcSuffixError { + /// Body length is not a whole number of headers. + NotHeaderMultiple { body_len: usize }, + /// More entries than the bitsets can address. + TooManyEntries { count: usize }, + /// An entry is not a valid `PrepareHeader` bit pattern. + MalformedHeader { index: usize }, + /// Entries are not consecutive descending from the head op. + OpOutOfOrder { + index: usize, + expected: u64, + found: u64, + }, + /// A bitset addresses an entry the body does not contain. + BitsetBeyondSuffix { count: usize }, + /// An entry's identity checksum does not match its own contents. + ChecksumMismatch { index: usize }, + /// A lower entry claims a view newer than the entry above it. + ViewRegressed { index: usize }, + /// A lower entry claims a timestamp at or after the entry above it. + TimestampNotDecreasing { index: usize }, + /// Consecutive entries do not hash-chain. + ChainBreak { index: usize }, } -/// Reset the DVC quorum array. -pub const fn dvc_reset(array: &mut DvcQuorumArray) { - *array = dvc_quorum_array_empty(); +impl std::fmt::Display for DvcSuffixError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NotHeaderMultiple { body_len } => write!( + f, + "do_view_change body of {body_len} bytes is not a whole number of {} -byte headers", + size_of::() + ), + Self::TooManyEntries { count } => write!( + f, + "do_view_change suffix of {count} entries exceeds the maximum {DVC_HEADERS_MAX}" + ), + Self::MalformedHeader { index } => { + write!( + f, + "do_view_change suffix entry {index} is not a prepare header" + ) + } + Self::OpOutOfOrder { + index, + expected, + found, + } => write!( + f, + "do_view_change suffix entry {index} carries op {found}, expected {expected}" + ), + Self::BitsetBeyondSuffix { count } => write!( + f, + "do_view_change bitset addresses an entry past the {count}-entry suffix" + ), + Self::ChecksumMismatch { index } => write!( + f, + "do_view_change suffix entry {index} does not match its own identity checksum" + ), + Self::ViewRegressed { index } => write!( + f, + "do_view_change suffix entry {index} claims a newer view than the entry above it" + ), + Self::TimestampNotDecreasing { index } => write!( + f, + "do_view_change suffix entry {index} does not predate the entry above it" + ), + Self::ChainBreak { index } => write!( + f, + "do_view_change suffix entry {index} does not chain to the entry above it" + ), + } + } } -/// Iterator over all stored DVCs. -// TODO: add #[must_use] -- pure iterator query, callers should not ignore. -pub fn dvc_iter(array: &DvcQuorumArray) -> impl Iterator { - array.iter().filter_map(|m| m.as_ref()) +impl std::error::Error for DvcSuffixError {} + +/// Read a suffix out of a `DoViewChange` body. +/// +/// `head_op` is the sender's `header.op`; entries run consecutively down from it, +/// blanks included, so slot `i` is unambiguously op `head_op - i`. An empty body +/// yields an empty suffix, which is what a sender with nothing to describe sends. +/// +/// `body` must carry [`PrepareHeader`]'s alignment: the cast below is checked, so an +/// unaligned body reports every entry as [`DvcSuffixError::MalformedHeader`] instead +/// of what is actually wrong. Real frames clear this because the body starts a whole +/// number of 256-byte headers into an aligned buffer; the debug assert catches a +/// hand-built one. +/// +/// # Errors +/// [`DvcSuffixError`] when the body is not a consecutive run of valid prepare +/// headers descending from `head_op`, or a bitset addresses a missing entry. +pub fn dvc_suffix_decode( + body: &[u8], + head_op: u64, + nack_bitset: u128, + present_bitset: u128, +) -> Result { + debug_assert!( + body.is_empty() + || body + .as_ptr() + .addr() + .is_multiple_of(align_of::()), + "suffix body must be aligned for PrepareHeader" + ); + let header_size = size_of::(); + if !body.len().is_multiple_of(header_size) { + return Err(DvcSuffixError::NotHeaderMultiple { + body_len: body.len(), + }); + } + let count = body.len() / header_size; + if count > DVC_HEADERS_MAX { + return Err(DvcSuffixError::TooManyEntries { count }); + } + if count < DVC_HEADERS_MAX { + let beyond = !((1u128 << count) - 1); + if nack_bitset & beyond != 0 || present_bitset & beyond != 0 { + return Err(DvcSuffixError::BitsetBeyondSuffix { count }); + } + } + + let mut headers = Vec::with_capacity(count); + // The entry above the current one, skipping blanks: high-to-low, so the child. + let mut child: Option = None; + for index in 0..count { + let chunk = &body[index * header_size..(index + 1) * header_size]; + let header = bytemuck::checked::try_from_bytes::(chunk) + .map_err(|_| DvcSuffixError::MalformedHeader { index })?; + let expected = head_op + .checked_sub(index as u64) + .ok_or(DvcSuffixError::OpOutOfOrder { + index, + expected: 0, + found: header.op, + })?; + if header.op != expected { + return Err(DvcSuffixError::OpOutOfOrder { + index, + expected, + found: header.op, + }); + } + + if matches!(dvc_header_kind(header), DvcHeaderKind::Valid) { + // Recompute rather than trust the field. Otherwise one bit flipped in + // transit becomes a canonical header no replica holds, honest senders + // read as disagreeing, and a corrupted frame turns into a nack quorum + // against a committed op. A pre-sealing peer's sentinel is skipped. + if header.checksum != CHECKSUM_UNSEALED && header.identity_checksum() != header.checksum + { + return Err(DvcSuffixError::ChecksumMismatch { index }); + } + if let Some(child) = child { + // Views never go backwards down the log, timestamps never forwards, + // and consecutive entries hash-chain. A frame breaking any of these + // describes a log that cannot exist. + if header.view > child.view { + return Err(DvcSuffixError::ViewRegressed { index }); + } + if header.timestamp >= child.timestamp { + return Err(DvcSuffixError::TimestampNotDecreasing { index }); + } + if header.op + 1 == child.op + && header.checksum != CHECKSUM_UNSEALED + && child.parent != header.checksum + { + return Err(DvcSuffixError::ChainBreak { index }); + } + } + child = Some(*header); + } + headers.push(*header); + } + + Ok(DvcSuffix::new(headers, nack_bitset, present_bitset)) } diff --git a/core/integration/tests/cluster/metadata_checkpoint_restart.rs b/core/integration/tests/cluster/metadata_checkpoint_restart.rs index 75ae60f6eb..d3d9a70e82 100644 --- a/core/integration/tests/cluster/metadata_checkpoint_restart.rs +++ b/core/integration/tests/cluster/metadata_checkpoint_restart.rs @@ -119,8 +119,10 @@ async fn await_checkpoint_on_all_nodes(harness: &TestHarness, generation: usize) // a checkpoint, so the transfer descriptor's `commit_op == snapshot_seq` and // the post-install tail repair has nothing to fetch (`commit_min == // commit_max` skips it). The below-floor retry then proves the reply ring -// rode the transferred table: request 191's reply was minted at op 192, which -// every node drained out of its WAL at that same checkpoint. +// rode the transferred table: request 191's reply was minted at op 192, and +// replay starts at `snapshot_seq + 1`, so no node re-executes it. The +// checkpoint drain keeps op 192's entry as the commit-point header a +// `DoViewChange` needs, but never replays it. #[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] async fn given_drained_journal_when_node_restarts_should_install_snapshot_only( harness: &mut TestHarness, diff --git a/core/journal/src/lib.rs b/core/journal/src/lib.rs index 9de342b6c1..b277d54ca5 100644 --- a/core/journal/src/lib.rs +++ b/core/journal/src/lib.rs @@ -46,6 +46,28 @@ where None } + /// Remove every entry at or above `from_op`, returning how many went, and + /// leave the snapshot watermark where it is. + /// + /// Not `drain` with a different range: `drain` advances the watermark past what + /// it removed, which would mark the removed ops evictable when a suffix + /// truncation needs them refillable. + /// + /// Defaults to `Unsupported` rather than a silent zero: "removed nothing" and + /// "cannot remove anything" demand different responses from the caller. + /// + /// # Errors + /// I/O error if the rewrite fails, or `Unsupported` if the implementation cannot + /// truncate. + fn truncate_from(&self, _from_op: u64) -> impl Future> { + async { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "this journal cannot truncate a suffix", + )) + } + } + /// Remove entries with ops in `ops` from the journal, /// returning the removed entries sorted by op. /// diff --git a/core/journal/src/prepare_journal.rs b/core/journal/src/prepare_journal.rs index 3f4f332ccd..c3e294fb5d 100644 --- a/core/journal/src/prepare_journal.rs +++ b/core/journal/src/prepare_journal.rs @@ -18,7 +18,7 @@ use crate::file_storage::FileStorage; use crate::{Journal, JournalHandle}; use compio::io::AsyncWriteAtExt; -use iggy_binary_protocol::consensus::{Command2, PrepareHeader}; +use iggy_binary_protocol::consensus::{CHECKSUM_UNSEALED, Command2, PrepareHeader}; use server_common::{MESSAGE_ALIGN, Message, iobuf::Owned}; use std::cell::{Cell, OnceCell, Ref, RefCell}; use std::fmt; @@ -412,7 +412,7 @@ impl PrepareJournal { Self::scan(storage, snapshot_op, slot_count).await } - #[allow(clippy::future_not_send)] + #[allow(clippy::future_not_send, clippy::too_many_lines)] async fn scan( storage: FileStorage, snapshot_op: u64, @@ -423,6 +423,8 @@ impl PrepareJournal { let mut offsets: Vec> = vec![None; slot_count]; let mut last_op: Option = None; let mut unsealed_entries: u64 = 0; + // Previous entry's `(op, checksum)`, for the parent-chain check. + let mut chain_previous: Option<(u64, u128)> = None; let mut pos: u64 = 0; let mut header_buf = vec![0u8; HEADER_SIZE]; // Reused 16-aligned scratch (PrepareHeader has u128 fields). Avoids @@ -477,14 +479,57 @@ impl PrepareJournal { // verify against and is skipped, not rejected: see // [`CHECKSUM_BODY_UNSEALED`]. // - // TODO(wal-integrity): the header `checksum` and its `parent` chain stay - // unverified, since the producer does not seal them yet (blocked on - // re-sealing re-stamped retransmits), so a bit-flip in a - // structurally-valid header field slips through. Recovery derives - // `commit_watermark = max(header.commit)`, so a flipped `commit` makes it - // apply prepared-but-uncommitted ops as committed, the very ops a view - // change may have truncated cluster-wide, diverging this replica. Seal - // and verify the header checksum + parent chain. + // The header's own integrity field is checked first, since a flipped + // header field is the more dangerous of the two: recovery derives + // `commit_watermark = max(header.commit)`, so a corrupted `commit` applies + // uncommitted ops as committed, diverging from the group. `size` and `op` + // are equally load-bearing for the scan itself. + if header.checksum != CHECKSUM_UNSEALED && header.identity_checksum() != header.checksum + { + if pos + entry_size < file_len { + return Err(JournalError::Io(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "interior WAL corruption at pos {pos} (op {}, operation {:?}): \ + prepare header checksum mismatch with {} bytes of entries \ + following; refusing to truncate and discard the committed suffix", + header.op, + header.operation, + file_len - (pos + entry_size), + ), + ))); + } + truncate_or_fail(&storage, pos, "prepare header checksum mismatch at tail").await?; + break; + } + + // The hash chain, checked only where meaningful: consecutive ops with both + // ends sealed. A gap means compaction dropped the predecessor, and an + // unsealed end has nothing to chain from, so neither is evidence of damage. + if let Some((previous_op, previous_checksum)) = chain_previous + && previous_op + 1 == header.op + && previous_checksum != CHECKSUM_UNSEALED + && header.parent != previous_checksum + { + if pos + entry_size < file_len { + return Err(JournalError::Io(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "interior WAL corruption at pos {pos}: op {} does not chain to op \ + {previous_op} (parent {} != {previous_checksum}) with {} bytes of \ + entries following; refusing to truncate and discard the committed \ + suffix", + header.op, + header.parent, + file_len - (pos + entry_size), + ), + ))); + } + truncate_or_fail(&storage, pos, "prepare parent chain break at tail").await?; + break; + } + chain_previous = Some((header.op, header.checksum)); + if header.checksum_body == CHECKSUM_BODY_UNSEALED { // Skip the body read too, so a WAL written entirely by a pre-sealing // build scans without touching its payload. @@ -697,6 +742,124 @@ impl PrepareJournal { clippy::future_not_send )] impl Journal for PrepareJournal { + /// Remove every entry at or above `from_op`, leaving the snapshot floor where + /// it is. Returns how many entries went. + /// + /// Deliberately not `drain`, which compacts a committed prefix and advances + /// `snapshot_op` past its range. Doing that to a suffix would declare everything + /// below the head snapshotted, letting `append` evict live entries repair cannot + /// put back, when those ops are exactly the ones that must stay refillable. + /// + /// For the one caller that needs it: a backup whose uncommitted entries disagree + /// with the log a view change settled on. They cannot be corrected in place, and + /// journal repair skips their ops as already-present, so dropping them is what + /// lets the primary's retransmission refill the range. + /// + /// # Errors + /// I/O error if the rewrite fails. Past the rename the journal is poisoned on any + /// failure, as in `drain`: serving a pre-truncation offset or appending at a stale + /// `write_offset` is worse than a hard stop. `from_op` must be at least 1. + async fn truncate_from(&self, from_op: u64) -> io::Result { + if let Some(state) = self.poisoned.get() { + return Err(Self::poisoned_io_error(state)); + } + if from_op == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "truncate_from: ops are 1-based, so 0 would discard the whole journal", + )); + } + // Shares the drain guard: both rewrite the same WAL through the same tmp + // path, so letting them overlap would race the swap. + if self.drain_in_flight.replace(true) { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + "drain or truncate already in flight: concurrent rewrites would race the WAL", + )); + } + let _guard = DrainInFlightGuard(&self.drain_in_flight); + + let mut removed = 0usize; + let mut live: Vec<(PrepareHeader, u64)> = Vec::new(); + { + let headers = self.headers.borrow(); + let offsets = self.offsets.borrow(); + for slot in 0..self.slot_count { + if let (Some(header), Some(offset)) = (&headers[slot], offsets[slot]) { + if header.op >= from_op { + removed += 1; + } else { + live.push((*header, offset)); + } + } + } + } + if removed == 0 { + return Ok(0); + } + live.sort_unstable_by_key(|(header, _)| header.op); + + let wal_path = self.storage.path(); + let tmp_path = wal_path.with_extension("wal.tmp"); + let tmp_guard = TmpFileGuard::new(tmp_path.clone()); + { + let mut tmp = compio::fs::File::create(&tmp_path).await?; + let mut write_pos: u64 = 0; + for (header, old_offset) in &live { + let size = header.size as usize; + let buf = vec![0u8; size]; + let buf = self.storage.read_at(*old_offset, buf).await?; + let (result, _buf) = tmp.write_all_at(buf, write_pos).await.into(); + result?; + write_pos += size as u64; + } + tmp.sync_all().await?; + } + + // COMMIT POINT, same as `drain`: past the rename the on-disk WAL is the new + // one while the in-memory index still describes the old layout, so every + // fallible step below poisons rather than serving stale offsets. + compio::fs::rename(&tmp_path, wal_path).await?; + tmp_guard.defuse(); + + if let Some(parent) = wal_path.parent() { + let dir = match compio::fs::File::open(parent).await { + Ok(dir) => dir, + Err(error) => { + return Err(self.poison("truncate_from: open parent dir for fsync", error)); + } + }; + if let Err(error) = dir.sync_all().await { + return Err(self.poison("truncate_from: parent dir fsync", error)); + } + } + if let Err(error) = self.storage.reopen().await { + return Err(self.poison("truncate_from: storage reopen after rename", error)); + } + + // `snapshot_op` is deliberately untouched. See the doc comment. + let mut headers = self.headers.borrow_mut(); + let mut offsets = self.offsets.borrow_mut(); + let mut pos: u64 = 0; + for (header, _) in &live { + let slot = slot_for_op(header.op, self.slot_count); + offsets[slot] = Some(pos); + pos += u64::from(header.size); + } + for slot in 0..self.slot_count { + if let Some(header) = &headers[slot] + && header.op >= from_op + { + headers[slot] = None; + offsets[slot] = None; + } + } + // Unlike a prefix drain, removing a suffix moves the head. + self.last_op.set(live.last().map(|(header, _)| header.op)); + + Ok(removed) + } + type Header = PrepareHeader; type Entry = Message; type HeaderRef<'a> = Ref<'a, PrepareHeader>; @@ -1047,6 +1210,257 @@ mod tests { Message::try_from(buffer).unwrap() } + /// A prepare with both integrity fields sealed and its parent chained, as a live + /// producer writes them. `make_entry` leaves `checksum` zero, read as unsealed. + fn make_identity_sealed_prepare( + op: u64, + body_size: usize, + parent: u128, + ) -> Message { + let mut message = make_prepare(op, body_size); + let bytes = message.as_mut_slice(); + let header = bytemuck::checked::from_bytes_mut::(&mut bytes[..HEADER_SIZE]); + header.parent = parent; + header.view = 1; + let checksum = header.identity_checksum(); + header.checksum = checksum; + message + } + + /// Byte offset of `field_offset` within the entry for `op`, at a fixed stride. + const fn header_field_offset(op: u64, body_size: usize, field_offset: usize) -> usize { + (op as usize - 1) * (HEADER_SIZE + body_size) + field_offset + } + + #[compio::test] + async fn truncate_from_removes_the_suffix_and_keeps_the_snapshot_floor() { + // The property that makes this not-a-drain: the floor must not move, or the + // removed ops become evictable and repair can never put them back. + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + for op in 1..=5u64 { + journal + .append(make_prepare(op, 64).deep_copy()) + .await + .unwrap(); + } + assert_eq!(journal.last_op(), Some(5)); + let floor_before = journal.snapshot_op(); + + let removed = journal.truncate_from(3).await.unwrap(); + assert_eq!(removed, 3, "ops 3, 4 and 5 must go"); + assert_eq!( + journal.snapshot_op(), + floor_before, + "truncating a suffix must not advance the snapshot floor" + ); + assert_eq!( + journal.last_op(), + Some(2), + "the head follows the truncation" + ); + for op in 1..=2u64 { + assert!( + journal.header(op as usize).is_some(), + "op {op} must survive" + ); + } + for op in 3..=5u64 { + assert!( + journal.header(op as usize).is_none(), + "op {op} must be gone" + ); + } + } + + #[compio::test] + async fn truncate_from_leaves_a_refillable_range() { + // The whole point: after truncation the ops can be appended again. A raised + // floor would either reject that or silently evict a live entry. + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + for op in 1..=4u64 { + journal + .append(make_prepare(op, 64).deep_copy()) + .await + .unwrap(); + } + journal.truncate_from(3).await.unwrap(); + + for op in 3..=4u64 { + journal + .append(make_prepare(op, 64).deep_copy()) + .await + .expect("a truncated op must be appendable again"); + } + assert_eq!(journal.last_op(), Some(4)); + assert!(journal.header(3).is_some()); + assert!(journal.header(4).is_some()); + } + + #[compio::test] + async fn truncate_from_survives_reopen() { + // The rewrite has to be durable, not just reflected in the index. + const BODY: usize = 64; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + let mut parent = 0u128; + for op in 1..=4u64 { + let entry = make_identity_sealed_prepare(op, BODY, parent); + parent = entry.header().checksum; + journal.append(entry.deep_copy()).await.unwrap(); + } + assert_eq!(journal.truncate_from(3).await.unwrap(), 2); + } + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + assert_eq!( + journal.last_op(), + Some(2), + "the truncation must be on disk, not only in the index" + ); + assert!(journal.header(3).is_none()); + } + + #[compio::test] + async fn scan_accepts_a_sealed_and_chained_wal() { + // Everything below only means something if the happy path still opens. + const BODY: usize = 64; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + let mut parent = 0u128; + for op in 1..=3u64 { + let entry = make_identity_sealed_prepare(op, BODY, parent); + parent = entry.header().checksum; + journal.append(entry.deep_copy()).await.unwrap(); + } + } + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + assert_eq!(journal.last_op(), Some(3)); + assert_eq!( + journal.unsealed_entry_count(), + 0, + "sealed entries must not be counted as unsealed" + ); + } + + #[compio::test] + async fn scan_truncates_tail_entry_with_header_checksum_mismatch() { + // A flipped `commit` leaves the header structurally valid, so only the identity + // checksum catches it. Recovery derives its watermark from `max(header.commit)`, + // so an undetected flip applies prepared-but-uncommitted ops as committed. + const BODY: usize = 64; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + let first = make_identity_sealed_prepare(1, BODY, 0); + let parent = first.header().checksum; + journal.append(first.deep_copy()).await.unwrap(); + journal + .append(make_identity_sealed_prepare(2, BODY, parent).deep_copy()) + .await + .unwrap(); + } + + let commit_offset = + header_field_offset(2, BODY, std::mem::offset_of!(PrepareHeader, commit)); + let mut bytes = std::fs::read(&path).unwrap(); + bytes[commit_offset] ^= 0xFF; + std::fs::write(&path, &bytes).unwrap(); + + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + assert_eq!( + journal.last_op(), + Some(1), + "a header-checksum mismatch on the tail entry must truncate it" + ); + assert!(journal.header(2).is_none()); + } + + #[compio::test] + async fn scan_refuses_boot_on_interior_header_checksum_mismatch() { + const BODY: usize = 64; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + let mut parent = 0u128; + for op in 1..=3u64 { + let entry = make_identity_sealed_prepare(op, BODY, parent); + parent = entry.header().checksum; + journal.append(entry.deep_copy()).await.unwrap(); + } + } + + let commit_offset = + header_field_offset(2, BODY, std::mem::offset_of!(PrepareHeader, commit)); + let mut bytes = std::fs::read(&path).unwrap(); + bytes[commit_offset] ^= 0xFF; + std::fs::write(&path, &bytes).unwrap(); + + let error = PrepareJournal::open(&path, 0).await.unwrap_err(); + let message = error.to_string(); + assert!( + message.contains("interior WAL corruption"), + "an interior header flip must refuse boot rather than discard the \ + committed suffix, got: {message}" + ); + } + + #[compio::test] + async fn scan_detects_a_parent_chain_break() { + // Both entries are individually well sealed; only the link is wrong. Catching + // this is what makes the log a chain rather than a bag of valid records. + const BODY: usize = 64; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + journal + .append(make_identity_sealed_prepare(1, BODY, 0).deep_copy()) + .await + .unwrap(); + // Op 2 chains to a parent that is not op 1's checksum. + journal + .append(make_identity_sealed_prepare(2, BODY, 0xDEAD_BEEF).deep_copy()) + .await + .unwrap(); + } + + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + assert_eq!( + journal.last_op(), + Some(1), + "op 2 does not chain to op 1 and must be truncated as a torn tail" + ); + } + + #[compio::test] + async fn scan_skips_verification_for_unsealed_entries() { + // A WAL from a pre-sealing build must still open: `checksum` reads as the + // unsealed sentinel, so neither the identity nor the chain is checked. + const BODY: usize = 32; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + for op in 1..=2u64 { + journal + .append(make_unsealed_prepare(op, BODY).deep_copy()) + .await + .unwrap(); + } + } + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + assert_eq!(journal.last_op(), Some(2)); + } + #[compio::test] async fn scan_truncates_entry_with_body_checksum_mismatch() { let dir = tempdir().unwrap(); diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 55d4fb042c..bafd51ceeb 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -34,6 +34,7 @@ use consensus::{ is_caught_up_primary, panic_if_hash_chain_would_break_in_same_view, peek_committable_head, pipeline_prepare_common, register_preflight, replicate_preflight, replicate_to_next_in_chain, request_preflight, send_eviction_to_client, send_prepare_ok as send_prepare_ok_common, + verify_prepare_integrity, }; use iggy_binary_protocol::WireIdentifier; use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; @@ -415,17 +416,29 @@ impl SnapshotCoordinator { Ok(checksum) } - /// Drain the snapshotted prefix `0..=last_op` to reclaim WAL space. Runs only - /// after the pairing is durable (see [`Self::persist_snapshot`]). + /// Drain the snapshotted prefix below `last_op` to reclaim WAL space. Runs + /// only after the pairing is durable (see [`Self::persist_snapshot`]). + /// + /// `last_op` itself is retained, one entry the snapshot has already + /// superseded. It is this replica's commit point, and a `DoViewChange` + /// carries a header for every op from there up. Draining it inclusively + /// leaves that entry blank, and blank at the commit point is the one slot + /// the merge can neither adopt nor discard: a quorum of senders that all + /// checkpointed at the same op deadlocks the view change + /// (`dvc_merge::merge_dvc_quorum`). Reclaiming one more entry is not worth + /// a group that cannot elect. #[allow(clippy::future_not_send)] async fn drain( &self, journal: &J, last_op: u64, ) -> Result<(), SnapshotError> { + let Some(drain_to) = last_op.checked_sub(1) else { + return Ok(()); + }; journal .handle() - .drain(0..=last_op) + .drain(0..=drain_to) .await .map_err(SnapshotError::Io)?; Ok(()) @@ -1041,6 +1054,23 @@ where let header = *message.header(); + // Before anything trusts `checksum` as an identity token, and before the WAL + // takes the bytes. Every live prepare travels this path: unverified, a frame + // corrupted between primary and backup is journaled as-is and re-served to + // peers, which the interior-corruption boot refusal turns into an unbootable + // node on the next restart. + if let Err(reason) = verify_prepare_integrity(&header, message.as_slice()) { + warn!( + target: "iggy.metadata.diag", + plane = "metadata", + replica_id = consensus.replica(), + view = consensus.view(), + op = header.op, + "discarding prepare: {reason}" + ); + return; + } + let current_op = match replicate_preflight(consensus, &header) { Ok(current_op) => current_op, Err(reason) => { @@ -3775,7 +3805,12 @@ where ..Default::default() }; - prepare + // Last, because the identity checksum covers every other field. Same contract as + // the wire path in `Project::project`; skipping it would leave the rewritten + // prepares (CreateTopic/CreatePartitions assignments, the UpdateTopic default-size + // rewrite, the PAT-cleaner delete) as the only ops the merge cannot tell apart + // from a competing prepare. + consensus::seal_prepare_checksum(prepare) } /// Eviction reason for a request `prepare_request` rejected as structurally @@ -4711,6 +4746,87 @@ mod tests { ); } + /// A checkpoint reclaims the WAL prefix the snapshot supersedes, but must + /// stop one op short of the checkpoint op itself. + /// + /// That op is the replica's commit point, and its `DoViewChange` suffix is + /// floored there. The merge scans the commit point and may not discard it, + /// so a sender with no header to put there is deferring to a peer; when + /// every sender has checkpointed at the same op the view change deadlocks + /// (`dvc_merge::merge_dvc_quorum`). Checkpoints fire on local journal + /// occupancy, which is symmetric across replicas seeing the same ops, so + /// "every sender" is the ordinary case, not a coincidence. + #[compio::test] + async fn checkpoint_drain_retains_the_commit_point_header() { + const CLIENT: u128 = 1; + const SESSION: u64 = 1; + const ACTING_USER: u32 = 7; + const OPS: u64 = 5; + const CHECKPOINT_OP: u64 = 3; + + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join(crate::impls::METADATA_DIR)).unwrap(); + let journal = + journal::prepare_journal::PrepareJournal::open(&dir.path().join("journal.wal"), 0) + .await + .unwrap(); + let consensus = VsrConsensus::new( + 1, + 0, + 1, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + NoopBus, + LocalPipeline::new(), + ); + consensus.init(); + let md: IggyMetadata<_, journal::prepare_journal::PrepareJournal, (), TestMux> = + IggyMetadata::new( + Some(consensus), + Some(journal), + None, + None, + TestMux::default(), + Some(dir.path().to_path_buf()), + ); + let consensus = md.consensus.as_ref().unwrap(); + md.client_table.borrow_mut().commit_register( + CLIENT, + ACTING_USER, + register_reply(CLIENT, SESSION), + ); + + for op in 1..=OPS { + let prepare = md + .prepare_request(create_stream_request(CLIENT, op, &format!("s{op}"))) + .expect("CreateStream is client-allowed"); + consensus.pipeline_message(PlaneKind::Metadata, &prepare); + md.on_replicate(prepare).await; + } + + let journal = md.journal.as_ref().unwrap(); + md.coordinator + .as_ref() + .expect("data_dir present arms the coordinator") + .drain(journal, CHECKPOINT_OP) + .await + .expect("drain the snapshotted prefix"); + + let header_at = |op: u64| journal.header(usize::try_from(op).expect("test ops fit usize")); + for op in 1..CHECKPOINT_OP { + assert!( + header_at(op).is_none(), + "op {op} is below the checkpoint and must be reclaimed" + ); + } + assert!( + header_at(CHECKPOINT_OP).is_some(), + "the checkpoint op is the commit point and must stay describable in a DVC" + ); + for op in CHECKPOINT_OP + 1..=OPS { + assert!(header_at(op).is_some(), "op {op} was never snapshotted"); + } + } + /// Reproduces the single-node "metadata prepare queue is full" wedge /// /// `checkpoint_if_needed` runs inside `on_replicate`, once per submit. diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs index 286d286e20..3b2e009afd 100644 --- a/core/metadata/src/impls/recovery.rs +++ b/core/metadata/src/impls/recovery.rs @@ -23,8 +23,9 @@ use consensus::{ ClientTable, ClientTableDecodeError, VsrState, VsrStateError, build_reply_message, build_reply_message_with, }; -use iggy_binary_protocol::consensus::{Operation, PrepareHeader}; +use iggy_binary_protocol::consensus::{CHECKSUM_UNSEALED, Operation, PrepareHeader}; use iggy_common::IggyError; +use journal::Journal as _; use journal::prepare_journal::{JournalError, PrepareJournal}; use journal::superblock::{ PingPongSuperblock, SLOT_FILE_NAMES, SuperblockContents, SuperblockStore, @@ -307,6 +308,13 @@ pub struct RecoveredMetadata { /// they stay journal-only until the recovered primary re-replicates them /// (or a backup sees the commit point advance past them). pub last_journaled_op: Option, + /// First op replay could not connect to its predecessor, `None` when the + /// replayed range is one unbroken chain. + /// + /// `Some(op)` means entries at and above `op` were truncated and must come back + /// from the cluster. `last_journaled_op` stops below it, which keeps the restored + /// head, the re-pipeline range, and the recovery barrier honest. + pub chain_break_op: Option, } /// Recover metadata state from disk. @@ -545,10 +553,35 @@ where let mut last_applied_op: Option = None; let mut last_journaled_op: Option = None; + let mut chain_break_op: Option = None; + let mut previous: Option = None; for header in &headers_to_replay { - // TODO: Check hash chain integrity against `previous_header`. On a - // same-view break, stop replay here and mark the remaining entries for - // repair via VSR instead of panicking. + // Stop at the first op that does not connect to the one before it. Applying + // across a hole replays effects onto a state machine that never saw the + // missing op, and nothing downstream re-checks it. + // + // The WAL scan does not cover this: it only fires on CONSECUTIVE ops with + // both ends sealed, so a gap reaches here. The first replayed op is exempt, + // since a snapshot records no checksum for its parent to chain to. + if let Some(previous) = previous { + let gap = previous.op + 1 != header.op; + let broken_chain = previous.checksum != CHECKSUM_UNSEALED + && header.checksum != CHECKSUM_UNSEALED + && header.parent != previous.checksum; + if gap || broken_chain { + tracing::error!( + op = header.op, + previous_op = previous.op, + gap, + broken_chain, + "metadata WAL does not connect at this op; stopping replay and dropping the \ + suffix for VSR repair" + ); + chain_break_op = Some(header.op); + break; + } + } + previous = Some(*header); last_journaled_op = Some(header.op); if header.op > commit_watermark { @@ -626,6 +659,22 @@ where last_applied_op = Some(header.op); } + // `truncate_from`, never `drain`: the removed ops must stay refillable, so the + // snapshot watermark stays put. Leaving them resident would make `append` refuse + // the slot, failing repair on exactly the ops it exists to fix. + if let Some(break_op) = chain_break_op { + let removed = journal + .truncate_from(break_op) + .await + .map_err(RecoveryError::Io)?; + tracing::warn!( + break_op, + removed, + last_journaled_op, + "dropped the disconnected metadata WAL suffix; the cluster re-supplies these ops" + ); + } + Ok(RecoveredMetadata { journal, snapshot, @@ -636,6 +685,7 @@ where client_table, last_applied_op, last_journaled_op, + chain_break_op, }) } @@ -976,6 +1026,130 @@ mod tests { assert_eq!(recovered.journal.last_op(), Some(3)); } + /// A prepare sealed the way a live primary seals one: `parent` chains to the + /// previous op's identity and `checksum` is that identity. + fn make_chained_prepare(op: u64, commit: u64, parent: u128) -> Message { + let mut message = make_prepare_with_commit(op, commit, 32); + let header = bytemuck::checked::from_bytes_mut::( + &mut message.as_mut_slice()[..HEADER_SIZE], + ); + header.parent = parent; + let checksum = header.identity_checksum(); + header.checksum = checksum; + message + } + + #[compio::test] + async fn recover_stops_at_a_gap_and_drops_the_disconnected_suffix() { + // Ops 1-3 then 5: op 4 never landed. Replaying 5 over a state machine that + // never saw 4 diverges silently, and the WAL scan waves this through -- + // its chain check only fires on CONSECUTIVE ops, since a gap is also what + // ordinary compaction leaves behind. + let dir = tempdir().unwrap(); + let metadata_dir = dir.path().join("metadata"); + std::fs::create_dir_all(&metadata_dir).unwrap(); + + { + let journal = PrepareJournal::open(&metadata_dir.join("journal.wal"), 0) + .await + .unwrap(); + for op in 1..=3u64 { + journal + .append(make_prepare_with_commit(op, op, 32)) + .await + .unwrap(); + } + journal + .append(make_prepare_with_commit(5, 5, 32)) + .await + .unwrap(); + journal.storage_ref().fsync().await.unwrap(); + } + + let recovered = recover::( + dir.path(), + CLUSTERED, + journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, + |_| {}, + ) + .await + .unwrap(); + + assert_eq!(recovered.chain_break_op, Some(5)); + assert_eq!( + recovered.last_applied_op, + Some(3), + "op 5 must not apply across the hole at op 4" + ); + assert_eq!( + recovered.last_journaled_op, + Some(3), + "the restored head stops below the break, so nothing re-pipelines it" + ); + assert_eq!( + recovered.journal.last_op(), + Some(3), + "the disconnected entry is dropped so repair can journal the cluster's op 5" + ); + assert_eq!( + recovered.journal.snapshot_op(), + 0, + "truncating a suffix must leave the watermark, or the ops stop being refillable" + ); + } + + #[compio::test] + async fn recover_stops_at_a_broken_chain_between_consecutive_ops() { + // Consecutive and sealed on both ends, but op 3 names a parent that is not + // op 2: a fork left by a crash mid view change. Ops are appended out of + // ascending file order so the scan's own chain check does not fire first. + let dir = tempdir().unwrap(); + let metadata_dir = dir.path().join("metadata"); + std::fs::create_dir_all(&metadata_dir).unwrap(); + + { + let journal = PrepareJournal::open(&metadata_dir.join("journal.wal"), 0) + .await + .unwrap(); + let first = make_chained_prepare(1, 1, 0); + let first_checksum = first.header().checksum; + journal.append(first).await.unwrap(); + let second = make_chained_prepare(2, 2, first_checksum); + journal.append(second).await.unwrap(); + // Parent of a prepare that is not op 2. + journal + .append(make_chained_prepare(3, 3, 0xdead_beef)) + .await + .unwrap(); + journal.storage_ref().fsync().await.unwrap(); + } + + let recovered = recover::( + dir.path(), + CLUSTERED, + journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, + |_| {}, + ) + .await; + + // The WAL scan reaches this first and refuses boot: consecutive ops, both + // sealed, chain broken, with no entry after it is only a tail. Either + // outcome is a refusal to apply the fork; what must never happen is a + // clean recovery that replayed op 3. + match recovered { + Err(RecoveryError::Journal(_) | RecoveryError::Io(_)) => {} + Ok(recovered) => { + assert!( + recovered.last_applied_op < Some(3), + "op 3 forks the chain and must not be applied" + ); + } + Err(other) => panic!("unexpected recovery error: {other:?}"), + } + } + #[compio::test] async fn recover_applies_only_the_committed_prefix() { let dir = tempdir().unwrap(); diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index dad3f70e34..bb7de4290f 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -39,6 +39,7 @@ use consensus::{ build_reply_message, drain_committable_prefix, emit_namespace_progress_event, emit_partition_diag, emit_sim_event, fence_old_prepare_by_commit, replicate_preflight, replicate_to_next_in_chain, send_prepare_ok as send_prepare_ok_common, + verify_prepare_integrity, }; use iggy_binary_protocol::requests::consumer_offsets::{ DeleteConsumerOffset2Request, DeleteConsumerOffsetRequest, StoreConsumerOffset2Request, @@ -1341,6 +1342,7 @@ where &mut self, consumer: PollingConsumer, args: &PollingArgs, + validate_checksum: bool, ) -> PollPlan { // Reads the durable commit frontier (`self.offset`, stored only on // commit). Also used below as the poll's high-water bound: this function @@ -1452,6 +1454,7 @@ where segments, start_position, namespace_raw: self.namespace().inner(), + validate_checksum, }; // Snapshot the resident journal tail now (on the pump, under the // borrow) so the straddle splice runs off-task on owned data with no @@ -2021,6 +2024,22 @@ where pub async fn on_replicate(&mut self, message: Message) { self.clear_pending_consumer_offset_commits_if_view_changed(); let header = *message.header(); + // Same reason as the metadata plane: `checksum` is compared as an opaque token + // downstream, so a corrupted frame passes whenever its flipped value satisfies + // those comparisons. + if let Err(reason) = verify_prepare_integrity(&header, message.as_slice()) { + emit_partition_diag( + tracing::Level::WARN, + &PartitionDiagEvent::new( + ReplicaLogContext::from_consensus(self.consensus(), PlaneKind::Partitions), + "discarding prepare that failed its own integrity check", + ) + .with_operation(header.operation) + .with_op(header.op) + .with_reason(reason), + ); + return; + } let current_op = { let consensus = self.consensus(); match replicate_preflight(consensus, &header) { @@ -4952,7 +4971,10 @@ mod tests { let path = format!("{dir}/{consumer_id}"); let read_disk = |p: &str| -> u64 { let bytes = std::fs::read(p).expect("offset file exists"); - u64::from_le_bytes(bytes.try_into().expect("offset file is 8 bytes")) + match crate::offset_storage::decode_offset_record(&bytes) { + crate::offset_storage::OffsetRecord::Value { offset, .. } => offset, + other => panic!("offset file must hold a readable value, got {other:?}"), + } }; // Reordered auto-commits: the later op (109) trails the earlier (114). @@ -5035,7 +5057,10 @@ mod tests { let path = format!("{dir}/{consumer_id}"); let read_disk = |p: &str| -> u64 { let bytes = std::fs::read(p).expect("offset file exists"); - u64::from_le_bytes(bytes.try_into().expect("offset file is 8 bytes")) + match crate::offset_storage::decode_offset_record(&bytes) { + crate::offset_storage::OffsetRecord::Value { offset, .. } => offset, + other => panic!("offset file must hold a readable value, got {other:?}"), + } }; // Simulate the previous process run: the file already holds 114. @@ -5198,6 +5223,7 @@ mod tests { // open exhausts retries -> the walk must fault-close before segment two. let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir), + validate_checksum: true, segments: vec![ DiskSegment { start_offset: 0, @@ -5283,6 +5309,7 @@ mod tests { let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir), + validate_checksum: true, segments: vec![ DiskSegment { start_offset: 0, @@ -5315,6 +5342,77 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// A segment whose bytes decode cleanly but do not match their own + /// `batch_checksum`: bit rot at rest, not a torn write. Unverified, the batch is + /// served and a consumer reads data provably not what was written. + /// + /// Detection only, per the operator knob: the poll fails closed and reports, with + /// no attempt to repair. + #[compio::test] + async fn read_disk_faults_closed_on_batch_checksum_mismatch() { + let namespace = IggyNamespace::new(1, 1, 0); + let dir = std::env::temp_dir().join(format!( + "iggy-read-disk-bitrot-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock after epoch") + .as_nanos(), + )); + compio::fs::create_dir_all(&dir) + .await + .expect("create temp partition dir"); + let partition_dir = dir.to_string_lossy().into_owned(); + + // Structurally valid with one payload byte flipped, so every length and + // offset still decodes and only the checksum disagrees. + let mut record = build_segment_record(namespace, 0); + let last = record.len() - 1; + record[last] ^= 0x01; + let record_len = record.len() as u64; + let path = format!("{partition_dir}/{:0>20}.log", 0u64); + { + let mut file = compio::fs::File::create(&path) + .await + .expect("create segment file"); + let (written, _) = file.write_all_at(record, 0).await.into(); + written.expect("write segment record"); + file.sync_all().await.expect("flush segment file"); + } + + let plan = |validate_checksum| DiskReadPlan { + partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum, + segments: vec![DiskSegment { + start_offset: 0, + persisted: record_len, + read_state: None, + }], + start_position: 0, + namespace_raw: namespace.inner(), + }; + let query = MessageLookup::Offset { + offset: 0, + count: 10, + ceiling: u64::MAX, + }; + + let outcome = plan(true).read_disk(query).await; + assert!( + matches!(outcome, DiskReadOutcome::Faulted), + "a batch that fails its own checksum must fault-close" + ); + + // The knob is really a knob: with verification off the same bytes serve. + let outcome = plan(false).read_disk(query).await; + assert!( + matches!(outcome, DiskReadOutcome::Matched { .. }), + "with verification off the corrupt batch is served" + ); + + let _ = std::fs::remove_dir_all(&dir); + } + /// A simulated (file-less) partition has no segment files by design, so a /// disk poll with no dir must stay `Empty`: the caller then serves the /// resident journal tier, the sim's only tier. @@ -5329,6 +5427,7 @@ mod tests { }], start_position: 0, namespace_raw: IggyNamespace::new(1, 1, 0).inner(), + validate_checksum: true, }; let outcome = plan @@ -5359,6 +5458,7 @@ mod tests { }], start_position: 0, namespace_raw: IggyNamespace::new(1, 1, 0).inner(), + validate_checksum: true, }; let outcome = plan @@ -5417,6 +5517,7 @@ mod tests { let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -5447,6 +5548,7 @@ mod tests { let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -5506,6 +5608,7 @@ mod tests { let handle = SealedSegmentHandle::default(); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -5589,6 +5692,7 @@ mod tests { let handle = SealedSegmentHandle::default(); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: log_len, @@ -5687,6 +5791,7 @@ mod tests { let handle = SealedSegmentHandle::default(); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: log_len, @@ -5764,6 +5869,7 @@ mod tests { let handle = Rc::clone(&partition.log.sealed_read_state()[0]); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -5811,6 +5917,7 @@ mod tests { // unlinked pre-purge inode. let resumed = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -5839,6 +5946,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: IggyByteSize::from(1024 * 1024), enforce_fsync: false, + validate_checksum: true, segment_size: IggyByteSize::from(1024 * 1024), encryptor: None, } diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs index a25aed657b..a316488758 100644 --- a/core/partitions/src/iggy_partitions.rs +++ b/core/partitions/src/iggy_partitions.rs @@ -412,8 +412,10 @@ where // `build_poll_plan` touches the partition's sealed-read-handle LRU, so it // needs `&mut`. Sound on the pump: it is fully synchronous (no `.await` // inside), so no sibling task can realloc the partitions vec under it. + // Read the knob first: the `&mut` borrow below covers `self.config` too. + let validate_checksum = self.config.validate_checksum; let partition = self.get_mut_by_ns(namespace)?; - Some(partition.build_poll_plan(consumer, args)) + Some(partition.build_poll_plan(consumer, args, validate_checksum)) } /// Read a consumer's stored offset + the partition commit offset. Fully @@ -654,6 +656,28 @@ mod tests { ) } + /// `build_partition` for a replicated group. The replica count is what + /// decides whether the journal retains evicted entries for repair, so a + /// single-replica partition cannot exercise anything that reads the ring. + fn build_replicated_partition() -> IggyPartition { + let namespace = IggyNamespace::new(1, 1, 0); + let consensus = VsrConsensus::new( + TEST_CLUSTER, + 0, + 3, + namespace.inner(), + IggyMessageBus::new(0), + LocalPipeline::new(), + ); + consensus.init(); + IggyPartition::with_in_memory_storage( + Arc::new(PartitionStats::default()), + consensus, + IggyByteSize::from(1024 * 1024), + false, + ) + } + /// One-message `SendMessages` journal entry stamped at `op` / `base_offset`. /// Reuses the production blob builder + checksum stamping so the entry /// decodes through `decode_prepare_slice` and indexes into `offset_to_op`, @@ -796,6 +820,67 @@ mod tests { ); } + /// A flush evicts the committed prefix up to and INCLUDING `commit_max`, so + /// a caught-up replica keeps no resident header at its own commit point. The + /// `DoViewChange` suffix is floored there and cannot nack it, so reading the + /// resident headers alone sends the commit point out blank, which a quorum of + /// senders turns into a view change that never starts. + /// + /// The entry is still servable (`repair_entry` answers from the evicted + /// ring), so the suffix reads through `repair_header`, over the same range. + #[compio::test] + async fn evicted_commit_point_still_answers_for_the_view_change_suffix() { + let namespace = IggyNamespace::new(1, 1, 0); + let partition = build_replicated_partition(); + + for offset in 0..=2u64 { + partition + .log + .journal() + .inner + .append(build_send_messages_entry(namespace, offset + 1, offset)) + .await + .expect("append journal entry"); + } + + let commit_max = 3; + let prefix = partition.log.journal().inner.committed_prefix(commit_max); + assert_eq!(prefix.len(), 3, "the whole log is committed and flushable"); + partition + .log + .journal() + .inner + .evict_prefix(prefix.len()) + .await; + + assert!( + partition + .log + .journal() + .inner + .header_by_op(commit_max) + .is_none(), + "the flush evicted the commit point from the resident headers", + ); + assert!( + partition + .log + .journal() + .inner + .repair_entry(commit_max) + .is_some(), + "yet the entry is still servable from the evicted ring", + ); + + let header = partition + .log + .journal() + .inner + .repair_header(commit_max) + .expect("the commit point must stay describable for the DVC suffix"); + assert_eq!(header.op, commit_max); + } + /// The resident journal holds replicated-but-uncommitted prepares ahead of /// the commit frontier. A poll must clamp at `ceiling` (the commit offset) /// so it never returns a dirty read of view-change-rollbackable data, even diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs index c3a5b96575..346a0f1cb1 100644 --- a/core/partitions/src/journal.rs +++ b/core/partitions/src/journal.rs @@ -359,6 +359,31 @@ impl PartitionJournal { .map(|(_, entry)| entry.clone()) } + /// The header at `op`, over exactly the range [`Self::repair_entry`] serves. + /// + /// NOT [`Self::header_by_op`], which reads the resident headers alone. The + /// committed prefix is evicted from those the moment its bytes reach a + /// segment, up to and including `commit_max`, so a `DoViewChange` built off + /// the resident headers reports its own commit point blank. The merge scans + /// the commit point and cannot discard it, so a quorum of such senders is + /// undecidable and the view never starts (`dvc_merge::merge_dvc_quorum`). + /// The entry is still servable from the evicted ring, which is what makes + /// the blank wrong rather than merely pessimistic. + /// + /// The ring drops from the front, so the highest evicted op -- the commit + /// point of the last flush -- is the last thing it forgets. + pub fn repair_header(&self, op: u64) -> Option { + if let Some(header) = self.header_by_op(op) { + return Some(header); + } + let ring = unsafe { &*self.evicted_ring.get() }; + let (_, entry) = ring.iter().find(|(ring_op, _)| *ring_op == op)?; + let header_bytes = entry.as_slice().get(..PREPARE_HEADER_SIZE)?; + bytemuck::checked::try_from_bytes::(header_bytes) + .ok() + .copied() + } + /// Oldest op this journal can still serve for repair (ring front, else /// resident head), or `None` when it holds nothing at all. pub fn repair_retained_from(&self) -> Option { diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs index 75b4c70ca8..2b62402cf6 100644 --- a/core/partitions/src/lib.rs +++ b/core/partitions/src/lib.rs @@ -25,7 +25,7 @@ mod iggy_partitions; mod journal; mod log; mod messages_writer; -mod offset_storage; +pub mod offset_storage; mod poll_plan; mod segment; pub mod state_transfer; diff --git a/core/partitions/src/offset_storage.rs b/core/partitions/src/offset_storage.rs index 3938435448..654d5b192a 100644 --- a/core/partitions/src/offset_storage.rs +++ b/core/partitions/src/offset_storage.rs @@ -19,11 +19,84 @@ use compio::{ fs::{OpenOptions, create_dir_all, remove_file}, io::{AsyncReadAtExt, AsyncWriteAtExt}, }; -use iggy_common::IggyError; +use iggy_common::{IggyError, calculate_checksum}; use std::path::Path; const OFFSET_SIZE: usize = core::mem::size_of::(); +const CHECKSUM_SIZE: usize = core::mem::size_of::(); +/// Bytes a consumer-offset file holds: the offset, then a checksum over it. +/// +/// The offset is a consumer cursor reloaded unchanged on every restart, so a +/// flipped bit silently rewinds the consumer into redelivery or skips it forward. +pub const OFFSET_RECORD_SIZE: usize = OFFSET_SIZE + CHECKSUM_SIZE; + +/// What a consumer-offset file was found to hold. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OffsetRecord { + /// A usable offset. `checksummed` is false for a bare offset predating the + /// checksum, read as-is and upgraded by the next write. + Value { offset: u64, checksummed: bool }, + /// Shorter than the value: a crash between `persist_offset`'s truncate and write. + Torn, + /// The checksum does not describe the value stored beside it. + Corrupt { + offset: u64, + expected: u64, + found: u64, + }, +} + +/// Encode a consumer offset for persistence. +#[must_use] +pub fn encode_offset_record(offset: u64) -> [u8; OFFSET_RECORD_SIZE] { + let mut record = [0u8; OFFSET_RECORD_SIZE]; + record[..OFFSET_SIZE].copy_from_slice(&offset.to_le_bytes()); + let checksum = calculate_checksum(&record[..OFFSET_SIZE]); + record[OFFSET_SIZE..].copy_from_slice(&checksum.to_le_bytes()); + record +} + +/// Decode whatever a consumer-offset file contained. +/// +/// A file of exactly one offset predates the checksum and is accepted. A partly +/// written checksum region reads as the bare offset for the same reason: the record +/// is written in one call, so the low bytes are the complete new value. +#[must_use] +pub fn decode_offset_record(bytes: &[u8]) -> OffsetRecord { + let Some(value) = bytes.first_chunk::() else { + return OffsetRecord::Torn; + }; + let offset = u64::from_le_bytes(*value); + let Some(stored) = bytes + .get(OFFSET_SIZE..) + .and_then(<[u8]>::first_chunk::) + else { + return OffsetRecord::Value { + offset, + checksummed: false, + }; + }; + let found = u64::from_le_bytes(*stored); + let expected = calculate_checksum(value); + if found == expected { + OffsetRecord::Value { + offset, + checksummed: true, + } + } else { + OffsetRecord::Corrupt { + offset, + expected, + found, + } + } +} + +/// Overwrite a consumer-offset file with `offset` and a checksum over it. +/// +/// # Errors +/// [`IggyError`] when the directory, file, or write cannot be created or completed. pub async fn persist_offset(path: &str, offset: u64, enforce_fsync: bool) -> Result<(), IggyError> { // No `exists()` probe first: that is a BLOCKING `std::path` stat on the pump // in front of every write, which serialises a batched fan-out on stats @@ -42,8 +115,7 @@ pub async fn persist_offset(path: &str, offset: u64, enforce_fsync: bool) -> Res .open(path) .await .map_err(|_| IggyError::CannotOpenConsumerOffsetsFile(path.to_owned()))?; - let buf = offset.to_le_bytes(); - file.write_all_at(buf, 0) + file.write_all_at(encode_offset_record(offset), 0) .await .0 .map_err(|_| IggyError::CannotWriteToFile)?; @@ -57,19 +129,23 @@ pub async fn persist_offset(path: &str, offset: u64, enforce_fsync: bool) -> Res Ok(()) } -/// Monotone counterpart of [`persist_offset`] for a server auto-commit op: -/// folds `max(current_on_disk, offset)` and returns the value now on disk, -/// skipping the write when the file already holds it. Disk-tier polls -/// replicate their auto-committed offsets in IO-completion order, so a -/// committed op can carry a lower offset than an earlier one; a plain -/// overwrite would leave the file rewound and a restart would reload the -/// stale value and re-deliver. The on-disk value is committed-only (this path -/// never writes the eager serving map), so the fold is identical on every -/// replica applying the same op order. +/// Monotone counterpart of [`persist_offset`] for a server auto-commit op. +/// +/// Folds `max(current_on_disk, offset)` and returns the value now on disk, skipping +/// the write when the file already holds it. Disk-tier polls replicate their +/// auto-committed offsets in IO-completion order, so a committed op can carry a lower +/// offset than an earlier one, and a plain overwrite would leave the file rewound for +/// a restart to reload and re-deliver. The on-disk value is committed-only, so the +/// fold is identical on every replica applying the same op order. +/// +/// The read makes this the cold-key path only: once the caller's persisted-offset +/// tracker knows the file's value, warm commits persist with a blind +/// [`persist_offset`] and skip covered offsets without reading. /// -/// The read makes this the cold-key path only: once the caller's -/// persisted-offset tracker knows the file's value, warm commits persist with -/// a blind [`persist_offset`] and skip covered offsets without any file read. +/// # Errors +/// [`IggyError`] when the file cannot be read or written, or when the value on disk +/// fails its checksum: folding against a cursor provably not the one written could +/// rewind or skip the consumer. pub async fn persist_offset_max( path: &str, offset: u64, @@ -89,6 +165,10 @@ pub async fn persist_offset_max( /// files, so the commit-path reader must agree or a torn file turns every /// later commit-apply into an error. Real I/O errors still propagate: mapping /// them to `None` would silently rewind a valid higher offset. +/// +/// A checksum mismatch is an error, not `None`. `None` means "no offset recorded", +/// which the caller folds as `max(absent, incoming)` and overwrites; doing that to a +/// failed-checksum file discards a cursor that may have been far ahead. async fn read_persisted_offset(path: &str) -> Result, IggyError> { if !Path::new(path).exists() { return Ok(None); @@ -98,17 +178,43 @@ async fn read_persisted_offset(path: &str) -> Result, IggyError> { .open(path) .await .map_err(|_| IggyError::CannotOpenConsumerOffsetsFile(path.to_owned()))?; - let buf = vec![0u8; OFFSET_SIZE]; - let compio::BufResult(read, buf) = file.read_exact_at(buf, 0).await; - match read { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), + // Read the whole record, falling back to a bare offset: a pre-checksum file is + // exactly `OFFSET_SIZE` long, so the first read reports EOF rather than failing. + let compio::BufResult(read, buf) = file.read_exact_at(vec![0u8; OFFSET_RECORD_SIZE], 0).await; + let bytes = match read { + Ok(()) => buf, + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => { + let compio::BufResult(read, legacy) = + file.read_exact_at(vec![0u8; OFFSET_SIZE], 0).await; + match read { + Ok(()) => legacy, + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => { + return Ok(None); + } + Err(_) => return Err(IggyError::CannotReadConsumerOffsets(path.to_owned())), + } + } Err(_) => return Err(IggyError::CannotReadConsumerOffsets(path.to_owned())), + }; + match decode_offset_record(&bytes) { + OffsetRecord::Value { offset, .. } => Ok(Some(offset)), + OffsetRecord::Torn => Ok(None), + OffsetRecord::Corrupt { + offset, + expected, + found, + } => { + tracing::error!( + path, + offset, + expected, + found, + "consumer offset file failed its checksum; refusing to fold a value that may \ + rewind or skip the consumer" + ); + Err(IggyError::CannotReadConsumerOffsets(path.to_owned())) + } } - let bytes: [u8; OFFSET_SIZE] = buf - .try_into() - .map_err(|_| IggyError::CannotReadConsumerOffsets(path.to_owned()))?; - Ok(Some(u64::from_le_bytes(bytes))) } /// Unlink a persisted consumer-offset file. A no-op if the file is absent. @@ -143,6 +249,96 @@ mod tests { dir } + #[test] + fn offset_record_round_trips() { + let record = encode_offset_record(114); + assert_eq!(record.len(), OFFSET_RECORD_SIZE); + assert_eq!( + decode_offset_record(&record), + OffsetRecord::Value { + offset: 114, + checksummed: true + } + ); + } + + #[test] + fn offset_record_accepts_a_bare_value_written_before_the_checksum() { + assert_eq!( + decode_offset_record(&114u64.to_le_bytes()), + OffsetRecord::Value { + offset: 114, + checksummed: false + } + ); + } + + #[test] + fn offset_record_rejects_either_half_flipped() { + // The point of the checksum: a flipped bit rewinds a consumer into redelivery + // or skips it forward, and nothing ever notices. + let mut value_flipped = encode_offset_record(114); + value_flipped[0] ^= 0x01; + assert!(matches!( + decode_offset_record(&value_flipped), + OffsetRecord::Corrupt { offset: 115, .. } + )); + + let mut checksum_flipped = encode_offset_record(114); + checksum_flipped[OFFSET_SIZE] ^= 0x01; + assert!(matches!( + decode_offset_record(&checksum_flipped), + OffsetRecord::Corrupt { offset: 114, .. } + )); + } + + #[test] + fn offset_record_partly_written_is_torn_below_the_value_and_bare_above_it() { + assert_eq!(decode_offset_record(&[]), OffsetRecord::Torn); + assert_eq!(decode_offset_record(&[0xAB; 7]), OffsetRecord::Torn); + + // One `write_all_at` writes the whole record, so a torn tail keeps the value. + let record = encode_offset_record(114); + assert_eq!( + decode_offset_record(&record[..OFFSET_SIZE + 3]), + OffsetRecord::Value { + offset: 114, + checksummed: false + } + ); + } + + #[compio::test] + async fn read_persisted_offset_rejects_a_corrupt_file() { + let dir = unique_temp_dir(); + let path = dir.join("42").to_string_lossy().into_owned(); + + persist_offset(&path, 114, false).await.expect("persist"); + let mut bytes = std::fs::read(&path).expect("offset file exists"); + bytes[0] ^= 0x01; + std::fs::write(&path, &bytes).expect("corrupt the file"); + + let result = read_persisted_offset(&path).await; + assert!( + matches!(result, Err(IggyError::CannotReadConsumerOffsets(_))), + "a corrupt cursor must not fold as absent, got {result:?}" + ); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[compio::test] + async fn read_persisted_offset_reads_a_legacy_bare_value() { + let dir = unique_temp_dir(); + let path = dir.join("42").to_string_lossy().into_owned(); + std::fs::write(&path, 114u64.to_le_bytes()).expect("write legacy file"); + + let read = read_persisted_offset(&path).await.expect("legacy file"); + assert_eq!(read, Some(114)); + + let _ = std::fs::remove_dir_all(&dir); + } + #[compio::test] async fn read_persisted_offset_absent_file_is_none() { let dir = unique_temp_dir(); diff --git a/core/partitions/src/poll_plan.rs b/core/partitions/src/poll_plan.rs index b618cd84ab..c7d8c50a39 100644 --- a/core/partitions/src/poll_plan.rs +++ b/core/partitions/src/poll_plan.rs @@ -39,13 +39,13 @@ use iggy_common::{ ConsumerGroupId, ConsumerGroupOffsets, ConsumerKind, ConsumerOffset, ConsumerOffsets, IggyError, }; use server_common::iobuf::{Frozen, Owned}; -use server_common::send_messages2::{COMMAND_HEADER_SIZE, decode_batch_slice}; +use server_common::send_messages2::{COMMAND_HEADER_SIZE, decode_batch_slice_verified}; use std::cell::{Cell, RefCell}; use std::hash::Hash; use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::Ordering; -use tracing::warn; +use tracing::{error, warn}; /// Byte cap for materializing a sealed segment's sparse index into its shared /// read-state handle. Index density is one entry per flush: at the default @@ -120,6 +120,9 @@ pub struct DiskReadPlan { pub(crate) segments: Vec, pub(crate) start_position: u64, pub(crate) namespace_raw: u64, + /// Whether to verify each batch's `batch_checksum` against the bytes read. + /// Detection only; a mismatch fails the poll closed and repairs nothing. + pub(crate) validate_checksum: bool, } pub struct DiskSegment { @@ -533,14 +536,23 @@ impl DiskReadPlan { faulted = true; break 'walk; }; - let consumed = walk_disk_chunk( + let ChunkWalk { consumed, corrupt } = walk_disk_chunk( &chunk, query, count, &mut matched, &mut fragments, &mut last_matching_offset, + self.validate_checksum, + self.namespace_raw, ); + if corrupt { + // A batch that does not match its own checksum. Fail closed like + // an IO fault: serving it hands a consumer data provably not what + // was written, and skipping ahead punches a silent gap. + faulted = true; + break 'walk; + } if consumed == 0 { if (len as u64) >= persisted - position { // The whole remainder fit yet no complete batch @@ -883,6 +895,7 @@ pub fn upsert_offset_max( /// chunk, pushing matching fragments. Returns bytes consumed: the start /// of the first batch that did not fully fit in the chunk (the caller /// re-reads from there), or the chunk end when everything decoded. +#[allow(clippy::too_many_arguments)] fn walk_disk_chunk( chunk: &Frozen<4096>, query: MessageLookup, @@ -890,15 +903,37 @@ fn walk_disk_chunk( matched: &mut u32, fragments: &mut PollFragments<4096>, last_matching_offset: &mut Option, -) -> usize { + validate_checksum: bool, + namespace_raw: u64, +) -> ChunkWalk { let bytes: &[u8] = chunk; let mut cursor = 0usize; while *matched < count && cursor + COMMAND_HEADER_SIZE <= bytes.len() { - let Ok(batch) = decode_batch_slice(&bytes[cursor..]) else { - // Incomplete tail batch (or corrupt data): hand the position - // back so the caller can re-read or bail. - break; + let batch = match decode_batch_slice_verified(&bytes[cursor..], validate_checksum) { + Ok(batch) => batch, + Err(IggyError::InvalidBatchChecksum(found, expected, base_offset)) => { + // Distinguished from the incomplete-tail case below: this batch is + // entirely present and fails its own checksum, so it is damaged at rest. + error!( + target: "iggy.partitions.diag", + plane = "partitions", + namespace_raw, + base_offset, + expected, + found, + position = cursor, + "disk poll: batch checksum mismatch; segment is corrupt at rest" + ); + return ChunkWalk { + consumed: cursor.min(bytes.len()), + corrupt: true, + }; + } + Err(_) => { + // Incomplete tail batch: hand the position back to re-read or bail. + break; + } }; let total_size = batch.header.total_size(); @@ -919,7 +954,17 @@ fn walk_disk_chunk( cursor += total_size; } - cursor.min(bytes.len()) + ChunkWalk { + consumed: cursor.min(bytes.len()), + corrupt: false, + } +} + +/// How far [`walk_disk_chunk`] got, and whether it stopped on corruption rather +/// than on a batch that simply did not fit in the chunk. +struct ChunkWalk { + consumed: usize, + corrupt: bool, } #[cfg(test)] diff --git a/core/partitions/src/types.rs b/core/partitions/src/types.rs index f052ff75f0..9636c29df5 100644 --- a/core/partitions/src/types.rs +++ b/core/partitions/src/types.rs @@ -269,6 +269,13 @@ pub struct PartitionsConfig { pub size_of_messages_required_to_save: IggyByteSize, /// Whether to enforce fsync after writes. pub enforce_fsync: bool, + /// Whether a disk poll verifies each batch's `batch_checksum` against the bytes + /// it just read. + /// + /// Detection only: a mismatch fails the poll closed and is reported, with no + /// attempt to repair. The alternative is serving bytes provably not the ones + /// written, which reads to a consumer as ordinary data. + pub validate_checksum: bool, /// Maximum size of a single segment before rotation. pub segment_size: IggyByteSize, /// Server-side at-rest encryption. Applied ONCE, on the primary at diff --git a/core/sdk/src/vsr.rs b/core/sdk/src/vsr.rs index 9bc5d1cd16..bc8513cadc 100644 --- a/core/sdk/src/vsr.rs +++ b/core/sdk/src/vsr.rs @@ -38,7 +38,7 @@ use iggy_binary_protocol::requests::consumer_offsets::{ use iggy_binary_protocol::requests::messages::SendMessagesHeader; use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; use iggy_binary_protocol::{WireIdentifier, WirePartitioning}; -use iggy_common::{IggyError, eviction_reason_to_error}; +use iggy_common::{IggyError, calculate_checksum, eviction_reason_to_error}; const NON_REPLICATED_CODE_RANGE: std::ops::Range = 0..4; @@ -122,6 +122,15 @@ pub(crate) fn encode_request_header( } } }; + // Stamped only for ops the server's `ClientTable` dedups. Partition ops are + // at-least-once with no reply cache to poison, and theirs are the large payloads, + // already covered client-side by `batch_checksum` over the same bytes. + // NonReplicated ops bypass dedup too. + let request_checksum = if operation.is_partition() || operation == Operation::NonReplicated { + 0 + } else { + u128::from(calculate_checksum(payload)) + }; let namespace = namespace_for_request(code, payload, operation)?; let total_size = HEADER_SIZE .checked_add(payload.len()) @@ -139,6 +148,11 @@ pub(crate) fn encode_request_header( request: request_id, session: session_id, namespace, + // Lets the client table tell a genuine retry from a `request` number reused + // for different arguments. Zero means unstamped, which is what an SDK + // predating this sends. A server that rewrites the body (PAT, password) + // carries it through untouched, so it keeps describing what the client sent. + request_checksum, // Zeroed: the field is "informational" -- the server copies it into // `ReplyHeader.timestamp` for RTT but nothing else reads it. Paying // a `clock_gettime` syscall per encoded request (formerly held the @@ -633,6 +647,27 @@ mod tests { assert_eq!(decode_request_header(&second).namespace, 0); } + #[test] + fn request_checksum_is_stamped_only_for_deduped_operations() { + // The stamp exists to stop a reused `request` number returning the wrong + // cached reply, so it is worth its hashing pass only where `ClientTable` + // dedups. Partition payloads are the large ones and carry `batch_checksum` + // over the same bytes already; hashing them again is pure cost. + let mut session = ConsensusSession::with_client_id(42); + session.bind(99); + let payload = Bytes::from_static(b"payload"); + + let deduped = + encode_contiguous_request(&mut session, CREATE_STREAM_CODE, &payload).unwrap(); + assert_eq!( + decode_request_header(&deduped).request_checksum, + u128::from(calculate_checksum(&payload)), + ); + + let ping = encode_contiguous_request(&mut session, PING_CODE, &Bytes::new()).unwrap(); + assert_eq!(decode_request_header(&ping).request_checksum, 0); + } + #[test] fn ping_uses_non_replicated_operation() { let mut session = ConsensusSession::with_client_id(42); diff --git a/core/server-ng/config.toml b/core/server-ng/config.toml index 10a529930a..e96dc16a55 100644 --- a/core/server-ng/config.toml +++ b/core/server-ng/config.toml @@ -917,6 +917,9 @@ key_file = "core/certs/iggy_key.pem" # Depth of the metadata prepare queue: how many uncommitted metadata ops # may be in flight at once. Submits beyond it are rejected with the # transient "metadata prepare queue is full" and retried by the SDK. +# Capped at 127 by the view-change wire format: a DoViewChange describes the +# uncommitted suffix with one nack bit and one present bit per entry in a u128 each, +# so a deeper queue produces entries a view change can neither adopt nor prove dead. prepare_queue_depth = 32 # Size of the metadata WAL's in-memory index, in slots (one committed but @@ -940,7 +943,9 @@ clients_table_max = 8192 # consumer-offset ops may be in flight at once for that partition. Submits past # it spill into a request queue of twice this depth; once both are full the # server drops the request without a reply and the client retries on its own -# request timeout. Must be > 0 and <= 256. +# request timeout. Must be > 0 and <= 127: the ceiling is the view-change wire, not +# memory. A DoViewChange describes the uncommitted suffix with one bit per op in a +# u128 bitset, and this depth bounds that suffix. prepare_queue_depth = 32 # Entries the evicted ring retains per multi-replica partition for journal diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs index 57994d9cc9..edbe4a7976 100644 --- a/core/server-ng/src/bootstrap.rs +++ b/core/server-ng/src/bootstrap.rs @@ -1677,6 +1677,7 @@ async fn build_shard_for_thread( .partition .size_of_messages_required_to_save, enforce_fsync: config.system.partition.enforce_fsync, + validate_checksum: config.system.partition.validate_checksum, segment_size: config.system.segment.size, encryptor, }, @@ -1959,6 +1960,22 @@ const _: () = assert!( configs::ng_cluster::STATE_CHUNK_HEADER_LEN == size_of::() as u64 ); +// Both prepare-queue ceilings are pinned by the view-change wire, not by memory: a +// `DoViewChange` carries the sender's suffix spanning `commit..=op` with one nack +// bit and one present bit per entry, each bitset a single `u128`. The depth bounds +// `op - commit`, so a depth at or above `DVC_HEADERS_MAX` produces entries the new +// primary can neither adopt nor prove dead. Strictly less than, because the head op +// needs the reserved slot. +const _: () = + assert!(configs::ng_metadata::MAX_METADATA_PREPARE_QUEUE_DEPTH < consensus::DVC_HEADERS_MAX); +const _: () = + assert!(configs::ng_partition::MAX_PARTITION_PREPARE_QUEUE_DEPTH < consensus::DVC_HEADERS_MAX); +// `DVC_HEADERS_MAX` is a bare literal in both the wire crate, which sizes the +// bitsets, and the consensus crate, which cannot depend on it the other way around. +// Same u128, so a drift lets one side address entries the other cannot. +const _: () = + assert!(consensus::DVC_HEADERS_MAX == iggy_binary_protocol::consensus::DVC_HEADERS_MAX); +const _: () = assert!(consensus::DVC_HEADERS_MAX == u128::BITS as usize); /// Convert a consensus-timer interval to whole ticks, floored at one tick so a /// sub-tick value still fires and saturated on overflow. fn duration_to_ticks(interval: Duration) -> u64 { diff --git a/core/server-ng/src/dispatch.rs b/core/server-ng/src/dispatch.rs index cbae3f2572..cd0ed34ed7 100644 --- a/core/server-ng/src/dispatch.rs +++ b/core/server-ng/src/dispatch.rs @@ -49,7 +49,7 @@ use crate::responses::{ use crate::session_manager::SessionManager; use crate::snapshot; use crate::users::maybe_rewrite_user_password_request; -use crate::wire::{request_body, usize_to_u32}; +use crate::wire::{request_body, usize_to_u32, verify_request_checksum}; use bytes::Bytes; use configs::server_ng::NgSystemConfig; use consensus::{ @@ -761,6 +761,37 @@ async fn handle_client_request( } }; + // The last point that still sees the body the CLIENT sent; every rewrite below + // substitutes server-chosen bytes and carries the stamp through unchanged. + if let Err(error) = verify_request_checksum(&request) { + warn!( + transport_client_id, + operation = ?request.header().operation, + request = request.header().request, + "dropping client request whose body does not match its own checksum" + ); + let commit = current_metadata_commit(shard); + let reply = build_deny_reply( + request.header(), + transport_client_id, + 0, + commit, + error.as_code(), + ); + if let Err(send_error) = shard + .bus + .send_to_client(transport_client_id, reply.into_generic().into_frozen()) + .await + { + warn!( + transport_client_id, + error = %send_error, + "failed to send request-checksum deny reply" + ); + } + return; + } + ensure_transport_connection(shard, sessions, transport_client_id); // Any request is liveness proof, not just PING: an idle-but-active client @@ -3098,12 +3129,13 @@ mod tests { client, request, user_id: 0, - checksum: 42, namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; } - message + // A real identity, not a placeholder: `on_replicate` recomputes it before the + // prepare reaches the WAL, so an arbitrary value reads as transit corruption. + consensus::seal_prepare_checksum(message) } /// Regression test for the production failure chain "CLI stream @@ -3167,6 +3199,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, + validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), encryptor: None, }, @@ -3285,6 +3318,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, + validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), encryptor: None, }, @@ -3408,6 +3442,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, + validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), encryptor: None, }, @@ -3470,6 +3505,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, + validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), encryptor: None, }, diff --git a/core/server-ng/src/offset_recovery.rs b/core/server-ng/src/offset_recovery.rs index 2e70291938..ebe6ce935b 100644 --- a/core/server-ng/src/offset_recovery.rs +++ b/core/server-ng/src/offset_recovery.rs @@ -20,12 +20,14 @@ //! Forked from `server::streaming::partitions::storage` (the legacy //! `load_consumer_offsets` / `load_consumer_group_offsets`) so server-ng //! owns the loaders for the offset files its own persistence path writes, -//! without depending on the legacy `server` crate. The on-disk format is -//! shared with the legacy server today: one file per consumer (numeric -//! file name = consumer id) holding a single little-endian `u64` offset. +//! without depending on the legacy `server` crate. One file per consumer (numeric +//! file name = consumer id) holding a little-endian `u64` offset then a checksum over +//! it; see [`partitions::offset_storage`]. The legacy server stays compatible both +//! ways: it reads the first eight bytes and stops, and a file it wrote itself decodes +//! here as unchecksummed. use iggy_common::{ConsumerGroupId, ConsumerKind, ConsumerOffset, IggyError}; -use std::io::Read; +use partitions::offset_storage::{OffsetRecord, decode_offset_record}; use std::sync::atomic::AtomicU64; use tracing::{error, trace, warn}; @@ -169,23 +171,38 @@ pub fn load_consumer_group_offsets( } fn read_offset_file(path: &str, offset_kind: &'static str) -> Option { - let mut file = match std::fs::File::open(path) { - Ok(file) => file, + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, Err(e) => { warn!( - "{COMPONENT} (error: {e}) - failed to open offset file, \ + "{COMPONENT} (error: {e}) - failed to read offset file, \ path: {path}, skipping." ); return None; } }; - let mut offset = [0; 8]; - if let Err(e) = file.read_exact(&mut offset) { - warn!( - "{COMPONENT} (error: {e}) - failed to read {offset_kind} from file \ - (truncated or corrupt?), path: {path}, skipping." - ); - return None; + match decode_offset_record(&bytes) { + OffsetRecord::Value { offset, .. } => Some(AtomicU64::new(offset)), + OffsetRecord::Torn => { + warn!( + "{COMPONENT} - failed to read {offset_kind} from file (truncated), \ + path: {path}, skipping." + ); + None + } + // Skipped rather than loaded: resuming from a cursor provably not the one + // written reads as ordinary redelivery or a gap, never as corruption. + OffsetRecord::Corrupt { + offset, + expected, + found, + } => { + error!( + "{COMPONENT} - {offset_kind} file failed its checksum \ + (offset: {offset}, expected: {expected}, found: {found}), \ + path: {path}, skipping." + ); + None + } } - Some(AtomicU64::new(u64::from_le_bytes(offset))) } diff --git a/core/server-ng/src/partition_reconciler.rs b/core/server-ng/src/partition_reconciler.rs index de8c036fc2..2706a6cea8 100644 --- a/core/server-ng/src/partition_reconciler.rs +++ b/core/server-ng/src/partition_reconciler.rs @@ -1434,6 +1434,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, + validate_checksum: true, segment_size: config.system.segment.size, encryptor: None, }, diff --git a/core/server-ng/src/wire.rs b/core/server-ng/src/wire.rs index e7a047089b..639819efcd 100644 --- a/core/server-ng/src/wire.rs +++ b/core/server-ng/src/wire.rs @@ -30,6 +30,23 @@ pub(crate) fn request_body(request: &Message) -> &[u8] { &request.as_slice()[std::mem::size_of::()..request.header().size as usize] } +/// Check a client's `request_checksum` against the body it stamps. +/// +/// Must run BEFORE any body rewrite: PAT / password / consumer-group paths +/// substitute server-chosen bytes. Zero is "unstamped" and skips the check, so an +/// SDK predating the stamp still works. +/// +/// # Errors +/// [`IggyError::InvalidFormat`] when the stamp disagrees with the body. +pub(crate) fn verify_request_checksum(request: &Message) -> Result<(), IggyError> { + let stamped = request.header().request_checksum; + if stamped == 0 || u128::from(iggy_common::calculate_checksum(request_body(request))) == stamped + { + return Ok(()); + } + Err(IggyError::InvalidFormat) +} + /// Map the transport kind to the legacy wire discriminant /// (`1=TCP, 2=QUIC, 4=WebSocket`); TLS variants report their base /// transport. `ClientTransportKind` is `#[non_exhaustive]`, so any other @@ -65,11 +82,78 @@ pub(crate) fn rewrite_request_body( .expect("zeroed bytes are a valid request header"); *header = *request.header(); header.size = size; + // Both describe the body just replaced, and nothing recomputes them for a + // `RequestHeader` -- the prepare projection derives its own `checksum_body` + // downstream. Clear rather than recompute; carrying them forward is a stale claim. + header.checksum = 0; + header.checksum_body = 0; + // `request_checksum` is deliberately NOT touched: it stamps what the CLIENT sent, + // already validated at admission. Re-stamping it over the substituted body would + // make the client-table reuse check compare a value no client ever produced. rewritten.as_mut_slice()[std::mem::size_of::()..].copy_from_slice(body); - // TODO(vsr): the body changed but `request_checksum` / `checksum` / - // `checksum_body` were copied verbatim from the original header. Safe - // today because the SDK initializes `request_checksum` to 0 and the - // server does not validate it; the moment integrity checking lands, - // recompute these here (or zero them and re-sign in a follow-up step). Ok(rewritten) } + +#[cfg(test)] +mod tests { + use super::{request_body, rewrite_request_body}; + use bytes::Bytes; + use iggy_binary_protocol::{Command2, Operation, RequestHeader}; + use server_common::Message; + use std::mem::size_of; + + fn request(body: &[u8], request_checksum: u128) -> Message { + let total_size = size_of::() + body.len(); + let mut message = Message::::new(total_size).transmute_header( + |_, header: &mut RequestHeader| { + header.command = Command2::Request; + header.operation = Operation::CreateStream; + header.client = 1; + header.session = 1; + header.request = 9; + header.size = u32::try_from(total_size).expect("fits u32"); + header.request_checksum = request_checksum; + header.checksum = 0xdead; + header.checksum_body = 0xbeef; + }, + ); + message.as_mut_slice()[size_of::()..].copy_from_slice(body); + message + } + + #[test] + fn given_a_body_rewrite_should_keep_the_client_stamp_and_clear_the_stale_seals() { + // The secret-bearing wire body is swapped for the hash-carrying replicated + // one. `request_checksum` describes what the client sent and admission has + // already checked it, so it must survive; the other two describe the body + // that just went away. + let original = request(b"plaintext-secret", 0x1234); + let rewritten = rewrite_request_body(&original, &Bytes::from_static(b"argon2-hash")) + .expect("the rewritten body fits a request message"); + + assert_eq!( + rewritten.header().request_checksum, + 0x1234, + "the client's stamp must not be re-signed over server-substituted bytes" + ); + assert_eq!(rewritten.header().checksum, 0); + assert_eq!(rewritten.header().checksum_body, 0); + assert_eq!(request_body(&rewritten), b"argon2-hash"); + assert_eq!( + rewritten.header().size as usize, + size_of::() + b"argon2-hash".len(), + "`size` follows the new body, so `request_body` bounds it correctly" + ); + } + + #[test] + fn given_an_unstamped_request_when_rewriting_should_stay_unstamped() { + // Zero means "unstamped" all the way through the client table, so a rewrite + // must not manufacture a stamp for a client that sent none. + let original = request(b"plaintext-secret", 0); + let rewritten = rewrite_request_body(&original, &Bytes::from_static(b"argon2-hash")) + .expect("the rewritten body fits a request message"); + + assert_eq!(rewritten.header().request_checksum, 0); + } +} diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs index 4fe65638be..1be05a27a7 100644 --- a/core/server_common/src/consensus_message.rs +++ b/core/server_common/src/consensus_message.rs @@ -237,6 +237,9 @@ where let bytes = >::header_storage(&self.backing); let typed = bytemuck::checked::try_from_bytes::(&bytes[..size_of::()]) .map_err(|_| ConsensusError::InvalidBitPattern)?; + // Before `validate`: a header that did not survive the link intact cannot + // have any of its fields believed, and `validate` reads them. + typed.verify_frame()?; typed.validate()?; Ok(Message { @@ -273,6 +276,9 @@ where let bytes = >::header_storage(&self.backing); let typed = bytemuck::checked::try_from_bytes::(&bytes[..size_of::()]) .map_err(|_| ConsensusError::InvalidBitPattern)?; + // Before `validate`: a header that did not survive the link intact cannot + // have any of its fields believed, and `validate` reads them. + typed.verify_frame()?; typed.validate()?; let typed_message = unsafe { &*std::ptr::from_ref(self).cast::>() }; @@ -654,7 +660,7 @@ where #[cfg(test)] mod tests { use super::*; - use iggy_binary_protocol::{Operation, ReplyHeader}; + use iggy_binary_protocol::{HEADER_SIZE, Operation, ReplyHeader, frame_checksum_bytes}; use smallvec::smallvec; // Field offsets via `offset_of!`: a field reorder fails to compile here @@ -684,10 +690,68 @@ mod tests { // `Register` needs session 0 and request 0, which zeroed bytes // already satisfy. buf[REQUEST_OPERATION_OFF] = Operation::Register as u8; + seal_header_bytes(buf); } o } + /// Seal a hand-built frame the way a real sender does. + /// + /// Control headers are rejected on the typed parse unless `checksum` covers the + /// rest of the header, so a fixture that skips this tests the rejection path. + fn seal_header_bytes(buf: &mut [u8]) { + let header: &[u8; HEADER_SIZE] = buf[..HEADER_SIZE].try_into().expect("frame is a header"); + let checksum = frame_checksum_bytes(header); + buf[..size_of::()].copy_from_slice(&checksum.to_le_bytes()); + } + + /// A `DoViewChange` frame carrying a one-entry suffix, sealed. + /// + /// One entry rather than none because a bitset bit is only legal within the + /// suffix, so an empty frame cannot express the attack this seals against. + fn sealed_do_view_change() -> Owned { + const DVC_SIZE: usize = HEADER_SIZE * 2; + let mut owned = Owned::::zeroed(DVC_SIZE); + { + let buf = owned.as_mut_slice(); + buf[SIZE_OFF..SIZE_OFF + 4].copy_from_slice(&(DVC_SIZE as u32).to_le_bytes()); + buf[COMMAND_OFF] = Command2::DoViewChange as u8; + seal_header_bytes(buf); + } + owned + } + + #[test] + fn given_a_sealed_do_view_change_when_dispatching_should_accept() { + let generic = Message::::try_from(sealed_do_view_change()) + .expect("a sealed DoViewChange frames correctly"); + assert!(matches!( + MessageBag::try_from(generic), + Ok(MessageBag::DoViewChange(_)) + )); + } + + #[test] + fn given_a_flipped_nack_bit_when_dispatching_should_reject_the_frame() { + // Why the header seal exists. `validate` accepts this frame: the bit sits + // inside the one-entry suffix, where a legitimate nack lives. Downstream the + // bitset goes to the merge unchanged and authorises truncating a committed op. + const NACK_OFF: usize = std::mem::offset_of!(DoViewChangeHeader, nack_bitset); + + let mut owned = sealed_do_view_change(); + owned.as_mut_slice()[NACK_OFF] ^= 0x01; + + let generic = + Message::::try_from(owned).expect("framing does not inspect the bitset"); + assert!( + matches!( + MessageBag::try_from(generic), + Err(ConsensusError::FrameChecksumMismatch { .. }) + ), + "a manufactured nack must not reach the merge" + ); + } + // MessageBag round-trip for the probe + repair command family. Locks // RangeEvicted delivery in particular: RepairDone and RangeEvicted share // one header layout and BOTH must survive the typed parse -- a strict @@ -712,6 +776,8 @@ mod tests { let buf = owned.as_mut_slice(); buf[FROM_OP_OFF..FROM_OP_OFF + 8].copy_from_slice(&1u64.to_le_bytes()); buf[TO_OP_OFF..TO_OP_OFF + 8].copy_from_slice(&1u64.to_le_bytes()); + // Re-seal: the range was written after `header_bytes` sealed. + seal_header_bytes(buf); } let generic = Message::::try_from(owned) .unwrap_or_else(|e| panic!("{command:?} failed generic framing: {e}")); diff --git a/core/server_common/src/send_messages2.rs b/core/server_common/src/send_messages2.rs index f0d9a7f88d..c7b0e60750 100644 --- a/core/server_common/src/send_messages2.rs +++ b/core/server_common/src/send_messages2.rs @@ -678,6 +678,24 @@ fn transcode_legacy_request( /// chunk and steps by `batch_length`. Callers whose buffer is meant to BE the /// batch must reject the surplus themselves - see [`convert_request_message`]. pub fn decode_batch_slice(body: &[u8]) -> Result, IggyError> { + decode_batch_slice_verified(body, true) +} + +/// [`decode_batch_slice`] with the checksum check made optional. +/// +/// `verify_checksum` exists for the disk-poll path, whose operator knob decides +/// whether a read pays for a full re-hash of every batch. The layout checks are not +/// optional either way: a short or self-inconsistent record is rejected regardless, +/// because the caller would otherwise index past it. +/// +/// # Errors +/// [`IggyError::InvalidCommand`] for a short or inconsistent record, and +/// [`IggyError::InvalidBatchChecksum`] when verification is on and the batch does not +/// match. Callers that must tell corruption from a partial tail need both. +pub fn decode_batch_slice_verified( + body: &[u8], + verify_checksum: bool, +) -> Result, IggyError> { if body.len() < COMMAND_HEADER_SIZE { return Err(IggyError::InvalidCommand); } @@ -690,13 +708,17 @@ pub fn decode_batch_slice(body: &[u8]) -> Result, IggyError let blob = &body[COMMAND_HEADER_SIZE..COMMAND_HEADER_SIZE + blob_len]; let batch = SendMessages2Ref { header, blob }; - let expected_checksum = verify_and_recompute_batch_checksum(&batch)?; - if header.batch_checksum != expected_checksum { - return Err(IggyError::InvalidBatchChecksum( - header.batch_checksum, - expected_checksum, - header.base_offset, - )); + if verify_checksum { + let expected_checksum = verify_and_recompute_batch_checksum(&batch)?; + if header.batch_checksum != expected_checksum { + return Err(IggyError::InvalidBatchChecksum( + header.batch_checksum, + expected_checksum, + header.base_offset, + )); + } + } else { + validate_batch_layout(&batch)?; } Ok(batch) @@ -1000,6 +1022,24 @@ fn verify_and_recompute_batch_checksum(batch: &SendMessages2Ref<'_>) -> Result) -> Result<(), IggyError> { + let mut framed = 0u32; + let mut covered = 0usize; + for message in batch.iter_with_offsets() { + framed += 1; + covered = message.end; + } + if framed != batch.message_count() || covered != batch.blob().len() { + return Err(IggyError::InvalidCommand); + } + Ok(()) +} + fn read_u32(bytes: &[u8], offset: usize) -> Result { bytes .get(offset..offset + 4) diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 5563f219cb..f5c5cf8379 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -28,18 +28,19 @@ pub use router::CONSENSUS_TICK_INTERVAL; #[cfg(any(test, feature = "simulator"))] use consensus::LocalPipeline; use consensus::{ - ChunkProgress, CommitOutcome, Consensus, ConsensusClock, MetadataHandle, MuxPlane, - PartitionsHandle, Pipeline, Plane, PlaneKind, STATE_TRANSFER_MAX_DECODE_RETRIES, - STATE_TRANSFER_MAX_STALL_RETRIES, Sequencer, VsrAction, VsrConsensus, - build_deny_reply_from_request_header, + ChunkProgress, CommitOutcome, Consensus, ConsensusClock, DVC_HEADERS_MAX, DvcHeaderKind, + DvcSuffix, MergedLog, MetadataHandle, MuxPlane, PartitionsHandle, Pipeline, Plane, PlaneKind, + STATE_TRANSFER_MAX_DECODE_RETRIES, STATE_TRANSFER_MAX_STALL_RETRIES, Sequencer, Status, + VsrAction, VsrConsensus, build_deny_reply_from_request_header, dvc_blank, dvc_header_kind, + encode_prepare_headers, verify_prepare_integrity, }; #[cfg(any(test, feature = "simulator"))] use crossfire::AsyncRxTrait; use crossfire::TrySendError; use futures::FutureExt; use iggy_binary_protocol::{ - Command2, CommitHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader, - PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, + Command2, CommitHeader, ConsensusHeader, DoViewChangeHeader, GenericHeader, Operation, + PrepareHeader, PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, @@ -3257,6 +3258,7 @@ where if let Some(ref consensus) = planes.0.consensus && consensus.namespace() == header.namespace { + refresh_metadata_dvc_suffix(consensus, planes.0.journal.as_ref()); let actions = consensus.handle_start_view_change(PlaneKind::Metadata, &header); let (local_actions, wire_actions) = split_local_actions(actions); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &local_actions).await; @@ -3275,6 +3277,7 @@ where ) else { return; }; + refresh_partition_dvc_suffix(partition); let consensus = partition.consensus(); let actions = consensus.handle_start_view_change(PlaneKind::Partitions, &header); let (local_actions, wire_actions) = split_local_actions(actions); @@ -3306,7 +3309,18 @@ where if let Some(ref consensus) = planes.0.consensus && consensus.namespace() == header.namespace { - let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &header); + refresh_metadata_dvc_suffix(consensus, planes.0.journal.as_ref()); + let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { + tracing::warn!( + shard = self.id, + from_replica = header.replica, + view = header.view, + "dropping do_view_change whose body failed its checksum" + ); + return; + }; + let actions = + consensus.handle_do_view_change(PlaneKind::Metadata, &header, suffix_body); let (local_actions, wire_actions) = split_local_actions(actions); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &local_actions).await; if planes.0.persist_superblock_if_needed(consensus).await { @@ -3334,8 +3348,18 @@ where ) else { return; }; + refresh_partition_dvc_suffix(partition); let consensus = partition.consensus(); - let actions = consensus.handle_do_view_change(PlaneKind::Partitions, &header); + let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { + tracing::warn!( + shard = self.id, + from_replica = header.replica, + view = header.view, + "dropping do_view_change whose body failed its checksum" + ); + return; + }; + let actions = consensus.handle_do_view_change(PlaneKind::Partitions, &header, suffix_body); let (local_actions, wire_actions) = split_local_actions(actions); // Locals go to the partition dispatcher ONLY: `RebuildPipeline` // executes there (`dispatch_vsr_actions` bails on `journal: None`) @@ -3357,8 +3381,7 @@ where } } - #[allow(clippy::future_not_send)] - #[allow(clippy::too_many_lines)] + #[allow(clippy::future_not_send, clippy::too_many_lines)] async fn on_start_view(&self, msg: Message) where B: MessageBus, @@ -3376,13 +3399,29 @@ where if let Some(ref consensus) = planes.0.consensus && consensus.namespace() == header.namespace { - let actions = consensus.handle_start_view(PlaneKind::Metadata, &header); + let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { + tracing::warn!( + shard = self.id, + from_replica = header.replica, + view = header.view, + "dropping start_view whose body failed its checksum" + ); + return; + }; + let actions = consensus.handle_start_view(PlaneKind::Metadata, &header, suffix_body); // Every rejection path (wrong primary, old view, stale incarnation, // below the commit floor, self-sent) returns no actions, and an // adopted StartView always emits at least `CommitJournal`. That // makes emptiness the adoption signal -- and the arms below must // not fire on a StartView this replica did not adopt. let adopted = !actions.is_empty(); + if adopted { + // First chance to spot a local entry disagreeing with the view's log. + // Ahead of the local dispatch below: it truncates the journal that + // `RebuildPipeline` reads back, so a rebuild before it would seed the + // pipeline from the entries this is about to drop. + self.reconcile_metadata_view_divergence().await; + } let (local_actions, wire_actions) = split_local_actions(actions); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &local_actions).await; if planes.0.persist_superblock_if_needed(consensus).await { @@ -3476,7 +3515,16 @@ where return; }; let consensus = partition.consensus(); - let actions = consensus.handle_start_view(PlaneKind::Partitions, &header); + let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { + tracing::warn!( + shard = self.id, + from_replica = header.replica, + view = header.view, + "dropping start_view whose body failed its checksum" + ); + return; + }; + let actions = consensus.handle_start_view(PlaneKind::Partitions, &header, suffix_body); let adopted = !actions.is_empty(); let (local_actions, wire_actions) = split_local_actions(actions); // Locals go to the partition dispatcher ONLY: `RebuildPipeline` @@ -3710,7 +3758,14 @@ where if let Some(ref consensus) = planes.0.consensus && consensus.namespace() == header.namespace { - if !consensus.is_normal() { + // Served in `ViewChange` too: the replicas holding a missing body are + // exactly those in `ViewChange`, so refusing would deadlock the repair + // the new primary waits on. Read-only; the requester decides. + if consensus.is_transferring() { + // A transfer rewrites local state wholesale; journal not stable yet. + return; + } + if !matches!(consensus.status(), Status::Normal | Status::ViewChange) { return; } let Some(journal) = planes.0.journal.as_ref() else { @@ -3719,7 +3774,11 @@ where let journal = journal.handle(); let cluster = consensus.cluster(); let self_id = consensus.replica(); - let to_op = header.to_op.min(consensus.commit_max()); + let to_op = repair_serve_ceiling( + header.to_op, + consensus.commit_max(), + consensus.sequencer().current_sequence(), + ); // Skip the compacted prefix (below the snapshot floor) in one // RangeEvicted notice, then serve contiguously until the range // ends or the WAL runs out. @@ -3876,7 +3935,7 @@ where /// Ingest one repaired prepare. Metadata journals it into the WAL (the /// commit walk at `RepairDone` applies it); partitions journal + stage it /// through the same apply path as live replication, minus fence and ack. - #[allow(clippy::future_not_send)] + #[allow(clippy::future_not_send, clippy::too_many_lines)] async fn on_repair_prepare(&self, msg: Message) where B: MessageBus, @@ -3928,7 +3987,47 @@ where let Some(session) = session else { return; }; - if header.op > session.to_op || header.op <= consensus.commit_min() { + if header.op > session.to_op { + return; + } + let pending = consensus.pending_view_log(); + if !repair_op_in_scope( + pending.as_ref(), + consensus.is_primary_for_view(consensus.view()), + consensus.commit_min(), + header.op, + ) { + return; + } + // Applies to both planes, and is why a backup parks a log at all. The + // view already decided which prepare belongs at this op; a different + // one forks the log. An op the parked log omits is unconstrained. + if let Some(pending) = &pending { + let expected = pending + .headers + .iter() + .chain(pending.committed_elsewhere.iter()) + .find(|expected| expected.op == header.op); + if let Some(expected) = expected + && expected.checksum != header.checksum + { + tracing::warn!( + shard = self.id, + op = header.op, + "discarding repaired prepare that disagrees with the merged log" + ); + return; + } + } + // Recompute both integrity fields before durable storage: everything + // above treats `header.checksum` as an opaque token, so a corrupted + // frame passes whenever its flipped value satisfies the comparisons. + if let Err(reason) = verify_prepare_integrity(&header, msg.as_slice()) { + tracing::warn!( + shard = self.id, + op = header.op, + "discarding repaired prepare: {reason}" + ); return; } let Some(journal) = planes.0.journal.as_ref() else { @@ -3989,6 +4088,18 @@ where else { return; }; + // The partition arm reaches the WAL via `apply_repaired_prepare` with no + // view fence and no ack, so this is its only integrity gate. Without it a + // repaired partition prepare is journaled on the serving peer's word alone. + if let Err(reason) = verify_prepare_integrity(&header, msg.as_slice()) { + tracing::warn!( + shard = self.id, + op = header.op, + namespace_raw = header.namespace, + "discarding repaired partition prepare: {reason}" + ); + return; + } partition.apply_repaired_prepare(msg).await; } @@ -4248,6 +4359,7 @@ where h.to_op = to_op; h.namespace = namespace; h.size = size_of::() as u32; + h.seal(); }); if self .bus @@ -4322,6 +4434,7 @@ where h.op = op; h.namespace = namespace; h.size = size_of::() as u32; + h.seal(); }); let _ = self .bus @@ -4329,130 +4442,509 @@ where .await; } - /// Start metadata tail journal-repair from `peer` when the commit walk - /// gap-stopped below the known frontier. Shared by `StartView` adoption - /// and the post-install step of a state transfer. + /// Partition-plane twin of [`Self::advance_pending_metadata_view`]. + /// + /// No `RequestPrepares` stream to arm: the partition journal is not durable + /// yet, so coverage either holds or a peer must retransmit. Same invariant + /// either way: the view does not start until this replica can serve its log. #[allow(clippy::future_not_send)] - async fn maybe_request_metadata_repair

(&self, consensus: &VsrConsensus, peer: u8) + async fn advance_pending_partition_view(&self, namespace: IggyNamespace) where B: MessageBus, - P: Pipeline, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, { - if consensus.is_normal() - && !consensus.is_transferring() - && consensus.commit_min() < consensus.commit_max() - && self.metadata_repair.borrow().is_none() - { - let nonce = iggy_common::random_id::get_uuid(); - let to_op = consensus.commit_max(); - let from_op = consensus.commit_min() + 1; - *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { - nonce, - to_op, - peer, - idle_ticks: 0, - }); - tracing::info!( - shard = self.id, - from_op, - to_op, - "metadata behind the group frontier; requesting repair" - ); - self.send_request_prepares( - consensus.cluster(), - consensus.replica(), - peer, - nonce, - from_op, - to_op, - consensus.namespace(), - ) - .await; + let partitions = self.plane.partitions(); + let started = { + let Some(partition) = partitions.get_by_ns(&namespace) else { + return; + }; + let consensus = partition.consensus(); + if !consensus.is_primary_for_view(consensus.view()) { + return; + } + let Some(pending) = consensus.pending_view_log() else { + return; + }; + let missing = { + let journal = partition.log.journal(); + (pending.commit_max.max(1)..=pending.op_head) + .find(|op| journal.inner.header_by_op(*op).is_none()) + }; + if let Some(missing_op) = missing { + tracing::debug!( + shard = self.id, + namespace_raw = namespace.inner(), + missing_op, + op_head = pending.op_head, + "partition view change waiting on op {missing_op} before starting the view" + ); + return; + } + + let actions = consensus.start_pending_view(PlaneKind::Partitions); + let (local_actions, wire_actions) = split_local_actions(actions); + // Locals go to the partition dispatcher ONLY: `RebuildPipeline` + // executes there (`dispatch_vsr_actions` bails on `journal: None`) + // and `CommitJournal` is a no-op in both. + dispatch_partition_journal_actions(consensus, partition, &local_actions).await; + // `start_pending_view` flips this replica into `Normal` for the new + // view, so the `StartView` it emits advertises a view the superblock + // must already record. Same gate as the `on_do_view_change` and + // `on_start_view` partition arms. + if partition.persist_superblock_if_needed().await { + dispatch_vsr_actions::(consensus, None, &wire_actions).await; + dispatch_partition_journal_actions(consensus, partition, &wire_actions).await; + } + local_actions + .iter() + .any(|action| matches!(action, VsrAction::CommitJournal)) + }; + if started { + let config = partitions.config(); + if let Some(partition) = partitions.get_mut_by_ns(&namespace) { + partition.commit_journal(config).await; + } } } - #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] - async fn send_request_state_transfer

( - &self, - consensus: &VsrConsensus, - target: u8, - nonce: u128, - ) where + /// Re-request the remaining repair window when the stream has gone quiet. + /// + /// Repair frames are fire-and-forget, so a lost one leaves the session armed + /// forever with the commit walk pinned below the frontier. + #[allow(clippy::future_not_send)] + async fn retry_stalled_metadata_repair

(&self, consensus: &VsrConsensus) + where B: MessageBus, P: Pipeline, { - let msg = - Message::::new(size_of::()) - .transmute_header(|_, h: &mut RequestStateTransferHeader| { - h.command = Command2::RequestStateTransfer; - h.cluster = consensus.cluster(); - h.replica = consensus.replica(); - h.nonce = nonce; - h.namespace = consensus.namespace(); - h.size = size_of::() as u32; - }); - let _ = self - .bus - .send_to_replica(target, msg.into_generic().into_frozen()) - .await; + // Stall retry (mirrors `tick_partitions`): a lost frame must not wedge it. + let repair_retry_ticks = self.repair_retry_ticks.get(); + let stalled = { + // `ViewChange` too: a parked view change repairs toward its merged log + // and cannot start until the window fills. Gating on `Normal` alone + // defers a dropped frame to the 500-tick escalation. + let repairing_view = consensus.pending_view_log().is_some() + && consensus.is_primary_for_view(consensus.view()); + let mut session = self.metadata_repair.borrow_mut(); + session.as_mut().and_then(|session| { + if !consensus.is_normal() && !repairing_view { + return None; + } + session.idle_ticks += 1; + if session.idle_ticks < repair_retry_ticks { + return None; + } + session.idle_ticks = 0; + Some((session.peer, session.nonce, session.to_op)) + }) + }; + if let Some((peer, nonce, to_op)) = stalled { + // Primary-elect only. Its window starts at the merged log's commit + // point, which can sit below local `commit_min` (the headers inherited + // from senders behind the canonical log_view live there), so + // `commit_min + 1` would skip them. A backup's parked `StartView` + // suffix is only a verification reference; resuming from its commit + // point would restart at the view's opening head, not at the gap. + let from_op = consensus + .pending_view_log() + .filter(|_| consensus.is_primary_for_view(consensus.view())) + .map_or_else( + || consensus.commit_min() + 1, + |pending| pending.commit_max.max(1), + ); + if from_op <= to_op { + tracing::info!( + shard = self.id, + from_op, + to_op, + peer, + "metadata repair stalled; re-requesting remaining window" + ); + self.send_request_prepares( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + from_op, + to_op, + consensus.namespace(), + ) + .await; + } + } } - /// Answer a `RequestStateTransfer`: `offer = None` sends a header-only - /// `available = 0` (the requester falls back to journal repair or - /// retries elsewhere); an offer ships its encoded state manifest as the - /// frame body. - #[allow( - clippy::future_not_send, - clippy::cast_possible_truncation, - clippy::too_many_arguments - )] - async fn send_state_transfer_target( - &self, - cluster: u128, - self_id: u8, - target: u8, - nonce: u128, - namespace: u64, - descriptor: TransferDescriptor<'_>, - ) where + /// Compare a backup's log against the headers the concluding `StartView` + /// published, and report where they disagree. + /// + /// Without this, divergence is silent and permanent: the backup acks with its + /// own checksum, the primary rejects the ack, and journal repair skips an op + /// it already has a header for. + /// + /// The split at the announced commit point is what matters. Above it a + /// disagreement is ordinary, so the entry is dropped and the primary's + /// retransmission refills the range. At or below it, this replica applied + /// something the view says was different, which only state transfer fixes, so + /// it is reported and left alone. + /// + /// Truncation uses `Journal::truncate_from`, not `drain`: `drain` advances + /// `snapshot_op` past what it removed, marking ops that must stay refillable + /// as evictable. + #[allow(clippy::future_not_send)] + async fn reconcile_metadata_view_divergence(&self) + where B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: MetadataStm, { - let manifest = descriptor - .offer - .map(|(entries, _)| consensus::encode_state_manifest(entries)); - let total_size = - size_of::() + manifest.as_ref().map_or(0, Vec::len); - let mut msg = Message::::new(total_size); - if let Some(manifest) = &manifest { - msg.as_mut_slice()[size_of::()..].copy_from_slice(manifest); + let metadata = self.plane.metadata(); + let Some(ref consensus) = metadata.consensus else { + return; + }; + // Backups only; a primary reconciles through the merge itself. + if consensus.is_primary_for_view(consensus.view()) { + return; } - let msg = msg.transmute_header(|_, h: &mut StateTransferTargetHeader| { - h.command = Command2::StateTransferTarget; - h.cluster = cluster; - h.replica = self_id; - h.nonce = nonce; - h.namespace = namespace; - h.size = total_size as u32; - // The serving replica's own progress travels with every descriptor, - // available or not: it is what lets a receiver refuse an offer from - // a replica that knows less than it does. - h.view = descriptor.view; - h.commit_max = descriptor.commit_max; - h.unavailable_transient = u8::from(descriptor.transient); - if let Some((_, commit_op)) = descriptor.offer { - h.available = 1; - h.commit_op = commit_op; - } - }); - let _ = self - .bus - .send_to_replica(target, msg.into_generic().into_frozen()) - .await; - } + let Some(pending) = consensus.pending_view_log() else { + return; + }; + let Some(journal) = metadata.journal.as_ref() else { + return; + }; - #[allow( - clippy::future_not_send, - clippy::cast_possible_truncation, + // Truncation is safe only above what this replica has *applied*, which is + // not the view's commit point: `pending.commit_max` is the new primary's + // number and a backup can sit above it. Splitting on the view's number + // would drop already-executed ops with no rollback, and silently. + let applied_floor = pending.commit_max.max(consensus.commit_min()); + + let mut repairable_from: Option = None; + for canonical in &pending.headers { + let Some(local) = usize::try_from(canonical.op) + .ok() + .and_then(|slot| journal.handle().header(slot)) + else { + continue; + }; + if local.checksum == canonical.checksum { + continue; + } + if canonical.op <= applied_floor { + tracing::error!( + shard = self.id, + op = canonical.op, + view = consensus.view(), + commit_max = pending.commit_max, + commit_min = consensus.commit_min(), + local_checksum = local.checksum, + canonical_checksum = canonical.checksum, + "committed op {} disagrees with the view that just started; this replica \ + applied a different op as committed and cannot be reconciled by log repair", + canonical.op + ); + continue; + } + repairable_from = Some(repairable_from.map_or(canonical.op, |op| op.min(canonical.op))); + } + + let Some(from_op) = repairable_from else { + return; + }; + match journal.handle().truncate_from(from_op).await { + Ok(removed) => { + // The snapshot's `(op, commit)` tag does not move when entries are + // removed under it, so the next `DoViewChange` would advertise the + // dropped headers and offer bodies this replica cannot serve. + consensus.invalidate_local_dvc_suffix(); + tracing::warn!( + shard = self.id, + from_op, + removed, + op_head = pending.op_head, + view = consensus.view(), + "dropped {removed} uncommitted entries from op {from_op} that disagreed with \ + the view's log; the primary's retransmission refills the range" + ); + } + Err(error) => { + tracing::error!( + shard = self.id, + from_op, + %error, + "could not drop the diverging uncommitted entries from op {from_op}; journal \ + repair skips ops it already holds a header for, so this replica will not \ + converge at those ops until it is restarted" + ); + } + } + } + + /// Drive a parked view change to completion. + /// + /// A DVC quorum decides the log before this replica necessarily holds it, so + /// the merged log parks in consensus and this replica stays in `ViewChange`, + /// announcing and preparing nothing: `StartView` promises it can serve every + /// op it names, and a backup adopting that head asks for the bodies at once. + /// + /// Check coverage, then start the view or pull missing bodies from a peer that + /// offered them in its DVC. Only those peers: a cleared present bit means the + /// body was never held or cannot be read back. + #[allow(clippy::future_not_send)] + async fn advance_pending_metadata_view(&self) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: MetadataStm, + { + let metadata = self.plane.metadata(); + let Some(ref consensus) = metadata.consensus else { + return; + }; + // Primary-elect only. A backup's parked `StartView` suffix is only what + // its ingest verifies bodies against; driving repair from it would put a + // rejoining node on the tail-repair path when its gap sits below every + // peer's retention floor, racing the view probe that picks state transfer. + if !consensus.is_primary_for_view(consensus.view()) { + return; + } + let Some(pending) = consensus.pending_view_log() else { + return; + }; + let Some(journal) = metadata.journal.as_ref() else { + return; + }; + + let held = |op: u64| { + usize::try_from(op) + .ok() + .and_then(|slot| journal.handle().header(slot)) + .is_some() + }; + // Floor on what this replica can be asked to hold before starting the view. + // Entries at or below the snapshot watermark are compacted, so no repair puts + // one back: demanding one parks the view change forever on an op already + // applied and durable in the snapshot. + let repair_floor = journal.handle().snapshot_op(); + // Every op the merged log names, including headers inherited from senders + // behind the canonical log_view: those sit below the canonical window and + // header repair cannot walk back across the gap later. + let missing = (pending.commit_max.max(1)..=pending.op_head) + .find(|op| !held(*op)) + .or_else(|| { + pending + .committed_elsewhere + .iter() + .map(|header| header.op) + .filter(|op| *op > repair_floor) + .find(|op| !held(*op)) + }); + + let Some(missing_op) = missing else { + let actions = consensus.start_pending_view(PlaneKind::Metadata); + tracing::info!( + shard = self.id, + view = consensus.view(), + op_head = pending.op_head, + commit_max = pending.commit_max, + "merged log is locally serveable; starting the view" + ); + if metadata.persist_superblock_if_needed(consensus).await { + dispatch_vsr_actions(consensus, metadata.journal.as_ref(), &actions).await; + } + if actions + .iter() + .any(|action| matches!(action, VsrAction::CommitJournal)) + && !consensus.is_transferring() + { + metadata.commit_journal().await; + } + return; + }; + + if self.metadata_repair.borrow().is_some() { + // Stream already running; the stall retry covers it drying up. + return; + } + let sources = consensus.pending_view_body_sources(missing_op); + let Some(peer) = sources.first().copied() else { + // The merge only returns a startable log when some replica offered each + // body, so an empty source list means that offer was withdrawn (peer + // restarted, or moved on). Let the view-change timeout escalate. + tracing::warn!( + shard = self.id, + missing_op, + "no replica offers op {missing_op} for the merged log; view change is stalled" + ); + return; + }; + + let nonce = iggy_common::random_id::get_uuid(); + *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { + nonce, + to_op: pending.op_head, + peer, + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + missing_op, + peer, + to_op = pending.op_head, + "repairing toward the merged log before starting the view" + ); + self.send_request_prepares( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + missing_op, + pending.op_head, + consensus.namespace(), + ) + .await; + } + + /// Start metadata tail journal-repair from `peer` when the commit walk + /// gap-stopped below the known frontier. Shared by `StartView` adoption + /// and the post-install step of a state transfer. + #[allow(clippy::future_not_send)] + async fn maybe_request_metadata_repair

(&self, consensus: &VsrConsensus, peer: u8) + where + B: MessageBus, + P: Pipeline, + { + if consensus.is_normal() + && !consensus.is_transferring() + && consensus.commit_min() < consensus.commit_max() + && self.metadata_repair.borrow().is_none() + { + let nonce = iggy_common::random_id::get_uuid(); + let to_op = consensus.commit_max(); + let from_op = consensus.commit_min() + 1; + *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { + nonce, + to_op, + peer, + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + from_op, + to_op, + "metadata behind the group frontier; requesting repair" + ); + self.send_request_prepares( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + from_op, + to_op, + consensus.namespace(), + ) + .await; + } + } + + #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] + async fn send_request_state_transfer

( + &self, + consensus: &VsrConsensus, + target: u8, + nonce: u128, + ) where + B: MessageBus, + P: Pipeline, + { + let msg = + Message::::new(size_of::()) + .transmute_header(|_, h: &mut RequestStateTransferHeader| { + h.command = Command2::RequestStateTransfer; + h.cluster = consensus.cluster(); + h.replica = consensus.replica(); + h.nonce = nonce; + h.namespace = consensus.namespace(); + h.size = size_of::() as u32; + h.seal(); + }); + let _ = self + .bus + .send_to_replica(target, msg.into_generic().into_frozen()) + .await; + } + + /// Answer a `RequestStateTransfer`: `offer = None` sends a header-only + /// `available = 0` (the requester falls back to journal repair or + /// retries elsewhere); an offer ships its encoded state manifest as the + /// frame body. + #[allow( + clippy::future_not_send, + clippy::cast_possible_truncation, + clippy::too_many_arguments + )] + async fn send_state_transfer_target( + &self, + cluster: u128, + self_id: u8, + target: u8, + nonce: u128, + namespace: u64, + descriptor: TransferDescriptor<'_>, + ) where + B: MessageBus, + { + let manifest = descriptor + .offer + .map(|(entries, _)| consensus::encode_state_manifest(entries)); + let total_size = + size_of::() + manifest.as_ref().map_or(0, Vec::len); + let mut msg = Message::::new(total_size); + if let Some(manifest) = &manifest { + msg.as_mut_slice()[size_of::()..].copy_from_slice(manifest); + } + let msg = msg.transmute_header(|_, h: &mut StateTransferTargetHeader| { + h.command = Command2::StateTransferTarget; + h.cluster = cluster; + h.replica = self_id; + h.nonce = nonce; + h.namespace = namespace; + h.size = total_size as u32; + // The serving replica's own progress travels with every descriptor, + // available or not: it is what lets a receiver refuse an offer from + // a replica that knows less than it does. + h.view = descriptor.view; + h.commit_max = descriptor.commit_max; + h.unavailable_transient = u8::from(descriptor.transient); + if let Some((_, commit_op)) = descriptor.offer { + h.available = 1; + h.commit_op = commit_op; + } + h.seal(); + }); + let _ = self + .bus + .send_to_replica(target, msg.into_generic().into_frozen()) + .await; + } + + #[allow( + clippy::future_not_send, + clippy::cast_possible_truncation, clippy::too_many_arguments )] async fn send_request_state_chunk( @@ -4479,6 +4971,7 @@ where h.offset = offset; h.len = len; h.size = size_of::() as u32; + h.seal(); }); let _ = self .bus @@ -5025,6 +5518,7 @@ where h.artifact = header.artifact; h.offset = header.offset; h.size = total_size as u32; + h.seal(); }, ))) }, @@ -5431,6 +5925,16 @@ where }; let consensus = partition.consensus(); + // Only while a view change is live. A `Normal` tick has no consumer: + // `start_election` records no DoViewChange, and every path that does + // either refreshes at its own call site (the SVC and DVC handlers, + // still `Normal` at that point) or runs in `ViewChange`. + // + // Ungated, this rebuilt a 128-entry window every 10 ms per advancing + // partition: a linear `header_by_op` scan per entry plus 32 KiB. + if consensus.status() != Status::Normal { + refresh_partition_dvc_suffix(partition); + } let actions = consensus.tick(PlaneKind::Partitions); // The tick emits view-scoped sends (heartbeats, view-change // retransmits), so it persists first like every dispatch site; @@ -5444,6 +5948,9 @@ where dispatch_partition_journal_actions(consensus, partition, &wire_actions).await; } + // Finish a view change whose quorum decided ahead of the local log. + self.advance_pending_partition_view(namespace).await; + // Stall retry: repair frames are fire-and-forget, so a lost // frame (or a peer that went silent mid-stream) would leave the // session armed forever with commit_min pinned below commit_max. @@ -7286,6 +7793,10 @@ where return; }; + // See the partition tick: no snapshot consumer on a `Normal` tick. + if consensus.status() != Status::Normal { + refresh_metadata_dvc_suffix(consensus, metadata.journal.as_ref()); + } let actions = consensus.tick(PlaneKind::Metadata); let (local_actions, wire_actions) = split_local_actions(actions); @@ -7309,6 +7820,9 @@ where // nothing is stranded. metadata.resume_stranded_commits().await; + self.advance_pending_metadata_view().await; + self.expire_idle_state_transfer_offers(); + // Stall retry for an in-flight state transfer: descriptor or chunk // frames are fire-and-forget, so a lost one must not wedge the // session (and the boot flow behind it) forever. @@ -7364,45 +7878,7 @@ where } } - // Stall retry, mirroring `tick_partitions`: a lost repair frame must - // not wedge the session forever. - let repair_retry_ticks = self.repair_retry_ticks.get(); - let stalled = { - let mut session = self.metadata_repair.borrow_mut(); - session.as_mut().and_then(|session| { - if !consensus.is_normal() { - return None; - } - session.idle_ticks += 1; - if session.idle_ticks < repair_retry_ticks { - return None; - } - session.idle_ticks = 0; - Some((session.peer, session.nonce, session.to_op)) - }) - }; - if let Some((peer, nonce, to_op)) = stalled { - let from_op = consensus.commit_min() + 1; - if from_op <= to_op { - tracing::info!( - shard = self.id, - from_op, - to_op, - peer, - "metadata repair stalled; re-requesting remaining window" - ); - self.send_request_prepares( - consensus.cluster(), - consensus.replica(), - peer, - nonce, - from_op, - to_op, - consensus.namespace(), - ) - .await; - } - } + self.retry_stalled_metadata_repair(consensus).await; } } @@ -7438,10 +7914,411 @@ where incarnation: 0, target: None, namespace: consensus.namespace(), + // Correcting a peer on a stale view, not concluding a view change: this + // publishes the settled frontier, which the peer reaches by repair. + suffix: Vec::new(), }; dispatch_vsr_actions::(consensus, None, &[action]).await; } +/// Rebuild the new primary's pipeline over `from_op..=to_op` from local journal +/// headers. +/// +/// A gap means the caller started the view before its journal could serve the +/// merged log: a bug in the transition, not a data condition. Nothing is +/// truncated, because truncating to the last findable op discards ops committed +/// on a quorum and already acknowledged. The pipeline is left short, the commit +/// walk stalls at the gap, and repair fills it in. +fn rebuild_pipeline_entries( + consensus: &VsrConsensus, + self_id: u8, + from_op: u64, + to_op: u64, + header_at: impl Fn(u64) -> Option, +) where + B: MessageBus, + P: Pipeline, +{ + let mut gap_at = None; + let entries: Vec<_> = (from_op..=to_op) + .map_while(|op| { + let header = header_at(op).or_else(|| { + gap_at = Some(op); + None + })?; + // Lift the monotonic timestamp floor to the rebuilt log so + // post-view-change prepares cannot stamp below committed ones. + consensus.observe_prepare_timestamp(header.timestamp); + let mut entry = consensus::PipelineEntry::new(header); + entry.add_ack(self_id); + Some(entry) + }) + .collect(); + + if let Some(missing_op) = gap_at { + tracing::error!( + replica = self_id, + missing_op, + range_start = from_op, + range_end = to_op, + rebuilt = entries.len(), + "RebuildPipeline: journal gap at op {missing_op} while starting a view; leaving the \ + sequencer at {to_op} and stalling the commit walk. Truncating here would discard ops \ + the view change proved recoverable." + ); + } + + let mut pipeline = consensus.pipeline().borrow_mut(); + for entry in entries { + pipeline.push(entry); + } +} + +/// Snapshot this replica's uncommitted suffix into consensus, if the journal has +/// moved since the last snapshot. +/// +/// Called before every handler that could start or join a view change: consensus +/// records its own `DoViewChange` there and has no journal to read. A stale +/// snapshot is never reused; consensus tags it with its `(op, commit)` and falls +/// back to an empty suffix, stalling the view change rather than nacking an op +/// since acquired. +fn refresh_metadata_dvc_suffix(consensus: &VsrConsensus, journal: Option<&MJ>) +where + B: MessageBus, + P: Pipeline, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, +{ + if !consensus.local_dvc_suffix_stale() { + return; + } + let op = consensus.sequencer().current_sequence(); + let commit = consensus.commit_max().min(op); + let pending = adopted_view_headers(consensus); + consensus.set_local_dvc_suffix(build_metadata_dvc_suffix( + journal, + commit, + op, + pending.as_ref().map(|pending| pending.headers.as_slice()), + )); +} + +/// The adopted view's headers, when they describe a log this replica has NOT itself +/// decided. +/// +/// `None` for the primary-elect holding the log its own merge produced: that log is +/// a proposal it is still repairing toward and may contain ops a later view +/// truncated, so stitching it into its own `DoViewChange` would re-assert them. +/// +/// A backup's parked log is the opposite: headers the view already decided and +/// announced, which this replica acknowledged and is repairing to hold. +fn adopted_view_headers(consensus: &VsrConsensus) -> Option +where + B: MessageBus, + P: Pipeline, +{ + if consensus.is_primary_for_view(consensus.view()) { + return None; + } + consensus.pending_view_log() +} + +/// Snapshot a partition's uncommitted suffix into its consensus. +/// +/// Same contract as [`Self::refresh_metadata_dvc_suffix`]. The partition journal +/// is in-memory only, so after a restart it reads empty and this replica votes +/// all-nack: correct, since the ops really are lost and the merge needs a peer +/// that still holds them. +/// +/// Read through `repair_header`, not the resident headers: the committed prefix +/// leaves those as soon as its bytes reach a segment, which on a caught-up +/// replica includes the commit point itself. +fn refresh_partition_dvc_suffix(partition: &partitions::IggyPartition) +where + B: MessageBus, + SB: SuperblockStore, +{ + let consensus = partition.consensus(); + if !consensus.local_dvc_suffix_stale() { + return; + } + let op = consensus.sequencer().current_sequence(); + let commit = consensus.commit_max().min(op); + let journal = partition.log.journal(); + let pending = adopted_view_headers(consensus); + let suffix = build_dvc_suffix( + commit, + op, + |entry_op| journal.inner.repair_header(entry_op), + pending.as_ref().map(|pending| pending.headers.as_slice()), + ); + consensus.set_local_dvc_suffix(suffix); +} + +/// The suffix headers a `DoViewChange` or `StartView` carries, as raw bytes. +/// +/// `size` is attacker-controlled, so it is clamped to what arrived; a short read +/// decodes as a malformed suffix and the DVC is dropped. +fn control_suffix_body(msg: &Message) -> &[u8] +where + H: iggy_binary_protocol::ConsensusHeader, +{ + let slice = msg.as_slice(); + let start = size_of::(); + let end = (msg.header().size() as usize).min(slice.len()); + if end <= start { + return &[]; + } + &slice[start..end] +} + +/// Seal a control-message body. Zero for an empty body, which is the unsealed +/// sentinel every other integrity field in this protocol uses. +fn control_body_checksum(body: &[u8]) -> u128 { + if body.is_empty() { + return 0; + } + u128::from(iggy_common::calculate_checksum(body)) +} + +/// The body of a control frame, once it matches the checksum its header carries. +/// +/// `None` means corruption in transit and the frame must be dropped whole: the +/// header numbers describe a body that did not arrive intact, so neither half is +/// trustworthy. This is what covers a body-carrying control message end to end. +/// +/// Keyed on whether a body is present, NOT on whether `checksum_body` looks +/// sealed: skipping the check when that field reads zero makes the layer +/// bypassable by clearing the one field that decides whether anything is checked. +/// A peer predating the suffix sends no body, so a non-empty body always came +/// from a sender that seals it, and a zero checksum there is corruption. +fn control_suffix_body_verified(msg: &Message, checksum_body: u128) -> Option<&[u8]> +where + H: iggy_binary_protocol::ConsensusHeader, +{ + let body = control_suffix_body(msg); + if body.is_empty() { + // Nothing to verify. `checksum_body` is irrelevant either way. + return Some(body); + } + if control_body_checksum(body) == checksum_body { + Some(body) + } else { + None + } +} + +/// Whether a repaired prepare at `op` falls inside the range this replica is +/// currently repairing. +/// +/// A parked log means two things depending on who parked it, and only one is a +/// repair window. The primary-elect parked the log its merge decided and repairs +/// toward exactly that range, so the range IS its scope, including ops at or +/// below `commit_min`: those are the headers inherited from senders behind the +/// canonical `log_view`, which the ordinary rule would reject and header repair +/// cannot walk back to. A backup's parked `StartView` suffix is only what its +/// ingest verifies bodies against, and its repair runs for the whole view, so +/// reading that range as a scope would discard every later op. +fn repair_op_in_scope( + pending: Option<&MergedLog>, + is_primary_elect: bool, + commit_min: u64, + op: u64, +) -> bool { + pending + .filter(|_| is_primary_elect) + .map_or(op > commit_min, |pending| { + (op >= pending.commit_max.max(1) && op <= pending.op_head) + || pending + .committed_elsewhere + .iter() + .any(|expected| expected.op == op) + }) +} + +/// Ceiling on the op range a repair request may ask this replica to walk. +/// +/// Not `commit_max` alone: a new primary repairing toward a merged log needs the +/// uncommitted suffix the view change kept, which sits above every commit point. +/// +/// Bounded by the local frontier all the same. `RequestPreparesHeader::validate` +/// accepts any `from_op <= to_op`, so `u64::MAX` is legal, and the metadata serve +/// path then walks op by op with no `.await` -- on a single-threaded shard pump +/// that ends the shard rather than merely serving slowly. Nothing above the +/// frontier is servable, so the clamp costs nothing. +fn repair_serve_ceiling(requested_to_op: u64, commit_max: u64, head: u64) -> u64 { + requested_to_op.min(commit_max.max(head)) +} + +/// Read this replica's uncommitted suffix out of the metadata journal, for the +/// window `commit..=op`. +/// +/// The nack bit is load-bearing, and is set only where absence *proves* this +/// replica never prepared the op: +/// * Above the commit point, a missing header is proof: the WAL refuses to boot +/// on interior corruption, so a hole in a journal that opened never arrived. +/// * At or below it, a checkpoint may have compacted the header away. Those slots +/// go out blank and un-nacked, read as "no information" rather than licence to +/// truncate an op this replica considers committed. +/// +/// Deriving the suffix on demand is also why it needs no durable record: the +/// merged log is in memory and bodies are fetched whole, so the WAL is the only +/// thing that ever backs a nack and recomputing after a restart gives the same +/// answer. A torn tail is the one exception, and it changes the answer correctly: +/// recovery truncates the incomplete append, which fsyncs before the ack, so no +/// replication quorum could have counted it. +fn build_metadata_dvc_suffix( + journal: Option<&J>, + commit: u64, + op: u64, + view_headers: Option<&[PrepareHeader]>, +) -> DvcSuffix +where + J: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, +{ + let Some(journal) = journal else { + return DvcSuffix::empty(); + }; + let handle = journal.handle(); + build_dvc_suffix( + commit, + op, + |entry_op| { + usize::try_from(entry_op) + .ok() + .and_then(|slot| handle.header(slot)) + .map(|header| *header) + }, + view_headers, + ) +} + +/// Plane-independent core of the suffix read. `header_at` answers "do I hold +/// this op, and what is its header". +fn build_dvc_suffix( + commit: u64, + op: u64, + header_at: impl Fn(u64) -> Option, + view_headers: Option<&[PrepareHeader]>, +) -> DvcSuffix { + // Stitch the adopted view's headers over the journal, high-to-low. + // + // Reading the journal alone is only correct for a replica whose journal IS its + // log. A backup that adopted a `StartView` is header-poor by design: the suffix + // went to `pending_view_log` and the bodies are still being repaired, so the + // journal holds nothing at those ops and would report them blank AND nacked, + // since a hole above the commit point is normally proof the op never arrived. + // Here it proves only unfinished repair, and enough such senders reach a nack + // quorum against ops the view just decided to keep. + // + // The head rises to the view's head too, so a later view change cannot let the + // op backtrack below what this replica already acknowledged. + let view_head = view_headers + .and_then(<[PrepareHeader]>::first) + .map_or(0, |header| header.op); + let op = op.max(view_head); + if op == 0 { + return DvcSuffix::empty(); + } + // Window runs from the commit point up, floored at 1 because ops are 1-based. + // That floor is a scan bound only: the lines below can raise it above the + // commit point, so no reader may read it back as one. See `merge_commit_max`. + let mut low = commit.max(1); + if low > op { + return DvcSuffix::empty(); + } + if op - low + 1 > DVC_HEADERS_MAX as u64 { + // Defensive: every plane's `prepare_queue_depth` is capped below + // `DVC_HEADERS_MAX` so `op - commit` cannot reach this. If it does, the + // clamped-away ops go out described by nobody and the merge stalls rather + // than deciding wrongly. Keep the highest entries, whose fate the view + // change decides, and log it rather than shipping a different window. + let clamped = op - DVC_HEADERS_MAX as u64 + 1; + tracing::warn!( + commit, + op, + window_from = clamped, + "uncommitted suffix wider than {DVC_HEADERS_MAX} entries; truncating the DVC window \ + from below. Ops {}..={} are now undecidable and will stall the view change", + commit + 1, + clamped - 1 + ); + low = clamped; + } + + let len = usize::try_from(op - low + 1).unwrap_or(DVC_HEADERS_MAX); + let mut headers = Vec::with_capacity(len); + let mut nack_bitset = 0u128; + let mut present_bitset = 0u128; + for (index, entry_op) in (low..=op).rev().enumerate() { + if let Some(header) = header_at(entry_op) { + headers.push(header); + // A header in the index means the entry is in the WAL at a known + // offset, the same condition `on_request_prepares` serves from. + present_bitset |= 1u128 << index; + } else if let Some(header) = + view_headers.and_then(|headers| view_header_at(headers, entry_op)) + { + // Held from the adopted view rather than from the journal, so the + // header is reported and the op is NOT nacked: this replica knows + // the op exists and simply cannot serve its body yet. No present + // bit for the same reason. + headers.push(*header); + } else { + headers.push(dvc_blank(entry_op)); + if entry_op > commit { + nack_bitset |= 1u128 << index; + } else { + // The commit point, the one slot that goes out blank AND + // un-nacked. The merge scans it and may not discard it, so a + // sender is asking the new primary to take the header from + // someone else; if every sender in the quorum does that, the + // op is undecidable and the view never starts. + // + // Every compaction path is supposed to leave this header behind + // (the metadata checkpoint drain stops one op short, a + // partition serves it from the evicted ring), so reaching here + // means a replica whose log genuinely starts above its own + // commit point: a state-transfer receiver that jumped its + // commit floor to a snapshot whose prepares it never held. + tracing::warn!( + op = entry_op, + commit, + "no header at this replica's commit point; the DVC reports it blank and \ + cannot nack it, so the view change stalls unless a peer supplies it" + ); + } + } + } + DvcSuffix::new(headers, nack_bitset, present_bitset) +} + +/// The adopted view's header at `op`, or `None` when the view says nothing about +/// it. +/// +/// Headers run high-to-low from the view's head, so the slot is arithmetic. The +/// op is re-checked rather than assumed: a mismatch means the range is not the +/// contiguous run this indexing needs, and inventing a header for the wrong op +/// is worse than reporting none. +fn view_header_at(view_headers: &[PrepareHeader], op: u64) -> Option<&PrepareHeader> { + let head = view_headers.first()?.op; + let index = usize::try_from(head.checked_sub(op)?).ok()?; + let header = view_headers.get(index)?; + if header.op != op || matches!(dvc_header_kind(header), DvcHeaderKind::Blank) { + return None; + } + Some(header) +} + /// Re-stamp a stored prepare with the current view before retransmission. /// After a view change the primary re-sends its uncommitted suffix as its /// own prepares (VSR), but the journal keeps the original view stamp and @@ -7548,6 +8425,7 @@ async fn dispatch_vsr_actions( h.view = *view; h.namespace = *namespace; h.size = size_of::() as u32; + h.seal(); }); broadcast(msg.into_generic().into_frozen()).await; } @@ -7558,20 +8436,38 @@ async fn dispatch_vsr_actions( op, commit, namespace, + suffix, } => { - let msg = Message::::new(size_of::()) - .transmute_header(|_, h: &mut DoViewChangeHeader| { - h.command = Command2::DoViewChange; - h.cluster = cluster; - h.replica = self_id; - h.view = *view; - h.log_view = *log_view; - h.op = *op; - h.commit = *commit; - h.namespace = *namespace; - h.size = size_of::() as u32; - }); - send(*target, msg.into_generic().into_frozen()).await; + let header_size = size_of::(); + let total_size = header_size + suffix.encoded_len(); + let mut msg = Message::::new(total_size); + // Body first: `transmute_header` zeroes only the header region, so + // anything past it survives. Same order as the manifest build. + suffix.encode_into(&mut msg.as_mut_slice()[header_size..total_size]); + let body_checksum = control_body_checksum(&msg.as_slice()[header_size..total_size]); + let nack_bitset = suffix.nack_bitset(); + let present_bitset = suffix.present_bitset(); + let msg = msg.transmute_header(|_, h: &mut DoViewChangeHeader| { + h.command = Command2::DoViewChange; + h.cluster = cluster; + h.replica = self_id; + h.view = *view; + h.log_view = *log_view; + h.op = *op; + h.commit = *commit; + h.namespace = *namespace; + h.nack_bitset = nack_bitset; + h.present_bitset = present_bitset; + h.checksum_body = body_checksum; + h.size = total_size as u32; + // Last: covers the bitsets a new primary truncates on. + h.seal(); + }); + // Broadcast, not unicast to `target`: a backup seeing a DVC for a + // newer view adopts it instead of waiting out its heartbeat + // timeout, which converges the view change in one round. + let _ = target; + broadcast(msg.into_generic().into_frozen()).await; } VsrAction::SendRequestStartView { view, namespace } => { // Stamp this replica's incarnation so the answering StartView can @@ -7587,6 +8483,7 @@ async fn dispatch_vsr_actions( h.incarnation = incarnation; h.namespace = *namespace; h.size = size_of::() as u32; + h.seal(); }); broadcast(msg.into_generic().into_frozen()).await; } @@ -7597,19 +8494,27 @@ async fn dispatch_vsr_actions( incarnation, target, namespace, + suffix, } => { - let msg = Message::::new(size_of::()) - .transmute_header(|_, h: &mut StartViewHeader| { - h.command = Command2::StartView; - h.cluster = cluster; - h.replica = self_id; - h.view = *view; - h.op = *op; - h.commit = *commit; - h.incarnation = *incarnation; - h.namespace = *namespace; - h.size = size_of::() as u32; - }); + let header_size = size_of::(); + let total_size = header_size + suffix.len() * size_of::(); + let mut msg = Message::::new(total_size); + // Body first: `transmute_header` zeroes only the header region. + encode_prepare_headers(suffix, &mut msg.as_mut_slice()[header_size..total_size]); + let body_checksum = control_body_checksum(&msg.as_slice()[header_size..total_size]); + let msg = msg.transmute_header(|_, h: &mut StartViewHeader| { + h.checksum_body = body_checksum; + h.command = Command2::StartView; + h.cluster = cluster; + h.replica = self_id; + h.view = *view; + h.op = *op; + h.commit = *commit; + h.incarnation = *incarnation; + h.namespace = *namespace; + h.size = total_size as u32; + h.seal(); + }); let frozen = msg.into_generic().into_frozen(); // A probe echo is addressed to its requester: the incarnation it // carries is that replica's freshness proof, and a peer recovering @@ -7650,6 +8555,7 @@ async fn dispatch_vsr_actions( h.operation = prepare_header.operation; h.namespace = *namespace; h.size = size_of::() as u32; + h.seal(); }); send(*target, msg.into_generic().into_frozen()).await; } @@ -7683,49 +8589,12 @@ async fn dispatch_vsr_actions( let Some(journal) = journal else { continue; }; - // Collect headers before borrowing the pipeline to avoid - // holding borrow_mut() across journal reads. - let mut gap_at = None; - let entries: Vec<_> = (*from_op..=*to_op) - .map_while(|op| { - let Some(header) = journal.handle().header(op as usize) else { - gap_at = Some(op); - return None; - }; - // New-primary path: lift the monotonic timestamp - // floor to the rebuilt log so post-view-change - // prepares cannot stamp below committed ones. - consensus.observe_prepare_timestamp(header.timestamp); - let mut entry = consensus::PipelineEntry::new(*header); - entry.add_ack(self_id); - Some(entry) - }) - .collect(); - if let Some(missing_op) = gap_at { - // A primary's own uncommitted suffix has no repair - // source: peers ack'd nothing above the gap or the DVC - // merge would have carried it, so the range is decided - // lost. Truncate the sequencer to the last op we could - // rebuild so the next client prepare chains correctly. - let rebuilt_up_to = missing_op.saturating_sub(1); - tracing::warn!( - replica = self_id, - missing_op, - range_start = from_op, - range_end = to_op, - rebuilt = entries.len(), - "RebuildPipeline: journal gap at op {missing_op}, \ - truncating sequencer from {to_op} to {rebuilt_up_to} \ - ({}/{} ops rebuilt)", - entries.len(), - to_op - from_op + 1, - ); - consensus.sequencer().set_sequence(rebuilt_up_to); - } - let mut pipeline = consensus.pipeline().borrow_mut(); - for entry in entries { - pipeline.push(entry); - } + rebuild_pipeline_entries(consensus, self_id, *from_op, *to_op, |op| { + usize::try_from(op) + .ok() + .and_then(|slot| journal.handle().header(slot)) + .map(|header| *header) + }); } // Handled by the caller (shard view change handlers) since it // requires access to the plane's commit_journal method. @@ -7746,6 +8615,7 @@ async fn dispatch_vsr_actions( h.namespace = *namespace; h.timestamp_monotonic = *timestamp_monotonic; h.size = size_of::() as u32; + h.seal(); }, ); broadcast(msg.into_generic().into_frozen()).await; @@ -7824,6 +8694,7 @@ async fn dispatch_partition_journal_actions( h.operation = prepare_header.operation; h.namespace = *namespace; h.size = size_of::() as u32; + h.seal(); }); send(*target, msg.into_generic().into_frozen()).await; } @@ -7867,42 +8738,9 @@ async fn dispatch_partition_journal_actions( } } VsrAction::RebuildPipeline { from_op, to_op } => { - let mut gap_at = None; - let entries: Vec<_> = (*from_op..=*to_op) - .map_while(|op| { - let Some(header) = journal.header_by_op(op) else { - gap_at = Some(op); - return None; - }; - // New-primary path: lift the monotonic timestamp - // floor to the rebuilt log so post-view-change - // prepares cannot stamp below committed ones. - consensus.observe_prepare_timestamp(header.timestamp); - let mut entry = consensus::PipelineEntry::new(header); - entry.add_ack(self_id); - Some(entry) - }) - .collect(); - if let Some(missing_op) = gap_at { - let rebuilt_up_to = missing_op.saturating_sub(1); - tracing::warn!( - replica = self_id, - missing_op, - range_start = from_op, - range_end = to_op, - rebuilt = entries.len(), - "RebuildPipeline: journal gap at op {missing_op}, \ - truncating sequencer from {to_op} to {rebuilt_up_to} \ - ({}/{} ops rebuilt)", - entries.len(), - to_op - from_op + 1, - ); - consensus.sequencer().set_sequence(rebuilt_up_to); - } - let mut pipeline = consensus.pipeline().borrow_mut(); - for entry in entries { - pipeline.push(entry); - } + rebuild_pipeline_entries(consensus, self_id, *from_op, *to_op, |op| { + journal.header_by_op(op) + }); } _ => {} } @@ -7935,6 +8773,7 @@ mod persist_gate_tests { incarnation: 0, target: None, namespace: 7, + suffix: Vec::new(), }, VsrAction::CommitJournal, rebuild(), @@ -7963,3 +8802,323 @@ mod persist_gate_tests { assert_eq!(wire.len(), 1); } } + +#[cfg(test)] +mod repair_scope_tests { + //! Who parked the log decides what it means. + + use super::{MergedLog, repair_op_in_scope, repair_serve_ceiling}; + use iggy_binary_protocol::{Command2, PrepareHeader}; + + fn header(op: u64) -> PrepareHeader { + PrepareHeader { + command: Command2::Prepare, + op, + ..Default::default() + } + } + + /// A view that started at op 100 with commit 98. + fn parked() -> MergedLog { + MergedLog { + op_head: 100, + commit_max: 98, + headers: (98..=100).rev().map(header).collect(), + committed_elsewhere: Vec::new(), + } + } + + #[test] + fn given_a_backup_with_a_parked_log_when_repairing_above_the_view_head_should_accept() { + // A backup keeps its parked `StartView` suffix for the whole view, so at + // op 200 the parked head is 100 ops stale. Reading it as a repair scope + // silently discards the served op: the retry loops, the commit walk + // freezes, checkpointing stops, and the backup stops acking. + assert!( + repair_op_in_scope(Some(&parked()), false, 149, 150), + "a backup repairs for the whole view, not just the view-start range" + ); + } + + #[test] + fn given_a_backup_with_a_parked_log_when_repairing_below_commit_min_should_reject() { + // A backup's parked log grants no licence to re-ingest committed ops. + assert!(!repair_op_in_scope(Some(&parked()), false, 149, 149)); + } + + #[test] + fn given_a_primary_elect_when_repairing_toward_its_merged_log_should_use_it_as_the_scope() { + let pending = parked(); + // Inside the merged range, including inherited headers below `commit_min`. + assert!(repair_op_in_scope(Some(&pending), true, 99, 98)); + assert!(repair_op_in_scope(Some(&pending), true, 99, 100)); + // Outside it: the primary-elect is not repairing toward these. + assert!(!repair_op_in_scope(Some(&pending), true, 99, 101)); + assert!(!repair_op_in_scope(Some(&pending), true, 99, 97)); + // With nothing parked, the ordinary commit-point rule applies. + assert!(!repair_op_in_scope(None, false, 149, 149)); + assert!(repair_op_in_scope(None, false, 149, 150)); + } + + #[test] + fn given_a_primary_elect_when_an_op_is_committed_elsewhere_should_accept_it() { + let mut pending = parked(); + pending.committed_elsewhere.push(header(42)); + assert!(repair_op_in_scope(Some(&pending), true, 99, 42)); + } + + #[test] + fn given_a_repair_request_when_serving_should_clamp_to_the_frontier_but_not_below_it() { + // `validate` accepts any `to_op >= from_op` and the serve path walks op by + // op with no `.await`, so an unclamped ceiling hangs the whole shard. + assert_eq!(repair_serve_ceiling(u64::MAX, 40, 90), 90); + assert_eq!(repair_serve_ceiling(50, 40, 90), 50); + // The suffix a new primary repairs toward sits above every commit point, + // so clamping to `commit_max` alone deadlocks the view change. + assert_eq!(repair_serve_ceiling(90, 40, 90), 90); + // `commit_max` above the local head still counts: heartbeats outrun prepares. + assert_eq!(repair_serve_ceiling(u64::MAX, 120, 90), 120); + } +} + +#[cfg(test)] +mod dvc_suffix_window_tests { + //! The suffix window's floor is a scan bound, not a commit point. + //! + //! Reading the lowest suffix op back as a proven commit point assumes suffix + //! generation stops at the sender's commit. These pin the two paths that break + //! that premise, so it cannot be quietly reintroduced. + + use super::{DVC_HEADERS_MAX, build_dvc_suffix}; + use iggy_binary_protocol::{Command2, Operation, PrepareHeader}; + + /// A real prepare at `op`. The operation must not be `Reserved`: that is + /// exactly `dvc_blank`, and `dvc_header_kind` classifies by equality with it. + fn held(op: u64) -> PrepareHeader { + PrepareHeader { + command: Command2::Prepare, + operation: Operation::CreateStream, + op, + ..Default::default() + } + } + + /// The lowest op the built window describes. + fn floor(suffix: &consensus::DvcSuffix) -> Option { + suffix.headers().last().map(|header| header.op) + } + + /// A view's headers for `low..=high`, high-to-low as the suffix carries them. + fn view_headers(low: u64, high: u64) -> Vec { + (low..=high).rev().map(held).collect() + } + + #[test] + fn given_an_adopted_view_when_the_journal_is_empty_should_report_its_headers_unnacked() { + // A backup that adopted a `StartView` put the suffix in `pending_view_log` + // and is still repairing bodies, so its journal holds nothing at those ops. + // Reading the journal alone reports them blank AND nacked, which reaches a + // nack quorum against ops the view had just decided to keep. + let view = view_headers(3, 5); + let suffix = build_dvc_suffix(2, 0, |_| None, Some(&view)); + + assert_eq!( + suffix.len(), + 4, + "the window rises to the view's head even with an empty journal" + ); + assert_eq!( + floor(&suffix), + Some(2), + "the floor is still the commit point" + ); + assert_eq!( + suffix.nack_bitset(), + 0, + "a header held from the adopted view is not a nack" + ); + assert_eq!( + suffix.present_bitset(), + 0, + "and its body is not servable, so no present bit either" + ); + } + + #[test] + fn given_no_adopted_view_when_the_journal_is_empty_should_nack() { + // The contrast: without an adopted view the same holes really are proof. + let suffix = build_dvc_suffix(2, 5, |_| None, None); + assert_eq!( + suffix.nack_bitset(), + 0b0111, + "ops 5, 4 and 3 nack; op 2 is the commit point" + ); + } + + #[test] + fn given_an_adopted_view_when_the_journal_covers_part_should_prefer_the_journal() { + // Journal first, so an op whose body this replica can serve keeps its + // present bit; the view fills only what the journal is missing. + let view = view_headers(3, 5); + let suffix = build_dvc_suffix(2, 5, |op| (op == 5).then(|| held(op)), Some(&view)); + + assert_eq!(suffix.len(), 4); + assert_eq!(suffix.present_bitset(), 0b0001, "only op 5 is servable"); + assert_eq!(suffix.nack_bitset(), 0, "the view covers ops 4 and 3"); + + // The head is the max of the two, never the view's alone. + let short_view = view_headers(3, 4); + let deeper = build_dvc_suffix(2, 6, |op| Some(held(op)), Some(&short_view)); + assert_eq!(deeper.headers().first().map(|header| header.op), Some(6)); + assert_eq!(deeper.present_bitset(), 0b1_1111, "ops 6 down to 2"); + } + + #[test] + fn given_a_blank_view_entry_should_not_report_it_as_held() { + // A blank is the view saying "no header here", not one this replica holds. + let mut view = view_headers(3, 5); + view[1] = consensus::dvc_blank(4); + let suffix = build_dvc_suffix(2, 0, |_| None, Some(&view)); + + assert_eq!(suffix.nack_bitset(), 0b010, "only the blank op nacks"); + } + + #[test] + fn given_no_header_at_the_commit_point_should_report_it_blank_and_undecidable() { + // The window's floor is the commit point, and a blank there is the one + // entry that goes out with neither a header nor a nack. The merge scans + // that op and may not discard it, so a quorum of these deadlocks the view + // change. Pinned here because both compaction paths are meant to keep the + // header alive precisely so this shape never leaves a healthy replica. + let suffix = build_dvc_suffix(5, 5, |_| None, None); + + assert_eq!(suffix.len(), 1); + assert_eq!(floor(&suffix), Some(5)); + assert_eq!( + suffix.nack_bitset(), + 0, + "the commit point is never nacked, whatever the journal says" + ); + assert_eq!(suffix.present_bitset(), 0); + } + + #[test] + fn given_a_window_at_the_depth_ceiling_when_building_should_floor_at_the_commit() { + // At the deepest legal prepare-queue depth the window still starts exactly + // at the commit point, so nothing is clamped and no op goes undescribed. + // Config ceilings and `LocalPipeline::with_capacities` enforce the depth. + let depth = DVC_HEADERS_MAX as u64 - 1; + let commit = 500; + let op = commit + depth; + let suffix = build_dvc_suffix(commit, op, |op| Some(held(op)), None); + + assert_eq!(suffix.len(), DVC_HEADERS_MAX, "the widest window that fits"); + assert_eq!( + floor(&suffix), + Some(commit), + "at the ceiling the floor is still the commit point" + ); + } + + #[test] + fn given_a_window_past_the_depth_ceiling_when_building_should_clamp_above_the_commit() { + // One op deeper and the window clamps: the floor sits 501 ops above the + // sender's commit, with no marker on the frame saying so. + let commit = 500; + let op = commit + DVC_HEADERS_MAX as u64; + let suffix = build_dvc_suffix(commit, op, |op| Some(held(op)), None); + + assert_eq!(suffix.len(), DVC_HEADERS_MAX); + assert_eq!( + floor(&suffix), + Some(op - DVC_HEADERS_MAX as u64 + 1), + "the clamped floor sits above the commit point" + ); + assert!(floor(&suffix) > Some(commit)); + + // Second path, at any depth: ops are 1-based, so commit 0 floors at op 1. + let from_zero = build_dvc_suffix(0, 3, |op| Some(held(op)), None); + assert_eq!(floor(&from_zero), Some(1)); + } +} + +#[cfg(test)] +mod control_frame_tests { + //! A control frame's body must be verified on a rule corruption cannot switch + //! off. Keying on `checksum_body` looking sealed is bypassable by zeroing it. + + use super::{control_body_checksum, control_suffix_body_verified}; + use iggy_binary_protocol::{Command2, DoViewChangeHeader, PrepareHeader}; + use server_common::Message; + use std::mem::size_of; + + /// A `DoViewChange` frame carrying `entries` blank suffix headers. + fn frame(entries: usize, checksum_body: u128) -> Message { + let header_size = size_of::(); + let total = header_size + entries * size_of::(); + let mut msg = Message::::new(total); + for (index, byte) in msg.as_mut_slice()[header_size..total] + .iter_mut() + .enumerate() + { + *byte = u8::try_from(index % 251).expect("modulus fits u8"); + } + msg.transmute_header(|_, header: &mut DoViewChangeHeader| { + header.command = Command2::DoViewChange; + header.checksum_body = checksum_body; + header.size = u32::try_from(total).expect("frame fits u32"); + }) + } + + #[test] + fn given_a_sealed_body_when_verifying_should_accept() { + let header_size = size_of::(); + let unsealed = frame(2, 0); + let sealed_value = control_body_checksum( + &unsealed.as_slice()[header_size..unsealed.header().size as usize], + ); + let msg = frame(2, sealed_value); + + assert!( + control_suffix_body_verified(&msg, msg.header().checksum_body).is_some(), + "a correctly sealed body must be accepted" + ); + } + + #[test] + fn given_a_body_with_a_zeroed_checksum_when_verifying_should_reject() { + // A peer predating the suffix sends no body, so a non-empty body always came + // from a sender that seals it; a zero here is corruption, not age. Treating + // it as "unsealed, skip" disables the layer by clearing one field. + let msg = frame(2, 0); + assert!( + control_suffix_body_verified(&msg, msg.header().checksum_body).is_none(), + "a non-empty body with a zeroed checksum must be rejected, not waved through" + ); + } + + #[test] + fn given_a_corrupted_body_when_verifying_should_reject() { + let header_size = size_of::(); + let unsealed = frame(2, 0); + let sealed_value = control_body_checksum( + &unsealed.as_slice()[header_size..unsealed.header().size as usize], + ); + let mut msg = frame(2, sealed_value); + msg.as_mut_slice()[header_size] ^= 0xFF; + + assert!( + control_suffix_body_verified(&msg, msg.header().checksum_body).is_none(), + "a body that does not match its checksum must be rejected" + ); + } + + #[test] + fn given_a_header_only_frame_when_verifying_should_accept() { + // Rolling upgrade: a peer predating the suffix sends numbers only, no body. + let msg = frame(0, 0); + let body = control_suffix_body_verified(&msg, msg.header().checksum_body) + .expect("a header-only frame has nothing to verify"); + assert!(body.is_empty()); + } +} diff --git a/core/simulator/src/deps.rs b/core/simulator/src/deps.rs index de7c2240ed..24734a8ebb 100644 --- a/core/simulator/src/deps.rs +++ b/core/simulator/src/deps.rs @@ -180,6 +180,33 @@ impl>> Journal for SimJournal { /// ever superseded by a snapshot. Answered explicitly (the trait has no /// default) so a simulated state transfer has to opt into a watermark /// rather than silently inherit one that never moves. + /// Drop the suffix, so a simulated backup whose entries disagree with a started + /// view reconciles the way a real one does. Mirrors + /// `PrepareJournal::truncate_from`, whose watermark stays put; here it never moves. + async fn truncate_from(&self, from_op: u64) -> std::io::Result { + if from_op == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "truncate_from: ops are 1-based, so 0 would discard the whole journal", + )); + } + #[cfg(debug_assertions)] + let _guard = JournalAccessGuard::new(&self.accessing); + let headers = unsafe { &mut *self.headers.get() }; + let offsets = unsafe { &mut *self.offsets.get() }; + let doomed: Vec = headers + .keys() + .copied() + .filter(|op| *op >= from_op) + .collect(); + for op in &doomed { + headers.remove(op); + offsets.remove(op); + } + self.last_op.set(headers.keys().copied().max()); + Ok(doomed.len()) + } + fn snapshot_op(&self) -> u64 { 0 } @@ -268,6 +295,24 @@ impl SimJournal { self.last_op.get() } + /// Forget one op, leaving a hole exactly where a lost prepare would. + /// + /// Tests only. The alternative is choreographing `Prepare`, `Commit` and + /// `RepairPrepare` drops on a directed link until a replica falls behind, which + /// is fragile to tune; the scenarios are about what a replica does with a hole, + /// not how it got one. + /// + /// `last_op` is deliberately left alone: a hole below the head must not look like + /// a shorter log, since that is the state a view change has to survive. + pub fn forget_op(&self, op: u64) -> bool { + #[cfg(debug_assertions)] + let _guard = JournalAccessGuard::new(&self.accessing); + let headers = unsafe { &mut *self.headers.get() }; + let offsets = unsafe { &mut *self.offsets.get() }; + offsets.remove(&op); + headers.remove(&op).is_some() + } + /// The committed watermark to restore after a restart, mirroring /// `metadata::recover`. On a solo cluster every appended op commits the instant /// it is durable, so the head IS the commit point; otherwise the highest diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index 64aecc440a..1e98b0ab0e 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs @@ -3097,3 +3097,162 @@ mod tests { assert_no_frame_drops(&sim); } } + +#[cfg(test)] +mod view_change_data_loss_tests { + //! A committed, client-acknowledged op must survive a view change even when + //! the replica that becomes primary is the one missing it. + //! + //! Without the sender's log suffix on the `DoViewChange`, the new primary adopts + //! the winner's op NUMBER, rebuilds its pipeline from its OWN journal, hits the + //! hole, and truncates the range as "decided lost" -- discarding an op journaled + //! on a quorum and already replied to. The next client op then reuses the number + //! and collides with the stale entry on the up-to-date backup. + //! + //! The hole here is punched at the commit point, so the assertion that catches a + //! regression is "the op came back", not "the head did not regress": with nothing + //! uncommitted there is no pipeline rebuild to truncate. The `dvc_merge` unit + //! tests cover the sequencer-truncation path directly. + + use super::*; + use consensus::{Sequencer, Status}; + use journal::Journal; + + /// Whether a replica's shard-0 metadata consensus is a settled primary in a + /// view past the one that crashed. + fn is_new_metadata_primary(sim: &Simulator, replica: u8) -> bool { + sim.replicas[replica as usize].shards[0] + .plane + .metadata() + .consensus + .as_ref() + .is_some_and(|consensus| { + consensus.view() > 0 + && consensus.status() == Status::Normal + && consensus.is_primary() + }) + } + + /// `(head op, commit_max)` of a replica's shard-0 metadata consensus. + fn metadata_progress(sim: &Simulator, replica: u8) -> (u64, u64) { + let consensus = sim.replicas[replica as usize].shards[0] + .plane + .metadata() + .consensus + .as_ref() + .expect("shard 0 owns metadata consensus"); + ( + consensus.sequencer().current_sequence(), + consensus.commit_max(), + ) + } + + /// Whether a replica's metadata journal holds `op`. + fn metadata_holds(sim: &Simulator, replica: u8, op: u64) -> bool { + let journal = sim.replicas[replica as usize].shards[0] + .plane + .metadata() + .journal + .as_ref() + .expect("shard 0 owns the metadata journal"); + let slot = usize::try_from(op).expect("op fits usize"); + Journal::header(journal.as_ref(), slot).is_some() + } + + /// Drop `op` from a replica's metadata journal, leaving a hole. + fn metadata_forget(sim: &Simulator, replica: u8, op: u64) -> bool { + sim.replicas[replica as usize].shards[0] + .plane + .metadata() + .journal + .as_ref() + .expect("shard 0 owns the metadata journal") + .forget_op(op) + } + + #[test] + fn given_committed_op_missing_on_next_primary_when_primary_crashes_should_survive_view_change() + { + server_common::MemoryPool::init_pool(&server_common::MemoryPoolConfigOther { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); + + let replica_count: u8 = 3; + let client_id: u128 = 1; + let network_opts = packet::PacketSimulatorOptions { + node_count: replica_count, + client_count: 1, + ..packet::PacketSimulatorOptions::default() + }; + let mut sim = Simulator::new( + replica_count as usize, + std::iter::once(client_id), + network_opts, + ); + let client = SimClient::new(client_id); + + // Commit some metadata ops so there is a log to lose. Registering binds + // a session, and seeding a stream/topic/partition commits several more. + sim.register_client_with_primary(&client); + sim.seed_stream_topic_partition(IggyNamespace::new(1, 1, 0)); + for _ in 0..200 { + sim.step(); + } + + // Replica 0 is primary for view 0, so replica 1 is primary-elect for view 1 + // (view % replica_count): the replica whose hole decides the outcome. + let next_primary: u8 = 1; + let (_, committed) = metadata_progress(&sim, next_primary); + assert!( + committed > 0, + "the test needs committed metadata ops to be able to lose one" + ); + + // Every replica must hold the op: the point is that it IS recoverable, and + // only the incoming primary lacks it. + for replica in 0..replica_count { + assert!( + metadata_holds(&sim, replica, committed), + "replica {replica} must hold op {committed} before the hole is punched" + ); + } + + // Punch the hole: the incoming primary forgets an op its peers still hold. + assert!( + metadata_forget(&sim, next_primary, committed), + "op {committed} must have been present to forget" + ); + + sim.replica_crash(0); + for _ in 0..1500 { + sim.step(); + } + + // A primary must emerge among the survivors. + let primary = (1..replica_count) + .find(|&replica| is_new_metadata_primary(&sim, replica)) + .expect("a metadata primary must be elected after the old one crashes"); + + let (head, commit_max) = metadata_progress(&sim, primary); + + // The committed op must not have been discarded. + assert!( + head >= committed, + "the new primary's head ({head}) regressed below the committed op ({committed}); \ + a committed, acknowledged op was discarded by the view change" + ); + assert!( + commit_max >= committed, + "commit_max ({commit_max}) regressed below the committed op ({committed})" + ); + + // And back in the new primary's journal: the view change repaired the hole + // from a peer that offered the body, rather than declaring the op lost. + assert!( + metadata_holds(&sim, primary, committed), + "op {committed} must be repaired back into the new primary's journal" + ); + } +} diff --git a/core/simulator/src/replica.rs b/core/simulator/src/replica.rs index 9abf73c325..a54a310a2e 100644 --- a/core/simulator/src/replica.rs +++ b/core/simulator/src/replica.rs @@ -283,6 +283,7 @@ pub fn new_shard( messages_required_to_save: 1000, size_of_messages_required_to_save: IggyByteSize::from(4 * 1024 * 1024), enforce_fsync: false, //Disable fsync for simulation + validate_checksum: true, segment_size: IggyByteSize::from(1024 * 1024 * 1024), encryptor: None, };