diff --git a/crates/hotblocks-harness/README.md b/crates/hotblocks-harness/README.md index 167ac9ff..aa26404c 100644 --- a/crates/hotblocks-harness/README.md +++ b/crates/hotblocks-harness/README.md @@ -75,6 +75,15 @@ block is emitted (RP-9); a filter-sparse query returning nothing tells the clien how far it got (GAP-8). Scanning with `include_all` sidesteps that, so the harness can always advance. Do not "optimize" it away. +**A source asked above its tip answers no-data, not a fork.** RP-5b confines the fork signal +to `from == tip + 1` — the one position where the parent assertion is evaluable against a block +the source actually holds. The simulator used to signal a fork at *any* position above its tip, +which made a source that is merely **behind** indistinguishable from one that **disagrees**; with +several endpoints per dataset that is the difference between a laggard and a reorg. The old +behavior is kept as an explicit fault (`SimFaults::fork_signal_above_tip`), because a real source +doing it wedges the service — one such endpoint out of three is enough +(`ct4_a_single_source_signalling_a_fork_above_its_tip_does_not_park_ingestion`). + **Numbering may be sparse.** Solana numbers blocks by time-based slots and a slot that produced nothing leaves a hole, so contiguous *numbering* is not an invariant — being parent and child is (`Block::parent_number`). The service agrees: it links batches and chunks by hash and never by @@ -124,7 +133,13 @@ Fixed in `crates/data-client/src/reqwest/lines.rs`; pinned by a unit test there matrix. - **CT-4 (fork/finality corpus)** — `Harness::fork()` and the model's `resolve_fork` / `Finalize::IntegrityFault` are implemented and unit-tested; the follower implements the - normative CONFLICT recovery of 04 §7. What is missing is the scripts. + normative CONFLICT recovery of 04 §7. What is missing is most of the scripts. + `ct4_lagging_source` covers the multi-endpoint shape production actually runs — several + sources per dataset, one of them behind (`HarnessConfig::sources`, `Harness::produce_ahead`). + Note `Harness::fork()` refuses to run with peers configured: reorging one endpoint of several + is a source *disagreement*, and what the service should do with it is the fork-consensus + question (`StandardDataSource::poll_next_event` — majority, or all-active, or a 2 s timeout). + That deserves a deliberate script, not an accidental one. - **CT-5 (error taxonomy)** — `ct5_error_soundness` covers unsupported-dialect containment, error classification, and mid-stream worker-panic abort; the anchored check across large sparse-number holes is deferred (GAP-21, test `#[ignore]`d). `Model::predict_query` supplies diff --git a/crates/hotblocks-harness/src/harness.rs b/crates/hotblocks-harness/src/harness.rs index 676198e6..d0c54b51 100644 --- a/crates/hotblocks-harness/src/harness.rs +++ b/crates/hotblocks-harness/src/harness.rs @@ -8,14 +8,14 @@ use std::{path::Path, sync::Arc, time::Duration}; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result, bail, ensure}; use crate::{ chain::Chain, compare::{self, Quiescence}, driver::{Client, Follower}, model::{Finalize, Model}, - sim::{Numbering, SimConfig, SimDataset, SourceSim}, + sim::{Numbering, SimConfig, SimDataset, SimStats, SourceSim}, sut::{DatasetSpec, Retention, Sut, SutConfig}, types::{Anchor, BlockNumber, block_hash} }; @@ -34,6 +34,11 @@ pub struct HarnessConfig { pub anchored: bool, /// How long the simulator holds a `/stream` request that has nothing to serve. pub source_poll: Duration, + /// How many independent simulators serve the dataset. Production configures several + /// endpoints per dataset, so anything above 1 exercises `StandardDataSource`'s + /// multi-endpoint path: the fixed-order poll race, the fork consensus, and the + /// per-endpoint `MaybeOnHead` flush trigger. + pub sources: usize, pub rust_log: String, pub quiescence: Quiescence, pub sut_args: Vec @@ -56,6 +61,7 @@ impl HarnessConfig { }, anchored: true, source_poll: Duration::from_millis(200), + sources: 1, rust_log: "info".to_string(), quiescence: Quiescence::default(), sut_args: sut.args @@ -64,7 +70,12 @@ impl HarnessConfig { } pub struct Harness { + /// The primary source — the one a script drives by default. pub sim: SourceSim, + /// The remaining endpoints serving the same dataset. Block hashes are a pure function of + /// ⟨number, fork⟩, so a peer fed the same script holds a byte-identical chain and one fed + /// less holds an exact prefix: a source that is *behind*, not one that disagrees. + pub peers: Vec, pub sut: Sut, pub client: Client, pub model: Model, @@ -72,23 +83,35 @@ pub struct Harness { pub dataset: String, pub start_block: BlockNumber, pub numbering: Numbering, - pub quiescence: Quiescence + pub quiescence: Quiescence, + /// Per peer, the blocks it owes the primary. Per peer because production's laggard is a + /// *minority* — one endpoint of three, not all of them. + peer_deficits: Vec } impl Harness { pub async fn start(cfg: HarnessConfig) -> Result { - let sim = SourceSim::start(SimConfig { - datasets: vec![SimDataset { - id: cfg.dataset.clone(), - chain: cfg.chain.clone(), - start_block: cfg.start_block, - base_timestamp_ms: cfg.base_timestamp_ms, - numbering: cfg.numbering - }], - poll_timeout: cfg.source_poll - }) - .await - .context("failed to start the source simulator")?; + ensure!(cfg.sources >= 1, "a dataset needs at least one source"); + + let mut sims = Vec::with_capacity(cfg.sources); + for _ in 0..cfg.sources { + sims.push( + SourceSim::start(SimConfig { + datasets: vec![SimDataset { + id: cfg.dataset.clone(), + chain: cfg.chain.clone(), + start_block: cfg.start_block, + base_timestamp_ms: cfg.base_timestamp_ms, + numbering: cfg.numbering + }], + poll_timeout: cfg.source_poll + }) + .await + .context("failed to start the source simulator")? + ); + } + let peers = sims.split_off(1); + let sim = sims.pop().expect("at least one source"); let mut sut_cfg = SutConfig::new( &cfg.bin, @@ -96,7 +119,10 @@ impl Harness { id: cfg.dataset.clone(), kind: cfg.chain.config_kind().to_string(), retention: cfg.retention.clone(), - sources: vec![sim.base_url(&cfg.dataset)] + sources: std::iter::once(&sim) + .chain(peers.iter()) + .map(|s| s.base_url(&cfg.dataset)) + .collect() }] ); sut_cfg.args = cfg.sut_args; @@ -108,7 +134,9 @@ impl Harness { let model = Model::new(Anchor::new(cfg.start_block - 1, anchor_hash)); Ok(Self { + peer_deficits: vec![0; peers.len()], sim, + peers, sut, client, model, @@ -120,21 +148,105 @@ impl Harness { }) } - /// The source mints `n` more blocks; the model EXTENDs by the same run. + /// Every source mints `n` more blocks; the model EXTENDs by the same run. pub fn produce(&mut self, n: u32) -> Result<()> { + self.produce_lagging(&[], n) + } + + /// Only the primary mints: *every* peer falls `n` blocks behind. + pub fn produce_ahead(&mut self, n: u32) -> Result<()> { + let all: Vec = (0..self.peers.len()).collect(); + self.produce_lagging(&all, n) + } + + /// The primary and every peer outside `behind` mint `n` blocks; those named hold and fall + /// `n` further back. + /// + /// A peer left behind holds an exact prefix of the canonical chain: healthy, agreeing, merely + /// not caught up — the one state `StandardDataSource` cannot name, since an endpoint leaves + /// the rotation by erroring and never by being slow. + pub fn produce_lagging(&mut self, behind: &[usize], n: u32) -> Result<()> { + for &i in behind { + ensure!( + i < self.peers.len(), + "peer {i} does not exist — {} configured", + self.peers.len() + ); + } let blocks = self.sim.produce(&self.dataset, n); - self.model.extend(&blocks, None) + self.model.extend(&blocks, None)?; + for i in 0..self.peers.len() { + if behind.contains(&i) { + self.peer_deficits[i] += n; + } else { + self.mint_on_peer(i, n)?; + } + } + Ok(()) + } + + /// Bring every peer back up to the primary's tip. + pub fn catch_up_peers(&mut self) -> Result<()> { + for i in 0..self.peers.len() { + let owed = std::mem::take(&mut self.peer_deficits[i]); + self.mint_on_peer(i, owed)?; + } + Ok(()) + } + + /// Mint on a peer and check it reproduced the primary's chain. Determinism is the premise + /// of every multi-source script: the moment it breaks, a lag test silently becomes a fork + /// test and stops testing what it claims to. + fn mint_on_peer(&self, i: usize, n: u32) -> Result<()> { + let minted = self.peers[i].produce(&self.dataset, n); + let canonical = self.model.blocks_in( + minted.first().map_or(0, |b| b.number), + minted.last().map_or(0, |b| b.number) + ); + // `zip` stops at the shorter side: without this a length mismatch passes vacuously. + ensure!( + minted.len() == canonical.len(), + "peer {i} minted {} blocks over a range the canonical chain fills with {}", + minted.len(), + canonical.len() + ); + for (got, want) in minted.iter().zip(canonical) { + ensure!( + got.hash == want.hash && got.number == want.number, + "peer {i} diverged from the primary at block {}: {} vs {}", + got.number, + got.hash, + want.hash + ); + } + Ok(()) } /// The source reorgs: the suffix at `from` is replaced by `len` freshly-minted blocks. + /// + /// Single-source only. Reorging one endpoint of several is not "a fork" but a disagreement, + /// and what the service should do with it is the fork-consensus question + /// (`StandardDataSource::poll_next_event`: majority, or all-active, or a 2 s timeout) — + /// that needs its own script, not an accidental one. pub fn fork(&mut self, from: BlockNumber, len: u32) -> Result<()> { + ensure!( + self.peers.is_empty(), + "fork() drives the primary only; with peers configured it would script a source \ + disagreement, not a reorg" + ); let blocks = self.sim.fork(&self.dataset, from, len)?; self.model.replace(from, &blocks, None) } - /// The source declares block `number` final. + /// The sources declare block `number` final. A peer that has not reached it yet stays + /// silent — a lagging source cannot declare a block it does not hold. pub fn finalize(&mut self, number: BlockNumber) -> Result<()> { let r = self.sim.finalize(&self.dataset, number)?; + for peer in &self.peers { + if peer.tip(&self.dataset).is_some_and(|t| t.number >= number) { + peer.finalize(&self.dataset, number)?; + } + } match self.model.finalize(&r) { Finalize::Applied | Finalize::Ignored => Ok(()), Finalize::IntegrityFault => bail!("the script declared a finality that contradicts the model: {r}") @@ -157,7 +269,16 @@ impl Harness { pub async fn settle(&self) -> Result<()> { compare::await_quiescence(&self.client, &self.model, &self.quiescence) .await - .with_context(|| format!("the source saw: {:?}", self.sim.stats(&self.dataset))) + .with_context(|| format!("the sources saw: {:?}", self.source_stats())) + } + + /// What each endpoint saw, primary first — in a multi-source script the endpoint that + /// explains a failure is rarely the driven one. + pub fn source_stats(&self) -> Vec { + std::iter::once(&self.sim) + .chain(self.peers.iter()) + .map(|s| s.stats(&self.dataset)) + .collect() } /// Diff every observable against the model. diff --git a/crates/hotblocks-harness/src/sim.rs b/crates/hotblocks-harness/src/sim.rs index 375fe46b..c37f8fb9 100644 --- a/crates/hotblocks-harness/src/sim.rs +++ b/crates/hotblocks-harness/src/sim.rs @@ -80,7 +80,12 @@ impl Numbering { #[derive(Clone, Copy, Debug, Default)] pub struct SimFaults { /// Serve a JSONL body whose final record is not newline-terminated. - pub unterminated_final_line: bool + pub unterminated_final_line: bool, + /// Answer a parent assertion naming a position above the tip with a fork signal instead of + /// no-data. RP-5b confines the signal to `from == tip + 1`, the one position where the + /// assertion is evaluable; a source doing it higher reports a divergence it cannot have + /// observed, and a source that is merely behind starts looking like a forked one. + pub fork_signal_above_tip: bool } /// Counters a test can assert on (how the SUT actually drove the source). @@ -179,6 +184,8 @@ impl SourceSim { /// Turn a source-side fault on or off (FM-SRC-*). pub fn inject_fault(&self, dataset: &str, f: impl FnOnce(&mut SimFaults)) { self.with(dataset, |d| f(&mut d.faults)); + // Without it a peer parked in the long poll serves one more request under the old behavior. + self.bump(); } fn with(&self, dataset: &str, f: impl FnOnce(&mut DatasetSim) -> T) -> T { @@ -352,14 +359,19 @@ impl DatasetSim { return Reply::Fork(self.hints_ending_at(parent_pos)); } None => { - // Above our tip: it claims blocks we do not have — disagree, hint at our tip. + // Strictly above our tip, so the assertion names a position we have not + // reached: not evaluable, and RP-5b confines the fork signal to the one + // position where it is (`from == tip + 1`, handled above). A source that is + // merely *behind* must say it has nothing yet — answering "you are on the + // wrong chain" is how a lagging endpoint gets mistaken for a forked one. + // Below our history: unverifiable, serve as-is (the `⊥`-anchor case). if let Some(tip) = self.tip_number() && parent_pos > tip + && self.faults.fork_signal_above_tip { self.stats.fork_signals += 1; return Reply::Fork(self.hints_ending_at(tip)); } - // Below our history: unverifiable, serve as-is (the `⊥`-anchor case). } } } @@ -570,6 +582,38 @@ mod tests { ); } + /// RP-5b confines the fork signal to `from == tip + 1`; above it the honest answer is "nothing + /// yet". Answering "wrong chain" makes a source that is *behind* look like one that + /// *disagrees* — the distinction every multi-source script rests on, so the default is pinned + /// here too, not just the fault. + #[tokio::test] + async fn answers_no_data_above_its_tip_unless_the_fault_is_injected() { + let sim = sim().await; + sim.produce(DS, 5); + let url = format!("{}/stream", sim.base_url(DS)); + let req = json!({"fromBlock": START + 10, "parentBlockHash": "0xnot-our-block"}); + + let res = reqwest::Client::new().post(&url).json(&req).send().await.unwrap(); + assert_eq!( + res.status(), + StatusCode::NO_CONTENT, + "a source below the asked position is behind, not disagreeing" + ); + assert_eq!(sim.stats(DS).fork_signals, 0); + + sim.inject_fault(DS, |f| f.fork_signal_above_tip = true); + + let res = reqwest::Client::new().post(&url).json(&req).send().await.unwrap(); + assert_eq!(res.status(), StatusCode::CONFLICT); + let body: serde_json::Value = res.json().await.unwrap(); + assert_eq!( + body["previousBlocks"].as_array().unwrap().last().unwrap()["number"], + START + 4, + "the fault reports a divergence at a tip it never reached past" + ); + assert_eq!(sim.stats(DS).fork_signals, 1); + } + #[tokio::test] async fn validates_parent_hash_across_a_sparse_numbering_hole() { let sim = SourceSim::start(SimConfig { diff --git a/crates/hotblocks/spec/12-conformance-tdd.md b/crates/hotblocks/spec/12-conformance-tdd.md index 388da896..2997225a 100644 --- a/crates/hotblocks/spec/12-conformance-tdd.md +++ b/crates/hotblocks/spec/12-conformance-tdd.md @@ -4,7 +4,7 @@ This document turns the spec into a test program: the reference model (oracle), harness architecture, the test-class taxonomy, the traceability matrix, and the dated gap register that seeds the hardening backlog. -Statuses and the gap register reflect the state of knowledge as of **2026-07-12** and are +Statuses and the gap register reflect the state of knowledge as of **2026-07-20** and are expected to change; everything else in this document is stable methodology. The harness described here exists: [`crates/hotblocks-harness`](../../hotblocks-harness). @@ -227,7 +227,7 @@ INV-21/22/23 (checks in parentheses): 6. anchored continuation across responses never breaks parent-hash chains (INV-23); 7. watermark coherence: `first ≤ fin ≤ head` whenever reported together (INV-5/30). -## 5. Traceability matrix (status @ 2026-07-15) +## 5. Traceability matrix (status @ 2026-07-20) Legend: **C** covered, **P** partial (some storage-layer or fixture coverage exists; service-level black-box coverage absent), **U** untested. Rows that changed with Phase 0 name @@ -271,7 +271,7 @@ the same property under forks, crashes and retention is the business of CT-2/CT- | LIV-5/6 startup/recovery | CT-2/6 | **U — known-violated** | GAP-7; `Sut::last_startup` records SLI-5 per boot | | LIV-7 reclamation | CT-7 | U | runtime reclaim on by default since 2026-07 (GAP-6 residual: boot-gated residue purge); convergence unmeasured | | LIV-8 isolation | CT-8 | **U — known-violated** | GAP-1/14 | -| LIV-9 fork convergence/alarm | CT-4 | **U — known-suspect** | GAP-5 | +| LIV-9 fork convergence/alarm | CT-4 | **U — known-violated** | GAP-5 now has a deterministic repro (`ct4_a_single_source_signalling_a_fork_above_its_tip_does_not_park_ingestion`): one bad source of three parks ingestion — no alarm, no recovery | | LIV-10 overload recovery | CT-6 | U | | | LIV-11 retention keep-up | CT-7 | U | | | LIV-12 shutdown | CT-2 | **U — known-suspect** | GAP-17; `Sut::stop` measures the drain | @@ -280,7 +280,7 @@ the same property under forks, crashes and retention is the business of CT-2/CT- | RS-8 boot maintenance | CT-2/7 | P | unlink/orphan-purge behaviors have storage-level tests | | RS-10/11 residue/deletion cost | CT-7 | P | GAP-6/13 | | FM-1 robustness | CT-9 | **P — known-violated** | GAP-12 open; unsupported query and query-worker panic classes are closed (§6.1), and the unterminated-record class is pinned by `ct9_source_faults` | -| FM-SRC-* corpus | CT-4 | U | one stale-pack crash-loop already occurred (GAP-5 class); no strike/quarantine substrate (GAP-30) | +| FM-SRC-* corpus | CT-4 | **P** | `ct4_lagging_source`: the multi-endpoint shape production runs — several sources per dataset, one far behind — is covered in both the minority and majority laggard shapes, and a *lagging* source is confirmed harmless to the head, independently of its slot in the fixed poll order. A source answering *wrongly* is not: one endpoint of three signalling a fork above its own tip parks ingestion (GAP-5's shape, GAP-41), pinned `#[ignore]`d. Still no strike/quarantine substrate (GAP-30) | | FM-STOR-2/3 disk pressure | CT-7 | U | incident-derived; no automated test | | FM-OP-1..5 | CT-5 | U | | | SLI-1..12 / PF-* | CT-6 | U | no scenario benchmark harness exists; the transaction-index ingest/lookup Criterion microbench is a component baseline only | @@ -288,7 +288,7 @@ the same property under forks, crashes and retention is the business of CT-2/CT- | OB-2..11 | all | P | query metrics exist; stall gauges pending on PR #83 (unmerged); OB-2 heartbeat, OB-6 debt accounting, OB-9 alarms, OB-11 forensics absent | | OB-12 index state | CT-1 | **P** | CF-wide estimated keys / live SST bytes are exported for both indexes; per-dataset enabled/count/bytes and lookup hit/miss/latency remain absent (GAP-40) | -## 6. Gap register (dated 2026-07-15, informative) +## 6. Gap register (dated 2026-07-20, informative) Known or strongly suspected divergences between this spec and the current system, from incident history, code-level review, and coverage analysis. Priorities: P0 = active @@ -301,7 +301,7 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-2 | Recovered anchor hash is reconstructed from the wrong value after restart — `WriteController::new` takes the first batch's *last*-block hash where the correct value sits in its `parent_block_hash` field; latent until the fork-fallback path consumes it, then ingestion resumes with a wrong expected parent (perpetual source-rejection loop) | INV-40, WP-19, LIV-6 | P1 | CT-2: restart, then force full-window fork fallback; assert resume position equals model | | GAP-3 | Fork floor at the window start is not enforced (marked-as-known in the system); a deeper-than-window divergence may be mishandled instead of becoming an explicit RESET per WP-6b | INV-14, WP-6/6b | P1 | CT-4 deep-fork case: hints strictly below `first(D)` | | GAP-4 | Finality reports strictly below the head are applied without verifying the hash against the stored block — in both the standalone FINALIZE path and the batch-composed path; the window floor is not checked either (GAP-27) | INV-6, WP §2.4 | P2 | CT-4: finality with corrupted hash below head; assert INTEGRITY_FAULT not acceptance | -| GAP-5 | Unapplicable divergence (fork below finality, finality conflicts) results in silent bounded-pause retry forever — no alarm state, no distinct observable; one such class already caused a crash-loop incident | LIV-9b, FM-SRC-5, OB-9 | P1 | CT-4: fork-below-finality script; assert alarmed state within `P-ALARM` while reads keep serving | +| GAP-5 | Unapplicable divergence (fork below finality, finality conflicts) results in silent bounded-pause retry forever — no alarm state, no distinct observable; one such class already caused a crash-loop incident. **Reproduced 2026-07-20** through a second trigger (GAP-41), and reachable from a *single* misbehaving endpoint of three: the head freezes at the position the divergence arrived at and never resumes, while the honest sources keep offering the chain — from the outside, indistinguishable from a hung service | LIV-9b, FM-SRC-5, OB-9 | P1 | `ct4_a_single_source_signalling_a_fork_above_its_tip_does_not_park_ingestion` (`#[ignore]`d); the fork-below-finality script still owes the `P-ALARM` assertion | | GAP-6 | ~~Default deployments never reclaim~~ — routine reclaim fixed 2026-07 (PR #79: 10 s point-delete sweep + deletion-collector + periodic-compaction backstop). REMAINING: the interrupted-build residue purge and the whole-file unlink are confined to the gated boot mode (off by default) — a torn build's residue leaks for good in default config and pins the boot-unlink watermark; SLI-8 under churn still unmeasured | RS-10, RS-8 (RS-6 residual) | P2 (was P0) | CT-7: churn soak in default config; assert SLI-8 bound + residue-age bound | | GAP-7 | Serving is gated on full initialization: tens of seconds of refused connections after deploy, scaling with state size and dataset count; readiness not observable per dataset | LIV-5, OB-8 | P1 | CT-6 S5: SLI-5 vs state-size regression curve | | GAP-8 | ~~Zero-emission responses do not convey the coverage end~~ — a de-facto carrier existed all along (the coverage-end block is always emitted, header-only when unmatched) and was adopted as normative RP-9 on 2026-07-12. REMAINING: (a) a zero-emission success now *asserts* full-range coverage, but under time-budget truncation over a blockless range (explicit `to` inside a hole run) the implementation can return an empty 200 having covered only part of it — the client then silently skips the rest; (b) the comparator must implement the INV-22 boundary-marker exemption; (c) in one reachable corner RP-9's clauses are jointly unsatisfiable — an effective range whose covered part contains no stored block and whose coverage cannot legally reach the range end (an availability boundary, RP-8, or a hard `P-QUERY-TIME` stop inside a long hole run): zero emission is legal only at full coverage, and the carrier (highest stored block ≤ `L`) lies below `from`, which INV-27 forbids emitting — here the *spec*, not just the implementation, owes an answer (candidate remedies: an explicit in-band terminal coverage record — a wire change — or forbidding coverage to end inside a hole except at the range end) | RP-9, INV-22, RP-8, INV-27 | P2 | CT-5: hole-range query with explicit `to` + tight budget; assert empty-200 only with full coverage | @@ -323,7 +323,7 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-27 | FINALIZE never checks `e ≥ first(D)`: with `fin = ⊥` (e.g. after a trim passed above it) a lagging source's report below the window commits `fin < first(D)` | WP §2.4, INV-5 | P2 | CT-4: finality below the window on a trimmed dataset; assert the report is ignored | | GAP-28 | The External retention instruction and the empty-dataset anchor are memory-only (the persisted label holds kind/version/fin) — a restart forgets the instructed bound (the dataset re-idles awaiting a new instruction) and resets an empty dataset's anchor | CN-9, INV-40, WP-11 | P2 | CT-2: SET-RETENTION, restart; assert the bound and anchor are recovered | | GAP-29 | A stalled-but-open connection pins the response's storage snapshot indefinitely (no overall response deadline / server write timeout; only disconnects release it); pinned snapshots block physical reclaim of point-deleted data | CN-7, RP-18, HZ-9, RS-6 | P2 | CT-3/CT-7: zombie client that stops reading; assert snapshot lifetime ≤ `P-QUERY-TIME` and reclaim proceeds | -| GAP-30 | No strike counting exists anywhere: a linkage-mismatch rejection resets the source session and re-requests in a zero-backoff hot loop; sources are never quarantined; cross-source equivocation never alarms. `P-SOURCE-STRIKES` has no substrate, so the WP-6b/FM-SRC-5 escalation rules are currently unimplementable | FM-SRC-3/4/6, WP-2 | P2 | CT-4: source serving a permanently mis-linked run; assert bounded request rate, quarantine after N strikes, alarm | +| GAP-30 | No strike counting exists anywhere: a linkage-mismatch rejection resets the source session and re-requests in a zero-backoff hot loop; sources are never quarantined; cross-source equivocation never alarms. `P-SOURCE-STRIKES` has no substrate, so the WP-6b/FM-SRC-5 escalation rules are currently unimplementable. **Slowness is likewise unnameable**: an endpoint leaves the rotation only by erroring (`Endpoint::on_error` → `Backoff`; `is_active` knows no other reason), so an endpoint minutes behind the tip keeps full standing and is streamed and parsed on every cycle, its blocks discarded below the shared cursor. Selection is per *block*, not per source — the first arrival that links onto the cursor wins, ties going to config order — so there is no convergence to a fast source and no memory of who wins. Confirmed harmless to the head on its own — in the minority shape production runs (`ct4_a_lagging_source_does_not_hold_the_head_back`), in the majority shape (`ct4_two_lagging_sources_of_three_do_not_hold_the_head_back`), and independently of the laggard's slot in the fixed poll order, since a laggard sits below the shared cursor and so has nothing to serve. The cost is ingest work paid for every endpoint regardless of standing, and the only lever is removing a source by hand | FM-SRC-3/4/6, WP-2 | P2 | CT-4: source serving a permanently mis-linked run; assert bounded request rate, quarantine after N strikes, alarm. Lag-aware demotion needs per-source observability first — PR #81 (`hotblocks_ingest_source_errors_total`) is the substrate and has been in draft since 2026-07-08 | | GAP-31 | A finality advance landing at/above the resume position while no block arrives is deferred (standalone reports are forwarded only when below the position; suppressed until the position is source-confirmed) — the finalized head stalls although the source declared the stored chain final | WP §2.4 clamping | P3 | CT-4: finality = head during a production stall; assert `fin` advances within a bound | | GAP-33 | The live-stream head-number header is computed from the *current* head, not the snapshot's: a head-lowering fork between snapshot and response start yields a header below blocks actually emitted (IB-6 requires ≥ the snapshot head) | IB-6, CN-5 | P3 | CT-3/CT-4: fork lowering the head during streaming; assert header ≥ snapshot head | | GAP-34 | OB-1 partially unimplemented: the commit version (present in the persisted label) and the retention policy in force are not exported | OB-1 | P3 | CT-1: scrape assertion once exported | @@ -332,6 +332,7 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-37 | PF-1's memory ceiling is not configuration-derivable on the read side: INV-25/RP-17 require emitting the first covered block whole even above `P-RESP-WEIGHT`, and nothing bounds a single block at ingest (`P-BATCH-BYTES` is a soft *batch* bound — one oversized block still stores), so per-response memory is bounded only by the largest block a source ever served. No `P-MAX-BLOCK-BYTES` exists | PF-1, RP-17/INV-25, FM-CLI-2 | P2 | CT-6/CT-9: ingest a pathological giant block, query it; assert bounded RSS and whole-block emission (INV-25) | | GAP-39 | A hash-lookup miss and an unknown dataset are both 404 with only a free-text body between them, and IB-7 forbids keying on text. This is worse than the GAP-36 family it belongs to: RP-19 makes "this hash is not indexed" a *deliberately uninformative* answer, so a client that cannot separate it from "this dataset does not exist" cannot tell a misconfiguration from a legitimate miss at all | INV-26, IB-7, RP-19 | P3 | CT-5: unknown dataset vs unknown hash; assert a structured discriminant | | GAP-40 | Hash-index CFs export only engine-wide estimated keys and live SST bytes. OB-12 still lacks per-dataset enabled state / entry count / bytes and hit-vs-miss lookup counts with latency. Because a miss is uninformative by design, an index empty for a structural reason — enabled after the window had filled, wrong kind — remains indistinguishable from one receiving only unknown hashes | OB-12, OB-6 | P3 | CT-1: scrape per-dataset state and exercise hit/miss counters once exported | +| GAP-41 | Fork consensus is credulous: `StandardDataSource::poll_next_event` counts fork-signalling endpoints without asking whether a signalling endpoint has any standing at the contested position, and `extract_fork` then adopts the *longest* hint chain among them. RP-5b confines a legitimate signal to `from == tip + 1`, so an in-spec source can only signal where it is; but FM-1 requires surviving one that is not, and a source signalling above its tip is what a source merely *behind* would look like if it answered wrongly. **One** such endpoint out of three suffices, and not by majority: `forks > endpoints.len() / 2` is false at 1-of-3, but the 2 s `fork_consensus_timeout` fires on any poll where every endpoint returned `Pending`, and `accept_new_block` clears that timer only on a commit — so on a chain with block time ≥ 2 s *every inter-block gap is a firing window*. Measured 2026-07-20, 5/5 runs, by instrumenting `poll_next_event`: `forks=1 endpoints=3 active=3 majority=false all_active=false timeout=true`; the adopted hints end at the liar's stale tip, `compute_rollback` rejects them as below `fin`, and ingestion parks per GAP-5 with two healthy sources still offering the chain. The endpoints' own position is known (`Endpoint::last_committed_block`) and unused | FM-1, WP-6, FM-SRC-4/5 | **P1** (was P2 on the premise that it took a majority — measurement overturned it: a lone bad source is enough, on the ordinary inter-block gap rather than a rare coincidence) | `ct4_a_single_source_signalling_a_fork_above_its_tip_does_not_park_ingestion`; the majority path is pinned separately by `ct4_a_fork_signal_majority_above_the_tip_does_not_park_ingestion` so a fix closing only that clause cannot read as green. Both `#[ignore]`d | ### 6.1 Closed diff --git a/crates/hotblocks/tests/ct4_lagging_source.rs b/crates/hotblocks/tests/ct4_lagging_source.rs new file mode 100644 index 00000000..61505b0a --- /dev/null +++ b/crates/hotblocks/tests/ct4_lagging_source.rs @@ -0,0 +1,195 @@ +//! CT-4 — several endpoints per dataset, one of them behind. +//! +//! Production configures a handful of sources per dataset and they do not stay in step: on +//! `base-mainnet` one endpoint has been observed minutes behind two that sit on the head. Every +//! script before this one ran exactly **one** source, so `StandardDataSource`'s multi-endpoint +//! path had never been executed by a test at all — not the fixed-order poll race +//! (`poll_next_event`), not the fork consensus, not the per-endpoint `MaybeOnHead` flush trigger. +//! `crates/data-source` carries no unit tests either. +//! +//! What makes lag worth a script of its own: the selector has no notion of *slow*. An endpoint +//! leaves the rotation only by erroring (`on_error` → `Backoff`); `is_active` knows `Backoff` and +//! nothing else. A healthy endpoint minutes behind the tip therefore keeps full standing in the +//! race forever, and two mechanisms could let it hold the served head back — +//! +//! - the flush at the tip is `DataEvent::MaybeOnHead`, and it fires only when the endpoint whose +//! stream just ended is *itself* the one that committed the current head block +//! (`ep.last_committed_block == Some(prev_block)`). `position` is shared across endpoints while +//! `last_committed_block` is per-endpoint, so which endpoint may trigger the flush depends on +//! who won the last race; +//! - blocks below the shared position are parsed and discarded with no early exit, and +//! `poll_endpoint` loops until the stream returns `Pending` — so an endpoint's ready backlog is +//! drained inside one poll of the single ingest task. +//! +//! Neither is asserted anywhere. These scripts assert the property both mechanisms would break: +//! **a source that is behind must not hold the head back.** +//! +//! It holds, in the minority and the majority alike and wherever the laggard sits in the fixed +//! order: a laggard is *below* the shared `position`, so it has nothing to serve and never wins +//! the race. What wedges the service is a source that answers **wrongly** rather than late — one +//! endpoint of three signalling a fork above its tip is enough. + +use std::{sync::Arc, time::Duration}; + +use anyhow::Result; +use sqd_hotblocks_harness::{ + Evm, + harness::{Harness, HarnessConfig} +}; + +fn config(sources: usize) -> HarnessConfig { + let mut cfg = HarnessConfig::from_block(env!("CARGO_BIN_EXE_sqd-hotblocks"), Arc::new(Evm), 1_000); + cfg.sources = sources; + cfg +} + +/// The baseline: with three endpoints all sitting on the head, the service must behave exactly as +/// it does with one. This is the first multi-source coverage the suite has ever had, and it has to +/// pass before a lag result means anything. +#[tokio::test(flavor = "multi_thread")] +async fn ct4_three_sources_in_step_conform() -> Result<()> { + let mut h = Harness::start(config(3)).await?; + + h.produce(40)?; + h.finalize_with_lag(5)?; + h.settle().await?; + h.assert_conforms().await?; + + Ok(()) +} + +/// Production's shape: one endpoint of three left far behind while the other two stay on the head. +/// The healthy majority carries the chain, so the served head must track the canonical tip +/// regardless of the laggard — INV-11/INV-30, LIV-1. +/// +/// A failure here is a stalled head, and `settle` reports it as the service not converging on the +/// script within the quiescence timeout. +#[tokio::test(flavor = "multi_thread")] +async fn ct4_a_lagging_source_does_not_hold_the_head_back() -> Result<()> { + let mut h = Harness::start(config(3)).await?; + + // All three on the head first, so the laggard starts from a state where it has committed + // blocks itself — that is what scatters `last_committed_block` across endpoints. + h.produce(20)?; + h.finalize_with_lag(5)?; + h.settle().await?; + + // Peer 0 holds while the primary and peer 1 advance. On a 2 s chain this is a ~4 min laggard. + for _ in 0..12 { + h.produce_lagging(&[0], 10)?; + h.finalize_with_lag(5)?; + tokio::time::sleep(Duration::from_millis(50)).await; + } + + h.settle().await?; + h.assert_conforms().await?; + + Ok(()) +} + +/// The margin: the laggards are the *majority* and one healthy endpoint carries the chain alone. +/// Not a shape production has been seen in — but if a healthy minority suffices, lag is not a +/// liveness input at all. +#[tokio::test(flavor = "multi_thread")] +async fn ct4_two_lagging_sources_of_three_do_not_hold_the_head_back() -> Result<()> { + let mut h = Harness::start(config(3)).await?; + + h.produce(20)?; + h.finalize_with_lag(5)?; + h.settle().await?; + + for _ in 0..12 { + h.produce_ahead(10)?; + h.finalize_with_lag(5)?; + tokio::time::sleep(Duration::from_millis(50)).await; + } + + h.settle().await?; + h.assert_conforms().await?; + + Ok(()) +} + +/// FM-SRC / FM-1 — **one** endpoint of three signals a fork above its own tip, which RP-5b +/// confines to `from == tip + 1`. A source defect, and two honest endpoints are still on the head. +/// +/// It wedges anyway, and takes no majority: `1 > 3/2` and `1 == 3` are both false, so the door is +/// the 2 s `fork_consensus_timeout` — reached only on a poll where *every* endpoint went +/// `Pending`, hence the 3 s lull. Measured 2026-07-20, 5/5 runs: +/// `forks=1 endpoints=3 active=3 majority=false all_active=false timeout=true`, then +/// `compute_rollback` rejects the liar's stale hints as below the finalized head and the epoch +/// parks for `P-EPOCH-RETRY`. +/// +/// At a 50 ms cadence the same fault signals repeatedly and the service conforms: +/// `accept_new_block` clears the timer on every commit, so on a chain with block time ≥ 2 s every +/// inter-block gap is a firing window. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "one fork signal above the tip parks ingestion — no alarm, no recovery"] +async fn ct4_a_single_source_signalling_a_fork_above_its_tip_does_not_park_ingestion() -> Result<()> { + let mut h = Harness::start(config(3)).await?; + + h.produce(20)?; + h.finalize_with_lag(5)?; + h.settle().await?; + + h.peers[0].inject_fault(&h.dataset, |f| f.fork_signal_above_tip = true); + + for _ in 0..8 { + h.produce_lagging(&[0], 10)?; + h.finalize_with_lag(5)?; + tokio::time::sleep(Duration::from_secs(3)).await; + } + + h.settle().await?; + h.assert_conforms().await?; + + Ok(()) +} + +/// The same defect on two of three — the other door: `forks > endpoints.len() / 2` carries +/// outright, no lull needed. Kept beside the single-liar script because a fix closing only the +/// majority path would leave that one green while a lone bad source still wedges. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "a fork-signal majority parks ingestion — no alarm, no recovery"] +async fn ct4_a_fork_signal_majority_above_the_tip_does_not_park_ingestion() -> Result<()> { + let mut h = Harness::start(config(3)).await?; + + h.produce(20)?; + h.finalize_with_lag(5)?; + h.settle().await?; + + for peer in &h.peers { + peer.inject_fault(&h.dataset, |f| f.fork_signal_above_tip = true); + } + + for _ in 0..12 { + h.produce_ahead(10)?; + h.finalize_with_lag(5)?; + tokio::time::sleep(Duration::from_millis(50)).await; + } + + h.settle().await?; + h.assert_conforms().await?; + + Ok(()) +} + +/// The laggard catches up. Nothing may change: it re-serves blocks the service already holds, and +/// re-offered known blocks must be ignored rather than reopen a settled window (INV-11). +#[tokio::test(flavor = "multi_thread")] +async fn ct4_a_caught_up_source_changes_nothing() -> Result<()> { + let mut h = Harness::start(config(2)).await?; + + h.produce(20)?; + h.produce_ahead(40)?; + h.finalize_with_lag(5)?; + h.settle().await?; + + h.catch_up_peers()?; + h.produce(5)?; + h.finalize_with_lag(5)?; + h.settle().await?; + h.assert_conforms().await?; + + Ok(()) +}