Consolidate the US survey graph build and enrichment pipeline

.gitattributes

@@ -0,0 +1,2 @@
+# Preserve exact unified-diff context markers in the archived measured experiment.
+experiments/us-acs-code-memo-not-adopted-20260910.patch -whitespace

.github/workflows/test.yml

@@ -207,7 +207,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.13", "3.14"]
+        python-version: ["3.13", "3.14.4"]
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v6
@@ -220,6 +220,70 @@ jobs:
           mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list shared-spec)
           /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider
 
+  spec-seed-diagnostics:
+    # Produce bounded diagnosis immediately, independently of the full suite.
+    # This remains required by ci-ok and never updates assertion goldens.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.14.4"
+      - name: Sync workspace with US and UK engines
+        run: uv sync --all-packages --locked --extra us --extra uk
+      - name: Check diagnostic bootstrap and failure reporting
+        timeout-minutes: 2
+        env:
+          PYTHONDONTWRITEBYTECODE: "1"
+          PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
+        run: >-
+          uv run --no-sync pytest --noconftest -p no:cacheprovider -q
+          packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py
+          packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py
+          packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py
+          packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py
+          packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py
+      - name: Derive candidate spec and seed identities (diagnostic only)
+        id: spec_seed_diagnostics
+        timeout-minutes: 15
+        env:
+          DIAG_WORKFLOW_SHA: ${{ github.workflow_sha }}
+          DIAG_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          DIAG_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          DIAG_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }}
+          PYTHONDONTWRITEBYTECODE: "1"
+          PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
+          OMP_NUM_THREADS: "1"
+          OPENBLAS_NUM_THREADS: "1"
+          MKL_NUM_THREADS: "1"
+          VECLIB_MAXIMUM_THREADS: "1"
+          NUMEXPR_NUM_THREADS: "1"
+          BLIS_NUM_THREADS: "1"
+          POPULACE_FIT_N_JOBS: "1"
+          POPULACE_FIT_PREDICT_WORKERS: "1"
+        run: |
+          DIAG_CHECKOUT_SHA="$(git rev-parse --verify HEAD)"
+          DIAG_UV_VERSION="$(uv --version)"
+          export DIAG_CHECKOUT_SHA DIAG_UV_VERSION
+          uv run --no-sync python tools/spec_seed_identity_diagnostics.py \
+            --output-dir "$RUNNER_TEMP/spec-seed-diagnostics"
+      - name: Retain bounded candidate identity evidence
+        if: ${{ always() && steps.spec_seed_diagnostics.outcome != 'skipped' }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: spec-seed-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-py3.14.4
+          path: |
+            ${{ runner.temp }}/spec-seed-diagnostics/candidate-digests.json
+            ${{ runner.temp }}/spec-seed-diagnostics/seed-protocol.json
+            ${{ runner.temp }}/spec-seed-diagnostics/seed-map.json
+            ${{ runner.temp }}/spec-seed-diagnostics/seed-bindings.json
+            ${{ runner.temp }}/spec-seed-diagnostics/environment-and-source.json
+            ${{ runner.temp }}/spec-seed-diagnostics/diagnostic-status.json
+          if-no-files-found: error
+          retention-days: 3
+          compression-level: 0
+          include-hidden-files: false
+
   engine-us:
     needs: changes
     if: github.event_name == 'push' || needs.changes.outputs.us == 'true' || needs.changes.outputs.shared == 'true'
@@ -336,6 +400,43 @@ jobs:
           PY
       - name: Spec identity section digests (cross-environment diffing aid)
         run: env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/spec_envelope_digests.py be uk
+      - name: Add the engine-free US extra the source runtimes require
+        # microunit is microcosm-frame's `us` extra: the sanctioned tax-unit
+        # constructor, and the only optional dependency the US source runtimes
+        # need without a country engine. Every ACS/CREATE stage names it in
+        # us_runtime.graph_implementation.STAGE_DEPENDENCIES, so an installed
+        # constellation without it cannot hash those stages: the suite then
+        # fails closed (SOURCE_PREPARATION_REFUSED / CATALOGUE_ISSUANCE_REFUSED)
+        # instead of skipping, which is a false packaging verdict.
+        #
+        # This runs after the boundary check above, so that check still proves
+        # the base wheels import with no optional dependency installed at all.
+        # Requesting the extra against the built wheel — rather than naming
+        # `microunit` here — keeps the extra declared only in
+        # packages/microcosm-frame/pyproject.toml and makes this gate prove the
+        # wheel carries resolvable extra metadata. policyengine-us stays absent:
+        # it is declared by microcosm-build[us] and microcosm-frame[policyengine],
+        # and neither is requested.
+        run: |
+          frame_wheel="$(ls dist/microcosm_frame-*.whl)"
+          uv pip install --python /tmp/wheels-venv/bin/python \
+            --constraint /tmp/wheel-constraints.txt \
+            "microcosm-frame[us] @ file://$(realpath "$frame_wheel")"
+          env -u PYTHONPATH /tmp/wheels-venv/bin/python -I - <<'PY'
+          from importlib import metadata, util
+
+          assert util.find_spec("microunit") is not None, (
+              "microcosm-frame[us] must install microunit into the wheel venv"
+          )
+          assert util.find_spec("policyengine_us") is None
+          try:
+              metadata.distribution("policyengine-us")
+          except metadata.PackageNotFoundError:
+              pass
+          else:
+              raise AssertionError("policyengine-us must be absent from the base wheel gate")
+          print("microunit", metadata.version("microunit"))
+          PY
       - name: Run the suite against the installed wheels
         run: |
           /usr/bin/time -v env -u PYTHONPATH /tmp/wheels-venv/bin/python -I -m pytest \
@@ -350,7 +451,7 @@ jobs:
     # dependency-skip lets a failed `changes` take the engine lanes down without
     # any required check going red: a lane skip is accepted only when
     # classification itself succeeded.
-    needs: [changes, lint, fast, engine-shared, engine-us, engine-uk, wheels]
+    needs: [changes, lint, fast, engine-shared, spec-seed-diagnostics, engine-us, engine-uk, wheels]
     if: always()
     runs-on: ubuntu-latest
     steps:
@@ -365,6 +466,7 @@ jobs:
           require_success lint           "${{ needs.lint.result }}"
           require_success fast           "${{ needs.fast.result }}"
           require_success engine-shared  "${{ needs['engine-shared'].result }}"
+          require_success spec-seed-diagnostics "${{ needs['spec-seed-diagnostics'].result }}"
           require_success wheels         "${{ needs.wheels.result }}"
           for lane in "engine-us:${{ needs['engine-us'].result }}" "engine-uk:${{ needs['engine-uk'].result }}"; do
             name="${lane%%:*}"; result="${lane#*:}"

CLAUDE.md

@@ -39,6 +39,12 @@ shard's real wheel, install into a clean uv-export-constrained venv, assert
 the wheel/import boundary and spec digests, and run the suite against installed
 wheels.
 
+The shared engine lane also runs a bounded source/seed identity diagnostic on
+Python 3.14.4 after its normal tests, including when those tests fail. Its six
+JSON artifacts retain candidate digests and their complete canonical seed
+records for review. They never replace the test assertions or certify coverage;
+review the captured source and dependency identities before updating any pins.
+
 New commits to a PR cancel older unfinished CI runs for that same PR.
 Each main-push run has a unique concurrency group, so all main-push runs
 remain independent and can finish validating their merged changes.

PROGRESS-908-target-snapshot-fixes.md

@@ -0,0 +1,145 @@
+# microcosm#908 — closing the four reviewed findings on the snapshot slice
+
+Lane journal for branch `calibration-target-snapshots-908-fixes-20260912`.
+Journals are history, not state (see CLAUDE.md): check git/GitHub for current truth.
+
+Historical note, 12 September 2026: this journal records the earlier repair
+lane. The lane closed at `9a325c8b0`; root independently repaired the remaining
+strict-null codec case in `b8380c475`, with 52 tests passing. Current scope and
+remaining integration work are in [the maintained guide](docs/calibration-target-snapshots.md).
+The earlier work-in-progress and not-pushed statements below are historical.
+
+## State
+
+Merged the completed slice (`bae1887ff`) onto `origin/main` `116d46ee9`, resolved
+the four overlapping spec-engine identity pins by recomputing them on the merge
+ref, and am now implementing the four reviewed findings. Not pushed, no PR.
+
+## Review basis
+
+`908-final-review-review-full-report.md` (head `bae1887ff`, merge base
+`35fc76dd1`) with `probe.py` / `reproductions.json`. Four open findings:
+
+1. **C1** — the aggregate-only claim is enforced by a denylist over arbitrary
+   nested JSON, so record-level vectors ride through `context` and a
+   mapping-valued `candidate_id`.
+2. **A1** — nested metadata is copied only by `dict(...)`, so a sink mutating a
+   delivered payload changes the caller's mapping and the next snapshot.
+3. **A2** — `_write_chunk` creates the final `history/<n>.json` name before
+   writing it, so a concurrent reader sees a partial chunk, and a failed write
+   leaves an invalid immutable chunk behind.
+4. **A3** — the public codec admits impossible metadata: `epoch > epochs`, a
+   non-timestamp `created_at`, a non-string `candidate_id`, and a
+   `best_retained` whose availability, epoch and loss contradict each other.
+
+## Done
+
+- Merged `bae1887ff` into this branch and recomputed every conflicting
+  identity pin on the merge ref (table below).
+- 14 red regressions in `packages/microcosm-calibrate/tests/test_target_snapshots.py`
+  (9 of them failing on the reviewed head), then the implementation:
+  - **C1** — a closed, typed, bounded aggregate metadata contract
+    (`normalize_metadata`, `normalize_best_retained`, `METADATA_LOCATIONS`,
+    `MAX_METADATA_*`) applied uniformly to `context`, `search`, `selection`
+    and `best_retained`, with string-only identifiers checked rather than
+    coerced. A list is not a scalar, so a record vector has no shape to ride
+    in at any depth. The record-level key-name rule is kept on top, because a
+    *scalar* `household_id` is still record-level identity.
+  - **A1** — every metadata container in a delivered payload is freshly built
+    from immutable scalars, pinned by a structural test asserting the payload
+    shares no mutable object with the caller, the observer, the bound view or
+    the next snapshot.
+  - **A2** — history chunks are written and `fsync`-ed to a hidden temporary
+    and then published atomically with `os.link`, which refuses rather than
+    overwrites an existing immutable chunk. Failed writes clean up their
+    temporary. Atomic `latest.json`, run ownership, duplicate refusal and
+    bounded retention are unchanged.
+  - **A3** — strict public-codec validation of `created_at` (timezone-aware
+    ISO-8601), identifiers, `epoch <= epochs`, `sequence >= 1`,
+    `non_finite_rows <= n_targets`, the closed `best_retained` triple's
+    availability/epoch/loss consistency, and a `selected` snapshot whose epoch
+    contradicts its own selection receipt — paired with normalization at the
+    emitting edge so honest nonfinite values still serialize as explicit nulls
+    and counts and the observer still cannot abort a run.
+- `experiments/908_review_findings_recheck.py` replays the review's own four
+  counterexamples against this branch; receipt in
+  `experiments/908-review-findings-recheck.json`. All four report closed.
+
+## Deliberately narrow choices
+
+- `context` has **no in-tree producer**. Rather than invent a wider shape for a
+  caller that does not exist, it takes the same flat scalar contract as the
+  seams `solve.py` really emits. A caller that needs structure should add a
+  typed seam and extend the contract, in the PR that adds the caller.
+- The metadata bounds (32 entries, 64-character keys, 256-character strings)
+  are the narrowest values that comfortably hold everything `solve.py` emits.
+
+## Verification (isolated interpreter, no installs, threads=1)
+
+`/Users/maxghenis/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910/.venv/bin/python -I -B -S`
+with this worktree's shard `src` roots ahead of that venv's site-packages and
+an import-path assertion, `OMP/MKL/OPENBLAS/NUMEXPR/VECLIB_NUM_THREADS=1`.
+
+| battery | result |
+| --- | --- |
+| `packages/microcosm-calibrate/tests` | 307 passed (12 s) |
+| `packages/microcosm-fit/tests` | 124 passed (13 s) — main's QRF pins intact |
+| `test_spec_engine_loader.py` + `test_us_multispine_pool_tool.py` | 197 passed (309 s) |
+| `packages/microcosm-build/tests -k "inventory or coverage or seed"` | 448 passed, 2 skipped (343 s) |
+| `packages/microcosm-graph/tests -k parity` | 25 passed (88 s) |
+| `tools/spec_engine_coverage.py --check` | 42156/42156 fields, 41/41 inventory |
+| `tools/ci_test_groups.py --verify` | `verification=ok`; the snapshot test file lands in fast `rest` / engine `us-am`, never `[defaulted]` |
+| `ruff check .` | clean; `ruff format --check` clean on both touched files |
+
+`test_target_snapshots.py` holds 52 tests, 16 of them new here and 2 existing
+ones updated to the closed contract's refusal points. Replaying the whole file
+against the reviewed head's `target_snapshots.py` (the only source that
+differs) gives **12 failed, 40 passed** — 10 new red regressions plus the 2
+updated tests. Every finding has at least one red regression:
+
+- **C1** — `test_context_refuses_the_record_vectors_the_review_smuggled_through`,
+  `test_metadata_refuses_arbitrary_nested_payloads`,
+  `test_identifier_fields_must_be_strings`,
+  `test_supported_scalar_metadata_survives_and_is_bounded`.
+- **A1** — `test_metadata_refuses_arbitrary_nested_payloads` is the review's
+  own aliasing counterexample; the detachment invariant itself is pinned by
+  `test_a_delivered_snapshot_shares_no_mutable_object_with_its_caller` and
+  `test_sink_mutation_cannot_reach_caller_metadata_or_the_next_snapshot`,
+  which pass on both heads because the flat case was never the defect.
+- **A2** — `test_a_history_chunk_is_published_only_after_its_bytes_are_complete`,
+  `test_a_failed_chunk_write_leaves_no_partial_or_leftover_file`.
+- **A3** — `test_codec_refuses_the_impossible_payloads_the_review_reproduced`,
+  `test_non_finite_diagnostics_stay_null_statuses_rather_than_aborting`,
+  `test_the_codec_is_stricter_than_the_emitting_edge_and_says_so`,
+  `test_created_at_is_a_timezone_aware_timestamp`.
+
+The remaining 6 new tests are preservation tests (the structured metadata
+`solve.py` really emits, the epochs the solver really selects, duplicate chunk
+refusal, and observer-off/observer-on weight parity on the L0, budget-search,
+refit and proximal paths); they pass on both heads by design.
+
+## Next
+
+- Independent re-review by main before integration/PR updates.
+
+## Identity pins recomputed on the merge ref
+
+`microcosm.calibrate.solve` is attested (`_DIRECT_KERNEL_MODULES`); main's #912
+edited the attested `microcosm.fit.qrf`, so both sides' pins were stale and the
+merge conflicted on all four. Each was recomputed on the merged tree, after
+first proving with the same interpreter that restoring **main's** `solve.py`
+into the merged tree reproduces **main's** committed pins exactly — so the
+drift is attributable to the slice's `solve.py` edit, not to an environment
+leak.
+
+| pin | main `116d46ee9` | slice `bae1887ff` | merged |
+| --- | --- | --- | --- |
+| `EXPECTED_HASHES["seed_protocol"]` | `553d5e0b…` | `f4dc507c…` | `d052fd87…` |
+| `EXPECTED_HASHES["seed_map"]` | `20058e54…` | `32dea304…` | `d5a9694a…` |
+| US resolved-spec `spec_sha256` | `1eeca53a…` | `35a3623b…` | `ff2c9703…` |
+| minimal-spec loader golden | `b4659890…` | `a67bb78c…` | `8a240898…` |
+
+`docs/evidence/spec-engine/us-f0-coverage.json` was regenerated with
+`tools/spec_engine_coverage.py` (42156/42156 fields, 41/41 inventory checks).
+The graph `calibrate.adam@1` parity pin merged cleanly and re-verified green
+(25 parity tests). `fit.qrf` and `simulate` pins are untouched.