diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..324641f92 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve exact unified-diff context markers in the archived measured experiment. +experiments/us-acs-code-memo-not-adopted-20260910.patch -whitespace diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0c721babe..86fcbf208 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -207,7 +207,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.13", "3.14"] + python-version: ["3.13", "3.14.4"] steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -220,6 +220,70 @@ jobs: mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list shared-spec) /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider + spec-seed-diagnostics: + # Produce bounded diagnosis immediately, independently of the full suite. + # This remains required by ci-ok and never updates assertion goldens. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.14.4" + - name: Sync workspace with US and UK engines + run: uv sync --all-packages --locked --extra us --extra uk + - name: Check diagnostic bootstrap and failure reporting + timeout-minutes: 2 + env: + PYTHONDONTWRITEBYTECODE: "1" + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + run: >- + uv run --no-sync pytest --noconftest -p no:cacheprovider -q + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py + packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + - name: Derive candidate spec and seed identities (diagnostic only) + id: spec_seed_diagnostics + timeout-minutes: 15 + env: + DIAG_WORKFLOW_SHA: ${{ github.workflow_sha }} + DIAG_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + DIAG_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + DIAG_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + PYTHONDONTWRITEBYTECODE: "1" + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + VECLIB_MAXIMUM_THREADS: "1" + NUMEXPR_NUM_THREADS: "1" + BLIS_NUM_THREADS: "1" + POPULACE_FIT_N_JOBS: "1" + POPULACE_FIT_PREDICT_WORKERS: "1" + run: | + DIAG_CHECKOUT_SHA="$(git rev-parse --verify HEAD)" + DIAG_UV_VERSION="$(uv --version)" + export DIAG_CHECKOUT_SHA DIAG_UV_VERSION + uv run --no-sync python tools/spec_seed_identity_diagnostics.py \ + --output-dir "$RUNNER_TEMP/spec-seed-diagnostics" + - name: Retain bounded candidate identity evidence + if: ${{ always() && steps.spec_seed_diagnostics.outcome != 'skipped' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: spec-seed-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-py3.14.4 + path: | + ${{ runner.temp }}/spec-seed-diagnostics/candidate-digests.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-protocol.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-map.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-bindings.json + ${{ runner.temp }}/spec-seed-diagnostics/environment-and-source.json + ${{ runner.temp }}/spec-seed-diagnostics/diagnostic-status.json + if-no-files-found: error + retention-days: 3 + compression-level: 0 + include-hidden-files: false + engine-us: needs: changes if: github.event_name == 'push' || needs.changes.outputs.us == 'true' || needs.changes.outputs.shared == 'true' @@ -336,6 +400,48 @@ jobs: PY - name: Spec identity section digests (cross-environment diffing aid) run: env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/spec_envelope_digests.py be uk + - name: Add engine-free source preparation extras + # Real source fixtures require microunit for tax-unit construction and + # PyTables for pandas HDF input, alongside h5py. Request the extras from + # built wheels so their metadata, not editable installs, supplies these + # dependencies. The preceding step still checks the base wheel boundary; + # neither source extra installs a country rules engine. + run: | + frame_wheel="$(ls dist/microcosm_frame-*.whl)" + build_wheel="$(ls dist/microcosm_build-*.whl)" + uv pip install --python /tmp/wheels-venv/bin/python \ + --constraint /tmp/wheel-constraints.txt \ + "microcosm-frame[us] @ file://$(realpath "$frame_wheel")" \ + "microcosm-build[source-io] @ file://$(realpath "$build_wheel")" + env -u PYTHONPATH /tmp/wheels-venv/bin/python -I - <<'PY' + from importlib import metadata, util + from pathlib import Path + from tempfile import TemporaryDirectory + + import h5py + import pandas as pd + import tables + + assert util.find_spec("microunit") is not None, ( + "microcosm-frame[us] must install microunit into the wheel venv" + ) + # Exercise the same HDF reader used by source preparation. Imports + # alone do not establish that the compiled PyTables backend works. + with TemporaryDirectory() as directory: + path = Path(directory) / "invented-source.h5" + expected = pd.DataFrame({"person_id": [1, 2], "amount": [0.0, 12.5]}) + expected.to_hdf(path, key="person") + pd.testing.assert_frame_equal(pd.read_hdf(path, key="person"), expected) + assert util.find_spec("policyengine_us") is None + try: + metadata.distribution("policyengine-us") + except metadata.PackageNotFoundError: + pass + else: + raise AssertionError("policyengine-us must be absent from the base wheel gate") + print("microunit", metadata.version("microunit")) + print("h5py", h5py.__version__, "tables", tables.__version__) + PY - name: Run the suite against the installed wheels run: | /usr/bin/time -v env -u PYTHONPATH /tmp/wheels-venv/bin/python -I -m pytest \ @@ -350,7 +456,7 @@ jobs: # dependency-skip lets a failed `changes` take the engine lanes down without # any required check going red: a lane skip is accepted only when # classification itself succeeded. - needs: [changes, lint, fast, engine-shared, engine-us, engine-uk, wheels] + needs: [changes, lint, fast, engine-shared, spec-seed-diagnostics, engine-us, engine-uk, wheels] if: always() runs-on: ubuntu-latest steps: @@ -365,6 +471,7 @@ jobs: require_success lint "${{ needs.lint.result }}" require_success fast "${{ needs.fast.result }}" require_success engine-shared "${{ needs['engine-shared'].result }}" + require_success spec-seed-diagnostics "${{ needs['spec-seed-diagnostics'].result }}" require_success wheels "${{ needs.wheels.result }}" for lane in "engine-us:${{ needs['engine-us'].result }}" "engine-uk:${{ needs['engine-uk'].result }}"; do name="${lane%%:*}"; result="${lane#*:}" diff --git a/CLAUDE.md b/CLAUDE.md index 68109e01d..445c9456e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,6 +39,17 @@ shard's real wheel, install into a clean uv-export-constrained venv, assert the wheel/import boundary and spec digests, and run the suite against installed wheels. +After checking the base wheel boundary, the wheel lane installs the built +`microcosm-frame[us]` and `microcosm-build[source-io]` extras for engine-free +source tests. An invented pandas HDF round trip checks the compiled reader; +the country rules engine must still be absent. + +The shared engine lane also runs a bounded source/seed identity diagnostic on +Python 3.14.4 after its normal tests, including when those tests fail. Its six +JSON artifacts retain candidate digests and their complete canonical seed +records for review. They never replace the test assertions or certify coverage; +review the captured source and dependency identities before updating any pins. + New commits to a PR cancel older unfinished CI runs for that same PR. Each main-push run has a unique concurrency group, so all main-push runs remain independent and can finish validating their merged changes. diff --git a/PROGRESS-908-target-snapshot-fixes.md b/PROGRESS-908-target-snapshot-fixes.md new file mode 100644 index 000000000..708880f6d --- /dev/null +++ b/PROGRESS-908-target-snapshot-fixes.md @@ -0,0 +1,145 @@ +# microcosm#908 — closing the four reviewed findings on the snapshot slice + +Lane journal for branch `calibration-target-snapshots-908-fixes-20260912`. +Journals are history, not state (see CLAUDE.md): check git/GitHub for current truth. + +Historical note, 12 September 2026: this journal records the earlier repair +lane. The lane closed at `9a325c8b0`; root independently repaired the remaining +strict-null codec case in `b8380c475`, with 52 tests passing. Current scope and +remaining integration work are in [the maintained guide](docs/calibration-target-snapshots.md). +The earlier work-in-progress and not-pushed statements below are historical. + +## State + +Merged the completed slice (`bae1887ff`) onto `origin/main` `116d46ee9`, resolved +the four overlapping spec-engine identity pins by recomputing them on the merge +ref, and am now implementing the four reviewed findings. Not pushed, no PR. + +## Review basis + +`908-final-review-review-full-report.md` (head `bae1887ff`, merge base +`35fc76dd1`) with `probe.py` / `reproductions.json`. Four open findings: + +1. **C1** — the aggregate-only claim is enforced by a denylist over arbitrary + nested JSON, so record-level vectors ride through `context` and a + mapping-valued `candidate_id`. +2. **A1** — nested metadata is copied only by `dict(...)`, so a sink mutating a + delivered payload changes the caller's mapping and the next snapshot. +3. **A2** — `_write_chunk` creates the final `history/.json` name before + writing it, so a concurrent reader sees a partial chunk, and a failed write + leaves an invalid immutable chunk behind. +4. **A3** — the public codec admits impossible metadata: `epoch > epochs`, a + non-timestamp `created_at`, a non-string `candidate_id`, and a + `best_retained` whose availability, epoch and loss contradict each other. + +## Done + +- Merged `bae1887ff` into this branch and recomputed every conflicting + identity pin on the merge ref (table below). +- 14 red regressions in `packages/microcosm-calibrate/tests/test_target_snapshots.py` + (9 of them failing on the reviewed head), then the implementation: + - **C1** — a closed, typed, bounded aggregate metadata contract + (`normalize_metadata`, `normalize_best_retained`, `METADATA_LOCATIONS`, + `MAX_METADATA_*`) applied uniformly to `context`, `search`, `selection` + and `best_retained`, with string-only identifiers checked rather than + coerced. A list is not a scalar, so a record vector has no shape to ride + in at any depth. The record-level key-name rule is kept on top, because a + *scalar* `household_id` is still record-level identity. + - **A1** — every metadata container in a delivered payload is freshly built + from immutable scalars, pinned by a structural test asserting the payload + shares no mutable object with the caller, the observer, the bound view or + the next snapshot. + - **A2** — history chunks are written and `fsync`-ed to a hidden temporary + and then published atomically with `os.link`, which refuses rather than + overwrites an existing immutable chunk. Failed writes clean up their + temporary. Atomic `latest.json`, run ownership, duplicate refusal and + bounded retention are unchanged. + - **A3** — strict public-codec validation of `created_at` (timezone-aware + ISO-8601), identifiers, `epoch <= epochs`, `sequence >= 1`, + `non_finite_rows <= n_targets`, the closed `best_retained` triple's + availability/epoch/loss consistency, and a `selected` snapshot whose epoch + contradicts its own selection receipt — paired with normalization at the + emitting edge so honest nonfinite values still serialize as explicit nulls + and counts and the observer still cannot abort a run. +- `experiments/908_review_findings_recheck.py` replays the review's own four + counterexamples against this branch; receipt in + `experiments/908-review-findings-recheck.json`. All four report closed. + +## Deliberately narrow choices + +- `context` has **no in-tree producer**. Rather than invent a wider shape for a + caller that does not exist, it takes the same flat scalar contract as the + seams `solve.py` really emits. A caller that needs structure should add a + typed seam and extend the contract, in the PR that adds the caller. +- The metadata bounds (32 entries, 64-character keys, 256-character strings) + are the narrowest values that comfortably hold everything `solve.py` emits. + +## Verification (isolated interpreter, no installs, threads=1) + +`/Users/maxghenis/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910/.venv/bin/python -I -B -S` +with this worktree's shard `src` roots ahead of that venv's site-packages and +an import-path assertion, `OMP/MKL/OPENBLAS/NUMEXPR/VECLIB_NUM_THREADS=1`. + +| battery | result | +| --- | --- | +| `packages/microcosm-calibrate/tests` | 307 passed (12 s) | +| `packages/microcosm-fit/tests` | 124 passed (13 s) — main's QRF pins intact | +| `test_spec_engine_loader.py` + `test_us_multispine_pool_tool.py` | 197 passed (309 s) | +| `packages/microcosm-build/tests -k "inventory or coverage or seed"` | 448 passed, 2 skipped (343 s) | +| `packages/microcosm-graph/tests -k parity` | 25 passed (88 s) | +| `tools/spec_engine_coverage.py --check` | 42156/42156 fields, 41/41 inventory | +| `tools/ci_test_groups.py --verify` | `verification=ok`; the snapshot test file lands in fast `rest` / engine `us-am`, never `[defaulted]` | +| `ruff check .` | clean; `ruff format --check` clean on both touched files | + +`test_target_snapshots.py` holds 52 tests, 16 of them new here and 2 existing +ones updated to the closed contract's refusal points. Replaying the whole file +against the reviewed head's `target_snapshots.py` (the only source that +differs) gives **12 failed, 40 passed** — 10 new red regressions plus the 2 +updated tests. Every finding has at least one red regression: + +- **C1** — `test_context_refuses_the_record_vectors_the_review_smuggled_through`, + `test_metadata_refuses_arbitrary_nested_payloads`, + `test_identifier_fields_must_be_strings`, + `test_supported_scalar_metadata_survives_and_is_bounded`. +- **A1** — `test_metadata_refuses_arbitrary_nested_payloads` is the review's + own aliasing counterexample; the detachment invariant itself is pinned by + `test_a_delivered_snapshot_shares_no_mutable_object_with_its_caller` and + `test_sink_mutation_cannot_reach_caller_metadata_or_the_next_snapshot`, + which pass on both heads because the flat case was never the defect. +- **A2** — `test_a_history_chunk_is_published_only_after_its_bytes_are_complete`, + `test_a_failed_chunk_write_leaves_no_partial_or_leftover_file`. +- **A3** — `test_codec_refuses_the_impossible_payloads_the_review_reproduced`, + `test_non_finite_diagnostics_stay_null_statuses_rather_than_aborting`, + `test_the_codec_is_stricter_than_the_emitting_edge_and_says_so`, + `test_created_at_is_a_timezone_aware_timestamp`. + +The remaining 6 new tests are preservation tests (the structured metadata +`solve.py` really emits, the epochs the solver really selects, duplicate chunk +refusal, and observer-off/observer-on weight parity on the L0, budget-search, +refit and proximal paths); they pass on both heads by design. + +## Next + +- Independent re-review by main before integration/PR updates. + +## Identity pins recomputed on the merge ref + +`microcosm.calibrate.solve` is attested (`_DIRECT_KERNEL_MODULES`); main's #912 +edited the attested `microcosm.fit.qrf`, so both sides' pins were stale and the +merge conflicted on all four. Each was recomputed on the merged tree, after +first proving with the same interpreter that restoring **main's** `solve.py` +into the merged tree reproduces **main's** committed pins exactly — so the +drift is attributable to the slice's `solve.py` edit, not to an environment +leak. + +| pin | main `116d46ee9` | slice `bae1887ff` | merged | +| --- | --- | --- | --- | +| `EXPECTED_HASHES["seed_protocol"]` | `553d5e0b…` | `f4dc507c…` | `d052fd87…` | +| `EXPECTED_HASHES["seed_map"]` | `20058e54…` | `32dea304…` | `d5a9694a…` | +| US resolved-spec `spec_sha256` | `1eeca53a…` | `35a3623b…` | `ff2c9703…` | +| minimal-spec loader golden | `b4659890…` | `a67bb78c…` | `8a240898…` | + +`docs/evidence/spec-engine/us-f0-coverage.json` was regenerated with +`tools/spec_engine_coverage.py` (42156/42156 fields, 41/41 inventory checks). +The graph `calibrate.adam@1` parity pin merged cleanly and re-verified green +(25 parity tests). `fit.qrf` and `simulate` pins are untouched. diff --git a/PROGRESS-908-target-snapshots.md b/PROGRESS-908-target-snapshots.md new file mode 100644 index 000000000..7a7f9dedc --- /dev/null +++ b/PROGRESS-908-target-snapshots.md @@ -0,0 +1,171 @@ +# microcosm#908 — per-target calibration estimate snapshots (bounded slice 1) + +Lane journal for branch `calibration-target-snapshots-908-20260912`. +Journals are history, not state (see CLAUDE.md): check git/GitHub for current truth. + +Historical note, 12 September 2026: the initial implementation below was +subsequently merged onto main `116d46ee9` and repaired through `b8380c475`. +Its original review and source-pin claims describe earlier revisions. +[The maintained guide](docs/calibration-target-snapshots.md) records current +component scope; country integration and issue 908 remain incomplete. + +## State + +Bounded first slice implemented, tested and committed on this branch. Not +pushed, no PR opened. #908 is NOT complete — see "Remaining #908 acceptance +scope" below. + +## Scope of this slice + +1. A shared, leaf-level typed snapshot codec in `microcosm-calibrate`: + schema name + version, aggregate-only per-target rows, ordered target + identity digest, finite-value and ordering validation, signed relative + error with the repo's existing zero-target convention. +2. Real emission from the actual Adam solver loop in + `microcosm.calibrate.solve`, at a configurable bounded cadence with an + explicit every-epoch option, opt-in (off by default). +3. Honest iterate labelling: current / best_retained / selected, with the + selected snapshot taken from the weights the solver actually returns. +4. Atomic local `latest.json` plus immutable, bounded history chunks. +5. A synthetic-only benchmark at US/UK-sized dimensions (invented matrices). + +## Out of scope for this slice (remaining #908 acceptance scope) + +- Version-2 staging upload / remote publication (PR896's country host). +- Calibration Diagnostics UI (different repository, by the issue's own text). +- Real UK/US native calibration measurements — this host is forbidden native + microdata, engine runs and installs, so no real-run cadence default is + announced here. + +## Done + +- `packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py`: + shared leaf codec. Schema name + version, aggregate-only validation + (unknown top-level keys and record-level key names refused at any depth), + compiled-order row check, finite-value check, signed relative error with + the solver's existing zero-target convention, ordered `(name, value)` + sha256 identity digest, `TargetSnapshotCadence` (bounded + `EVERY_EPOCH`), + and `TargetSnapshotWriter` (atomic `latest.json`, write-once history + chunks, bounded retention with recorded drops). +- Solver integration in `solve.py`: emission from the Adam loop and the + proximal loop off the estimate tensor the epoch's loss was already + computed from (no second forward pass, so the hard-concrete gates' RNG + stream is untouched); budget-search probes carry their own search + identity; `l0_selection` / `post_l0_refit` carry their phase; one monotone + sequence counter spans every phase; the closing `selected` snapshot comes + off the float64 estimates the final diagnostics use. +- Public opt-in parameter `target_snapshots=` on `calibrate`, + `refit_l0_selection` and `calibrate_l0_refit`; exports in the shard + `__init__`. Off by default. +- 24 new tests in `packages/microcosm-calibrate/tests/test_target_snapshots.py` + (flat path; `tools/ci_test_groups.py --verify` = ok, lands in fast `rest` + and engine `us-am`, never `[defaulted]`). +- Synthetic benchmark + receipts under `experiments/`, explicitly labelled + synthetic. +- changelog.d fragment `908-calibration-target-snapshots.added.md`. + +## Remaining #908 acceptance scope (NOT done here) + +- Version-2 staging persistence/upload of the snapshot and its history. + PR #896 is open, draft, CONFLICTING and absent from `main`; its + `calibration_progress` filters on `kind == "calibration_epoch"` and its + per-event schema is `additionalProperties: false`, so it needs a new + schema name, not a widened one. Nothing here claims remote publication. +- US host wiring (`tools/build_us_fiscal_refresh_release.py` still builds v1 + `StagingTelemetry`) and UK host cadence flags. +- Exact-k ladder phases in `microcosm.build.us_runtime.exact_k_ladder` and + the UK size-search phases in `uk_runtime.dataset_size` are not wired. +- A test asserting intermediate snapshots never substitute for the canonical + `calibration_diagnostics.json`. +- Real UK/US target/epoch counts, wall-clock, and upload time, and therefore + the production default cadence. Forbidden on this host; the default stays + off. +- Calibration Diagnostics UI (different repository, by the issue's own text). + +## Adversarial review round (same lane, before hand-off) + +Three independent review lenses (determinism, label honesty, codec/store) +raised 21 findings; each was handed to an adversarial verifier told to refute +it, and 10 survived. All 10 are fixed on this branch, with a regression test +each: + +- Enabling the observer aborted runs that succeed without it, in two ways — + duplicate compiled row labels (`row_name` is the lossy `f"{name}@{period}"`, + so `("income", 2024)` and `("income", "2024")` collide) and a non-finite + float32 in-loop estimate (the capped loss absorbs it and the run returns + weights). The digest now carries the row index instead of refusing + duplicates, and a non-finite estimate or target serializes as null with a + `non_finite_rows` count, following `diagnostics._finite`. +- The `selected` snapshot stamped the closing epoch on a retained-best + iterate, and claimed `best_retained.available: false` on a run that + retained and returned a best. Both now read the selection receipt. +- The store silently collided across runs sharing a directory (sequences + restart at 1 per observer) and could prune another run's chunks, or the + chunk it had just written. Construction now refuses a populated directory + unless explicitly adopted, a store refuses a second run's snapshots, and + pruning is scoped to the chunks that writer wrote. +- The aggregate-only scan covered a hand-listed subset of keys; it now walks + the whole payload. Numeric fields are type-checked, so a stringified number + no longer validates. + +## Source identity re-pins (disclosed) + +`microcosm.calibrate.solve` is an attested module: it is in +`_DIRECT_KERNEL_MODULES` (`spec_engine/seeds.py`) and it defines the graph +shard's `calibrate.adam@1` kernel. Editing it legitimately moves several +pinned identities. Each was re-pinned to a value computed from this branch, +after first confirming with the *same* interpreter that pristine `origin/main` +reproduces the committed pins exactly — so the drift is attributable to the +source edit, not to an environment leak: + +- `packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json` + (`implementation_hash`, `node_key`). `fit.qrf` and `simulate` untouched; the + regeneration asserted `graph.json`, `inputs.csv` and `direct.csv` come back + byte-identical, so the kernel's numbers did not change. +- `EXPECTED_HASHES["seed_protocol"]` and `["seed_map"]` in + `spec_engine/inventory_coverage.py`, plus the regenerated + `docs/evidence/spec-engine/us-f0-coverage.json`. +- The resolved-spec golden vectors in + `test_spec_engine_loader.py` and `test_us_multispine_pool_tool.py`. + +No other pin was touched. + +**These pins are correct for this branch's base (0c3f4f651) only.** +`origin/main` has since advanced to 116d46ee9 (#912, +`amend-keyed-seed-and-uniform-draws`), which itself edited the attested +`microcosm.fit.qrf` and re-pinned the same loader golden vector and pool-tool +spec digest. Every identity pin above must therefore be recomputed on the +merge ref before this branch merges, per CLAUDE.md's "CI tests the merge ref, +so merge main and re-pin". Do not treat the values here as final. + +## How this lane ran the tests (no installs, no uv sync) + +An isolated interpreter with this worktree's shard sources ahead of a shared +venv's site-packages, plus an import-path assertion so `microcosm.*` can never +resolve outside this worktree: + +```python +# run.py — invoke as: /bin/python -I -B -S run.py +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VENV_SITE = Path("/site-packages") +sys.path[:0] = [ + *sorted(str(p) for p in ROOT.glob("packages/*/src")), + str(VENV_SITE), + str(ROOT), +] +import microcosm.calibrate as _cal +assert str(ROOT) in str(Path(_cal.__file__).resolve()), _cal.__file__ +import pytest +raise SystemExit(pytest.main(sys.argv[1:])) +``` + +Note pytest `addopts` already carries `-q`, so gate on the exit code rather +than adding another `-q` (which hides the summary line). + +## Next + +- Independent review of this slice, then the staging/host wiring as a + separate slice once #896 lands. diff --git a/PROGRESS-current-asec-income-routing.md b/PROGRESS-current-asec-income-routing.md new file mode 100644 index 000000000..9f76d23d9 --- /dev/null +++ b/PROGRESS-current-asec-income-routing.md @@ -0,0 +1,175 @@ +# Current ASEC income reporting/routing source qualifier — lane progress + +Branch `current-asec-income-routing-20260912`, worktree +`_worktrees/microcosm-current-asec-income-routing-20260912`, base +`d5cbe60b2c6402648565f5139dbf7d94be209def` (reviewed US integration). + +Journal, not state. Check git/GitHub for current truth. + +## Scope + +Deliver a thin **source qualifier** plus a **pure reporting/routing projection** +for the five original-channel ASEC income families the current graph does not +yet supply: pension/annuity, IRA distributions, net property income, farm, and +other income. Consumed later by the already-owned `graph_us_survey_enrichment` +host, which root's active amount owner owns. This lane creates **no** host, no +issuer, no engine execution, no native cell writes, no release claim. + +## State + +- [x] Read the actual-current-graph trace + (`_recovered/scratch-backup/893/codex-takeover-20260912/original-tax-graph-gap-source-trace.md`) + and confirmed the current host does not call `derive_cps_carried_current_leaves` + or the preclone gap-fill. +- [x] Verified the official 2025 dictionary bytes locally: + SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +- [x] Read the accepted UC pattern + (`current_asec_unemployment_source.py`, reviewed hash + `624c76ae058e7b2a466a017eba348327c94751088d97011d3474cd8314187519`) read-only + in the successor worktree; not mutated. +- [x] Read the in-base siblings `current_social_security_source.py`, + `current_asec_demographics.py`, `current_survey_predictors.py`. +- [x] Extracted every target field's printed literal (length/position/range, + verbatim Universe and Values) from the verified dictionary, and + cross-checked each against `asec_current_money_domains_v1.json`: exact match + on position, length, universe and values for all nine money fields. +- [x] Module + `packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py` + (commit 12ac3a75b, corrected in aa7bde7d4 and 7e2c7e77b). +- [x] Tests `packages/microcosm-build/tests/test_us_current_asec_income_routing.py` + — 64 passing, no engine, no PUF fixture. +- [x] Source contract note `docs/us-current-asec-income-routing-source.md` + plus `changelog.d/us-current-asec-income-routing-source.added.md`. +- [x] `ruff check .` and `ruff format --check` clean; + `tools/ci_test_groups.py --verify` reports `verification=ok` and the new + file lands in `[fast] rest` and `[engine] us-am`, not `[defaulted]`. +- [x] Mutation-checked the regressions: default-zero completion of an ambiguous + recipient zero, reading NIU from the raw number instead of the parent + status axis, collapsing receipt-with-net-zero into known nonreceipt, and + resolving an unknown slot account into a known non-IRA zero each turn the + suite red; reverting each returns it green. + +## Second review round: corrections after adversarial review + +An independent five-lens adversarial review of the finished lane (source +fidelity, projection semantics, owner pattern, tests, repository fit) found a +further set of defects, every one of which is fixed on this branch: + +1. **Fail-closed roster (CI-breaking).** `test_us_spine_blindness.py` globs + every `us_runtime` module and fails closed on an unregistered one. The + qualifier is now classified there and listed among the reviewed source-spine + provenance owners. Three of that file's tests remain red for four modules + already unclassified at the reviewed base `d5cbe60b2` — + `graph_fiscal_dense_calibration.py`, `graph_fiscal_measurement.py`, + `graph_puf55_route_attachment.py`, `graph_survey_puf55.py` — which this lane + neither owns nor touched. **Root should route those to their owners.** +2. **Recipient zeros were completed on an unprinted rationale.** `RNT_VAL` and + `FRSE_VAL` print the same `0 = none or niu` label the gross entries print, so + a signed net measure's recipient zero is no longer a known amount. Only + `ANN_VAL`, whose printed zero is `valid_zero_dollars`, resolves; that test is + read from the pinned domains artifact rather than decided in code. +3. **Off-route distribution evidence yielded an affirmative zero.** Off-route + dollars or an answered off-route recipiency now contradict the row, and an + applicable slot declaring an account whose amount is a "none or niu" zero + leaves both the total and the regular-IRA share unresolved. +4. **Other-income routing ignored the printed receipt universe**, so an + out-of-universe row read as reported alimony. +5. **Allocation origins asserted publisher-confirmed non-allocation** from flags + whose printed universes are conditional and unevaluated. +6. **The printed zero receipt label was collapsed** to one shared `niu_or_none` + across nine entries that print three different labels. +7. **`asec_literals` escaped with no digest**, outside the final identity check. +8. **`I_FRMYN` prints an empty `Values:` block**; its codes were described as + following `I_ANNVAL`. **`FARM_AMOUNT_SCOPE` dropped its composite clause.** + The claim that every other family prints a 15+ floor was false — the farm + family prints none. +9. **The doc called `TAXABLE_PENSION_FRACTION` archived**; it is live in + `cps_carried_current.py` under the prepared-ASEC stage, so a second split + attached here would double-count. +10. Printed entry tables are `NamedTuple`s now, the account code domain is + compared against `retirement_distributions._VALID_ACCOUNT_CODES` rather than + a re-typed literal, and `project_income_routing` validates its routing token + arrays. + +## Corrections made after independent review of the first draft + +1. The money owner normalizes `ANN_VAL`'s printed `-1` to a stored zero and + records `DECLARED_NIU` (`asec_current_money.py:974-977`). The first draft + re-derived NIU from the stored number, which would have read that cell as a + zero dollar annuity and also failed the literal-identity join. The amount + reading now comes from the parent's status axis. +2. The nine printed money entries were retyped by hand; they are already + attested per vintage in `asec_current_money_domains_v1.json`. They are now + read from that packaged artifact under `money.RESOURCE_PINS[0]`. +3. The published-allocation roster wrongly listed `DST_VAL1`, `DST_VAL2`, + `DST_YN`, `DST_SC1`, `DST_SC2` and `FRMOTR` as unflagged; all six are + flagged. `OI_YN` is unflagged and was missing from the list. +4. Three printed universes were transcribed with ASCII `>=` where the + dictionary prints U+2265; they now carry the printed character. + +## Next + +Root review and integration into `graph_us_survey_enrichment`. The open +decisions are listed under "Remaining work" in the contract note: canonical +attachment and clone policy for these PUF-overlapping leaves, the ACS clone0 +conditional model and its reconciliation against the ACS aggregate anchors, the +unobserved pension and distribution tax composition, the net property +decomposition, and any other-income residual rule. + +## Verified source literals (2025 dictionary, pages 43-49 + allocation pages) + +All quoted verbatim from the pinned PDF. + +| Field | Len | Pos | Range | Universe as printed | +| --- | --- | --- | --- | --- | +| PNSN_VAL | 7 | 571 | (0:9999999) | `PEN_YN = 1` | +| PEN_YN | 1 | 570 | (0:2) | `All Persons aged 15+` | +| ANN_VAL | 6 | 438 | (-1:999999) | `ANN_YN = 1` | +| ANN_YN | 1 | 444 | (0:2) | `All Persons aged 15+` | +| DST_VAL1 | 6 | 495 | (000000:999999) | `DST_SC1 = 1` | +| DST_VAL1_YNG | 6 | 501 | (000000:999999) | `DST_SC1_YNG = 1` | +| DST_VAL2 | 6 | 507 | (000000:999999) | `DST_SC2 = 1` | +| DST_VAL2_YNG | 6 | 513 | (000000:999999) | `DST_SC2_YNG = 1` | +| DST_SC1 | 1 | 491 | (0:7) | `DST_VAL1 > 0 and a_age >= 58` | +| DST_SC1_YNG | 1 | 492 | (0:7) | `DST_YN_YNG = 1 and a_age < 58` | +| DST_SC2 | 1 | 493 | (0:7) | `DST_VAL2 > 0 and a_age >= 58` | +| DST_SC2_YNG | 1 | 494 | (0:7) | `DST_VAL_YNG > 0 and a_age < 58` | +| DST_YN | 1 | 519 | (0:2) | `Persons aged 58 and over (a_age >= 58)` | +| DST_YN_YNG | 1 | 520 | (0:2) | `Persons under age 58 (a_age < 58)` | +| RNT_VAL | 6 | 621 | (-9999:999999) | `RNT_YN = 1` | +| RNT_YN | 1 | 627 | (0:2) | `All Persons aged 15+` | +| FRSE_VAL | 7 | 390 | (-9999999:9999999) | `ERN_YN=1 or FRMOTR=1` | +| FRSE_YN | 1 | 397 | (0:2) | `ERN_YN=1 or FRMOTR=1` | +| ERN_YN | 1 | 381 | (0:2) | `WORKYN=1 OR WTEMP=1` | +| FRMOTR | 1 | 389 | (0:2) | `ERN_OTR = 1` | +| OI_VAL | 6 | 549 | (0:999999) | `OI_YN = 1` | +| OI_OFF | 2 | 547 | (0:20) | `OI_YN = 1` | +| OI_YN | 1 | 555 | (0:2) | `All Persons aged 15+` | + +Source-level questions that stay open (evidence, not modeling judgments): + +- `PNSN_VAL` is printed as "total combined amount of pension income received + from **all** pension sources". It is not an observed private/taxable amount. + The legacy 0.590 split is a modeled assumption and is not applied here. +- `RNT_YN`'s printed question covers rent, royalties, roomers/boarders **and** + estates or trusts; `RNT_VAL`'s printed question asks only about "income from + rent after expenses". The receipt and amount questions do not have the same + printed coverage, so the total is not independently labelled rental. +- `DST_SC*` code 4 is `Regular IRA`. Account identity does not observe a taxable + fraction, so no taxable component is derived. +- `DST_VAL1`'s printed universe is `DST_SC1 = 1` (401k account) although its + label is the source-1 distribution amount; preserved verbatim as an + unresolved printed-universe ambiguity rather than silently corrected. +- `DST_SC2_YNG`'s printed universe names `DST_VAL_YNG`, a field with no + dictionary entry; preserved verbatim. +- `OI_YN`'s printed `0` label is `none or niu`, unlike `PEN_YN`/`ANN_YN`/ + `RNT_YN`/`DST_YN` whose `0` is `niu`. Zero receipt literals are therefore not + interchangeable across the five families. +- Published allocation flags exist for `ANN_VAL` (`I_ANNVAL`), `ANN_YN` + (`I_ANNYN`), `PEN_YN` (`I_PENYN`), `RNT_VAL` (`I_RNTVAL`), `RNT_YN` + (`I_RNTYN`), `OI_VAL` (`I_OIVAL`), `ERN_YN` (`I_ERNYN`), and the DST + composites (`I_DSTSC`, `I_DSTSCCOMP`, `I_DSTVAL1COMP`, `I_DSTVAL2COMP`, + `I_DSTYNCOMP`). The dictionary publishes **no** direct flag for `PNSN_VAL`, + `FRSE_VAL`, `FRSE_YN`, `OI_OFF`, `DST_VAL*`, `DST_YN` or `FRMOTR`; their + allocation provenance is therefore recorded as unresolved, not as "raw". +- `I_DSTVAL1COMP`'s printed `Universe:` line is empty in the dictionary. diff --git a/PROGRESS.md b/PROGRESS.md index ae184206e..2af0074f8 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,3 +1,147 @@ +# Grouped solver x calibration target snapshots integration - 2026-09-12 + +Historical note, 12 September 2026: this grouped-lane journal was subsequently +integrated with PR #914's corrected iteration identities and the actual fiscal +host observer. Its 443-test evidence and statements that host wiring is absent +describe that earlier lane. Current source and acceptance are recorded in +[the fiscal-host evidence](experiments/fiscal-target-snapshot-host-20260912.md). + +Lane: `microcosm-grouped-target-snapshot-integration-20260912`, branch +`grouped-target-snapshot-integration-20260912`. Base: fresh `origin/main` +`116d46ee9dc2aafdc68259b7c06e4c3462522e8b` (unmoved; it is exactly the shared +base both reviewed heads were cut from). Integrates the exact reviewed heads +`536f1ceefcdafda3cc619c14b4da18e012a7be57` (G, US grouped/fixed-zero Adam) and +`b43369dc49e175803f62020cc1e72fa53926aed8` (S, calibration target snapshots, +PR #914) against the read-only checklist +`grouped-snapshot-integration-review.md`. + +Everything below this section is prior-lane history and was accurate when +written; see "Root journals are history, not state" in `CLAUDE.md`. + +## State + +Complete and local. Four commits: the merge, the grouped instrumentation, the +identity recomputation, and the cross-product controls. Nothing pushed, no PR, +no release action. Root reviews and integrates. + +Scope is the shared solver seam only. `graph_fiscal_dense_calibration.py` still +calls `calibrate` without an observer, so this branch emits no snapshot for the +real US fiscal path; that host wiring is a separate step, and no observer +registry, `Node.params` callback or replay-reruns-the-optimizer claim was +invented here. See +[the lane experiment](experiments/grouped-target-snapshot-integration-20260912.md) +for scope, evidence and residual risks. + +## Done + +- Fetched `origin/main`; confirmed it is still `116d46ee9`, so no main drift + had to be preserved. Recorded as the integration base. +- Created this worktree on a new branch from that base; fast-forwarded to G, + then merged S. Verified all four pinned source hashes + (`solve.py` and `calibrate/__init__.py` on both heads) match + `grouped-snapshot-integration-review-pins.json` byte for byte before merging. +- `calibrate/__init__.py` auto-merged as a true union: `GroupedUpperBounds` + export retained alongside every snapshot export. +- `solve.py`: the single textual conflict was the `calibrate()` signature; + resolved as a union of S's `target_snapshots` and G's + `grouped_upper_bounds` / `grouped_preserve_zeros` / + `_post_projection_observer`. +- `test_us_multispine_pool_tool.py`: resolved semantically. G's unrelated US + integration delta (schema_version 2 PUMA-ladder fixture with joint + PUMA/tract/CD overlap arrays and per-layer `source` labels) merged cleanly + and is retained; the only textual conflict was the `spec_sha256` pin. +- The five source-derived identity conflicts + (`inventory_coverage.py` EXPECTED_HASHES, `us-f0-coverage.json`, + `test_spec_engine_loader.py` golden, the multispine `spec_sha256`, and the + calibrate parity `pins.json`) carry a placeholder in this merge commit. None + of them is an ours/theirs decision: both sides' values describe their own + tree, and neither describes the merged one. They are recomputed against the + final merged sources in a later commit on this branch. + +## Done (continued) + +- Instrumented `_optimize_grouped`: the grouped early return in `_optimize` + bypassed every hook S added, so grouped runs emitted only a closing snapshot. + The in-loop emission sits after the progress callback and before `backward()` + and reads the exact float32 estimate tensor the epoch's loss was computed + from. Returned weights, trajectory and RNG state are bit-identical with the + observer on and off, and no evaluation is added. +- Separated retain-best detection from the receipt for grouped runs, so a later + change that populated a grouped receipt cannot make the reused final emitter + claim a retained best. The receipt stays empty. +- Recomputed six source-derived identity artifacts against this checkout (the + five conflicted ones plus the country-bundle digests, which were not + conflicted because only G had touched them but which move for the same + reason). No value equals either branch's. The simulate and fit.qrf pins were + deliberately left alone. +- Added the checklist's cross product to the three existing grouped test files + rather than a new one, so every case reuses fixtures already there. + +## Next + +Root's independent review and integration. Open follow-ups, none owned here: +the host observer seam for the US fiscal dense calibration path, a country-scale +cadence choice backed by a real measurement, and the dashboard consumer. + +--- + +# #893 reconciliation to main's amended graph interface (amendments 19 and 20) + +Lane: `microcosm-us-launch-verified-lanes-20260910` (the live integration +worktree for PR #893, branch `microcosm-us-launch-integration-20260909`). +Started 2026-09-12 at `069d5ed9a`. Everything below the `---` rule at the end +of this section is prior-lane history; see "Root journals are history, not +state" in `CLAUDE.md`. + +## State + +Merges done, four graph pieces re-applied, graph package green (518 passed), +fit green (191), spec check / groups verify / ruff clean. Build-side consumer +suites running at the time of this entry; final results in `out.md` §5 and +`experiments/893-reconciliation-amendments-19-20-20260912.md`. Nothing +pushed, no PR, no new branch, `uv.lock` untouched. + +## Done + +- Read `CLAUDE.md`, the charter's "Interface freeze" at `23ba24770`, PR + #893's body, the Amendment 19 lane's "Scope" list; diffed the graph package + against `23ba24770` per file; AST-scanned every non-graph consumer. +- `uv sync --all-packages --locked --extra us --extra uk` exit 0. Baseline + graph suite: 5 failed / 362 passed (the five the brief names). +- `3010b7788` merge `origin/main`: graph package, fit sources, lock and + charter resolved to main's bytes; branch-only `attachments.py`, + `availability.py`, `schema.py` and the two branch-only graph tests removed + in the merge, to return only where consumed. +- `051357909` merge `23ba24770` (amendment 20, merged from the branch head + because #912 was still on CI; the dispatcher re-runs `git merge origin/main` + after it lands — expected no-op for graph/fit/lock/charter). +- `dc621c14c` seed digests re-pinned (the branch's `acs_transfer` and + `housing_inputs` plus amendment 20's `fit.qrf` move them); coverage report + regenerated; `--check` exit 0. +- `cff8fbf32` calibrate/simulate H1 pins re-recorded (the branch's solver and + `Frame.__reduce__` changes move them); parity files 27 passed. +- Pieces re-applied on main's files, each with graph-level tests the branch + never had: `752ab840f` raw-byte codec (13 US consumers; 8 codec tests), + `db1b7821a` Frame-metadata store (3 named consumers; 274 passed across the + store/population/executor/manifest files), `d1019762b` execution states + (the post-clone geography gate's typed artifact; replaces the amendment-19 + gate refusal and its test; whole graph package 518 passed), `a16eeacf8` + population observer (4 consumers; 94 passed on the executor + B/F files). +- Dropped for lack of a consumer: lazy retention (`attachments.py`, layer + 08), `schema.py`, `keys._stream_file`, the `_write_node` refactor. +- Layer map computed from `git log --name-only` attribution plus an AST + import scan with hard / name-hard / soft link classes; written to `out.md`. + +## Next + +- Record the build-side consumer results in `out.md` §5, commit the + `experiments/` copy of the report, leave `out.md` uncommitted (it is + another lane's tracked report; see the memory note). +- For Max: piece C supersedes amendment 19's gate refusal (`out.md` §8.1); + the dropped lazy retention / `_stream_file` / `_write_node` pieces (§8.2–3). + +--- + # Amendment 19 — typed opaque artifacts on the graph interface Lane: `amend-typed-artifacts`, off `origin/main` at `3094bfe84`. Started @@ -960,3 +1104,65 @@ The still-earlier PolicyEngine-US 1.819.0 lock-bump lane merged into `origin/main` at `7b90bb18` on 2026-08-24; its final state remains at commit `05d254aa` and its detailed receipts remain in the historical section of `_LANE-NOTES.md`. + +## US launch integration staging — 2026-09-09 + +### State +Source-only staging in progress. Execution and source/data admission remain root-owned. + +### Done +Verified requested clean branch, base HEAD, main ancestry and preservation pins. + +### Next +Apply thirteen explicit source layers, commit each, then separately review isolated ordinary execution. Existing journal history above is retained. + +Layer 1: SAFE-ADDITIVE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 2: GRAPH-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 3: ACCEPTED-SHARED-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 4: PUF-SUPPORT-MERGE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 5: SOLVE-MERGE-PROPOSAL.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 6: J-GRAPH-COMPATIBILITY.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 7: F-CATALOGUE-OPTIMIZATION.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 8: GRAPH-ATTACHMENT-METADATA.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 9: F-JOINT-GEOGRAPHY-GATE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 10: SOURCE-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 11: PLACEMENT-ADDITIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 12: ORDINARY-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 13: INTEGRATION-REGRESSIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +### State +All thirteen source layers staged and committed; behavioral qualification pending. + +### Done +Per-layer pins and actual commit messages checked; store/current-main preservation retained. + +### Next +Root reviews exact ordinary guard/source/resource admissions before execution. Full65 findings and survey/SCF lanes remain separately owned. No remote action is authorized. + +### Integration source-resource closure — 2026-09-09 + +Root preflight found seven JSON source definitions declared by the ordinary guard but omitted by the staged patch delivery. Added the exact previously reviewed resource bytes from the accepted full65 source projection; no new resource admission or genuine payload. Preserved source staging and earlier evidence. + +## Layer14 full65 replay correction — 2026-09-09 + +State: exact accepted two-file correction staged; successor57 integration execution pending. +Done: verified clean84243 preimages, exact r2 postimages, fixed store/current-main sources and frozen36 evidence. +Next: root admits the separate exact-allowlist57-case guard and final source identities before execution; no new resources. + +## Source review publication — 2026-09-09 + +The user explicitly authorized pushing the current source work and creating PRs for Anthony to review. This supersedes earlier source-only local restrictions for source publication; it does not authorize a data release, merge or deployment. + +Integration controls passed 36 cases; the exact replay correction subsequently passed all 57 cases at dfa7f872cd3eba3c42adf5758cde8b3ca38f3d17. Source/control, model-declaration and resource hashes matched externally after both runs. A final formatting/import cleanup and explicit test-observer loop binding are included for CI; no release result is claimed. See docs/us-launch-review.md for current scope, evidence and related PRs. diff --git a/changelog.d/20260906-frame-metadata-storage.fixed.md b/changelog.d/20260906-frame-metadata-storage.fixed.md new file mode 100644 index 000000000..dfa6dd022 --- /dev/null +++ b/changelog.d/20260906-frame-metadata-storage.fixed.md @@ -0,0 +1 @@ +Preserve complete Frame metadata in graph content-store format v2, including nested source evidence, and refuse older frame caches that cannot restore it or reuse of one frame key with different metadata. diff --git a/changelog.d/893-age-count-fixture.internal.md b/changelog.d/893-age-count-fixture.internal.md new file mode 100644 index 000000000..1e691423f --- /dev/null +++ b/changelog.d/893-age-count-fixture.internal.md @@ -0,0 +1 @@ +Use the maintained Frame store API in the national age-count cold and fresh-process replay tests after removal of the transfer example module. diff --git a/changelog.d/893-asec-prepared-resources.fixed.md b/changelog.d/893-asec-prepared-resources.fixed.md new file mode 100644 index 000000000..95dc74d99 --- /dev/null +++ b/changelog.d/893-asec-prepared-resources.fixed.md @@ -0,0 +1 @@ +Restore three pinned ASEC resource files omitted during integration so prepared and composed source stages can compute their implementation manifests. Preserve the existing empty engine-default allowlist, source declarations, and reported-income contract. diff --git a/changelog.d/893-diagnostic-current-lock.fixed.md b/changelog.d/893-diagnostic-current-lock.fixed.md new file mode 100644 index 000000000..cff257d2e --- /dev/null +++ b/changelog.d/893-diagnostic-current-lock.fixed.md @@ -0,0 +1 @@ +Align the source/seed diagnostic with the lockfile already approved for the primary QRF worker after the UK openpyxl dependency declaration. diff --git a/changelog.d/893-diagnostic-refusal-context.internal.md b/changelog.d/893-diagnostic-refusal-context.internal.md new file mode 100644 index 000000000..c6e330a2f --- /dev/null +++ b/changelog.d/893-diagnostic-refusal-context.internal.md @@ -0,0 +1 @@ +Record bounded code-path context for refused CI spec diagnostics while preserving the existing read restrictions and incomplete result status. diff --git a/changelog.d/893-diagnostic-source-io-lock.fixed.md b/changelog.d/893-diagnostic-source-io-lock.fixed.md new file mode 100644 index 000000000..f192b5b45 --- /dev/null +++ b/changelog.d/893-diagnostic-source-io-lock.fixed.md @@ -0,0 +1 @@ +Update the source identity diagnostic's exact lockfile pin after adding the engine-free source reader extra, and verify that the checked-in lock passes while an incorrect pin still refuses. diff --git a/changelog.d/893-historical-zero-provenance.internal.md b/changelog.d/893-historical-zero-provenance.internal.md new file mode 100644 index 000000000..5c7d83751 --- /dev/null +++ b/changelog.d/893-historical-zero-provenance.internal.md @@ -0,0 +1 @@ +Keep the exact historical ASEC zero-origin citations in the dependency scan while continuing to reject other incumbent data-package references in that module. diff --git a/changelog.d/893-population-observer-isolation.fixed.md b/changelog.d/893-population-observer-isolation.fixed.md new file mode 100644 index 000000000..710af6485 --- /dev/null +++ b/changelog.d/893-population-observer-isolation.fixed.md @@ -0,0 +1 @@ +Give graph population observers detached snapshots so callback mutations and retained references cannot change subsequent calculations or cached populations. Preserve the complete frame, provenance, weights and mass accounting on both cold runs and cache hits. diff --git a/changelog.d/893-population-only-block-source.added.md b/changelog.d/893-population-only-block-source.added.md new file mode 100644 index 000000000..09da0ddba --- /dev/null +++ b/changelog.d/893-population-only-block-source.added.md @@ -0,0 +1 @@ +Normalize pinned population-only Census API responses into atomic block support. Reconcile every positive block against independent state totals and CD/PUMA mappings, retain exact source bytes and request descriptions, and keep publisher qualification separate from byte integrity. diff --git a/changelog.d/893-source-demographic-conditioning.added.md b/changelog.d/893-source-demographic-conditioning.added.md new file mode 100644 index 000000000..88bc7fb66 --- /dev/null +++ b/changelog.d/893-source-demographic-conditioning.added.md @@ -0,0 +1 @@ +Add optional source-qualified sex and state predictors to survey financial imputation. Preserve observed-year distinctions and default behavior, refuse unresolved source demographics, and verify complete population preservation through fitting, attachment and required replay. diff --git a/changelog.d/893-spec-identity-diagnostics.internal.md b/changelog.d/893-spec-identity-diagnostics.internal.md new file mode 100644 index 000000000..3eb49f5a0 --- /dev/null +++ b/changelog.d/893-spec-identity-diagnostics.internal.md @@ -0,0 +1 @@ +Capture bounded spec and seed identity evidence after shared CI tests so changed source fingerprints can be reviewed without weakening coverage assertions. diff --git a/changelog.d/893-survey-atomic-prefix.added.md b/changelog.d/893-survey-atomic-prefix.added.md new file mode 100644 index 000000000..30658e355 --- /dev/null +++ b/changelog.d/893-survey-atomic-prefix.added.md @@ -0,0 +1 @@ +Connect qualified survey geography to shared Census-block assignment before cloning and age calibration. Preserve raw sampling allocation separately, reconstruct geography in calibration budgets, and verify complete geography inheritance through cold execution and required replay. Add bounded source-byte normalization controls and record the remaining native-source and release checks. diff --git a/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md b/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md new file mode 100644 index 000000000..9614ce4df --- /dev/null +++ b/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md @@ -0,0 +1 @@ +Add supplied-source UK atomic-area assignment keyed to stable post-clone household identity, and load UK runtime exports only when requested. diff --git a/changelog.d/893-worker-resource-trace.fixed.md b/changelog.d/893-worker-resource-trace.fixed.md new file mode 100644 index 000000000..a48093a90 --- /dev/null +++ b/changelog.d/893-worker-resource-trace.fixed.md @@ -0,0 +1 @@ +Update the worker resource identity regression to follow the actual lazy import closure, and verify that opened JSON resources affect identity while unlisted resources do not. diff --git a/changelog.d/908-calibration-target-snapshots.added.md b/changelog.d/908-calibration-target-snapshots.added.md new file mode 100644 index 000000000..127a2b385 --- /dev/null +++ b/changelog.d/908-calibration-target-snapshots.added.md @@ -0,0 +1 @@ +Shared microcosm-calibrate can emit aggregate-only per-target estimate snapshots from real solver iterations at a configurable bounded or every-epoch cadence, with a versioned schema, stable ordered target identity digest, honest current/best-retained/selected iterate labelling, and an atomic latest-snapshot plus immutable bounded history store (opt-in; off by default). diff --git a/changelog.d/908-target-snapshot-review-fixes.fixed.md b/changelog.d/908-target-snapshot-review-fixes.fixed.md new file mode 100644 index 000000000..373ab23d0 --- /dev/null +++ b/changelog.d/908-target-snapshot-review-fixes.fixed.md @@ -0,0 +1 @@ +Close the four #908 target-snapshot review findings: a closed typed aggregate metadata contract, detached snapshot delivery, atomic history publication, and strict public codec validation. diff --git a/changelog.d/914-snapshot-iterate-identity.fixed.md b/changelog.d/914-snapshot-iterate-identity.fixed.md new file mode 100644 index 000000000..2563244d6 --- /dev/null +++ b/changelog.d/914-snapshot-iterate-identity.fixed.md @@ -0,0 +1 @@ +Use completed optimizer updates consistently in target snapshots, retain the selected budget probe, and encode overflowing relative errors as counted nulls. diff --git a/changelog.d/asec-farm-allocation-meaning.fixed.md b/changelog.d/asec-farm-allocation-meaning.fixed.md new file mode 100644 index 000000000..ab570a7c3 --- /dev/null +++ b/changelog.d/asec-farm-allocation-meaning.fixed.md @@ -0,0 +1 @@ +Keep farm allocation provenance unresolved when only I_FRMYN has a nonzero code: the published dictionary supplies its range but no code meanings. Preserve allocation established by documented flags. diff --git a/changelog.d/asec-source-domain-contract.fixed.md b/changelog.d/asec-source-domain-contract.fixed.md new file mode 100644 index 000000000..e6b385f9c --- /dev/null +++ b/changelog.d/asec-source-domain-contract.fixed.md @@ -0,0 +1,3 @@ +Prevent returned ASEC amount metadata from mutating cached definitions. Require +unique current-vintage entries, supported interest and child-support domains, +and agreement with the retained current-money owner before source capture. diff --git a/changelog.d/atomic-property-financial-host.added.md b/changelog.d/atomic-property-financial-host.added.md new file mode 100644 index 000000000..460a98daf --- /dev/null +++ b/changelog.d/atomic-property-financial-host.added.md @@ -0,0 +1 @@ +Add an explicit property-income extension to the checked US financial host, retaining complete legacy populations and validating model receipts, reconstructed outputs and required cache replay. diff --git a/changelog.d/atomic-property-tax-host.added.md b/changelog.d/atomic-property-tax-host.added.md new file mode 100644 index 000000000..f3ef5b12a --- /dev/null +++ b/changelog.d/atomic-property-tax-host.added.md @@ -0,0 +1,5 @@ +Add an explicit optional 38-node survey financial host that retains its complete +19- and 35-node populations, rebases property tax leaves with strict unknowns, +and verifies the new full population and numerical artifacts. Add a conditional +typed PUF gate dependency and refuse PUF qualification when required rebased +inputs remain unknown. The existing 19- and 35-node options remain unchanged. diff --git a/changelog.d/cbo-projection-assertion.fixed.md b/changelog.d/cbo-projection-assertion.fixed.md new file mode 100644 index 000000000..3a8c7d0f3 --- /dev/null +++ b/changelog.d/cbo-projection-assertion.fixed.md @@ -0,0 +1 @@ +Allow the explicitly mapped CBO income projections to retain their source-projection assertion when compiling US fiscal targets, while preserving observation-only checks for other sources. diff --git a/changelog.d/common-frame-export-contract.added.md b/changelog.d/common-frame-export-contract.added.md new file mode 100644 index 000000000..ea5fadddb --- /dev/null +++ b/changelog.d/common-frame-export-contract.added.md @@ -0,0 +1 @@ +Add supplied-parent comparison for US full, pruned and local frame exports, preserving retained inputs and missingness through checkpoint readback. diff --git a/changelog.d/country-bundle-shared-source-pins.fixed.md b/changelog.d/country-bundle-shared-source-pins.fixed.md new file mode 100644 index 000000000..089ac9751 --- /dev/null +++ b/changelog.d/country-bundle-shared-source-pins.fixed.md @@ -0,0 +1 @@ +Update the AM, BE and UK country-bundle identity checks after the reviewed calibration snapshot metadata changes. Their authored resources and draw-site bindings remain unchanged; their resolved spec digests include the shared solver source attestation. diff --git a/changelog.d/current-asec-child-support-source.added.md b/changelog.d/current-asec-child-support-source.added.md new file mode 100644 index 000000000..4b951e9a3 --- /dev/null +++ b/changelog.d/current-asec-child-support-source.added.md @@ -0,0 +1 @@ +Qualify retained ASEC child-support amounts, receipt and obligation answers, preserving paid NIU and unknown voluntary payments alongside original source and allocation metadata. diff --git a/changelog.d/current-asec-interest-source.added.md b/changelog.d/current-asec-interest-source.added.md new file mode 100644 index 000000000..73de9e322 --- /dev/null +++ b/changelog.d/current-asec-interest-source.added.md @@ -0,0 +1 @@ +Qualify ordinary and retirement-account ASEC interest from retained source owners, preserving published components, combined totals, knownness and allocation metadata without assigning tax treatment or balancing observations. diff --git a/changelog.d/current-survey-property-graph.added.md b/changelog.d/current-survey-property-graph.added.md new file mode 100644 index 000000000..45b5f9bd0 --- /dev/null +++ b/changelog.d/current-survey-property-graph.added.md @@ -0,0 +1 @@ +Add an opt-in 16-node survey property-income fragment with original DESIGN donor selection, joint ACS draws, signed reconciliation, explicit unknownness, and exact clone attachment while preserving legacy tax leaves and their stated capital-gains limitation. diff --git a/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md new file mode 100644 index 000000000..8f4a31c03 --- /dev/null +++ b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md @@ -0,0 +1 @@ +Bind fiscal measurement caches to target arithmetic and engine table materialization, refuse missing consumed predicate inputs, and count boolean terms arithmetically in shared sum expressions. diff --git a/changelog.d/graph-scalar-float-packing.changed.md b/changelog.d/graph-scalar-float-packing.changed.md new file mode 100644 index 000000000..0cb0b67fe --- /dev/null +++ b/changelog.d/graph-scalar-float-packing.changed.md @@ -0,0 +1 @@ +Pack an exact Python float straight into the graph executor's scalar digest framing with `struct.pack("=d", ...)` instead of a one-element float64 array, preserving the payload bytes, the length prefix and every other scalar's behaviour. diff --git a/changelog.d/graph-signed-reconciliation.added.md b/changelog.d/graph-signed-reconciliation.added.md new file mode 100644 index 000000000..96be9e554 --- /dev/null +++ b/changelog.d/graph-signed-reconciliation.added.md @@ -0,0 +1 @@ +Expose signed-total component reconciliation as a graph node with explicit scales, bounds, and row-level adjustment diagnostics. diff --git a/changelog.d/graph-storage-parts-whole-series.changed.md b/changelog.d/graph-storage-parts-whole-series.changed.md new file mode 100644 index 000000000..afe3d3e1c --- /dev/null +++ b/changelog.d/graph-storage-parts-whole-series.changed.md @@ -0,0 +1 @@ +Select whole series with `slice(None)` instead of an all-True mask when folding physical storage parts, and skip the null recomputation that masked storage never uses, preserving the emitted value bytes, null bytes and subset-selection behaviour exactly. diff --git a/changelog.d/grouped-target-snapshot-integration.added.md b/changelog.d/grouped-target-snapshot-integration.added.md new file mode 100644 index 000000000..c5df41593 --- /dev/null +++ b/changelog.d/grouped-target-snapshot-integration.added.md @@ -0,0 +1 @@ +Instrument the grouped/fixed-zero Adam solver with the opt-in calibration target snapshot observer, reusing the estimate tensor each epoch's loss was computed from and the closing float64 diagnostics estimate. diff --git a/changelog.d/property-completion-host.added.md b/changelog.d/property-completion-host.added.md new file mode 100644 index 000000000..d6d8c1b15 --- /dev/null +++ b/changelog.d/property-completion-host.added.md @@ -0,0 +1 @@ +Add opt-in property completion diagnostics to the existing source projection, with a private typed row artifact, allowlisted aggregate graph receipts and independent replay checks. Preserve default options and numerical property behavior. diff --git a/changelog.d/property-completion-routing.added.md b/changelog.d/property-completion-routing.added.md new file mode 100644 index 000000000..b7e842805 --- /dev/null +++ b/changelog.d/property-completion-routing.added.md @@ -0,0 +1 @@ +Add deterministic original-person property completion routing, independent interest/dividend knownness, exact clone identity diagnostics and original DESIGN support summaries. The pure operation assigns no amounts and grants no source or release authority; host artifact wiring remains separate. diff --git a/changelog.d/property-independent-observation-preservation.internal.md b/changelog.d/property-independent-observation-preservation.internal.md new file mode 100644 index 000000000..cb1fc6a60 --- /dev/null +++ b/changelog.d/property-independent-observation-preservation.internal.md @@ -0,0 +1 @@ +Add a focused invented-source regression confirming that joint property-donor exclusions preserve independently qualified ASEC interest and dividends on both initial clones and through the existing tax split. Production behavior is unchanged. diff --git a/changelog.d/property-model-receipts.added.md b/changelog.d/property-model-receipts.added.md new file mode 100644 index 000000000..de28e20d2 --- /dev/null +++ b/changelog.d/property-model-receipts.added.md @@ -0,0 +1 @@ +Add source-blind verification of the four property fit/apply receipt pairs against authenticated original donor/recipient branches, with complete model history checks and deterministic application replay without refitting. diff --git a/changelog.d/property-survivor-scope.fixed.md b/changelog.d/property-survivor-scope.fixed.md new file mode 100644 index 000000000..4b9258e1c --- /dev/null +++ b/changelog.d/property-survivor-scope.fixed.md @@ -0,0 +1 @@ +Exclude positive survivor receipts from the initial property donor bridge because its two qualified source slots do not cover the additional sources included in the published aggregate. Preserve visible route diagnostics and report the excluded design-weight mass. diff --git a/changelog.d/puf55-canonical-donor.added.md b/changelog.d/puf55-canonical-donor.added.md new file mode 100644 index 000000000..10e06ae48 --- /dev/null +++ b/changelog.d/puf55-canonical-donor.added.md @@ -0,0 +1 @@ +Add a canonical59 artifact adapter for the explicit PUF55 survey Social Security profile, retaining return identity and weights and validating the total-carrier convention. Record the scoped passing native seven-node age-development run. diff --git a/changelog.d/puf55-checked-output.added.md b/changelog.d/puf55-checked-output.added.md new file mode 100644 index 000000000..c5fe0d86c --- /dev/null +++ b/changelog.d/puf55-checked-output.added.md @@ -0,0 +1 @@ +Retain PUF55 output handles for source, producer, store, and complete-population rechecks before downstream consumption, revoking a handle after a failed check without rerunning imputation. diff --git a/changelog.d/puf55-survey-host.added.md b/changelog.d/puf55-survey-host.added.md new file mode 100644 index 000000000..f0e410b85 --- /dev/null +++ b/changelog.d/puf55-survey-host.added.md @@ -0,0 +1 @@ +Add the PUF55 host over the combined ACS/ASEC support-clone population, with source-qualified Social Security routes, complete population preservation and required graph replay. diff --git a/changelog.d/puf55-survey-social-security.added.md b/changelog.d/puf55-survey-social-security.added.md new file mode 100644 index 000000000..ba625b57e --- /dev/null +++ b/changelog.d/puf55-survey-social-security.added.md @@ -0,0 +1 @@ +Add an explicit 55-output PUF profile that preserves survey Social Security components and requires a separate total as its ninth conditioning predictor. Validate canonical donor projection and whole-Population attachment/replay while retaining source identities, weights and Social Security unknownness. diff --git a/changelog.d/shared-atomic-geography.added.md b/changelog.d/shared-atomic-geography.added.md new file mode 100644 index 000000000..4eed5d1a9 --- /dev/null +++ b/changelog.d/shared-atomic-geography.added.md @@ -0,0 +1 @@ +Add shared graph operators for atomic-area assignment and geographic derivation. Country declarations choose observed constraints, stable draw identities, sampling stages and versioned mapping relations. Preserve observed locations, reject ambiguous support, and verify mapping consistency on cloned or pruned populations. Existing country builds retain their current geography paths pending native support integration. Normalize US block mappings, qualify observed ACS/ASEC geography against source receipts, and compose shared assignment before the combined-survey clone, with an inherited-mapping gate afterward. diff --git a/changelog.d/signed-income-reconciliation.added.md b/changelog.d/signed-income-reconciliation.added.md new file mode 100644 index 000000000..428496f2c --- /dev/null +++ b/changelog.d/signed-income-reconciliation.added.md @@ -0,0 +1,2 @@ +Add a deterministic weighted projection onto a signed total with explicit +nonnegative component bounds, adjustment diagnostics and floating-point checks. diff --git a/changelog.d/source-io-wheel-extra.fixed.md b/changelog.d/source-io-wheel-extra.fixed.md new file mode 100644 index 000000000..9e8232f55 --- /dev/null +++ b/changelog.d/source-io-wheel-extra.fixed.md @@ -0,0 +1 @@ +Declare an engine-free HDF source extra and install it in the isolated wheel test environment. Verify an invented pandas HDF round trip before source tests, while retaining the base wheel import boundary and excluding the US rules engine. diff --git a/changelog.d/survey-final-observation-seal.fixed.md b/changelog.d/survey-final-observation-seal.fixed.md new file mode 100644 index 000000000..bfa186d97 --- /dev/null +++ b/changelog.d/survey-final-observation-seal.fixed.md @@ -0,0 +1 @@ +Recheck the complete detached survey population observations after final source and artifact I/O, so late mutations cannot escape through the values retained for downstream graph composition. diff --git a/changelog.d/survey-social-security-source.added.md b/changelog.d/survey-social-security-source.added.md new file mode 100644 index 000000000..11d76e2dd --- /dev/null +++ b/changelog.d/survey-social-security-source.added.md @@ -0,0 +1 @@ +Add current survey Social Security source qualification with explicit reporting units, question-universe unknownness, allocation literals and component mapping judgments. diff --git a/changelog.d/uk-atomic-household-lineage.added.md b/changelog.d/uk-atomic-household-lineage.added.md new file mode 100644 index 000000000..b40a2a605 --- /dev/null +++ b/changelog.d/uk-atomic-household-lineage.added.md @@ -0,0 +1 @@ +Add a pure UK household-lineage adapter that derives stable atomic-geography keys from explicit source identities and selection/expansion records, preserving exact IDs and refusing ambiguous ancestry without granting source authority or changing the full-build host. diff --git a/changelog.d/us-asec-retirement-annuity-niu.fixed.md b/changelog.d/us-asec-retirement-annuity-niu.fixed.md new file mode 100644 index 000000000..0ddba37c7 --- /dev/null +++ b/changelog.d/us-asec-retirement-annuity-niu.fixed.md @@ -0,0 +1 @@ +Match published annuity NIU literals to the authenticated money owner's normalized zero and NIU status while preserving raw retirement source values. Reject mismatched dollar-zero and NIU encodings before qualification. diff --git a/changelog.d/us-atomic-financial-composition.added.md b/changelog.d/us-atomic-financial-composition.added.md new file mode 100644 index 000000000..a28385f6c --- /dev/null +++ b/changelog.d/us-atomic-financial-composition.added.md @@ -0,0 +1 @@ +Compose survey atomic geography and current financial imputation in a checked twenty-node graph, retaining geography through cloning and financial attachment. Add native Delaware population-only source normalization and complete support readback controls. diff --git a/changelog.d/us-current-acs-income-anchor-source.added.md b/changelog.d/us-current-acs-income-anchor-source.added.md new file mode 100644 index 000000000..d8d6fce32 --- /dev/null +++ b/changelog.d/us-current-acs-income-anchor-source.added.md @@ -0,0 +1,3 @@ +Qualify original ACS property and retirement income anchors through the retained +survey source owner, preserving literal missingness, allocation, age universe and +signed values while verifying raw identities and adjusted amount bits. diff --git a/changelog.d/us-current-asec-dividend-source.added.md b/changelog.d/us-current-asec-dividend-source.added.md new file mode 100644 index 000000000..a7d453233 --- /dev/null +++ b/changelog.d/us-current-asec-dividend-source.added.md @@ -0,0 +1 @@ +Qualify current ASEC dividend observations and survivor source routes from the retained survey owner, preserving ambiguous zeros, allocation-codebook conflicts and source flags without assigning tax treatment or selecting donors. diff --git a/changelog.d/us-current-asec-income-routing-source.added.md b/changelog.d/us-current-asec-income-routing-source.added.md new file mode 100644 index 000000000..b4d469ed8 --- /dev/null +++ b/changelog.d/us-current-asec-income-routing-source.added.md @@ -0,0 +1 @@ +Qualify the current ASEC pension/annuity, retirement distribution, net property, farm and other-income source families into a typed reporting/routing projection, keeping printed totals, receipt universes, account and category routing, and allocation provenance distinct from any modelled taxable, private or residual component. diff --git a/changelog.d/us-current-asec-property-basis.added.md b/changelog.d/us-current-asec-property-basis.added.md new file mode 100644 index 000000000..908dff17d --- /dev/null +++ b/changelog.d/us-current-asec-property-basis.added.md @@ -0,0 +1 @@ +Add a pure ASEC property donor basis with separate reported and component totals, explicit retirement-slot routing derivations, exact identity alignment and original-design-weight exclusion diagnostics. diff --git a/changelog.d/us-current-asec-retirement-basis.added.md b/changelog.d/us-current-asec-retirement-basis.added.md new file mode 100644 index 000000000..d7da7254e --- /dev/null +++ b/changelog.d/us-current-asec-retirement-basis.added.md @@ -0,0 +1,3 @@ +Add a pure ASEC retirement candidate ledger with explicit measurement assumptions, source routing and knownness, conservative accounting intervals, preserved unresolved survivor/distribution scope, and original DESIGN-weight support diagnostics. It produces no fiscal inputs or source authority. + +Require exact applicable distribution aggregate balance and readable off-route evidence before admitting candidate intervals; preserve account evidence and signed residuals separately from interval availability. diff --git a/changelog.d/us-current-asec-retirement-detail-source.added.md b/changelog.d/us-current-asec-retirement-detail-source.added.md new file mode 100644 index 000000000..799cbc4b2 --- /dev/null +++ b/changelog.d/us-current-asec-retirement-detail-source.added.md @@ -0,0 +1 @@ +Qualify current ASEC pension, disability and survivor source details with exact original-person joins, preserved unknownness and unallocated aggregate comparisons. New literal fields remain separate from retained money; no retirement regularity, taxability or ACS component mapping is inferred. diff --git a/changelog.d/us-current-property-income-sources.added.md b/changelog.d/us-current-property-income-sources.added.md new file mode 100644 index 000000000..ca64a9350 --- /dev/null +++ b/changelog.d/us-current-property-income-sources.added.md @@ -0,0 +1 @@ +Compose source-qualified original ASEC property donor and ACS adult-anchor recipient branches, preserving DESIGN weights, full origin axes, knownness diagnostics and final source-lifetime checks without fitting or attaching tax leaves. diff --git a/changelog.d/us-current-survey-health-coverage.added.md b/changelog.d/us-current-survey-health-coverage.added.md new file mode 100644 index 000000000..5443947b8 --- /dev/null +++ b/changelog.d/us-current-survey-health-coverage.added.md @@ -0,0 +1,4 @@ +Add a source-qualified health coverage graph fragment for the common US survey +frame and its PUF clones. Preserve literal source and allocation codes, explicit +unknowns, and incoming amount fields; keep broader ACS concepts as documented +gaps instead of manufacturing narrower coverage flags. diff --git a/changelog.d/us-declared-fiscal-measurement.added.md b/changelog.d/us-declared-fiscal-measurement.added.md new file mode 100644 index 000000000..58d36e459 --- /dev/null +++ b/changelog.d/us-declared-fiscal-measurement.added.md @@ -0,0 +1 @@ +Add a candidate-only US graph measurement stage using declared target bindings, household-aligned sparse measures, explicit national/state/district support checks, and complete input-closure checks for optional model evaluation. diff --git a/changelog.d/us-fiscal-dense-calibration.added.md b/changelog.d/us-fiscal-dense-calibration.added.md new file mode 100644 index 000000000..497b5415f --- /dev/null +++ b/changelog.d/us-fiscal-dense-calibration.added.md @@ -0,0 +1 @@ +Add a numerical fiscal graph calibration stage using existing grouped Adam, explicit original weight and origin bounds, exact household alignment, and target and origin diagnostics without claiming source or release admission. diff --git a/changelog.d/us-fiscal-leaf-policy.added.md b/changelog.d/us-fiscal-leaf-policy.added.md new file mode 100644 index 000000000..dee0da8d5 --- /dev/null +++ b/changelog.d/us-fiscal-leaf-policy.added.md @@ -0,0 +1 @@ +Add optional pinned fiscal input policies with strict producer defaults, complete-population collision checks and private-copy preparation. Literal assumptions remain non-executable pending checked parent integration; sampled traces cannot exempt inactive inputs. diff --git a/changelog.d/us-fiscal-target-snapshots.added.md b/changelog.d/us-fiscal-target-snapshots.added.md new file mode 100644 index 000000000..61514455b --- /dev/null +++ b/changelog.d/us-fiscal-target-snapshots.added.md @@ -0,0 +1 @@ +Allow host-owned target snapshots on the US dense fiscal calibration kernel while preserving result and cache identity. Align grouped snapshot epochs with completed optimizer updates and keep required cache replay silent. diff --git a/changelog.d/us-launch-integration.fixed.md b/changelog.d/us-launch-integration.fixed.md new file mode 100644 index 000000000..170634a55 --- /dev/null +++ b/changelog.d/us-launch-integration.fixed.md @@ -0,0 +1 @@ +Preserve grouped calibration closing-state results alongside ordinary best-iterate selection, retain the combined US runtime facade and joint geography validation, and publish complete metadata for lazy graph snapshots. diff --git a/changelog.d/us-launch-store-grouped-validation.fixed.md b/changelog.d/us-launch-store-grouped-validation.fixed.md new file mode 100644 index 000000000..c341c76e2 --- /dev/null +++ b/changelog.d/us-launch-store-grouped-validation.fixed.md @@ -0,0 +1 @@ +Reject nonfinite stored JSON as a store-corruption error and normalize the deprecated calibration method alias before validating grouped constraints. diff --git a/changelog.d/us-national-atomic-support.added.md b/changelog.d/us-national-atomic-support.added.md new file mode 100644 index 000000000..085c1627a --- /dev/null +++ b/changelog.d/us-national-atomic-support.added.md @@ -0,0 +1,4 @@ +Add an opt-in national Census population-only source control covering the fifty +states and DC, with independent block/state reconciliation, complete CD119 and +PUMA joins, and full support serialization/readback. Record the accepted native +normalization separately from household assignment and dataset release checks. diff --git a/changelog.d/us-population-seal-streaming.changed.md b/changelog.d/us-population-seal-streaming.changed.md new file mode 100644 index 000000000..60002223e --- /dev/null +++ b/changelog.d/us-population-seal-streaming.changed.md @@ -0,0 +1 @@ +Stream the US full-PUF placement's in-process population seal into its digest instead of expanding every part into one tuple first, preserving each part, its order, its length prefix and its bytes exactly. diff --git a/changelog.d/us-property-income-graph.added.md b/changelog.d/us-property-income-graph.added.md new file mode 100644 index 000000000..ac92df734 --- /dev/null +++ b/changelog.d/us-property-income-graph.added.md @@ -0,0 +1 @@ +Add a source-blind property-income model graph with original-design QRF fitting, joint draw columns, signed reconciliation and explicit diagnostics. diff --git a/changelog.d/us-property-tax-leaves.added.md b/changelog.d/us-property-tax-leaves.added.md new file mode 100644 index 000000000..fe4905618 --- /dev/null +++ b/changelog.d/us-property-tax-leaves.added.md @@ -0,0 +1,6 @@ +Add a deterministic three-node property tax rebase with explicit receiving +version, separate interest/dividend knownness, maintained fraction splits, +subtraction complements and typed numerical completeness evidence. Preserve +unknowns and retirement-account earnings; refuse receiving shapes that the +current FILTER operation cannot preserve. Country and PUF host integration +remains separate. diff --git a/changelog.d/us-puf55-canonical-and-output-seals.added.md b/changelog.d/us-puf55-canonical-and-output-seals.added.md new file mode 100644 index 000000000..0e6679803 --- /dev/null +++ b/changelog.d/us-puf55-canonical-and-output-seals.added.md @@ -0,0 +1,4 @@ +Add a source-bound PUF55 canonical donor graph node with cold/replay controls, +stable live-code identity checks, and finalized-candidate seals for the shared +two-route numerical finalizer. Complete native survey attachment and release +validation remain separate build steps. diff --git a/changelog.d/us-puf55-survey-ss-measurement.added.md b/changelog.d/us-puf55-survey-ss-measurement.added.md new file mode 100644 index 000000000..ebe159455 --- /dev/null +++ b/changelog.d/us-puf55-survey-ss-measurement.added.md @@ -0,0 +1,5 @@ +Add an authenticated filer/joint-spouse Social Security report-sum measurement +for PUF55 conditioning, with an explicit eight-predictor fallback for incomplete +reports and preservation of all survey Social Security cells and masks. Record +31 passing invented controls and the remaining recipient integration and +measurement limitations. diff --git a/changelog.d/us-source-string-seal-efficiency.changed.md b/changelog.d/us-source-string-seal-efficiency.changed.md new file mode 100644 index 000000000..a0c33268e --- /dev/null +++ b/changelog.d/us-source-string-seal-efficiency.changed.md @@ -0,0 +1 @@ +Reuse bounded, column-local encodings for repeated source strings during ASEC frame hashing while preserving the exact digest byte stream and checking all values on every validation. diff --git a/changelog.d/us-survey-catalogue-digest.changed.md b/changelog.d/us-survey-catalogue-digest.changed.md new file mode 100644 index 000000000..96d1c8d13 --- /dev/null +++ b/changelog.d/us-survey-catalogue-digest.changed.md @@ -0,0 +1 @@ +Stream eligible ACS catalogue seals through bounded record JSON encoding while preserving canonical digest bytes, full-value fallback, and repeated source and owner validation. Bind the additional JSON provider calls into the existing preparation authority checks. diff --git a/changelog.d/us-survey-copy-and-identity.fixed.md b/changelog.d/us-survey-copy-and-identity.fixed.md new file mode 100644 index 000000000..76d9520fa --- /dev/null +++ b/changelog.d/us-survey-copy-and-identity.fixed.md @@ -0,0 +1 @@ +Keep frame metadata copying and pickling from changing producer identity on Python 3.14. Refresh ACS source pins after verified formatting-only changes, and reduce per-cell survey identity encoding overhead while preserving canonical bytes, mutation checks, and the current-wage projection. diff --git a/changelog.d/us-survey-current-amount-enrichment.added.md b/changelog.d/us-survey-current-amount-enrichment.added.md new file mode 100644 index 000000000..dab37b769 --- /dev/null +++ b/changelog.d/us-survey-current-amount-enrichment.added.md @@ -0,0 +1 @@ +Add a checked US survey enrichment host that preserves the PUF population while attaching source-qualified unemployment, medical costs and current health coverage. Keep reporting unknowns, original observations and source-to-clone provenance explicit. diff --git a/changelog.d/us-survey-diagnostic-options.fixed.md b/changelog.d/us-survey-diagnostic-options.fixed.md new file mode 100644 index 000000000..e8668b7e0 --- /dev/null +++ b/changelog.d/us-survey-diagnostic-options.fixed.md @@ -0,0 +1 @@ +Reconstruct the grouped calibration solver's closing-state selection options during survey diagnostic verification, preserving exact diagnostic comparison and late-mutation checks. diff --git a/changelog.d/us-survey-interest-conservation.fixed.md b/changelog.d/us-survey-interest-conservation.fixed.md new file mode 100644 index 000000000..b58d850b5 --- /dev/null +++ b/changelog.d/us-survey-interest-conservation.fixed.md @@ -0,0 +1 @@ +Retain the tax-exempt remainder when the survey financial graph splits observed ASEC or modeled ACS interest totals, preserving both parts on the original survey support before PUF enrichment. diff --git a/changelog.d/us-whole-series-selection-sites.changed.md b/changelog.d/us-whole-series-selection-sites.changed.md new file mode 100644 index 000000000..e14368e80 --- /dev/null +++ b/changelog.d/us-whole-series-selection-sites.changed.md @@ -0,0 +1 @@ +Take the whole series by slice at the remaining US in-process seal sites (atomic geography, input coverage, origin budget, current survey predictors and PUF55 recipients), preserving each seal's emitted bytes exactly. diff --git a/changelog.d/weights-copy-protocol.fixed.md b/changelog.d/weights-copy-protocol.fixed.md new file mode 100644 index 000000000..4a29fdf93 --- /dev/null +++ b/changelog.d/weights-copy-protocol.fixed.md @@ -0,0 +1 @@ +Reconstruct base `Weights` through validation during shallow copy, deep copy and pickle round trips, preserving exact values and read-only storage without changing class metadata. Preserve existing subclass copy protocols. diff --git a/changelog.d/worker-identity-current-lock.fixed.md b/changelog.d/worker-identity-current-lock.fixed.md new file mode 100644 index 000000000..66e497e13 --- /dev/null +++ b/changelog.d/worker-identity-current-lock.fixed.md @@ -0,0 +1 @@ +Update the primary QRF worker's approved lock identity for the reviewed source-I/O packaging metadata change while preserving strict rejection of other locks and the explicit legacy campaign boundary. diff --git a/docs/calibration-target-snapshots.md b/docs/calibration-target-snapshots.md new file mode 100644 index 000000000..b48fd32a9 --- /dev/null +++ b/docs/calibration-target-snapshots.md @@ -0,0 +1,155 @@ +# Calibration target snapshots + +An opt-in observer exposes target estimates already computed during an Adam +solve. It also records the estimates on the weights actually returned. Use it +to inspect fit over time without adding another model evaluation or changing +the optimization steps. + +```python +from pathlib import Path + +from microcosm.calibrate import ( + TargetSnapshotCadence, + TargetSnapshotObserver, + TargetSnapshotWriter, + calibrate, +) + +observer = TargetSnapshotObserver( + sink=TargetSnapshotWriter(Path("runs/example/targets"), history_limit=256), + run_id="example", + cadence=TargetSnapshotCadence(every=25), +) +result = calibrate(frame, targets, method="adam", target_snapshots=observer) +``` + +The caller supplies its existing frame and targets. No observer is enabled by +default. The example cadence is a caller choice, not a measured country-build +default. A sink exception propagates, matching the existing progress callback. + +Each snapshot has an ordered target identity, target values, achieved estimates, +relative errors, sequence, phase and iterate labels. Duplicate target names +remain distinguishable by ordered position. Honest nonfinite intermediate +estimates are represented by nulls and counts. Intermediate float32 estimates +and final float64 estimates are labeled with their actual iterate semantics. +Budget probes and subsequent selection/refit phases share one emission sequence. +`epoch` counts completed optimizer updates: zero is the starting state. An +in-loop estimate precedes that iteration's update; its selected counterpart uses +the same convention. Cadence counts loss evaluations starting at one, so an +every-25 cadence can emit epoch 24. The selected budget-search snapshot retains +the winning probe's identity, even when another probe ran afterward. If computing +a relative error overflows, the error is null and `non_finite_rows` counts it; +finite operands remain visible. + +Metadata accepts bounded flat scalar mappings and checked string identifiers; +it does not accept nested payloads or record vectors. `best_retained` must be a +complete `{available, epoch, loss}` mapping in a serialized snapshot. The emitter +can fill that mapping when no best iterate exists; the public validator rejects +a serialized null in its place. Each delivered payload is detached from the +caller's metadata and subsequent snapshots. + +The local writer atomically replaces `latest.json`. History chunks become +visible only after their complete bytes are written and synced, and an existing +chunk cannot be overwritten. Retention and dropped-history counts are explicit. +Use a fresh run directory unless intentionally adopting existing history. +Atomic history publication requires a filesystem that supports hard links and +directory synchronization. Unsupported filesystem errors propagate to the caller; +the writer does not expose partially written history as a fallback. + +## The grouped US solver + +The grouped/fixed-zero Adam path (`grouped_upper_bounds`, optionally +`grouped_preserve_zeros`) is instrumented at the same two seams and with the +same guarantees: the in-loop snapshot reads the exact float32 estimate tensor +the epoch's loss was computed from, and the closing `selected` snapshot reads +the same float64 `problem.estimates` the final diagnostics use. Enabling the +observer on a grouped run adds no matrix evaluation, no model evaluation and no +RNG advance, and returns bit-identical weights and trajectory. + +Two grouped specifics a consumer must read correctly: + +- Grouped Adam is a **closing-state** algorithm. It never runs the retain-best + rule, so every grouped snapshot carries + `best_retained = {"available": false, "epoch": null, "loss": null}` and the + closing snapshot is stamped `epoch == epochs`. That is not the same as an + ordinary run whose retain-best rule was switched off, so grouped snapshots + also carry bounded `selection` labels — `rule: "closing_state"`, + `constraint_mode: "grouped_upper_bounds"` and `grouped_preserve_zeros` — + which mirror the run's `options["iterate_selection"]`. +- In-loop snapshots are **pre-update loss evaluations**, the same convention the + ungrouped Adam loop uses. They are deliberately not the post-update projected + accepted vector that the private `_post_projection_observer` proof seam + reports after the next completed update: those are different vectors, and + producing target totals for the accepted one would require an extra matrix evaluation. + That private seam carries record-length weights, household IDs, the group map + and the absolute-bound vector; none of it reaches a snapshot, and the public + codec rejects record-level key names outright. + +Grouped runs keep every existing guard: fixed zeros stay in full-population +coordinates, the accepted-weight byte equality and ordered-household-ID checks +still run *after* the closing snapshot sink, and the unsupported grouped modes +(scalar cap, conserved mass, prox, L0/exact-k, gate initialization) are still +refused before the optimizer is constructed. Snapshot support does not turn any +of them into a permitted numeric path. + +## US fiscal host + +The actual dense fiscal kernel accepts the shared observer through an optional, +keyword-only constructor argument: + +```python +from microcosm.build.us_runtime.graph_fiscal_dense_calibration import ( + FiscalDenseCalibrationKernel, +) +from microcosm.calibrate import TargetSnapshotCadence, TargetSnapshotObserver + +observer = TargetSnapshotObserver( + sink=host_owned_sink, + run_id="fiscal-run", + candidate_id="candidate-under-review", + cadence=TargetSnapshotCadence(every=25), +) +kernels.register(FiscalDenseCalibrationKernel(target_snapshots=observer)) +``` + +The host owns the sink and any local writer directory. The kernel retains the +observer only on its instance and forwards it to the existing `calibrate` call. +Omitting the argument or passing `None` disables observation. Observer values, +callbacks and paths are absent from node parameters, implementation identity, +cache keys, graph artifacts and receipts. Changing this source implementation +changes its fingerprint normally; changing observer configuration does not. + +The solver binds snapshot identity to the actual compiled target order and +values from the fiscal measurement. The existing final matrix/target, household +ID, weight and measurement checks run after the sink completes. Sink exceptions +propagate, as with the ordinary progress callback. Each delivered dictionary is +detached: mutating its aggregate metadata or target rows cannot change the next +payload or optimizer result. This does not make an arbitrary callback safe to +mutate unrelated live application state. + +A required graph cache hit does not execute the optimizer and emits no target +snapshots, even when a new observer is supplied. Hosts should retain the previous +run's diagnostics and history with their identities; a replay must not be shown +as a new optimizer run. Complete-population and source admission remain the +country host's responsibility. + +This implements the solver, local storage and dense fiscal-host portions of +[issue 908](https://github.com/PolicyEngine/microcosm/issues/908). Staging upload, +a dashboard consumer and native cadence/performance acceptance remain separate +work. See [the fiscal-host integration evidence](../experiments/fiscal-target-snapshot-host-20260912.md). + +The reviewed repair branch passed 307 calibration tests before the final strict +null correction. All 52 snapshot tests then passed after that one-line correction, +including passive solver parity tests. Independent review closed the metadata, +detachment, partial-publication and codec findings. See the +[invented benchmark evidence](../experiments/908-target-snapshot-bench-receipts.md) +for overhead measurements; those are not native US or UK benchmarks. + +Fable's later PR review identified a mixed epoch convention, a missing winning +budget-probe label and a relative-error overflow case. Four new counterexamples +failed first; all 56 snapshot tests and all 311 calibration tests then passed +after those corrections. The 26 affected build identity and graph parity checks +also passed after recalculating the calibration source pins; fit and simulation +pins were preserved. These changes affect diagnostics only. The public writer +still validates each incoming payload because callers can supply serialized +snapshots independently of an observer. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md new file mode 100644 index 000000000..a4351354a --- /dev/null +++ b/docs/current-survey-puf59-progress.md @@ -0,0 +1,72 @@ +# PUF59 and current survey predictor implementation status + +The PUF59 profile fits and transfers 56 person outcomes and three tax-unit outcomes. Six detailed mortgage fields remain owned by the separate SCF producer. The historical full65 default remains available. PUF59 conditions on the source-specific PUF2015 filing-status class and capped return-size proxy, plus six monetary predictors. Its self-employment predictor includes ordinary and SSTB Schedule C income. Return-level Boolean outputs use an explicit incidence capacity of one; that is not a physical person count. + +The local PUF59/full65 regression run passed 128 ordinary tests. Those tests execute real weighted QRF fits, sequential target conditioning, draws, finalization, whole-Population preservation and required cache replay. They include excluded mortgage incumbent preservation and source-specific predictor checks. The run took 93.9 seconds and peaked at 622.7 MB RSS. These are invented mechanism fixtures, not held-out fit-quality or launch acceptance. + +Current survey financial completion passed 26 ordinary tests, including 15 ASEC demographic projection controls. The financial graph uses original ASEC design weights to fit interest, dividends and capital-gains outcomes for native ACS people. It preserves ACS observed wage/self-employment amounts and source-owned age-15 universe zeros. A draw is shared with its clone by source identity. The graph retains typed preparation, allocation and context evidence while separating the donor branch from survey allocation, avoiding a structural dependency cycle. Cold execution and required replay passed with the actual fitted model artifacts. The combined run took 121.4 seconds and peaked at 524.0 MB RSS. + +The source projection records the distinct survey observation, income and price periods. ASEC sex and state come from the retained original person and household sources, with allocation flags and unknownness preserved. Attaching those demographic projections to the graph is a separate pending integration step. The current financial model conditions on age and two earnings fields; sex/state conditioning and held-out fit quality remain explicit modeling work. + +Social Security handling remains a launch blocker. The historical PUF59 donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. Its reconciliation code can treat unknown recipient components as zero and use equal component shares. The explicit `PUF55_SURVEY_SS` profile now excludes these four outcomes from PUF enrichment and conditions the remaining 55 outcomes on an additional Social Security total predictor. It preserves existing survey component fields, including unknownness. This profile is not yet wired into the native build. + +The new profile requires a known, finite, nonnegative `tax_unit.puf_conditioning_social_security_total` supplied by an upstream source owner. It does not infer that total from unknown components or move it between entity grains. All eight historical PUF59 predictors retain their order, with the total appended as predictor nine. The profile has a distinct train/apply phase and retains return-level incidence-capacity validation. Historical full65/PUF59 defaults and their 65/59 outcome contracts remain unchanged. + +Eleven new controls pass with real 55-target QRF fits, finalization, raw-draw validation and required cache replay. They check exact Social Security incumbent bit patterns, including signed zero and NaN, and refuse changes to the conditioning total after fitting. The run took 28.5 seconds and peaked at 631.6 MB RSS; receipt SHA256 is `8d5beacf15d8eb7bf568705d4e12bd71d810f3e748571b669a6ec44921ebf0cf`. All 128 historical profile cases also pass against the identical production module. An earlier combined run retained a failure in the new test's comparison of Boolean storage types; the corrected eleven-case run preserves exact values and missingness while treating the requested PUF Boolean output's nullable storage explicitly. Native recipient total qualification, beneficiary completion and whole-population PUF55 attachment remain separate pending checks. The native donor projection now passes as described below. + +The additive current-source qualifier now verifies original ASEC total, reason and allocation literals against the live survey preparation and retained original member. It also binds the ACS 2024 issuance and SSP price adjustment. Both surveys preserve under-15 question-universe unknownness. ASEC may record combined family payments on one person's report; a reason code is not a separately observed component amount or an assignment to individual beneficiaries. See [Social Security source semantics](survey-social-security-source.md) for the source contract, judgment boundaries and unresolved model work. + +The qualifier and numerical basis pass 52 controls: 50 numerical/literal cases and two actual maintained source issuances on invented original inputs. They include reordered source identities, ambiguous reasons, allocation flags, exact requalification, unknownness and no input mutation. The corrected run took 67.6 seconds and peaked at 428.9 MB RSS; the receipt SHA256 is `2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. An earlier run's two source tests exposed an incorrect capture-budget argument; the corrected caller uses the source owner's existing decoded-body bound while retaining exact CSV byte and digest checks. No native Social Security completion or beneficiary allocation has been accepted. + +The combined current source also passes all **225 component integration controls**: 34 shared geography cases, 52 survey Social Security cases, 11 PUF55 cases and all 128 historical full65/PUF59 cases. The exact combined run completed in 205.6 seconds at 687.8 MB peak RSS with no skips or unexpected access refusals. Before/after/current source, model-source and code-resource checks all agree; the maintained checkout matches the tested source map. Receipt SHA256: `63ef0ad0602a66d40f0736490d4c1e29f57723ba938c58d317802b387f947a87`. This remains an invented-input component integration, not a native population certification. + +The consolidated source passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. A subsequent diagnostic-option reconstruction correction passes all 16 current seven-node age-development tests, including late-mutation refusals. All source, model-source and resource pins survived external postchecks for each run. See the [review guide](us-launch-review.md) for exact revisions, receipts and exclusions. + +The new canonical59-to-PUF55 donor adapter also passes the maintained invented-source construction and artifact wrapper on 64 and 2,048 returns. It verifies the supplied artifact identity and the exact Social Security carrier convention before projecting the grown total into predictor nine. All 55 retained outputs, source RECIDs and weights survive unchanged. Internally rehashed artifacts with altered carrier semantics refuse. The corrected wrapper passed in 9.4 seconds at 486.9 MB peak RSS; receipt SHA256: `db3e9d8aa91863973de5d3201efcbabe31cbc1067c8c3f7fd72be79d887ffc27`. The initial knownness-column mismatch is preserved as a failed control; the fix supplies the owner's exact nonstructural column roster. This adapter grants no source authority. A subsequent separately guarded native projection passes for all 207,692 returns, retaining every target, source RECID, weight and incidence capacity and appending the modeled Social Security total as predictor nine. It took 11.05 seconds at 1.43 GB peak RSS. The original canonical artifact remained unchanged; source/control/model/code checks agree, with no unexpected access refusals. Receipt SHA256: `ce09455947f0efbca16014a23baa19037ac6450876c639568af6584c81bfc45c`. The earlier attempt passed its value assertions but lacked collection accounting and was refused by the guard; that failure is preserved. This native projection ran in memory and produced metadata only. It did not fit a recipient model or attach results to the survey population. + +The genuine seven-node survey age-development run now passes at the unchanged 1/1000 sampling fraction. It prepares 1,584 households and 3,464 people (ACS: 1,529 households/3,324 people; ASEC: 55/140), then retains 3,168 household records and 6,928 person records after the support clone. Export/readback, final source-owner and target checks completed under the frozen `a9895d8a5` source; the final manifest was written only after those checks. All 473 source/control/runtime and 12 code-resource postchecks agree, with no unexpected access refusals. The run took 4,268.8 seconds and peaked at 8.26 GB RSS. Receipt SHA256: `2aa76b5daca7bb0df1ac85d4db23d6b5482073d31d437418d46442ef01b002ca`. This run does not include the subsequent Social Security, PUF55 or atomic-geography integration. + +This is accepted native age-development evidence, not a national or congressional-district release, native enrichment fit-quality result or country-engine result. Complete enrichment, geography, calibration and release-export acceptance remain pending. + +The PUF55 profile now also passes three invented whole-Population attachment +controls. They execute all 55 fitted outcomes through the actual graph, typed +artifact loader and materialized population verifier, followed by Frame storage +and required replay. Every non-owned field, Social Security bit pattern and +unknown cell, membership, design anchor and household weight remains intact. +Two negative cases reject changed Social Security values and unknown-to-zero +conversions even when predictor matrices remain identical. The run took 21.03 +seconds at 631.3 MB peak RSS; receipt SHA256: +`ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30`. +All 482 source/control, 5,983 model-source and 12 code-resource postchecks agree, +with no skips or unexpected access refusals. A subsequent independent review +required a direct finalizer comparison, because the attachment and its verifier +share a column-construction helper. The strengthened three-case run also passes: +all 55 attached outputs agree with the direct finalizer after independent entity +ID alignment, on cold and required replay. It took 20.73 seconds at 630.5 MB; +receipt SHA256 is +`e3a4d548f0209cfacd7d9e3beddb63c1b46204a3168eb9869d2b2eafe9cf2291`. +All 486 source/control, 5,983 model-source and 12 code-resource postchecks agree. +Native recipient qualification, measurement comparability and actual survey +attachment remain pending. + +The observed-geography source qualifier passes 11 invented original-source +controls. It binds current ASEC state and ACS state/PUMA to literal source +household keys, preserves unknown locations, and keeps draw keys stable across +sampling fractions. The initial ten-case run correctly refused a smaller sample +with no positive ASEC support; its corrected fixture supplies positive support +before issuance. Review also added a receipt-digest check against mutation of +the detached ASEC projection. The passing run took 182.50 seconds at 430.3 MB; +receipt SHA256 is +`1900801b1bbcea89a3747f656410e500b1328519694c3351100562bba61cab08`. +The 484 source/control, 5,983 model-source and 12 code-resource postchecks agree, +with no skips or unexpected refusals. + +The observed-geography graph node separately passes 14 controls over the same +kind of invented original sources. Actual three-node execution and required +replay preserve inputs, weights and unknown geography. Declaration, source +receipt, typed-artifact and final-mutation cases refuse invalid output. The run +took 319.44 seconds at 439.1 MB; receipt SHA256 is +`28a6b5c58a258c7b3129a48054a3360647c788f8b76d781cc3bc8fc35a73f997`. +All 486 source/control, 5,983 model-source and 12 code-resource postchecks agree. +Full geography-before-clone composition, native block-source integration and +complete calibration remain separate work. diff --git a/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json b/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json new file mode 100644 index 000000000..79974b85d --- /dev/null +++ b/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json @@ -0,0 +1,447 @@ +{ + "schema": "microcosm.us.puf2015_target2024_growth/1", + "input_money_year": 2015, + "output_money_year": 2024, + "output_profile": "puf59_scf_mortgage_v1", + "ordered_targets": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "tax_exempt_interest_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", + "taxable_private_pension_income", + "taxable_ira_distributions", + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", + "alimony_income", + "alimony_expense", + "salt_refund_income", + "charitable_cash_donations", + "charitable_non_cash_donations", + "real_estate_taxes", + "home_mortgage_interest", + "investment_interest_expense", + "investment_income_elected_form_4952", + "student_loan_interest", + "educator_expense", + "qualified_tuition_expenses", + "casualty_loss", + "unreimbursed_business_employee_expenses", + "traditional_ira_contributions_desired", + "self_employed_pension_contributions_desired", + "rental_income", + "estate_income", + "farm_income", + "farm_operations_income", + "farm_rent_income", + "miscellaneous_income", + "partnership_income", + "s_corp_income", + "partnership_self_employment_net_earnings", + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified", + "business_is_sstb", + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_self_employment_income_before_lsr", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "health_savings_account_ald" + ], + "money_fields": { + "employment_income_before_lsr": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "w2_wages_from_qualified_business": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "sstb_w2_wages_from_qualified_business": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "self_employment_income_before_lsr": { + "positive_family": "business_profit", + "negative_family": "business_loss", + "positive_factor": "1.222517420922031096142334753675412822692", + "negative_factor": "1.718898116890934040328182671660914217539", + "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition." + }, + "sstb_self_employment_income_before_lsr": { + "positive_family": "business_profit", + "negative_family": "business_loss", + "positive_factor": "1.222517420922031096142334753675412822692", + "negative_factor": "1.718898116890934040328182671660914217539", + "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition." + }, + "taxable_interest_income": { + "positive_family": "taxable_interest", + "negative_family": "taxable_interest", + "positive_factor": "2.599750625636417399355437523686408804619", + "negative_factor": "2.599750625636417399355437523686408804619", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "tax_exempt_interest_income": { + "positive_family": "tax_exempt_interest", + "negative_family": "tax_exempt_interest", + "positive_factor": "0.9295395824924337172788943063597248917385", + "negative_factor": "0.9295395824924337172788943063597248917385", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "qualified_dividend_income": { + "positive_family": "qualified_dividends", + "negative_family": "qualified_dividends", + "positive_factor": "1.409891045424454325551259680558160347719", + "negative_factor": "1.409891045424454325551259680558160347719", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "non_qualified_dividend_income": { + "positive_family": "nonqualified_dividends", + "negative_family": "nonqualified_dividends", + "positive_factor": "2.483795459689869824933006427197633652403", + "negative_factor": "2.483795459689869824933006427197633652403", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "non_sch_d_capital_gains": { + "positive_family": "non_schedule_d_distributions", + "negative_family": "non_schedule_d_distributions", + "positive_factor": "1.120350061633613230084419890940102481805", + "negative_factor": "1.120350061633613230084419890940102481805", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "taxable_private_pension_income": { + "positive_family": "taxable_pensions", + "negative_family": "taxable_pensions", + "positive_factor": "1.327588034973940867230285994253866499704", + "negative_factor": "1.327588034973940867230285994253866499704", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "taxable_ira_distributions": { + "positive_family": "ira_distributions", + "negative_family": "ira_distributions", + "positive_factor": "1.510871073557979130794209158828995676924", + "negative_factor": "1.510871073557979130794209158828995676924", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "alimony_income": { + "positive_family": "alimony_income", + "negative_family": "alimony_income", + "positive_factor": "1.542033706590010885068081742929926580179", + "negative_factor": "1.542033706590010885068081742929926580179", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "salt_refund_income": { + "positive_family": "state_tax_refund", + "negative_family": "state_tax_refund", + "positive_factor": "0.9296526775720357466732616002244135476521", + "negative_factor": "0.9296526775720357466732616002244135476521", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "short_term_capital_gains": { + "positive_family": "short_current_gain", + "negative_family": "short_current_loss", + "positive_factor": "2.042865587775988756504710527637911842775", + "negative_factor": "1.738139193325432816803529458382722438642", + "assumption": "Current-year gross gains/loss components per all return proxy for signed net PUF amount; carried losses are excluded." + }, + "long_term_capital_gains_before_response": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "long_term_capital_gains_on_collectibles": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "unrecaptured_section_1250_gain": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "social_security_retirement": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_disability": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_dependents": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_survivors": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "rental_income": { + "positive_family": "rental_royalty_profit", + "negative_family": "rental_royalty_loss", + "positive_factor": "1.476663829685885394438774259073237903799", + "negative_factor": "1.690793641559507564235995133214906035185", + "assumption": "Related rental/royalty reporting-return mean proxy; published total family includes farm rent and does not exactly equal E25850/E25860 component definitions." + }, + "estate_income": { + "positive_family": "estate_profit", + "negative_family": "estate_loss", + "positive_factor": "1.476331638358104936021886589313386353228", + "negative_factor": "1.582810027948011789708036602150526302352", + "assumption": "" + }, + "farm_operations_income": { + "positive_family": "farm_profit", + "negative_family": "farm_loss", + "positive_factor": "1.444720208215047581256304812994467826979", + "negative_factor": "1.728383897337776476443532509645504817056", + "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income." + }, + "farm_income": { + "positive_family": "farm_profit", + "negative_family": "farm_loss", + "positive_factor": "1.444720208215047581256304812994467826979", + "negative_factor": "1.728383897337776476443532509645504817056", + "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income." + }, + "farm_rent_income": { + "positive_family": "farm_rent_profit", + "negative_family": "farm_rent_loss", + "positive_factor": "1.629452810459537374541656575493532799084", + "negative_factor": "1.150378978298943195689490419607276484593", + "assumption": "" + }, + "miscellaneous_income": { + "positive_family": "other_property_gain", + "negative_family": "other_property_loss", + "positive_factor": "1.738242164922618767195190248113271033066", + "negative_factor": "1.411645744283618865226902643798237455207", + "assumption": "Canonical source E01200 is other property gains/losses, not Table1.4 other income." + }, + "partnership_income": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "s_corp_income": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "partnership_self_employment_net_earnings": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "qualified_bdc_income": { + "positive_family": "nonqualified_dividends", + "negative_family": "nonqualified_dividends", + "positive_factor": "2.483795459689869824933006427197633652403", + "negative_factor": "2.483795459689869824933006427197633652403", + "assumption": "Modeled BDC component follows its parent nonqualified dividend pool; no independent source observation." + }, + "qualified_reit_and_ptp_income": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real proxy for the combined modeled REIT/PTP output, which mixes nonqualified-dividend and pass-through components. No claim that this is a pure dividend pool." + }, + "alimony_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "charitable_cash_donations": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "charitable_non_cash_donations": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "real_estate_taxes": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "home_mortgage_interest": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "investment_interest_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "investment_income_elected_form_4952": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "student_loan_interest": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "educator_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "qualified_tuition_expenses": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "casualty_loss": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "unreimbursed_business_employee_expenses": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "traditional_ira_contributions_desired": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "self_employed_pension_contributions_desired": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "domestic_production_ald": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "health_savings_account_ald": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "unadjusted_basis_qualified_property": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "sstb_unadjusted_basis_qualified_property": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + } + }, + "return_incidence_fields": [ + "business_is_sstb", + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified" + ], + "source_pins": { + "NATIONAL-GROWTH-EXTRACT.json": "775d89d3fe7a3b1eb9a09c50f6077d41bf838cdb80d9e4aaa9f0ab80ada4e0c8", + "INDEX-VALUES.json": "221f3786a462fe105d3da15df3624de5ad3e3f921b4d0205e096067685fd59b3", + "extract_national.py": "b6c3a8e4fea52653c01d801d6efe9620cdba4a489d6d7ae8c95cf00ee6dfbe1b", + "ACQUISITION.json": "e9a2c34c2ca06a229b2a790ed2987f25d31996653490f342b4b54af37d5f098c" + }, + "sensitivity_cpi_factor": "1.323487344789614247079323424058189918867", + "provenance": "All growth applications are modeled distribution transport from observed national series; source observed/derived/modeled flags remain separate.", + "operation_order": "Decode source statistical2015 \u2192 source canonical transformations \u2192 QBI2015 model (all16 outputs) \u2192 this growth transform once \u2192 2024 canonical donor. Monetary recipient predictors remain 2024.", + "noninterference": "RECID, S006, raw FLPDYR/FLPDMO, raw money, count source fields and actual survey membership are separate inputs and are not accepted or mutated by this transform.", + "sensitivities": "cpi_only uses exactly the same source cohort, zeros, signs and return incidence; no new policy evaluation or calibration authority." +} diff --git a/docs/evidence/puf2015-target2024/INDEX-VALUES.json b/docs/evidence/puf2015-target2024/INDEX-VALUES.json new file mode 100644 index 000000000..5dd19e557 --- /dev/null +++ b/docs/evidence/puf2015-target2024/INDEX-VALUES.json @@ -0,0 +1,8 @@ +{ + "schema": "microcosm.public_index_value_transcript/1", + "retrieved_date": "2026-09-09", + "retrieval": "Official URLs successfully read with web retrieval; exact values transcribed and independently checkable. These are not hashes of original publisher HTML/PDF bytes. Direct SSA urllib acquisition returned HTTP403 before writing any document.", + "awi": {"url": "https://www.ssa.gov/oact/cola/AWI.html", "series": "National Average Wage Index", "2015": "48098.63", "2024": "69846.57", "measurement": "Wages subject to federal income tax and deferred compensation; average-worker index, used as an explicit proxy for PUF wage amount growth."}, + "cola": {"url": "https://www.ssa.gov/oact/cola/colaseries.html", "cash_timing_url": "https://www.ssa.gov/cola/", "december_payable_percent": {"2015": "0.0", "2016": "0.3", "2017": "2.0", "2018": "2.8", "2019": "1.6", "2020": "1.3", "2021": "5.9", "2022": "8.7", "2023": "3.2"}, "timing": "December benefits are received in January of the following year. Product covers January2016 through January2024 cash increases relative to calendar2015 cash benefits. This transports a fixed entitlement; it does not forecast the recipient mix or new awards."}, + "cpi": {"url": "https://www.bls.gov/cpi/tables/supplemental-files/historical-cpi-u-202412.pdf", "series": "CPI-U, US city average, all items, annual average, 1982-84=100", "table": "Historical CPI-U index averages, annual average column", "2015": "237.017", "2023": "304.702", "2024": "313.689", "pages_one_based": [4,5], "monthly_2024_check": ["308.417","310.326","312.332","313.548","314.069","314.175","314.540","314.796","315.301","315.664","315.493","315.605"]} +} diff --git a/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json b/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json new file mode 100644 index 000000000..e24f8d37d --- /dev/null +++ b/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json @@ -0,0 +1,4439 @@ +{ + "schema": "microcosm.us.puf_observed_growth_sources/1", + "scope": "exact headers and national row only; no AGI rows used", + "tables": { + "15in14ar.xls": { + "url": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "sha256": "9f41bee263f5a8bb971df39a9673b55136c94954131a95bfd83b73a22801ec89", + "sheet": "TBL14", + "year": 2015, + "title": "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, \nby Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 9, + "all_returns": { + "cell": "B9", + "value": 150493263, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns" + ] + }, + "families": { + "taxable_interest": { + "count": { + "cell": "H9", + "value": 42636696, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I9", + "value": 95881223, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "tax_exempt_interest": { + "count": { + "cell": "J9", + "value": 5827038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K9", + "value": 61871455, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Amount", + "Amount" + ] + } + }, + "ordinary_dividends": { + "count": { + "cell": "L9", + "value": 27607044, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M9", + "value": 260252720, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "qualified_dividends": { + "count": { + "cell": "N9", + "value": 25755976, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "O9", + "value": 203187788, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Amount", + "Amount" + ] + } + }, + "state_tax_refund": { + "count": { + "cell": "P9", + "value": 20256512, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Q9", + "value": 31110732, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "alimony_income": { + "count": { + "cell": "R9", + "value": 414420, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "S9", + "value": 10077086, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "business_profit": { + "count": { + "cell": "T9", + "value": 18791200, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 391975736, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "business_loss": { + "count": { + "cell": "V9", + "value": 5935725, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 60161435, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "non_schedule_d_distributions": { + "count": { + "cell": "X9", + "value": 4323250, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 11563203, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "other_property_gain": { + "count": { + "cell": "AD9", + "value": 1063576, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 33036562, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "other_property_loss": { + "count": { + "cell": "AF9", + "value": 1153117, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 21093508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "ira_distributions": { + "count": { + "cell": "AH9", + "value": 14159018, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 253213041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "taxable_pensions": { + "count": { + "cell": "AL9", + "value": 28199160, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AM9", + "value": 689991999, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_profit": { + "count": { + "cell": "AZ9", + "value": 6768234, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA9", + "value": 103058883, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_loss": { + "count": { + "cell": "BB9", + "value": 4531666, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC9", + "value": 46245560, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_rent_profit": { + "count": { + "cell": "AV9", + "value": 394273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AW9", + "value": 5205330, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_rent_loss": { + "count": { + "cell": "AX9", + "value": 101041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 710953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "estate_profit": { + "count": { + "cell": "BH9", + "value": 630256, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 32453002, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "estate_loss": { + "count": { + "cell": "BJ9", + "value": 57802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 5033199, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_profit": { + "count": { + "cell": "BL9", + "value": 520802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 13533867, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_loss": { + "count": { + "cell": "BN9", + "value": 1278825, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 27497651, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "other_income_profit": { + "count": { + "cell": "BX9", + "value": 6121770, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 46769290, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "other_income_loss": { + "count": { + "cell": "BZ9", + "value": 332708, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 6693961, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "partnership_s_corp_profit": { + "count": { + "cell": "BD9", + "value": 6044409, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE9", + "value": 755622761, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "partnership_s_corp_loss": { + "count": { + "cell": "BF9", + "value": 2699817, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG9", + "value": 126618186, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + } + }, + "23in14ar.xls": { + "url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "sha256": "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af", + "sheet": "TBL14", + "year": 2023, + "title": "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, \nby Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 9, + "all_returns": { + "cell": "B9", + "value": 160602107, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns" + ] + }, + "families": { + "taxable_interest": { + "count": { + "cell": "T9", + "value": 55260238, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 313812674, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "tax_exempt_interest": { + "count": { + "cell": "V9", + "value": 6893860, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 66091992, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Amount", + "Amount" + ] + } + }, + "ordinary_dividends": { + "count": { + "cell": "X9", + "value": 33718460, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 504224981, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "qualified_dividends": { + "count": { + "cell": "Z9", + "value": 31106344, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AA9", + "value": 336070296, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Amount", + "Amount" + ] + } + }, + "state_tax_refund": { + "count": { + "cell": "AB9", + "value": 3109096, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AC9", + "value": 4311977, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "alimony_income": { + "count": { + "cell": "AD9", + "value": 183582, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 6686429, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "business_profit": { + "count": { + "cell": "AF9", + "value": 21411713, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 530380019, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "business_loss": { + "count": { + "cell": "AH9", + "value": 9067174, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 153441387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "non_schedule_d_distributions": { + "count": { + "cell": "AJ9", + "value": 3209131, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AK9", + "value": 9340820, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "other_property_gain": { + "count": { + "cell": "AP9", + "value": 945122, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AQ9", + "value": 49567896, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "other_property_loss": { + "count": { + "cell": "AR9", + "value": 775387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AS9", + "value": 19448929, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "ira_distributions": { + "count": { + "cell": "AT9", + "value": 16694154, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AU9", + "value": 438147938, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "taxable_pensions": { + "count": { + "cell": "AX9", + "value": 29541284, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 932130236, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_profit": { + "count": { + "cell": "BL9", + "value": 6346357, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 138609251, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_loss": { + "count": { + "cell": "BN9", + "value": 3196282, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 53570273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_rent_profit": { + "count": { + "cell": "BH9", + "value": 325953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 6811206, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_rent_loss": { + "count": { + "cell": "BJ9", + "value": 67356, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 529586, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "estate_profit": { + "count": { + "cell": "BX9", + "value": 648583, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 47892046, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "estate_loss": { + "count": { + "cell": "BZ9", + "value": 36592, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 4898828, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_profit": { + "count": { + "cell": "CB9", + "value": 484162, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CC9", + "value": 17656301, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_loss": { + "count": { + "cell": "CD9", + "value": 1258869, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CE9", + "value": 45444491, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "other_income_profit": { + "count": { + "cell": "CN9", + "value": 7468434, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CO9", + "value": 60816508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "other_income_loss": { + "count": { + "cell": "CP9", + "value": 318839, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CQ9", + "value": 12213628, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "partnership_profit": { + "count": { + "cell": "BP9", + "value": 3266345, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ9", + "value": 452077038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "partnership_loss": { + "count": { + "cell": "BR9", + "value": 2103634, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BS9", + "value": 168884100, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "s_corp_profit": { + "count": { + "cell": "BT9", + "value": 4138040, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BU9", + "value": 797450861, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "s_corp_loss": { + "count": { + "cell": "BV9", + "value": 1415974, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BW9", + "value": 89543715, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + } + }, + "15in14acg.xls": { + "url": "https://www.irs.gov/pub/irs-soi/15in14acg.xls", + "sha256": "7e128eb06e14371a15baa72060e91d74e4c55ba8596610ddd047c9ef168cbbd1", + "sheet": "Sheet1", + "year": 2015, + "title": "Table 1.4A Returns with Income or Loss from Sales of Capital Assets\nReported on Form 1040, Schedule D: Selected Items, by Size of\nAdjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 10, + "families": { + "short_term_gain": { + "count": { + "cell": "F10", + "value": 3900677, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "G10", + "value": 38164189, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Amount", + "Amount" + ] + } + }, + "short_term_loss": { + "count": { + "cell": "H10", + "value": 7820586, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I10", + "value": 250592206, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Amount", + "Amount" + ] + } + }, + "long_term_gain": { + "count": { + "cell": "BJ10", + "value": 11735160, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK10", + "value": 733313255, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Amount", + "Amount" + ] + } + }, + "long_term_loss": { + "count": { + "cell": "BL10", + "value": 6375392, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM10", + "value": 312723716, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Amount", + "Amount" + ] + } + }, + "short_assets_gain": { + "count": { + "cell": "J10", + "value": 3923409, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K10", + "value": 22196394, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_assets_loss": { + "count": { + "cell": "L10", + "value": 6269981, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M10", + "value": 68624843, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_other_gain": { + "count": { + "cell": "AZ10", + "value": 231076, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA10", + "value": 5948464, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "short_other_loss": { + "count": { + "cell": "BB10", + "value": 336867, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC10", + "value": 1980907, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_gain": { + "count": { + "cell": "BD10", + "value": 483349, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE10", + "value": 17194108, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_loss": { + "count": { + "cell": "BF10", + "value": 725964, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG10", + "value": 16056081, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_carryover": { + "count": { + "cell": "BH10", + "value": 2000156, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI10", + "value": 171105171, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Amount", + "Amount" + ] + } + }, + "long_assets_gain": { + "count": { + "cell": "BN10", + "value": 8440065, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO10", + "value": 282493545, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_assets_loss": { + "count": { + "cell": "BP10", + "value": 4441176, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ10", + "value": 62343837, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_other_gain": { + "count": { + "cell": "DD10", + "value": 2177243, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DE10", + "value": 226671889, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "long_other_loss": { + "count": { + "cell": "DF10", + "value": 296456, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DG10", + "value": 2562723, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_gain": { + "count": { + "cell": "DH10", + "value": 1721361, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DI10", + "value": 216227585, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_loss": { + "count": { + "cell": "DJ10", + "value": 552552, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DK10", + "value": 10694019, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_distributions": { + "count": { + "cell": "DL10", + "value": 9733033, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DM10", + "value": 62496866, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Amount", + "Amount" + ] + } + }, + "long_carryover": { + "count": { + "cell": "DN10", + "value": 4371225, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DO10", + "value": 291699768, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Amount", + "Amount" + ] + } + } + } + }, + "23in14acg.xls": { + "url": "https://www.irs.gov/pub/irs-soi/23in14acg.xls", + "sha256": "5b35c2f007f16503c0dcf6cc085d3a6b049ca523d99b7d7ae29afdbdd2d02e70", + "sheet": "Sheet1", + "year": 2023, + "title": "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets\nReported on Form 1040, Schedule D: Selected Items, by Size of\nAdjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 10, + "families": { + "short_term_gain": { + "count": { + "cell": "F10", + "value": 6215519, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "G10", + "value": 70482448, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Amount", + "Amount" + ] + } + }, + "short_term_loss": { + "count": { + "cell": "H10", + "value": 11370982, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I10", + "value": 673910232, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Amount", + "Amount" + ] + } + }, + "long_term_gain": { + "count": { + "cell": "BJ10", + "value": 11888516, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK10", + "value": 971279947, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Amount", + "Amount" + ] + } + }, + "long_term_loss": { + "count": { + "cell": "BL10", + "value": 11230240, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM10", + "value": 450065138, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Amount", + "Amount" + ] + } + }, + "short_assets_gain": { + "count": { + "cell": "J10", + "value": 7140149, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K10", + "value": 58927145, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_assets_loss": { + "count": { + "cell": "L10", + "value": 8377604, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M10", + "value": 135549749, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_other_gain": { + "count": { + "cell": "AZ10", + "value": 174491, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA10", + "value": 8619769, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "short_other_loss": { + "count": { + "cell": "BB10", + "value": 401474, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC10", + "value": 3833687, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_gain": { + "count": { + "cell": "BD10", + "value": 495780, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE10", + "value": 28464220, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_loss": { + "count": { + "cell": "BF10", + "value": 560758, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG10", + "value": 16759596, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_carryover": { + "count": { + "cell": "BH10", + "value": 5151589, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI10", + "value": 543302365, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Amount", + "Amount" + ] + } + }, + "long_assets_gain": { + "count": { + "cell": "BN10", + "value": 8891406, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO10", + "value": 405281941, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_assets_loss": { + "count": { + "cell": "BP10", + "value": 9496262, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ10", + "value": 149558927, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_other_gain": { + "count": { + "cell": "DD10", + "value": 1979380, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DE10", + "value": 316779349, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "long_other_loss": { + "count": { + "cell": "DF10", + "value": 338620, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DG10", + "value": 5211519, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_gain": { + "count": { + "cell": "DH10", + "value": 1219800, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DI10", + "value": 282281744, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_loss": { + "count": { + "cell": "DJ10", + "value": 766861, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DK10", + "value": 22526709, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_distributions": { + "count": { + "cell": "DL10", + "value": 9487999, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DM10", + "value": 44095015, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Amount", + "Amount" + ] + } + }, + "long_carryover": { + "count": { + "cell": "DN10", + "value": 5622923, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DO10", + "value": 349944357, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Amount", + "Amount" + ] + } + } + } + } + }, + "factors": { + "taxable_interest": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "H9", + "value": 42636696, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I9", + "value": 95881223, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "T9", + "value": 55260238, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 313812674, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "2.525269343625908636956987730976528139607", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.599750625636417399355437523686408804619" + }, + "tax_exempt_interest": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "J9", + "value": 5827038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K9", + "value": 61871455, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "V9", + "value": 6893860, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 66091992, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "0.9029088360274333448808012169263853497016", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "0.9295395824924337172788943063597248917385" + }, + "ordinary_dividends": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "L9", + "value": 27607044, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M9", + "value": 260252720, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "X9", + "value": 33718460, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 504224981, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.586285145427430620786783837166923431342", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.633071660126895406016322292262784767590" + }, + "qualified_dividends": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "N9", + "value": 25755976, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "O9", + "value": 203187788, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "Z9", + "value": 31106344, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AA9", + "value": 336070296, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.369498520263452278862567470282453558367", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.409891045424454325551259680558160347719" + }, + "state_tax_refund": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "P9", + "value": 20256512, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Q9", + "value": 31110732, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AB9", + "value": 3109096, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AC9", + "value": 4311977, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "0.9030186910014518713848306957259555062394", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "0.9296526775720357466732616002244135476521" + }, + "alimony_income": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "R9", + "value": 414420, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "S9", + "value": 10077086, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AD9", + "value": 183582, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 6686429, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.497855374161636196047724476262267688168", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.542033706590010885068081742929926580179" + }, + "business_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "T9", + "value": 18791200, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 391975736, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AF9", + "value": 21411713, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 530380019, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.187493036701270108472919624578501757792", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.222517420922031096142334753675412822692" + }, + "business_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "V9", + "value": 5935725, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 60161435, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AH9", + "value": 9067174, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 153441387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.669652726148833347538733957583542565766", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.718898116890934040328182671660914217539" + }, + "non_schedule_d_distributions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "X9", + "value": 4323250, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 11563203, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AJ9", + "value": 3209131, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AK9", + "value": 9340820, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.088252710423015210712466518141315463440", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.120350061633613230084419890940102481805" + }, + "other_property_gain": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AD9", + "value": 1063576, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 33036562, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AP9", + "value": 945122, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AQ9", + "value": 49567896, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.688442578911762234575993608257254511052", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.738242164922618767195190248113271033066" + }, + "other_property_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AF9", + "value": 1153117, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 21093508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AR9", + "value": 775387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AS9", + "value": 19448929, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.371202948062275806522918206792748706765", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.411645744283618865226902643798237455207" + }, + "ira_distributions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AH9", + "value": 14159018, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 253213041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AT9", + "value": 16694154, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AU9", + "value": 438147938, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.467585531705808482641269279807429143993", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.510871073557979130794209158828995676924" + }, + "taxable_pensions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AL9", + "value": 28199160, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AM9", + "value": 689991999, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AX9", + "value": 29541284, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 932130236, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.289553441251142788324750319651443404751", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.327588034973940867230285994253866499704" + }, + "rental_royalty_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AZ9", + "value": 6768234, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA9", + "value": 103058883, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BL9", + "value": 6346357, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 138609251, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.434358304667835504133977902598222238470", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.476663829685885394438774259073237903799" + }, + "rental_royalty_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BB9", + "value": 4531666, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC9", + "value": 46245560, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BN9", + "value": 3196282, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 53570273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.642353427026338423846026443645930519505", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.690793641559507564235995133214906035185" + }, + "farm_rent_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AV9", + "value": 394273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AW9", + "value": 5205330, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BH9", + "value": 325953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 6811206, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.582769973612852083106490319603270854083", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.629452810459537374541656575493532799084" + }, + "farm_rent_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AX9", + "value": 101041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 710953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BJ9", + "value": 67356, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 529586, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.117421316799902418041369349372073484912", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.150378978298943195689490419607276484593" + }, + "estate_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BH9", + "value": 630256, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 32453002, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BX9", + "value": 648583, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 47892046, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.434035630420548027555129084975779987827", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.476331638358104936021886589313386353228" + }, + "estate_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BJ9", + "value": 57802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 5033199, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BZ9", + "value": 36592, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 4898828, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.537463478591264240530009559622650667952", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.582810027948011789708036602150526302352" + }, + "farm_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BL9", + "value": 520802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 13533867, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CB9", + "value": 484162, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CC9", + "value": 17656301, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.403329848619305835091312061082920777637", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.444720208215047581256304812994467826979" + }, + "farm_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BN9", + "value": 1278825, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 27497651, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CD9", + "value": 1258869, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CE9", + "value": 45444491, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.678866744726831887395787683833365558775", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.728383897337776476443532509645504817056" + }, + "other_income_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BX9", + "value": 6121770, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 46769290, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CN9", + "value": 7468434, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CO9", + "value": 60816508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.065879634464584874641519579136196913601", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.097317105419594110775195552571543585636" + }, + "other_income_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BZ9", + "value": 332708, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 6693961, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CP9", + "value": 318839, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CQ9", + "value": 12213628, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.903940252832176698123261573865488200400", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.960095811549227363973941096035769795063" + }, + "short_current_gain": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "short_assets_gain", + "short_other_gain", + "short_pass_through_gain" + ], + "amounts_thousands_usd": [ + 45338966, + 96011134 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.984338725063739328074935070060878814103", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.042865587775988756504710527637911842775" + }, + "short_current_loss": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "short_assets_loss", + "short_other_loss", + "short_pass_through_loss" + ], + "amounts_thousands_usd": [ + 86661831, + 156143032 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.688342557388515472795249540239320768338", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.738139193325432816803529458382722438642" + }, + "long_current_gain": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "long_assets_gain", + "long_other_gain", + "long_pass_through_gain", + "long_distributions" + ], + "amounts_thousands_usd": [ + 787889885, + 1048438049 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.246932856321931105568396313543555115004", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.283710381837894882785950440755768785471" + }, + "long_current_loss": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "long_assets_loss", + "long_other_loss", + "long_pass_through_loss" + ], + "amounts_thousands_usd": [ + 75600579, + 177297155 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "2.197568722014698372473641296575486125444", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.262384673681395979556694950087192296750" + }, + "partnership_s_corp_profit": { + "method": "combined_dollars_per_all_return", + "source": { + "count": { + "cell": "BD9", + "value": 6044409, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE9", + "value": 755622761, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023_components": [ + { + "count": { + "cell": "BP9", + "value": 3266345, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ9", + "value": 452077038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + { + "count": { + "cell": "BT9", + "value": 4138040, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BU9", + "value": 797450861, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + } + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "observed_2015_2023_factor": "1.549554109090802831376907083449129785317", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.595257264233857504616939193376066035754" + }, + "partnership_s_corp_loss": { + "method": "combined_dollars_per_all_return", + "source": { + "count": { + "cell": "BF9", + "value": 2699817, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG9", + "value": 126618186, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023_components": [ + { + "count": { + "cell": "BR9", + "value": 2103634, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BS9", + "value": 168884100, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + { + "count": { + "cell": "BV9", + "value": 1415974, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BW9", + "value": 89543715, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "observed_2015_2023_factor": "1.912533224223542778513579904641714927631", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.968942227400735507640732147170530268700" + }, + "nonqualified_dividends": { + "method": "residual_dollars_per_ordinary_dividend_return", + "observed_2015_2023_factor": "2.412636223005660751243227924409122950293", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.483795459689869824933006427197633652403", + "source_families": [ + "ordinary_dividends", + "qualified_dividends" + ] + } + }, + "modeled_bridge": { + "method": "constant real conditional amounts after 2023", + "cpi_2023": "304.702", + "cpi_2024": "313.689" + }, + "capital_carryover_reconciliation_thousands_usd": { + "short": { + "2015": { + "net_including_carried_loss": -212428017, + "carried_loss": 171105171, + "current_year_net_by_removing_carryover": -41322846, + "current_year_gross_component_gain": 45338966, + "current_year_gross_component_loss": 86661831, + "current_year_net_from_component_difference": -41322865, + "reported_less_component_net_residual": 19, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + }, + "2023": { + "net_including_carried_loss": -603427784, + "carried_loss": 543302365, + "current_year_net_by_removing_carryover": -60125419, + "current_year_gross_component_gain": 96011134, + "current_year_gross_component_loss": 156143032, + "current_year_net_from_component_difference": -60131898, + "reported_less_component_net_residual": 6479, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + } + }, + "long": { + "2015": { + "net_including_carried_loss": 420589539, + "carried_loss": 291699768, + "current_year_net_by_removing_carryover": 712289307, + "current_year_gross_component_gain": 787889885, + "current_year_gross_component_loss": 75600579, + "current_year_net_from_component_difference": 712289306, + "reported_less_component_net_residual": 1, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + }, + "2023": { + "net_including_carried_loss": 521214809, + "carried_loss": 349944357, + "current_year_net_by_removing_carryover": 871159166, + "current_year_gross_component_gain": 1048438049, + "current_year_gross_component_loss": 177297155, + "current_year_net_from_component_difference": 871140894, + "reported_less_component_net_residual": 18272, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + } + } + }, + "notes": [ + "Reporting-return means mix intensive-margin growth and changing composition. This is a distribution transport assumption, not panel growth.", + "Positive/loss magnitudes are distinct series. Source zeros and sign states are retained.", + "No total calibration or reporting-incidence claim; source design weights are unchanged." + ] +} diff --git a/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json b/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json new file mode 100644 index 000000000..23975da76 --- /dev/null +++ b/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json @@ -0,0 +1,34 @@ +[ + { + "url": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "bytes": 83456, + "sha256": "9f41bee263f5a8bb971df39a9673b55136c94954131a95bfd83b73a22801ec89", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "15in14ar.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "bytes": 115712, + "sha256": "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "23in14ar.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/15in14acg.xls", + "bytes": 64000, + "sha256": "7e128eb06e14371a15baa72060e91d74e4c55ba8596610ddd047c9ef168cbbd1", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "15in14acg.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/23in14acg.xls", + "bytes": 74752, + "sha256": "5b35c2f007f16503c0dcf6cc085d3a6b049ca523d99b7d7ae29afdbdd2d02e70", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "23in14acg.xls" + } +] diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index a759198bc..26f82300b 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "20058e544f6034cee2e76d3b864cf86b6230c8dce042931dfe9247d8ac1329c4", - "protocol_sha256": "553d5e0bd5afa93146dfbf8dca684926ee773dbfc89a559434528af5b4eb1d97" + "map_sha256": "da07a54ab2bc4e297ac7d0693a8559b359de230787757e4b6e26dcb619c8e5a0", + "protocol_sha256": "5f93b3ec98ada30338b06ad978a6e1b47d0f9f916af89418500cea435235ad06" }, "failures": [], "observed": { - "map_sha256": "20058e544f6034cee2e76d3b864cf86b6230c8dce042931dfe9247d8ac1329c4", - "protocol_sha256": "553d5e0bd5afa93146dfbf8dca684926ee773dbfc89a559434528af5b4eb1d97" + "map_sha256": "da07a54ab2bc4e297ac7d0693a8559b359de230787757e4b6e26dcb619c8e5a0", + "protocol_sha256": "5f93b3ec98ada30338b06ad978a6e1b47d0f9f916af89418500cea435235ad06" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "553d5e0bd5afa93146dfbf8dca684926ee773dbfc89a559434528af5b4eb1d97", + "implementation_sha256": "5f93b3ec98ada30338b06ad978a6e1b47d0f9f916af89418500cea435235ad06", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "553d5e0bd5afa93146dfbf8dca684926ee773dbfc89a559434528af5b4eb1d97", + "implementation_sha256": "5f93b3ec98ada30338b06ad978a6e1b47d0f9f916af89418500cea435235ad06", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "1eeca53aa80da949a292fbd8cb0afefde95c68888ed35f3f477f3c962e6bc644" + "spec_sha256": "54aa5d96b062a66207dfe49d4a03817e9658bd0ac11acb37546b1bf2a01f533d" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "1eeca53aa80da949a292fbd8cb0afefde95c68888ed35f3f477f3c962e6bc644" + "spec_sha256": "54aa5d96b062a66207dfe49d4a03817e9658bd0ac11acb37546b1bf2a01f533d" }, "status": "pass" } diff --git a/docs/fiscal-leaf-policy.md b/docs/fiscal-leaf-policy.md new file mode 100644 index 000000000..823603506 --- /dev/null +++ b/docs/fiscal-leaf-policy.md @@ -0,0 +1,114 @@ +# Fiscal input policies + +The declared fiscal measurement stage requires an input producer for every +leaf in its static model closure. Missing and nonfinite values still refuse +evaluation. The optional leaf policy records more precise intent without +weakening that default. + +An explicit policy can describe producer ownership or a proposed literal +scenario assumption. **Assumption-bearing executions currently refuse with +`FISCAL_MEASUREMENT_ASSUMPTION_PARENT_ADMISSION_UNSUPPORTED`.** No US assumption +resource is enabled, and no dataset column or producer requirement changes. + +## Exact policy identity + +`load_fiscal_leaf_policy(payload, expected_sha256=...)` accepts bounded, +canonical JSON bytes and an explicit SHA-256 pin. It performs no file or +network I/O. The same `FiscalLeafPolicy` goes to `fiscal_measurement_node` and +`FiscalMeasurementKernel`. Its pin is rechecked when used; the parsed document +is detached. Omitting the argument, or passing `None`, preserves the previous +all-producer declaration shape and runtime behavior. + +A policy has schema version 1, artifact kind +`microcosm.us.fiscal_leaf_policy`, one positive integer `period`, an exact +ordered list of model `roots`, and an `entries` mapping covering the static +closure exactly. Missing entries and stale entries refuse. Each entry's +entity must agree with the independently derived model metadata. + +| Kind | Required entry fields | Behavior | +| --- | --- | --- | +| `producer` | `kind`, `entity`, `producer` | The declared input column must exist and every cell must be known and finite. | +| `assumption` | `kind`, `entity`, `value`, `interpretation`, `period`, `roots`, `affected_roots`, `affected_programs`, `rationale`, `reviewer`, `source_issue`, `reform_sensitivity` | Validated and inspectable; execution remains unsupported. | +| `inactive` | No supported shape | Always refused. | + +An assumption's interpretation is explicitly `baseline_behavior` or +`scenario_parameter`. Neither category means the program is insensitive to +the choice. The affected roots must include every declared root whose static +closure contains that leaf, in declaration order. Scope must exactly match +the policy's period and roots; program names and review fields cannot be +empty. These fields describe claimed review and intent. They do not prove +reviewer authorization, establish a source observation, or make an observed +attribute an acceptable assumption. + +V1 supports finite Boolean, signed 64-bit integer and floating-point literals +with explicit engine type checks. Strings, enums, null, containers and an +`engine default` reference refuse. The actual engine metadata and whether a +default is available (with its literal value when present) are recorded +beside the proposed value. The host never chooses a value by default lookup. +The exact policy and engine/default record enter `model_contract`, so policy, +scope, ownership, review or engine-default changes alter the declared node and +its cache key. The same declaration is embedded in a successful measurement +artifact. In this version, only producer policies can produce that artifact. + +A column constant at an engine default remains admissible as a producer at +this boundary. The label does not establish observation or modeled variation. +The source owner and existing release input-coverage/variation gates retain +that responsibility. Behavioral take-up and filing propensities are not +automatically classified as innocuous scenario settings. No prior wages are +added by this infrastructure. + +## Complete-parent admission still required + +The graph kernel receives only its declared input slices. It cannot prove a +column is absent from the complete parent when a caller omitted that column +from the slice. Declaration checks reject an assumption name present on any +declared entity, but this is insufficient to enable execution. + +The future country host must retain its independently checked complete +population and use `validate_fiscal_leaf_policy_population(frame, policy)` +before projection, around relevant I/O, and during cache admission. The +validator rejects any incumbent assumption name on any entity, including +wholly unknown or partially unknown columns. Passing a projection or a +caller-authored column roster is not absence proof. The validator itself +does not authenticate or issue a parent. + +`private_fiscal_assumption_frame(frame, policy)` is a detached preparation +helper for that future integration. It repeats the collision check and +creates engine-only tables. It preserves the supplied frame's values, +membership, weights, strata and metadata. The current fiscal kernel does not +call it. A future host must validate the exact policy against the admitted +engine, retain source and complete-parent checks before and after I/O, bind +the engine/default record, evaluate only the private copy, and keep assumption +columns out of the population and measurement adapter. A cached artifact +cannot supply missing source or parent authority. + +## Why inactivity is unsupported + +`validation_input_coverage._provision_input_leaves` traces a single empty +one-person simulation. It observes the leaves read by that execution. A +conditional formula or `defined_for` branch may read additional leaves for +other households. Absence from that trace cannot establish universal +inactivity, even at the same period and roots. + +Future inactivity support needs exhaustive engine-, period- and root-scoped +dependency proof, including conditional paths and declared state roots. The +static closure remains conservative; neither it nor a sampled trace supplies +this exemption today. + +## Bounded implementation evidence + +The local source/mock test run passed 91 cases, including the existing +invented fiscal measurement, calibration and observer graphs. The new controls +check strict-default compatibility, malformed policies, exact scope and +closure coverage, type mismatches, private-copy isolation, omitted-column +collisions, late policy mutation, graph key changes and required replay with +a forbidden model call. There were no country engine imports, network +connections or subprocess operations; no native population or source data +was used. This is infrastructure validation, not actual engine-policy +admission or candidate fiscal validation. + +The source and selected test files were sealed before and after the run. +Receipt SHA-256: +`c7aed643333638460d7464091be0c068b910baba2eefbb76cfa1208af6f8265f`. +JUnit SHA-256: +`54215b02c7a8a85488db48b2fee339b0ddbad907a0985502019c185b9a1c3adb`. diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md new file mode 100644 index 000000000..5793b0b1f --- /dev/null +++ b/docs/geography-assignment.md @@ -0,0 +1,261 @@ +# Geography assignment in the population graph + +Design decision, corrected 10 September 2026: finish constructing the full survey +multispine, including its initial support and PUF clones, then assign one small +census area to each resulting household. Derive larger geographies from that +area using identified, versioned mappings. Distinct clones may receive different +areas subject to their observed source constraints. The assigned location then +remains fixed through subsequent enrichment, calibration and analysis views. + +The earlier block-before-clone implementation and its passing controls remain +historical evidence. They do not demonstrate this corrected ordering. Geographic +assignment must use a stable post-clone identity, including a clone discriminator +where original-source identity alone is shared by multiple households. + +The corrected order now passes 70 controls on invented inputs, plus a separate +default-path compatibility check. The nine-node survey prefix expands the initial +clones before assignment; its nineteen-node financial and twelve-node age +extensions retain the assigned geography. Draw identity combines +`survey_geography_origin_key` with `household_support_clone_index`, so renumbering +households does not redraw their location. A typed validation artifact orders +financial enrichment after the geography gate. See the +[scoped acceptance](../experiments/us-postclone-geography-70-controls-20260910.json). +Native execution of this corrected order and population-quality acceptance +remain pending. The older results below retain their original source scope. + +## Declarative country capability + +Geography is a declarative country capability backed by shared graph operators. +A country declares its atomic area type, code system and vintage, source mapping +artifacts, observed-geography constraints and sampling-weight convention. The UK +declaration can select an area system by nation. Adding a country should not +require writing a country-specific assignment or derivation kernel. + +The shared assignment operator selects the atomic area using the declared +support and stable household identity. Shared lookup/join operations derive +larger geographies from identified mappings. If an input already has a qualified +atomic area, validate and retain it. Source acquisition and normalization may +still require publisher-specific adapters; they produce the common support and +mapping contracts, rather than owning a separate assignment algorithm. + +Sampling and derivation remain executable graph operations with typed inputs, +outputs and replay identity. A country configuration does not itself execute +them. Mapping edges retain their relation type, including exact nesting or an +explicit best-fit convention. Prefer existing graph lookup/join primitives where +they support these contracts; introduce only the shared behavior they lack. + +The existing country-spec geography declaration is a starting point, but its +legacy clone-and-assign contract and country-specific runtime references do not +yet implement this shared atomic-area contract. Preserve those compatibility +paths while the shared operators and country adapters receive their own checks. + +```mermaid +flowchart LR + S[Complete survey multispine including clones] --> A[Assign one small area per resulting household] + C[Observed survey geography constraints] --> A + L[Versioned small-area support] --> A + A --> D[Derive larger geographies] + M[Versioned geographic mappings] --> D + D --> E[Subsequent enrichment retains location] + E --> R[Rules and calibration] + R --> P[Pruned analysis file retains location] +``` + +## Country anchors + +| Country | Assigned household anchor | Derived geographies | +| --- | --- | --- | +| US | Census block, with census vintage | Block group, tract, county, state, PUMA and congressional district using the declared mapping/boundary vintages | +| England and Wales | 2021 Output Area | LSOA, MSOA, local authority, ward, constituency and region | +| Scotland | 2022 Output Area | Scottish statistical areas, council area, ward, constituency and region | +| Northern Ireland | 2021 Data Zone | Super Data Zone, local government district, constituency and nation | + +Northern Ireland's 2021 Data Zones replaced the 2011 Small Areas. Scottish +Output Areas use the 2022 census. These are different source systems; the graph +must retain the original area type and vintage rather than imply identical units. +See [NISRA census output geography](https://www.nisra.gov.uk/statistics/census-2021-results/census-output-geography) +and [Scotland's census general report](https://www.scotlandscensus.gov.uk/about/scotlands-census-2022-general-report/). + +Some larger boundaries do not nest exactly. England and Wales small-area estimates +use best-fit mappings for wards and parliamentary constituencies. The derived +edge must identify whether its mapping is exact, best-fit or another explicit +approximation, with its source and vintage. A mapping convention cannot be +presented as observed household location. See the +[ONS methodology](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationestimates/methodologies/smallareapopulationestimatesqmi). + +For US congressional districts, the Census Bureau's CD119 block equivalency +file assigns whole original 2020 tabulation blocks to districts. Colorado's +block `080010096072000` crosses the enacted CD07/CD08 boundary but is assigned +to CD08 for tabulation. Represent this as an official tabulation mapping, +not exact spatial containment. The same source distinguishes original 2020 +blocks from subsequently adjusted geometry and includes undefined `ZZ` areas. +See the [Census Bureau's CD119 documentation](https://www.census.gov/geographies/mapping-files/2025/dec/rdo/119-congressional-district-bef.html). +By contrast, the same-vintage tract-to-PUMA relation is nested, as described in +the [Census PUMA guidance](https://www.census.gov/programs-surveys/geography/guidance/geo-areas/pumas.html). + +## Required behavior + +- Respect observed source constraints, such as ACS PUMA and FRS region. Record + inferred geographic detail as modeled. Preserve observed source geography + separately from the assigned location. +- Bind each draw to a stable household identity, seed and assignment definition. + Reordering rows or selecting an existing household must not redraw its location. + Changed source support or boundary mappings produce a new identified revision. +- Assign household members consistently with their resulting household. Initial + clones have distinct draw identities and may receive different anchors. + Materialized county, district and other fields must agree with that household's + selected anchor. Source geography remains a separate observed constraint. +- Show assignment and derivation as separate operations in the graph. Expose the + source constraints, support, sampling weights, mapping conventions and judgment + annotations in the inspector. +- Refuse missing, ambiguous or incompatible mappings unless an explicit, + source-supported resolution rule is part of the assignment definition. +- Verify inheritance and mapping consistency again on every pruned export. + Engine input profiles and geographic build invariants are separate contracts. + +## Existing implementation and next changes + +The shared implementation now lives in `microcosm.build.atomic_geography` and +`microcosm.build.graph_atomic_geography`. It provides four country-neutral nodes: +`geography.support_import@1`, `geography.assign_atomic@1`, +`geography.derive@1` and `geography.gate@1`. Countries supply declarations and +normalized, pinned support files. The graph interfaces and legacy country +operators are unchanged. Country configuration and native support integration +remain necessary before these nodes can assign locations in a release build. + +The support contract uses deterministic NPZ bytes with no object arrays. It has +one unique string code per atomic area; every mapping column identifies its +source, vintage and relation, and every integer sampling-weight column identifies +its source and basis. Nonnegative weights have a bounded exact total. A source +adapter must resolve any nonfunctional crosswalk before admission; the shared +lookup never multiplies households through a many-to-many join. + +Assignment conditions on all nonmissing observed constraints jointly. Declared +stages can use different weights—for example, household counts to select a +constituency followed by population counts to select its small area. Each draw +uses the existing `keyed_uniform` protocol with stable source identity, the +declared stream, system, stage, assignment definition and support digest. +Selection uses an integer inverse CDF. Reordering, subsetting or adding unrelated +households preserves draws. The numerical declaration remains `platform_bitwise`; +cross-platform equivalence has not been established. + +The version-1 declaration lists `identity`, `stream`, three assignment `outputs` +(`area`, `system`, `basis`), and `systems`. Each system supplies its atomic +identity and source, a selector, observed constraints, an optional observed-area +column, sampling stages and derived layers. Geographic codes use nullable +strings throughout, retaining leading zeroes. An engine that needs integer +storage must declare that conversion separately. A system without a particular +layer produces missing values for that layer, which the integrity gate checks. +The graph stores the canonical declaration in its normative `definition` +parameter and exposes the stream separately; no graph parameter grammar changed. + +The 34 invented-data controls include real graph execution, cold and warm reuse, +source-change invalidation, exact lookup metadata, independent sampling-boundary +checks, three-system routing, observed-area retention, row/subset stability and +clone/prune validation. All passed under the isolated control runner on +9 September 2026. These checks establish shared operator behavior; they do not +admit a Census/ONS source or certify any population's geographic fit. The gate +checks mapping integrity, while the executor and build own structural lineage +and population-quality acceptance. The country's chosen identity still needs +verification across actual sampling rungs. + +The US development composed graph attaches geography after harmonization, but +`us_runtime/graph_geography.py` selects a joint tract/congressional-district cell +and emits PUMA, county and district. It does not assign a Census block. Its prior +national source and joint-support checks therefore do not establish block-first +acceptance. The separate atomic survey graph now provides that block assignment +and derivation connection before enrichment. Its population-only national block +support and source review are recorded below. + +The UK already has an area-based ladder in `uk_runtime/geography_ladder.py`. +Its sampler selects a constituency within FRS region using household counts, +then an OA within constituency using population; one selected ladder row supplies +the other geographies. It currently consumes a shared seeded random stream, so +stable household-keyed assignment under reordering and subsets still needs work. +The graph should expose the final OA/Data Zone assignment and the mapping +derivations explicitly, including that sampling convention. + +The Northern Ireland source builder currently infers Data Zone constituencies +using the modal active-postcode constituency. Review the official +[NISRA constituency aggregation and lookup resources](https://www.nisra.gov.uk/publications/census-2021-output-geography-information-papers) +before choosing or replacing that approximation. Source existence is not +acceptance of a particular lookup or its application to the population. + +The US block adapter in `us_runtime/atomic_block_support.py` now preserves each +supplied block, leading zeroes, population weight and the source-labeled +tract-to-PUMA and block-to-district mappings. Its declaration uses the shared +operators, preserves observed state/PUMA separately, and labels CD119 as +`official_tabulation`. Its sampling proxy is 2020 persons; this does not imply +household counts or support for subsequent construction in unpopulated 2020 +blocks. All 13 invented tests passed, including missing/inconsistent mapping +refusals and subset-stable assignment. Receipt SHA256: +`404deb7666b3ac9b8a654181228c545ab95c131f30a1fbe65a2e15d5a08a4c4c`. +These initial adapter tests do not establish native geographic fit. Subsequent +national support and native pilot results are recorded below. + +The earlier `us_runtime/graph_atomic_survey_clone.py` declared this sequence: +shared import, assignment, derivation and integrity gate, then the existing +combined-survey support clone and an inherited-mapping gate. The graph compiler +makes the clone depend on every member of its base version, including the +pre-clone gate. The post-clone gate never draws another location. An actual +executor control passes cold execution and required replay: four invented +households become eight, every location column is inherited, all entity rows +are copied, and each household weight is split equally across the pair. +Subset mapping verification also passes. Receipt SHA256: +`1bb75a5ee53edcbdd199a6d0163cbe75c32e54c80513cb3eb8d13021c3c2b1c3`. +A separate source-only Codex review found no actionable defects. + +The current survey path now qualifies observed state and ACS PUMA from the +retained original-source preparation. Eleven source controls and fourteen graph +controls passed, covering missing geography, source changes, replay and stable +household identity across sampling rungs. These controls use invented originals +through the actual source issuers. Their receipts are recorded in +`experiments/us-survey-geography-source-controls-20260909.json` and +`experiments/us-survey-geography-graph-controls-20260909.json`. + +The earlier `us_runtime/graph_atomic_survey_population.py` connected that projection +to block assignment, geographic derivation, the integrity gate and the combined +survey clone. It retains the raw allocation separately from the enriched +pre-clone population, even though both share the allocation version identifier. +The complete ten-node prefix passed five controls: fresh execution, required +replay, full geography inheritance, support-byte mismatch and late mutation +refusals. The test population contains six survey households before cloning and +twelve afterward. The exact tested revisions and remaining checks are recorded +in `experiments/us-atomic-survey-population-controls-20260909.json`. + +The earlier calibration budget and age runner consumed this optional prefix +and independently reconstructed it from the raw allocation. Eight budget +controls passed, as did the complete thirteen-node age graph through fresh +execution and required replay. A separate compatibility control verifies that +the existing predictor and PUF-host qualifiers still accept the default survey +prefix. See the corresponding `us-atomic-budget-semantic-8`, +`us-atomic-age-v2-1`, and `us-budget-predictor-compatibility-1` records in +`experiments/`. + +Replay can canonicalize the hidden backing values of missing numeric cells. +Persistent budget identities therefore bind logical Frame values, population +version, complete ownership, weight kinds, mass ledger and exact design-weight +bytes. They exclude only the helper receipt's physical population stamp. +Same-object mutation checks retain physical identity. The initial physical-stamp +replay failure remains recorded; the corrected budget controls pass. + +The byte-source adapter passed thirty-six invented-source controls through the +maintained PL, CD and PUMA parsers. This does not authorize acquisition of native +PL archives, which also contain housing segments. Native support preparation +will instead request only Census `P1_001N` block populations and independent +state totals, then join CD119 and tract-to-PUMA mappings. Delaware is the first +source control; national survey assignment requires complete admitted support +for the fifty states and DC. No native acquisition is accepted by these tests. + +`us_runtime/atomic_block_api_sources.py` implements that population-only response +adapter. All thirty-five invented controls pass, including independent state +totals, exact block preservation, missing mappings, malformed response fields, +byte limits and late source-record mutation. Request descriptors preserve the +repeated Census `in` parameters and omit credentials. The adapter does not infer +the response's origin from its contents; acquisition and publisher qualification +remain separate. Evidence: +`experiments/us-atomic-block-api-sources-35-controls-20260909.json`. + +Publisher-qualified native block support, national and district calibration, +and pruned export verification remain necessary for a release. Source integrity +checks and successful invented-data runs do not establish geographic fit. diff --git a/docs/graph-signed-reconciliation.md b/docs/graph-signed-reconciliation.md new file mode 100644 index 000000000..8cba5aceb --- /dev/null +++ b/docs/graph-signed-reconciliation.md @@ -0,0 +1,57 @@ +# Signed component reconciliation in the graph + +`microcosm.fit.graph_signed_reconciliation.signed_reconciliation_node` exposes +the existing weighted Euclidean projection as an ordinary graph calculation. +Its parameters name the anchor, input draws, output components, nonnegative +bounds, positive scales and numerical tolerances. The country model must +declare and justify those choices. The operator supplies no default scale or +tax interpretation. + +For the proposed ACS property-income family, the components are ordinary +interest, retirement-account interest, dividends and signed property income. +The first three have nonnegative bounds; property income can offset them. +A zero reported total therefore does not imply that every component is zero. +Retirement-account interest remains a separate survey component and is not +relabeled as taxable or tax-exempt interest. Source qualification, a defensible +ASEC-to-ACS measurement bridge, fitting and tax conversion remain separate +upstream/downstream operations; this numeric node alone does not complete them. + +The graph retains the anchor and raw draw columns. It adds the reconciled +components, each component's adjustment and bound activity, the sum residual +and the projection objective. A typed summary artifact records the complete +rule, row count, adjusted-row count, maximum absolute residual and bound counts. +It describes the numerical result and grants no source or release authority. +These declarations and columns make the rule inspectable in the shared graph +viewer without another viewer implementation. + +All input amounts must be finite float64 values. Unknown anchors or draws +refuse; the country host must declare how it selects or models those cases. +Inputs cannot share names with the new outputs, and no original observation +is overwritten. The kernel never reads survey origin, weights or an RNG and +returns no membership or weight changes. Its numerical contract is bitwise on +one platform; the explicit projection tolerances test constraints within each +run and are not a claim of cross-platform equality. + +## Verification on 12 September 2026 + +Twenty-four guarded tests pass, including direct comparison to the independently +tested pure projection; negative and zero-net anchors; exact IDs above 2^53; +row permutation; missing, infinite and incorrectly typed inputs; invalid +bounds/scales and column collisions; and declaration/context disagreement. +An actual two-node graph executes cold, retains the complete input Frame, +metadata and zero-weight record, then reopens the store under required replay. +The replay hits both nodes and preserves artifact identity. Changing scales +reuses the source node and invalidates the reconciliation node. + +The closed run used 3.69 seconds wall time and 417,775,616 bytes peak RSS. +All 1,000 source and eight owned hashes, plus 15 resource hashes, were unchanged; +there were no unexpected denied operations or child processes. These are +invented-data graph checks, not native Microcosm acceptance. Receipt SHA-256: +`7e0ed8fefc911b9d7969f48e0d2c382c189e40a0427a120a163a8ce9c8ffae01`. +JUnit SHA-256: +`914745d61c768ae4f445df408db89601d4a4268374ba4f0e6caba6085ea6be6c`. + +The first run preserved 22 passing tests and two fixture failures: the test +attempted an empty `Weights` object and constructed unsorted group IDs. +Correcting those fixture inputs produced the result above; production source +was unchanged between the two runs. Independent review is recorded separately. diff --git a/docs/puf2015-canonical59-and-growth.md b/docs/puf2015-canonical59-and-growth.md new file mode 100644 index 000000000..36733c8fe --- /dev/null +++ b/docs/puf2015-canonical59-and-growth.md @@ -0,0 +1,74 @@ +# PUF 2015 canonical donor and 2024 transport + +These source owners turn the statistical 2015 PUF into an explicit return-level donor with 59 canonical outputs. They preserve raw fields, source status and documentary decisions separately from modeled values. The six detailed mortgage balance, interest and origination-year outputs belong to the downstream SCF stage. + +```mermaid +flowchart LR + A[Authenticated PUF typed source] --> B[Observed and derived return fields] + B --> C[Eleven explicit baseline models] + C --> D[QBI model in 2015 money] + D --> E[Source-specific 2024 monetary transport] + E --> F[Canonical donor with 59 outputs typed donor] + F --> G[Eight-feature PUF59 donor interface] +``` + +The source codec retains all ordinary returns, including those missing the demographic supplement. Disclosure aggregates retain their source lexemes but are excluded from individual donors. Neither a demographic gap nor zero reported exemptions means that a source return is dropped. + +## Count measurements + +PUF 2015 MARS 2 combines joint and qualifying widow(er) returns. The matching predictor maps the survey's separate widow category into that combined class. It is named `puf_2015_filing_status_code`; generic model filing status is unchanged. + +The corresponding `puf_2015_capped_return_size` equals one plus an extra unit for MARS 2 plus the sum of the four reported dependent-category counts. The source owner validates their disclosure caps by filing class. On the survey side, the matching measurement caps the reported dependent total before adding the filing-class units. The result is a statistical predictor, not a count of physical people or co-residents. Actual survey membership remains unchanged. + +The separate `puf_person_incidence_capacity` is one for return-level modeled zero/one QBI incidence. It is used only to validate the canonical outcome representation and is excluded from the eight fitted predictors. + +## Observed fields and explicit models + +The decoder emits 33 directly mapped or algebraically derived baseline fields. Eleven additional baseline outputs explicitly model source gaps: a normalized interest split, a component-neutral Social Security carrier, desired contributions proxied by realized deductions, incomplete tuition and employee-expense proxies, and active-partnership earnings. These transformations retain their assumptions in the receipt. + +The interest model uses the existing [IRS 2015 Table 2.1 workbook](https://www.irs.gov/pub/irs-soi/15in21id.xls). It normalizes the four published component amounts within the source AGI band. Mortgage interest, points and mortgage insurance share the available home-interest leaf; the investment component alone enters investment expense. Complementary rounding preserves the source total exactly. This does not create a mortgage balance, origination year or observed loan structure. + +The QBI owner reproduces the archived data model's assumptions with RECID-keyed random draws, retaining a reusable full-cohort employee calibration. Its 15 added leaves and Schedule C replacement are modeled inputs. The statutory engine owns deductions and limits. Ordinary and SSTB Schedule C branches are summed for the donor's total self-employment predictor, matching the survey measurement. + +## Source-specific monetary transport + +The publisher's statistical 2015 convention is the input basis. Raw filing-year/month fields remain intact; the transform does not apply another row-year CPI adjustment. + +Wages and modeled W2 amounts use the [SSA average wage index](https://www.ssa.gov/oact/cola/AWI.html). Social Security carriers use [SSA COLAs](https://www.ssa.gov/oact/cola/colaseries.html) received from January 2016 through January 2024. This transports fixed entitlements and does not predict changes in recipient composition. + +Other income families use the observed 2015 and 2023 national amount/count cells of [IRS Table 1.4 and 1.4A](https://www.irs.gov/statistics/soi-tax-stats-individual-statistical-tables-by-size-of-adjusted-gross-income), with positive and loss magnitudes kept separate. The 2023→2024 step uses the [BLS annual CPI-U series](https://www.bls.gov/cpi/tables/supplemental-files/historical-cpi-u-202412.pdf), explicitly assuming stable real conditional amounts for that year. Expenses, UBIA and selected mixed model outputs use an explicitly named CPI proxy. Every monetary field has a declared rule; incidence fields are unchanged. + +Partnership and S-corporation reporting counts overlap, so their dollar components are summed with the common all-return denominator. Dividend residuals use the ordinary-dividend reporting universe and do not subtract overlapping recipient counts. + +The PUF short- and long-term capital fields remove the effect of carried losses. Published net-loss means that include carryovers cannot be substituted directly. The model uses current-year gross gain/loss components per all return as a declared proxy for signed net flows. The evidence retains both the carryover-adjusted reported net and component-difference net, including their unresolved residual. It does not attribute that difference to rounding without evidence. + +The observed series are source facts. Their application to individual donor amounts, the related-family substitutions, and the last-year bridge are modeling decisions. They do not establish forecast accuracy or current reporting incidence. A same-cohort CPI-only sensitivity is available. Source design weights are divided from S006 hundredths once and are never grown. + +## Validation status + +The ordinary verification executes real source decoding, the QBI model, growth, the deterministic typed envelope and the PUF59 donor interface at 64 and 2,048 invented records. It checks all 59 outputs, sign/zero preservation, physical type and knownness refusal, exact interest conservation, SSTB/base identities, source immutability, stable full-fit subset replay, artifact byte/value replay and malformed-input refusal. + +The 59 invented controls passed. The separately reviewed genuine construction then produced all 59 canonical outcomes for 207,692 ordinary returns in 23.07 seconds. The 64-return subset reproduced the full-cohort outputs when it reused the full QBI employee fit, and the typed donor passed byte/value replay through the canonical donor interface. + +Independent review subsequently found that the version 1 envelope could attach +an old descriptive receipt to altered, domain-valid values. The version 2 codec +now checks the row-ordered identity/weight/source-feature digest and all 59 output +values against that receipt on encode and decode. It also pins the actual +interest-band facts, verifies the embedded growth recipe, requires canonical +JSON and physical integer row metadata, and makes CPI-sensitivity acceptance +explicit. The corrected source passed 80 invented controls, including the 21 +new integrity cases. Formatting followed that run; the composed integration +checkout still needs its own execution checks. + +The original genuine donor and evidence remain on version 1. A version 2 genuine +construction or repacking requires separate verification, and the new decoder +refuses old magic. Serialized receipt consistency does not authenticate an +issuer, establish execution or confer release status. + +This result verifies source donor construction. Social Security component reconciliation, downstream SCF mortgage semantics, conditional-fit quality, recipient placement and calibration remain open release requirements. It does not certify a calibrated population or policy-analysis file. + +## Evidence bundled with the source + +The [national extract](evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json) records the exact public workbook cells, units, hashes, family calculations and unresolved capital residuals. The [index transcript](evidence/puf2015-target2024/INDEX-VALUES.json) identifies the SSA and BLS values and retrieval sources; it is a transcription of retrieved values, not a hash of the original web pages. The [growth recipe](evidence/puf2015-target2024/GROWTH-RECIPE.json) assigns each monetary field a factor and interpretation. + +[Public workbook locators](evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json) provide the four official URLs and exact sizes and hashes without local cache paths. The recipe also preserves hashes of the original local acquisition receipt and extraction script. Those two local files are outside this publication bundle; their recorded hashes do not imply that a reader received them. diff --git a/docs/puf55-survey-ss-measurement-decision.md b/docs/puf55-survey-ss-measurement-decision.md new file mode 100644 index 000000000..fc12e6b85 --- /dev/null +++ b/docs/puf55-survey-ss-measurement-decision.md @@ -0,0 +1,107 @@ +# PUF55 survey Social Security conditioning decision + +On 2026-09-09 the build owner adopted +`social_security_filer_joint_spouse_report_sum_proxy` for PUF matching. This +supersedes the pending measurement judgment in +[survey-social-security-source.md](survey-social-security-source.md). The +decision authorizes this conditioning measurement and its explicit fallback; +it does not assert donor/recipient equivalence, allocate beneficiaries, complete +Social Security components, approve a fit, or admit a release. + +For each modeled return, include exactly one `HEAD` and, only when the actual +`filing_status_input` is `JOINT`, exactly one `SPOUSE`. Sum their available +nonnegative source report totals. Preserve combined family and child reports +without splitting, deduplicating, or subtracting amounts. Exclude dependents +from this return feature without changing any person's observed or unknown +Social Security cells or component masks. + +The public helper requires the actual authenticated survey preparation and +qualifies current source reports. It uses the exact modeled return roles +retained by that preparation. It does **not** rerun tax-unit construction using +current money. That limitation is accepted for this conditioning stage; later +production integration must qualify the chosen role/current-money relationship +explicitly. A caller-made role table or decoded receipt grants no authority. +The PUF disclosure-capped return-size predictor is not a person count, and the +coarsened PUF filing class 2 cannot identify an actual spouse: surviving-spouse +returns also map to that class. + +## Availability and refusal + +An available report-sum means every included reporter has an in-universe, +finite source amount. It does not mean those reporters legally own all the +reported benefits. Known zero reports are valid. Unknown components do not +invalidate a known total. Family/child ambiguity remains an explicit limitation +and does not itself trigger missingness. + +An unavailable included report, including an under-15 filer outside the source +question universe, leaves the proxy unknown and selects the separately named +`PUF55_SURVEY_SS_NO_TOTAL` profile. This profile has the same 55 outputs and +eight predictors. The existing `PUF55_SURVEY_SS` retains nine predictors. +Neither produces the four Social Security components. There is no partial sum, +dependent-zero convention, component-sum fallback, silent row deletion, or +implicit change to the 59/65-output profiles. + +Missing/invalid roles, duplicate or orphan membership, conflicting source +values, nonnumeric amounts, negative/nonfinite observations, and known amounts +outside the reporting universe refuse. They cannot route to eight predictors: +the first two PUF predictors still require valid roles. A source qualifier's own +refusals remain refusals; the fallback does not override them. Graph integration +must bind both disjoint recipient routes, account for each recipient exactly +once, authenticate inherited clone membership, and execute the corresponding +complete fit/apply/attachment chain. This numerical helper does none of that +graph admission or fitting. + +## Measurement limits + +[IRS Publication 915 (2015), pages 2–6](https://www.irs.gov/pub/irs-prior/p915--2015.pdf#page=2) +assigns benefits to the legal beneficiary, including a child whose parent +receives the check; joint filers combine spouses' benefits. The return amount +uses benefits net of repayments and includes the Social Security equivalent +part of tier-1 Railroad Retirement, excluding other pension portions. Form +1040EZ omits benefits, and some recipients need not file. These rules motivate +the selected return members but do not establish survey report ownership or +PUF editing conventions. [SSA's representative-payee guidance](https://www.ssa.gov/payee/faqrep.htm) +likewise distinguishes the person managing a payment from its beneficiary. + +The [2025 ASEC dictionary, PDF pages 48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=48) +permits combined family payments and identifies payments on behalf of children. +Its source total covers calendar 2024 for a 2025 interview. The +[ACS 2024 questionnaire, question 43](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +uses the prior twelve months, permits jointly received income entirely on one +person's record, and combines Social Security with Railroad Retirement. +[ACS SSP metadata, PDF page 45](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf#page=45) +defines the age-15 universe, rounded/topcoded amounts and ADJINC price +adjustment. The helper preserves those distinct observation windows; it does +not calendarize ACS or estimate repayments/Railroad adjustments. + +The ninth donor predictor remains the existing transported 2015 `E02400` +carrier under its declared 2024 money-growth convention. Carrier zeros do not +prove absent benefits. Exact 2015 publisher editing/omission rules remain +unverified; the [IRS PUF page](https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files) +reports the 2012–2015 files unavailable. Matching is therefore conditional on +this documented proxy assumption, not a newly authenticated observed 2024 +return total. Using eight predictors does not remove donor coverage concerns +for nonfilers or resolve assumptions in the other donor outcomes. + +## Diagnostics required before relying on a fit + +Family payments on selected reporters may overstate their own benefits; +payments reported elsewhere may understate them. An all-member sum includes +dependent reports and cannot solve this attribution problem. Topcoding, +allocation, return editing and transported donor money can distort zeros and +tails. Growth of fixed 2015 amounts does not predict 2024 cohort composition. + +Report route counts, known zero/positive distributions and tails by survey, +actual filing status, age, child-report reasons, allocation provenance, and +multiple-return household status. Preserve original-household holdouts across +clones. Compare design-weighted held-out performance and recipient overlap for +eight versus nine predictors; donor-only validation cannot establish cross-source +measurement validity. An adult in-universe all-member report sum may be a +labelled sensitivity diagnostic, with under-15 unknownness retained. It is not +the default feature. Benefit-component modeling and release checks remain +separate work. + +This change provides source-bound numerical values, explicit route metadata, +and an isolated graph profile declaration. Actual source guards, route-aware +graph integration, donor projection for the fallback, fits and diagnostics are +subsequent verification steps, not results claimed by this decision. diff --git a/docs/survey-social-security-source.md b/docs/survey-social-security-source.md new file mode 100644 index 000000000..ea9cfa1ce --- /dev/null +++ b/docs/survey-social-security-source.md @@ -0,0 +1,124 @@ +# Survey Social Security source semantics + +The source-report Social Security total precedes PUF enrichment. The current +qualifier verifies the original observations and preserves their limits; it +does not fit a completion model, assign individual beneficiaries, or certify a +population. The explicit `PUF55_SURVEY_SS` profile has 55 outputs and uses an +upstream-qualified tax-unit total as its ninth conditioning input. Historical +full65 and PUF59 behavior is unchanged. The new profile's native source and +whole-population attachment integration remain pending. + +## Source measurements + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +describes `SS_YN` and `SS_VAL` on PDF page 49. The question covers people aged +15 or older and permits combined family payments on a person's report. A +positive report is therefore not necessarily one individual beneficiary's +payment. The qualifier preserves the source report's grain. + +The retained ASEC original member supplies `PERIDNUM`, household/line/age +coordinates, `SS_VAL`, `SS_YN`, `RESNSS1`, `RESNSS2`, `RESNSSA`, `I_SSVAL` +and `I_SSYN`. Source keys, the complete current cohort, raw member size/hash, +live source issuance and exact current money values must all agree. Source +year and income year are 2024; interview year is 2025. Carried reason fields +are not a substitute for these original literals. + +The [2024 ACS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +defines combined `SSP`, its age-15 question universe, and allocation flag +`FSSP`. The qualifier binds the retained 2024 ACS issuance and verifies +`SSP * (ADJINC / 1000000)` against the prepared amount. This remains a rolling +income window expressed in the survey's price basis. `FSSP` is not in the +current native projection, so publisher allocation provenance stays explicitly +unresolved; no observed-value claim is inferred from a missing flag. + +## Mapping judgments and unknownness + +The reason mapping is an explicit judgment at source-report grain. Retirement +and disability map to their corresponding component. Widowed and surviving +child reasons map to survivors; spouse and dependent child reasons map to +dependents. A single resolved component receives the report total. Repeated +reasons in the same component do not duplicate the amount. + +Distinct component reasons, missing reasons, code 7's combined child categories +and code 8's residual category leave the component amounts unknown. The +published reasons restrict possible components where they can; there is no +reason priority, age fallback or equal-share default. Separately observed +component amounts and individual beneficiary assignments are not claimed. + +An ASEC under-15 `SS_VAL=0` is a not-in-universe sentinel. Both ASEC and ACS +under-15 beneficiary totals remain unknown in the projection, with the raw +ASEC zero retained in the separate literal table. An in-universe zero remains +a known report total of zero. Contradictory universe/recipiency combinations +and invalid nonempty codes refuse; blank literals remain distinguishable +from invalid codes. + +An eventual component model may normalize nonnegative scores within allowed +components, preserving the source total and already resolved source basis. +It must supply positive allowed mass. The numerical helper does not authenticate +such a model; the graph must bind real fit/apply artifacts. A reporting-unit +convention, allocation to individual beneficiaries where necessary, explicit +income periods, design-weighted training, native-to-clone inheritance and +held-out checks remain separate required work. + +## Conditioning measurement judgment, 2026-09-09 + +The recipient measurement for predictor nine remains undecided. Passing the +numerical `PUF55_SURVEY_SS` mechanism does not establish donor/recipient +comparability or authenticate a beneficiary total. + +[IRS Publication 915 (2015), pages 3 and 5–6](https://www.irs.gov/pub/irs-prior/p915--2015.pdf#page=3) +combines spouses' benefits on a joint return and assigns benefits to the person +legally entitled to receive them. A child's payment remains the child's even +when the check names the parent. Form 1040 line 20a reports net benefits after +repayments, including equivalent tier-1 railroad benefits; Form 1040EZ does not +report benefits. Thus a sum over all modeled tax-unit members is a different +measurement from the filer/joint-spouse return amount. + +Restricting survey records to the filer and joint-return spouse would still +produce a **report-sum proxy**, not an observed beneficiary total. The +[ASEC dictionary, PDF pages 48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=48) +permits combined family payments and includes reasons for payments on behalf +of children. The [ACS 2024 questionnaire, question 43, page 18](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +also permits jointly received income to appear entirely on one person's +record, with no receipt recorded for the other. Its income category includes +Social Security or Railroad Retirement. A parent may therefore report child +payments, or another household member may report an included person's income. +Multiple tax units within one household make reporter membership insufficient +to establish benefit ownership; household totals must not be duplicated across +returns or redistributed without a separate supported model. + +If adopted, a measurement such as +`social_security_filer_joint_spouse_report_sum_proxy` would need an explicit +upstream contract before supplying `puf_conditioning_social_security_total`. +That contract would bind actual return roles, source reporting records, +knownness and income periods; the coarsened PUF filing-class predictor alone +cannot establish joint-return membership. ASEC calendar-year income, ACS's +rolling window, transported donor money, repayment treatment and railroad +coverage remain comparability questions. Component completion is separate. + +Excluding a dependent from a parent's return feature does not assign that +dependent zero benefits. Both surveys' under-15 amounts remain unknown. If an +included filer has no in-universe report, an empty or missing report set cannot +silently become a known zero. Family-payment ambiguity also survives an +age-universe restriction. + +The exact 2015 `E02400` definition and publisher editing/omission conventions +remain unauthenticated by this lookup. The [live IRS PUF page](https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files), +checked on 2026-09-09, says the 2012–2015 files are unavailable. The verified +Form 1040 rules do not independently establish how every PUF record encodes +them. This record adopts no proxy, allocates no beneficiaries and grants no +dataset or release acceptance. + +## Verification scope + +The 52-case control run executes 50 numerical/literal tests and two complete +maintained source issuances from invented original CSV/checkpoint fixtures. +It verifies source-identity joins despite reordered rows, repeat qualification, +defensive returned data, exact source totals, unknown components and unchanged +input frames. Source/control files, installed model source and the 12 admitted +code resources were unchanged before, after and in the external postcheck. + +The run completed in 67.6 seconds with 428.9 MB peak RSS. Receipt SHA256: +`2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. +These controls are not native-source execution, a fitted component model, +native PUF55 integration, policy-engine validation or release acceptance. diff --git a/docs/uk-atomic-household-lineage.md b/docs/uk-atomic-household-lineage.md new file mode 100644 index 000000000..067353f48 --- /dev/null +++ b/docs/uk-atomic-household-lineage.md @@ -0,0 +1,28 @@ +# UK household identity after cloning + +The shared atomic-area operators need a stable household key after initial cloning. `uk_runtime.atomic_household_lineage` projects supplied original-source identities through explicit selection and expansion records, using the existing `household_draw_key` encoding. It adds no assignment algorithm, data reader, issuance registry or full-build host. + +A key includes the declared source, source vintage, original household ID and ordered clone path. It excludes final offset IDs, row order, sample size, calibration weights and financial values. Exact int64 IDs above `2**53` remain distinct. Original households keep their previous path; a new child appends its explicitly supplied branch ordinal. An original branch does not gain an inferred zero ordinal. + +## Contract + +`project_atomic_household_keys(roots, *, steps, final_ids)` returns a detached two-column table: int64 `household_id` and nullable-string `geography_household_key`, in the requested final order. Roots have exactly `household_id`, `source`, `source_vintage` and `source_household_id`. An empty selection is a valid description, not evidence that an empty dataset is useful or releasable. + +Steps are immutable descriptions: + +- `HouseholdExpansion(branch, before_ids, after_ids, parent_pairs, child_ordinals)` carries exact household axes, the household part of the shared executor's `receipt["expand"]`, and explicit child ordinals. The existing ordered branches are SPI support, CGT incidence, CGT band donors and geographic support. Each arriving household needs one known preceding parent and one ordinal. Two children of the same parent cannot reuse an ordinal in the same branch. +- `HouseholdSelection(before_ids, after_ids)` explicitly removes households while preserving their keys. This supports the source-family sample between the completed source spine and geographic expansion. It does not check that the sample respected the host's family-sampling rule. + +Each before-axis must have exactly the previous step's household IDs. Expansion cannot remove incumbents; selection cannot add households. Unknown or null parents, repeated or reordered branches, incomplete parent/ordinal coverage, duplicate keys and a mismatched final axis refuse. Axis ordering may change, because ordering is not identity. The function never guesses ancestry from ID arithmetic, support flags or model values. + +## Host responsibilities + +These checks establish internal consistency of supplied descriptions. A dataclass, DataFrame or serialized receipt does not prove its source. The receiving host must retain and check the actual original source owner, complete before/after populations and source/EXPAND/FILTER evidence. It must authenticate the complete structural stage roster, the explicit branch-ordinal convention, and the exact final receiving household axis before and after relevant I/O. A coherent invented history can pass this pure helper; it is not an admission gate. + +The host can extract the household pairs from its verified shared EXPAND receipt without decoding household ID offsets. The helper returns the key column for the existing `uk_atomic_assignment_definition` and `atomic_geography_nodes` APIs. Support-byte provenance, observed-region constraints and versioned mapping relations remain separate responsibilities. Larger geographies continue to derive from the assigned atomic area. + +No production UK host is changed here. María's full-build graph currently uses its existing ladder draw after geographic expansion. Adopting this key and shared assignment must be coordinated in that host. The existing source stage order also interleaves enrichment before later SPI/CGT cloning; moving atomic assignment before every enrichment stage requires a separate staged migration. This helper does not claim that migration, native FRS qualification or a new UK release. + +## Verification scope + +Invented tests cover the full four-branch path, explicit selection, exact large IDs, source and ordinal changes, stable keys under row reordering and pool growth, unknown/colliding ancestry refusal and detached output. A real shared CREATE/EXPAND/FILTER graph supplies the household lineage receipt and replays from cache. Its projected keys then feed the existing seven-node atomic graph, including three system imports, assignment, derivation and its gate. That graph also completes required replay and preserves the supplied entity tables, weights, strata and metadata. No native source, private archive, country engine or external resource is used. diff --git a/docs/us-asec-source-domain-contract.md b/docs/us-asec-source-domain-contract.md new file mode 100644 index 000000000..597ec5a7b --- /dev/null +++ b/docs/us-asec-source-domain-contract.md @@ -0,0 +1,43 @@ +# ASEC amount-domain contracts + +The current income-routing, interest and child-support qualifiers read monetary +definitions from the pinned `asec_current_money_domains_v1.json` artifact. +Their public amount-entry functions return detached mappings. Routing and +interest cache immutable tuples privately; child support caches immutable JSON +bytes and reconstructs its nested evidence on each call. Replacing an entry, +clearing a returned mapping or editing nested child-support metadata cannot +change a later caller's definitions. The public mapping and evidence shapes +remain unchanged. + +Each relevant field must occur exactly once and contain exactly one vintage for +the current income year. Its dictionary spelling, URL and digest must match the +qualifier's source pin. Missing or duplicate entries refuse qualification. + +The interest total and child-support parsers accept unsigned integer literals. +They explicitly require the corresponding artifact domains to describe +nonnegative integer dollars, with the supported zero and NIU meanings, no +additional missing-value codes or excluded dollar values, and consistent +encoded, dollar and printed ranges. Bounds remain artifact-derived. Re-pinning +an artifact with an unsupported domain requires a parser change and its own +review; it cannot silently change the interpretation of those literals. + +Before capturing the original source member, each qualifier compares its +relevant amount definitions with the actual retained current-money owner's +`MoneyDomain` entries. Names must be unique; entity, grain, native column, +bounds and zero semantics must agree. Interest checks the retained `INT_VAL` +total. Its additional ordinary and retirement-account components retain their +separate published definitions. Child support checks both `CSP_VAL` and +`CHSP_VAL`; routing checks its existing nine fields. + +These checks preserve the existing source meanings. In particular, a +`CHSP_YN` answer describes an obligation, and neither a no-obligation answer nor +an NIU paid amount establishes zero voluntary payments. Receipt, reporting, +allocation and top-code classifications are unchanged. The changes add no +model, canonical engine input, source issuer or native-data admission. + +Validation used invented mutations of the public domain artifact and actual +invented source-owner fixtures: 73 new domain/cache controls, 88 routing tests, +40 interest tests, 10 allocation tests and 40 child-support tests passed under +the existing bounded source guard. The new controls first reproduced 65 +failures against the prior implementation. These results certify the tested +source contracts, not a native dataset or release. diff --git a/docs/us-atomic-property-financial.md b/docs/us-atomic-property-financial.md new file mode 100644 index 000000000..4380c0f91 --- /dev/null +++ b/docs/us-atomic-property-financial.md @@ -0,0 +1,117 @@ +# Current survey property-income host + +`run_atomic_survey_financial` can append the current property-income graph to +the existing financial composition. Pass an explicit `PropertyIncomeOptions` +through `property_income`; omitting it retains the nineteen-node path. + +```python +from microcosm.build.us_runtime.graph_atomic_survey_financial import ( + run_atomic_survey_financial, +) +from microcosm.build.us_runtime.graph_current_survey_property import ( + PropertyIncomeOptions, +) + +options = PropertyIncomeOptions( + scales=(1.0, 1.0, 1.0, 1.0), + atol=1e-10, + rtol=1e-12, + n_estimators=2, +) +run = run_atomic_survey_financial( + **source_arguments, + geography_config=atomic_geography, + property_income=options, + n_estimators=2, + return_values=True, +) +checked = run.checked_view() +``` + +The small tree counts above are development settings. The caller supplies the +existing authenticated source arguments and atomic geography configuration. +They are not replaced by a downloaded graph, a detached Frame or a receipt. + +The thirty-five-node composition consists of the nine-node survey, clone and +atomic-geography prefix; ten existing financial nodes; and sixteen property +nodes. The new branches select eligible original ASEC donors and original ACS +recipients before allocation. Four ordered, DESIGN-weighted conditional models +produce raw component draws, followed by reconciliation to a known adult ACS +property-income anchor. An explicit attachment carries each original result to +both initial clones. Original ASEC components remain separately qualified even +when that person is excluded from fitting. See the +[fragment contract](us-current-survey-property-graph.md). + +The output adds 24 component, raw-draw, diagnostic and knownness columns. It +preserves the complete preceding financial population: other columns, entity +membership, geography, schema, metadata, weights, strata, ownership and mass +history. Missing source anchors and unresolved components retain missing +values and explicit knownness. This mode does not infer property-income zeros +for children outside the survey question's universe. + +The runner retains both the preceding and extended populations. Before issuance, +execution and required cache replay verify property model training against the +original donor and replay four applications against actual recipient features. +Later issued-run checks requalify the existing source owners, validate the legacy +financial output, reconstruct the complete extended population and check the +verified model artifacts' byte identities. They do not refit or reapply the +models. Copied public result dataclasses have no run authority. See the +[model receipt contract](us-property-model-receipts.md). + +The thirty-five-node option leaves the eight existing financial leaves unchanged. +Its run document therefore records `tax_split_rebased: false` and the legacy +capital-gain model's conditioning on its earlier interest and dividend draws. +It does not itself replace tax inputs. Optional tax rebasing is a separate +three-node extension, described below. + +The PUF host derives its upstream node count and checks the actual final +financial writer. Its recipient nodes read all columns of that population, +including the new property fields, so the dependency is present in the graph. +This interface does not establish that a native PUF run, tax rebase, calibration +or release has passed. Candidate execution evidence belongs in `experiments/`. + +## Optional tax rebase + +Pass `rebase_property_taxes=True` together with `property_income=options` to +append the deterministic receiving version, tax split and numerical gate. The +default is `False`; requesting a rebase without property options refuses before +source I/O. This path has 38 nodes. It retains the complete legacy financial +population and the complete 35-node property population separately from the +final `survey_property.tax_receiving` version. + +The rebase uses ordinary interest and dividends with the maintained fractions +and subtraction complements. Separate missing O/D inputs produce missing tax +leaves, replacing earlier predictions; retirement-account interest remains +auxiliary. A new FILTER conservation record is intentional, while existing +Frame mass history and all non-rebased cells remain unchanged. See the +[numerical fragment contract](us-property-tax-leaves.md). + +The host constructs the declarations from the checked clone Frame and the +existing financial/property output descriptors. It never supplies fake values +for future columns. After the real 35-node output is available, it independently +re-declares and executes the three deterministic operations, checks stored +artifact bytes and complete node receipts, and compares every expected full +population with actual cold or cached observations. The issued run retains +the source, declaration, model, artifact, intermediate population and final +population seals. Later checks reconstruct the same tax operations before the +final source requalification and pure lifetime fence. + +An incomplete numerical gate is valid development evidence. The run document +reports `tax_split_rebased: true`, three extra nodes and the actual +`tax_leaf_complete` value; it remains `release_eligible: false`. PUF recipient +qualification explicitly requires a complete independently checked numerical +gate before it reads its additional source inputs. Both PUF recipient graph +nodes carry a typed edge to that gate on the actual receiving version. The +19- and 35-node paths do not add that edge or claim a tax rebase. + +No child zero completion or missing-adult model is introduced. Consequently the +invented source fixture with under-15 and missing-anchor cases must remain a +development run and refuse full PUF qualification. Capital gains retain their +earlier conditioning; rebasing interest and dividends does not update that model. + +The bounded invented-source acceptance covers 11 host controls, including actual +cold and required-cache 38-node runs, issued-view parity, intermediate/final +population preservation, mutation refusals and incomplete PUF qualification. +The numerical fragment has a separate 40-control acceptance, including a +complete-input positive gate and descriptor-only declaration checks. These +checks do not use native survey data, run a country engine or execute full PUF. diff --git a/docs/us-current-asec-child-support-source.md b/docs/us-current-asec-child-support-source.md new file mode 100644 index 000000000..c55593637 --- /dev/null +++ b/docs/us-current-asec-child-support-source.md @@ -0,0 +1,79 @@ +# Current ASEC child-support observations + +`qualify_current_asec_child_support(preparation)` in +`microcosm.build.us_runtime.current_asec_child_support_source` qualifies the +original received- and paid-support observations from the retained 2025 ASEC +person source. The dollar amounts refer to the current 2024 income cohort. + +The source is the [2025 ASEC public-use dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA-256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The qualifier reuses the exact packaged monetary domains and temporal +authority for `CSP_VAL` and `CHSP_VAL`. It adds source questions and flags from +the same dictionary, with their positions, widths and printed universes. + +| Fields | Source meaning | One-based PDF page | +| --- | --- | --- | +| `CSP_VAL`, `CSP_YN` | Amount received and actual receipt answer; receipt universe age 15+ | 52 | +| `CHSP_VAL`, `CHSP_YN` | Annual amount paid and whether payment is required; the latter is an obligation question | 52 | +| `CHELSEW_YN` | Whether a child lives outside the household; age 15+ | 52 | +| `I_CHELSEWYN`, `I_CHSPVAL`, `I_CHSPYN`, `I_CSPVAL`, `I_CSPYN` | Published allocation flags, using the shared I_ANNVAL code meanings | 54 | +| `TCHSP_VAL`, `TCSP_VAL` | Published top-code indicators for positive amounts | 59 | + +## Received amounts + +The printed zero for `CSP_VAL` means none or NIU. An in-universe receipt-no +answer plus zero establishes known nonreceipt. A receipt-yes answer plus zero +remains ambiguous; the qualifier does not complete it with a zero-dollar +observation. Positive receipt amounts are retained, including valid allocated +observations. Under-15 NIU, missing answers, malformed literals and +contradictory receipt/amount pairs remain distinguishable and unknown. + +## Payments and obligations + +`CHSP_YN` asks whether the person is required to pay child support. A no +answer does not rule out voluntary payment. `CHSP_VAL` describes actual annual +payments and prints zero as NIU, so every paid zero remains a published code +with an unknown canonical payment amount. No obligation answer turns it into +an observed zero. + +The paid amount's universe is `CHSP_YN = 1`. The obligation question itself +prints only the bare field `CHELSEW_YN`, while the allocation flag `I_CHSPYN` +prints `CHELSEW_YN = 1`. That discrepancy remains in the evidence. The +qualifier conservatively accepts positive payments on the observed +intersection age 15+, child-elsewhere yes and required-to-pay yes. Other +routes keep the published positive amount and an explicit unresolved status. +This intersection is a qualification choice, not a correction of the printed +universe or proof that voluntary payments do not occur. + +The obligation and child-elsewhere entries do not independently state a +reference year. Their source context is retained; the qualifier does not +issue a current-at-interview or eligibility input from those answers. + +## Provenance and consumption + +The result has two complete tables: selected ASEC people on the common +preparation's person identities, and the original current-cohort literals on +native identities. Source amount validity, statuses and zero origin are +retained alongside the descriptive projection. Allocation/top-code literal +values and parse statuses remain separate from amount knownness. All-zero +allocation flags describe the published codes; they are not a blanket claim +of unallocated or respondent-reported data. + +The qualifier takes the actual authenticated preparation, captures the +pinned original member once through the shared bounded literal reader, +checks its bytes and complete native-key roster, verifies household/line/raw +age coordinates, and compares both source amounts to their retained +current-money values with exact validity and float64 bits. Source row order +does not define person identity. + +`CurrentAsecChildSupportValues` is descriptive transport, not an issuer. +`child_support_values_seal` covers both complete tables, axes, nullable masks +and backing storage, float bits and detached evidence. The actual preparation +is requalified after source I/O, followed by final owner and physical-value +checks. A consumer must retain the actual owner, requalify immediately before +consumption and after its own last relevant I/O, and compare complete value +seals before returning or exporting a successor. Copied data or a matching +digest cannot grant source authority. + +This module assigns no engine leaf, changes no clone, fits no ACS model and +issues no new source admission or release eligibility. diff --git a/docs/us-current-asec-dividend-source.md b/docs/us-current-asec-dividend-source.md new file mode 100644 index 000000000..de977cd18 --- /dev/null +++ b/docs/us-current-asec-dividend-source.md @@ -0,0 +1,114 @@ +# Current ASEC dividend and survivor routes + +`current_asec_dividend_source.qualify_current_asec_dividend(preparation)` +qualifies current ASEC dividend observations and the recorded survivor-income +source routes that could overlap with a broad property-income concept. It +reads the original 2025 person member, whose annual money amounts refer to +2024. It returns source observations and provenance, without modeling ACS +values, assigning tax treatment or authorizing a dataset release. + +The [2025 ASEC public-use dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +is pinned to SHA-256 +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The dividend domain also comes from the existing pinned +`asec_current_money_domains_v1.json`. Page numbers below are one-based PDF +pages; positions and widths refer to the printed ASCII layout. + +| Field | Position / width | Page | Printed universe | +| --- | --- | --- | --- | +| `DIV_VAL` | 478 / 6 | 45 | `DIV_YN = 1` | +| `DIV_YN` | 484 / 1 | 45 | Age 15+ | +| `SUR_SC1`, `SUR_SC2` | 648 / 2, 650 / 2 | 50 | `SUR_YN = 1` | +| `SUR_YN` | 664 / 1 | 50 | Age 15+ | +| `I_DIVVAL` | 819 / 1 | 54 | `DIV_YN = 1` | +| `I_DIVYN` | 820 / 1 | 54 | Age 15+ | +| `TDIV_VAL` | 905 / 1 | 59 | `DIV_VAL > 0` | +| `TRNT_VAL` | 917 / 1 | 60 | `RNT_VAL > 0` | + +## Dividend values and source flags + +The projection preserves the exact `DIV_VAL_literal`, parsed +`DIV_VAL_published_amount`, literal and reporting statuses, and a separate +`DIV_VAL_amount_known` mask. `DIV_VAL_amount` contains only the values whose +reporting status is resolved. An in-universe receipt-yes and positive amount +is known; receipt-no and zero establish known nonreceipt. Receipt-yes and +zero remain ambiguous because the dictionary combines none and NIU in its +zero code. Missing literals, NIU and contradictory combinations retain +their source evidence and an unknown canonical amount. No analytical zero +is inferred solely from age. + +Source flags never change this knownness or select donors. A known published +cell can still have publisher allocation or topcoding metadata. It is not a +claim about an uncensored latent amount or a respondent-only report. +`I_DIVVAL` refers to the allocation meanings under `I_ANNVAL` on page 53. +`TDIV_VAL` and `TRNT_VAL` retain codes 0 (not topcoded) and 1 (topcoded), +including missing or malformed flags. `TRNT_VAL` is retained only for future +diagnostics: this qualifier does not read `RNT_VAL` or evaluate that flag's +positive-rent universe. + +The dictionary is internally inconsistent for `I_DIVYN`: its header prints +0–1, while its Values section refers to the `I_ANNVAL` codes 0–9. The result +preserves the literal and code, physical syntax status, separate +`I_DIVYN_header_range_status` and `I_DIVYN_referenced_values_status`, and an +explicit `I_DIVYN_codebook_status`. Codes 2–9 therefore retain the conflict; +they are not silently dropped or admitted under a preferred interpretation. + +## Survivor source routes + +`SUR_YN`, `SUR_SC1` and `SUR_SC2` each retain raw literals, parsed codes, +literal status and source labels. Code 8 identifies regular payments from +estates or trusts. It signals a possible property-income overlap, without +establishing an amount or tax classification. Code 10 means other or don't +know: its literal is valid, but its property route remains unspecified. + +`survivor_property_route_clear` is nullable. It is true only for an +in-universe known receipt-no with both source slots NIU, or receipt-yes with +every slot readable, at least one active slot, and neither code 8 nor code +10. Receipt-yes with an explicit code 8 sets it false. Missing or unreadable +receipts, unresolved slots, contradictory nonreceipt routes, NIU and code 10 +leave it unknown. Under-15 rows remain unknown; incomplete under-15 literals +are labeled unresolved rather than contradictory. Independent booleans +retain the presence of codes 8 and 10 even when the receipt is unresolved. + +This clearance describes only the two recorded source slots. The dictionary's +`SRVS_VAL` entry (PDF page 49, printed page 6C-28) also includes unedited third +and fourth sources. Their types are not qualified here, so two clear visible +slots cannot establish complete absence of survivor estate/trust income. +The property donor bridge conservatively excludes positive survivor receipts +until that additional scope is qualified. This clearance does not establish +that the person has no property income. No survivor amount is captured, no +donor exclusion is applied, and a missing slot is never an observed zero +amount. A later measurement bridge must explicitly choose and validate how +these routes affect donor eligibility. + +## Retained source boundaries + +The qualifier borrows the original authenticated survey preparation and its +retained native ASEC/current-money owner. Before capture it checks the live +money domain against the pinned dividend contract. The domain reader +requires a unique field and current vintage, refuses unsupported negative, +missing-code or zero semantics, and caches only immutable values. Public +entry mappings are detached on every call. + +The exact member's archive and member digests, row count and byte count +must agree with the retained owner. A single bounded capture is checked +again after reading. All current-year native keys, original household and +person-line coordinates, and source ages are joined before selection. +`DIV_VAL` validity and every valid float64 bit must match the actual money +owner, including people omitted from the selected survey. Original +`DIV_VAL_parent_statuses`, `DIV_VAL_parent_validity` and +`DIV_VAL_parent_zero_origin` remain in the projection. + +`CurrentAsecDividendValues(person, asec_literals, evidence)` is descriptive +transport. `person` is indexed by `person_id` and includes `native_person_id` +and `source_age`; `asec_literals` retains the complete current-year literal +table indexed by native person ID. `dividend_values_seal(values)` covers +both tables, their axes, exact float bits, nullable masks and backing +storage, and detached evidence. The qualifier rechecks the original +preparation after source I/O and verifies the complete seal before return. + +Consumers must retain and requalify the actual preparation before +consumption and after their last relevant I/O, and compare the complete +values seal. Matching JSON, a copied dataclass or a digest does not grant +source authority. No host, graph attachment, tax leaf, source issuer, +original/PUF clone change or release gate is added here. diff --git a/docs/us-current-asec-income-routing-source.md b/docs/us-current-asec-income-routing-source.md new file mode 100644 index 000000000..22968e633 --- /dev/null +++ b/docs/us-current-asec-income-routing-source.md @@ -0,0 +1,272 @@ +# Current ASEC income reporting and routing source + +`microcosm.build.us_runtime.current_asec_income_routing_source` qualifies the +exact retained 2025 ASEC person member against the authenticated current-money +owner and projects five original-channel income families the current survey +graph does not yet supply: + +| Family | Printed amounts | Printed receipt / routing | +| --- | --- | --- | +| `pension_annuity` | `PNSN_VAL`, `ANN_VAL` | `PEN_YN`, `ANN_YN` | +| `retirement_distribution` | `DST_VAL1`, `DST_VAL1_YNG`, `DST_VAL2`, `DST_VAL2_YNG` | `DST_YN`, `DST_YN_YNG`, `DST_SC1`, `DST_SC1_YNG`, `DST_SC2`, `DST_SC2_YNG` | +| `net_property` | `RNT_VAL` | `RNT_YN` | +| `farm` | `FRSE_VAL` | `FRSE_YN`, `ERN_YN`, `FRMOTR` | +| `other_income` | `OI_VAL` | `OI_YN`, `OI_OFF` | + +Thirteen published allocation flags are read alongside them. + +Both returned frames carry a digest: `evidence["projection_sha256"]` over +`person` and `evidence["literals_sha256"]` over `asec_literals`. Both are +re-checked at the end of the qualifier, so a host comparing the documented +digests detects a corrupted transport on either frame. + +`qualify_current_asec_income_routing(preparation)` takes a live +`AuthenticatedSurveyPopulationPreparation`, requalifies it, captures the pinned +2024 member once, joins it to the money parent by exact native keys, and returns +`CurrentAsecIncomeRoutingValues(person, asec_literals, evidence)`. The returned +values are descriptive transport. Constructing or mutating them issues no source +authority: a consuming host re-runs this qualifier and compares the result +around its own I/O, exactly as the accepted unemployment qualifier requires. + +## What the qualifier establishes + +- **The exact source member.** The 2024 pin is matched against the retained + native coverage receipt (member name, archive digest, member digest, row count + and byte size) before the capture, and the captured file's stat identity and + SHA-256 are re-checked after it. +- **Row-for-row parent correspondence.** `PH_SEQ`, `A_LINENO` and `A_AGE` from + the member must equal the money parent's `source_household_id`, `A_LINENO` and + `A_AGE` at the 2024 scope positions. The join is by native key, so member row + order cannot change the projection. +- **Literal/amount identity.** Every projected amount is the money owner's own + value, checked against the member literal, with the parent's `statuses`, + `validity` and `zero_origin` carried through as `amount_status_*`, + `amount_validity_*` and `zero_origin_*` columns. +- **A single bounded pass.** One capture, one read, one projection. There is no + per-family re-capture, no nested run issuer, and no PUF borrow. + +## What it deliberately does not establish + +- `PNSN_VAL` is printed as the *total combined amount of pension income received + from all pension sources* — `PEN_SC1`/`PEN_SC2` enumerate company, union, + federal, state, local, military and railroad pensions. It is not an observed + private or taxable amount. The archived `TAXABLE_PENSION_FRACTION = 0.590` + split is a modelled assumption and is not applied or referenced here; + `pension_annuity_private_share_applied` and + `pension_annuity_taxable_amount_known` are constant `False`. +- Account code 4 is `Regular IRA`. That identity does not observe any taxable + fraction, so `retirement_distribution_taxable_amount_known` is constant + `False` and no taxable component is derived. +- `RNT_YN` asks about land, property rented to others, royalties, roomers or + boarders, and estates or trusts, while `RNT_VAL` asks only about *income from + rent after expenses*. Both scopes are carried on every row and + `net_property_component_split_known` is constant `False`, so the total is not + independently labelled rental and no subdivision is separately modelled. +- `FRSE_VAL` is farm self-employment, not nonfarm `SEMP_VAL`; + `farm_is_nonfarm_self_employment` is constant `False`. Its printed label + includes the composite clause naming `ERN_VAL` (when `ERN_SRCE=3`) and + `FRM_VAL`, so the total already contains those components. +- `retirement_distribution_regular_ira_amount` is published only when every + applicable slot is fully resolved on both axes. An unreadable account code + could itself be a regular IRA, and a declared account whose amount is a + "none or niu" zero does not observe a zero dollar distribution, so both leave + the regular-IRA share unknown rather than at zero. +- `OI_OFF` code 20 is the reported alimony category. Nothing maps any other + code — including 19, `anything else` — onto alimony or onto a miscellaneous + residual; `other_income_residual_rule_applied` is constant `False`. + `other_income_routing_status` separates `reported_category` from + `receipt_without_category`, `category_without_receipt`, + `unresolved_receipt_routing`, `missing_category_literal`, + `unrecognized_category_literal` and `niu_category`, and + `other_income_is_reported_alimony` is set only on a `reported_category` row, + so an unreadable receipt literal never yields a reported alimony receipt. +- ACS-channel people are not projected at all. `acs_components_modeled` is + `False` in the evidence and the projection covers only ASEC-channel rows. + +## Reporting status vocabulary + +`receipt_status` is the single classifier. Only `known_receipt`, +`known_recipient_zero` and `known_nonreceipt` establish a dollar reading +(`KNOWN_AMOUNT_STATUSES`); every other status leaves the canonical amount +unknown rather than completing it with zero. + +| Status | Meaning | +| --- | --- | +| `known_receipt` | in universe, receipt yes, amount non-zero (a signed loss included) | +| `known_recipient_zero` | in universe, receipt yes, amount zero on an entry whose printed zero is valid dollars — `ANN_VAL` alone, whose NIU is the separate `-1` code | +| `receipt_with_net_zero` | in universe, receipt yes, amount zero on a signed net measure (`RNT_VAL`, `FRSE_VAL`). Distinct from a gross entry's recipient zero, but **not** a known amount: both print `0 = none or niu` | +| `ambiguous_recipient_zero` | in universe, receipt yes, amount zero on a gross entry whose printed zero reads "none or niu" | +| `known_nonreceipt` | in universe, receipt no, amount zero | +| `niu` | in universe, receipt 0, amount zero or a declared non-money code | +| `missing_amount` / `missing_receipt_literal` | one side of the pair is absent | +| `unrecognized_receipt_literal` | the receipt literal is malformed or outside the printed range | +| `contradictory_no_nonzero`, `contradictory_niu_nonzero`, `contradictory_declared_niu_amount`, `contradictory_outside_reporting_universe` | retained source contradictions, excluded from every canonical amount | +| `contradictory_offroute_evidence` | distributions only: the route that does not apply carries dollars or an answered recipiency | +| `unresolved_slot_composition` | distributions only: an applicable slot declares an account whose amount is a "none or niu" zero | +| `outside_reporting_universe` | the printed universe excludes the row and the literals agree | +| `unresolved_reporting_universe` | the printed universe cannot be resolved from the retained literals | + +Which recipient zeros resolve is read from the pinned domains artifact, not +decided here: `_zero_is_dollars` compares each entry's `zero_semantics` against +`valid_zero_dollars`. Eight of the nine entries record +`none_or_niu_not_distinguishable_from_amount_alone`, including both signed net +measures, so a signed zero is separated by label but never completed. + +The receipt code labels are per entry, not shared: `receipt_codes(field)` reads +the printed zero label from `RECEIPT_ENTRIES`, so `OI_YN` reports `none or niu` +where `PEN_YN` reports `niu` and `FRSE_YN` reports `Niu` exactly as printed. + +Other-income routing (`other_income_routing_status`) follows the same universe: +`OI_OFF` is printed for `OI_YN = 1` and `OI_YN` for persons aged 15+, so a row +outside or unresolved on that universe reports +`outside_reporting_universe_routing` or `unresolved_reporting_universe_routing` +and never a reported category. + +Universes are per family and never reduced to age alone. `PEN_YN`, `ANN_YN`, +`RNT_YN` and `OI_YN` print `All Persons aged 15+`. `FRSE_YN` prints +`ERN_YN=1 or FRMOTR=1`, so the farm universe is `True` when either literal is +`1`, `False` when both are known and neither is, and unresolved otherwise — +a nonfiler is never treated as a non-recipient. + +## Source-level questions preserved as evidence + +These are printed-dictionary facts, kept distinguishable from modelling +judgments. The first four are recorded under +`evidence["dictionary"]["printed_universe_questions"]`; the rest under +`evidence["dictionary"]["printed_scope_and_code_questions"]`: + +- `DST_VAL1`'s printed universe is `DST_SC1 = 1` although its label names the + source-1 distribution amount; taken literally that would restrict it to 401k + accounts. Retained verbatim, not silently corrected. +- `DST_SC2_YNG`'s printed universe names `DST_VAL_YNG`, a field with no + dictionary entry. +- `I_DSTVAL1COMP`'s printed `Universe:` line is empty. +- `DST_YN` and `DST_YN_YNG` print only the age-58 split. They print no 15+ floor + as the four age-universe families (`PEN_YN`, `ANN_YN`, `RNT_YN`, `OI_YN`) do, + and they are not gated on other literals as the farm family is — `FRSE_YN` + prints `ERN_YN=1 or FRMOTR=1` and so carries no age floor either. Coverage + below age 15 is therefore recorded as unresolved rather than resolved either + way. +- `OI_YN`'s printed zero label is `none or niu` where `PEN_YN`, `ANN_YN`, + `DST_YN` and `RNT_YN` print `niu`, so a zero receipt literal is not + interchangeable across families. +- `DST_SC1` is gated on `DST_VAL1 > 0 and a_age ≥ 58` while `DST_SC1_YNG` is + gated on `DST_YN_YNG = 1 and a_age < 58` — the two routes are not symmetric. + +Both printed recipiency literals are retained per row +(`retirement_distribution_receipt_58_*` and `..._receipt_young_*`) alongside all +four slots. The age route selects which pair applies; +`retirement_distribution_offroute_receipt` and +`retirement_distribution_offroute_nonzero` flag an answered recipiency or a +non-zero amount on the route that does not apply, so the complete slot/receipt +pattern stays inspectable rather than being reduced to the applicable half. + +## Amount semantics from the money owner + +The authenticated money owner normalizes `ANN_VAL`'s printed `-1` to a stored +zero and records `CodebookStatus.DECLARED_NIU`. `amount_state` therefore reads +the dollar meaning of a cell from the parent's status axis, never from the +stored number: re-deriving NIU from the number would read that cell as a zero +dollar annuity. `evidence["declared_niu_normalized_to_zero"]` (a top-level key) lists the +affected entries. + +The money owner restates non-2024 cohorts to the pinned price basis, so the +literal-identity join is taken only on `person_years == 2024`; widening it to +the pooled 2022/2023 cohorts would require handling that restatement first. +`evidence["restatement_note"]` records this. + +## Allocation provenance + +`*_allocation_origin` is derived from published flags only: + +- `publisher_allocated` — some applicable flag is non-zero. +- `published_flags_all_zero` — every applicable flag is populated and reads + zero, and every field in the family carries a published flag. This is **not** + an assertion of non-allocation: each flag prints a conditional universe (for + example `I_RNTVAL` is printed for `RNT_VAL > 0`) and those universes are not + evaluated here, so a non-recipient row sits outside its own flag's universe. +- `published_flags_all_zero_with_unflagged_fields` — the same, but the family + also holds a field the dictionary does not flag. +- `allocation_flag_not_populated` — a flag literal is absent. Every flag here + prints a conditional universe, so this is not a defect and not "no + allocation". +- `unresolved_allocation_provenance` — a flag literal is malformed or outside + its printed range. + +`PUBLISHED_ALLOCATION_FLAG_BY_FIELD` names which flag covers which field. +`UNFLAGGED_FIELDS` records the ten fields for which the 2025 dictionary +publishes no flag: `PNSN_VAL`, `FRSE_VAL`, `FRSE_YN`, `OI_OFF`, `OI_YN`, +`DST_VAL1_YNG`, `DST_VAL2_YNG`, `DST_YN_YNG`, `DST_SC1_YNG`, `DST_SC2_YNG`. +`AMBIGUOUS_FLAG_COVERAGE` records three printed ambiguities: `I_DSTSCCOMP`'s +label names `DST_SC(2)` while its universe names both routes, so its coverage of +the under-58 source codes is unresolved; `I_DSTSC`'s label uses the same +`DST_SC(2)` notation, so mapping it to both `DST_SC1` and `DST_SC2` reads the +parenthesis as a slot count that the dictionary does not state; and `I_FRMYN` +prints an empty `Values:` block, so only its `(0:9)` range header is published +and no meaning is claimed for its codes. A nonzero `I_FRMYN` alone yields +`allocation_code_meaning_unpublished`; a documented nonzero allocation flag +such as `I_ERNYN` can still establish `publisher_allocated`. All-zero readings +remain descriptions of the published codes, not assertions of nonallocation. +The original flag values and parse statuses remain unchanged. No row is ever labelled a raw +respondent value. + +## Constants + +The nine printed money entries are **not** retyped here: they are read on demand +from the packaged `asec_current_money_domains_v1.json` under its pinned digest +(`money.RESOURCE_PINS[0]`), which already attests printed position, length, +page, range, universe and values per vintage. The routing code systems +(`RECEIPT_CODES`, `ACCOUNT_CODES`, `OTHER_INCOME_CATEGORIES`, +`ALLOCATION_ENTRIES`) are not in that artifact and are defined here from the +pinned dictionary. + +The printed entry tables (`RECEIPT_ENTRIES`, `ACCOUNT_ENTRIES`, +`ALLOCATION_ENTRIES`, `OTHER_INCOME_CATEGORY_ENTRY`) are `NamedTuple`s, so every +use site and the emitted receipt reach their fields by name rather than by +position. + +Two adjacent definitions exist in tree and are deliberately **not** imported, so +this qualifier keeps the accepted source-only import neighbourhood +(`asec_coverage_authentication`, `asec_current_money`, `source_csv_builtin`, +`survey_population_preparation`, `support_provenance`) and stays free of the +modelled stage machinery: + +- `retirement_distributions.US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS` + — the same eight `DST_SC*`/`DST_VAL*` column names. +- `alimony._ASEC_ALIMONY_OTHER_INCOME_CODE` and + `alimony._ASEC_STRIKE_BENEFITS_OTHER_INCOME_CODE` — the same category codes 20 + and 12. +- `retirement_distributions._VALID_ACCOUNT_CODES` — the same 0-7 account code + domain that `ACCOUNT_CODES` gives printed labels for. + +`test_routing_code_systems_agree_with_the_existing_domain_constants` asserts +both agreements, per `docs/shared-constants.md` rule 5. If a later change makes +the modelled modules importable from a source qualifier, import those +definitions directly and drop the local copies. + +`RINT_SC1`/`RINT_SC2` reuse the identical 0:7 retirement-account enumeration for +retirement *interest*. They are out of this slice; a future consumer mapping +account codes to outputs must decide whether they share this code system. + +## Remaining work + +This lane delivers source qualification only. Still open, and explicitly not +decided here: + +1. **Canonical attachment.** Which projected columns become graph leaves, on + which clone, and under what name. These families overlap PUF-owned values, so + the unemployment/health both-clone policy cannot simply be reused. Root's + active amount owner owns `graph_us_survey_enrichment`. +2. **ACS clone0 components.** ACS publishes aggregate anchors + (`acs_retirement_income`, `acs_interest_dividend_rental_income`) rather than + these components. A conditional model plus a documented reconciliation rule + is needed, and the existing interest/dividend draws and ACS `INTP` overlap + mean rent cannot be added independently without checking that aggregate. +3. **Tax composition.** The pension private/taxable split and the distribution + taxable fraction remain unobserved. Any split must be approved explicitly + against the source, not inherited from the archived assumptions. +4. **Net property decomposition.** The receipt/amount scope mismatch must be + resolved before rent, royalties, roomers/boarders and estate/trust income are + modelled as separate components. +5. **Other-income residual.** Categories other than 20 are preserved but + unrouted; a residual-to-miscellaneous rule is a separate, reviewed decision. diff --git a/docs/us-current-asec-interest-source.md b/docs/us-current-asec-interest-source.md new file mode 100644 index 000000000..d4cbfbfd8 --- /dev/null +++ b/docs/us-current-asec-interest-source.md @@ -0,0 +1,91 @@ +# Current ASEC interest observations + +`current_asec_interest_source.qualify_current_asec_interest(preparation)` +qualifies ordinary and retirement-account interest from the original 2025 ASEC +person member, referring to income in 2024. It preserves the already qualified +`INT_VAL` combined total and exposes the published components and their +reporting, account, allocation and disclosure metadata. + +The source is the [2025 ASEC public-use data dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA-256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The exact total-income domain is reused from the pinned +`asec_current_money_domains_v1.json`; the additional fields use this same +dictionary. Pages below are one-based PDF pages. + +| Fields | Meaning and printed universe | Page | +| --- | --- | --- | +| `INT_VAL`, `INT_YN` | Combined interest; amount universe `INT_YN = 1`, receipt universe age 15+ | 46 | +| `TRDINT_VAL` | Interest excluding retirement-account interest; `INT_YN = 1` | 50 | +| `RINT_YN` | Retirement-account interest receipt; age 15+ | 49 | +| `RINT_SC1`, `RINT_SC2` | Account identities, including 401k, 403b, Roth IRA, Regular IRA, KEOGH, SEP and other; `RINT_YN = 1` | 49 | +| `RINT_VAL1`, `RINT_VAL2` | Interest in the two reported account slots; corresponding account code greater than zero | 49 | +| `I_INTVAL`, `I_INTYN` | Combined-interest allocation flags, with distinct composite code systems | 56 | +| `I_RINTSC`, `I_RINTVAL1`, `I_RINTVAL2`, `I_RINTYN` | Retirement-account allocation flags; `I_RINTSC` names slot 1 only | 57 | +| `TRINT_VAL1`, `TRINT_VAL2`, `TTRDINT_VAL` | Published top-code indicators for the respective positive component amounts | 60 | + +The account, receipt and applicable allocation code systems reuse the income +routing qualifier. `I_INTVAL` accepts the printed codes 0 and 11–15; its header +interval does not make codes 1–10 valid. There is no individually published +allocation flag for `TRDINT_VAL` or `RINT_SC2` in this dictionary. Allocation +labels describe publisher processing, not observation validity or tax status. + +## Amounts and knownness + +The result keeps raw source literals, parsed published amounts, parse status, +reporting status and canonical amount knownness separately. Missing and +malformed components remain unknown. Interest amounts must be nonnegative +integers within each field's published encoding; these disclosure ranges are +not latent-income bounds. + +`TRDINT_VAL` describes its range as a dollar value, so a published zero with +combined-interest receipt `yes` is an observed zero ordinary component. +`INT_VAL` and `RINT_VAL1/2` describe zero as none or NIU, so zero with receipt +`yes` remains ambiguous. A retirement-interest recipient with account code 0 +and slot amount 0 has an `unreported_account_slot`, not an imputed account +amount. Receipt `no` plus account code 0 and amount 0 establishes known +nonreceipt. Under-15 NIU values never become analytical zeros. + +The diagnostic `combined_minus_published_components` subtracts the three +parsed published component cells from `INT_VAL`. Its knownness means that all +four published numbers are readable; it does not establish that all three +components are reported observations. `all_component_amounts_observed` +separately records that stronger condition. The diagnostic may be negative; +the qualifier never balances, clips or replaces any published number. + +## Source lifetime and composition + +Qualification borrows the actual `AuthenticatedSurveyPopulationPreparation` +and its retained native ASEC/current-money owners. It matches the original +member's archive digest, member digest, byte count and row count, captures that +member once, and verifies the captured file after reading. Current-cohort +`PERIDNUM` keys must form an exact roster; household coordinates, person line +numbers and raw ages must match the retained parent. Source row order is +irrelevant. The original `INT_VAL` validity and float64 bits must agree with +the current-money owner before selected ASEC people are projected. + +`CurrentAsecInterestValues(person, asec_literals, evidence)` is descriptive +transport. Its seal covers both complete tables, axes, nullable masks and +backing storage, exact float bits and evidence. The qualifier rechecks the +actual preparation after its last source I/O and checks the complete value +seal before returning. A consumer must retain the actual preparation, +requalify it immediately before consumption and after its own last relevant +I/O, and compare the full value seal before returning or exporting a +successor. A copied dataclass, evidence dictionary or matching digest cannot +grant source authority. + +The only existing helper change is an optional explicit column/amount-grammar +roster on the private income-routing literal reader. Its original defaults +remain in effect for the income-routing qualifier. This module adds no host, +run issuer, graph execution or model fitting. + +## Bridge work still required + +[ACS response guidance](https://www.census.gov/programs-surveys/acs/respond/get-help.html) +includes interest credited to retirement accounts in its broad +interest/dividend/property item. The live guide currently refers to the 2025 +questionnaire; it is not an archived 2024 instruction artifact. This +ASEC qualification therefore keeps ordinary interest and retirement-account +interest separate for a later, reviewed measurement bridge. It does not +declare either component taxable, tax-exempt, regularly withdrawn, or equal +to an ACS component. It does not alter tax leaves, original or PUF clones, +financial imputation, calibration or release eligibility. diff --git a/docs/us-current-asec-property-basis.md b/docs/us-current-asec-property-basis.md new file mode 100644 index 000000000..40a4be5ea --- /dev/null +++ b/docs/us-current-asec-property-basis.md @@ -0,0 +1,132 @@ +# ASEC property donor basis + +`build_asec_property_basis` is a pure operation over the selected, qualified ASEC +person descriptions. It does not qualify a source, issue a handle, read files, +fit a model or assign tax leaves. The owning country host retains the actual +preparation and source owners, checks them before consumption and requalifies +and checks their complete seals after its last relevant I/O before return or +export. Reconstructing the descriptive input or output dataclasses grants no +source authority. + +## Inputs and outputs + +The keyword-only inputs are the `person` tables from the current interest, +income-routing and dividend/survivor qualifiers, named `interest`, +`income_routing` and `dividend`. Every table must have the same exact ordered +`person_id` index, matching unique `native_person_id` values and matching source +ages. Both ID axes are int64. Source ages may use different numeric storage +but must be finite, integer-valued ages 0–99 and equal without tolerance. No +sorting or inner join repairs a mismatch. A common row permutation is valid; +permuting one source alone refuses. + +`original_household_membership` is an int64 Series on that exact person index. +`original_household_design_weights` is a finite, nonnegative Series indexed by +unique int64 `household_id`, with exactly the membership's household set. +Extra, missing or duplicated household keys refuse. These inputs must be the +original household design mapping, before importance allocation or cloning; +this pure operation cannot authenticate that origin. Zero-weight persons are +retained. No engine or microsimulation aggregation occurs. + +The descriptive `AsecPropertyBasis` contains: + +- `person`: the component values, reported total, discrepancies, named retirement + derivations, original household mapping and the two eligibility masks. +- `provenance`: detached copies of all three supplied qualified tables, with + `interest.`, `income_routing.` and `dividend.` prefixes. Published amounts, + unknownness, allocation and disclosure indicators are preserved independently. +- `exclusions`: independent reasons for exclusion on the complete person axis. + A person can have multiple reasons; reason totals must not be summed as a + disjoint partition. +- `summary`: counts and original design-weight mass for the whole cohort, each + eligibility mask, the joint-fit complement and each exclusion reason. + +The shared names live in `property_income_constants.py`. `PROPERTY_COMPONENTS` +is, in order, `property_ordinary_interest`, `property_retirement_interest`, +`property_dividends`, `property_broad_receipts`. `PROPERTY_REPORTED_TOTAL` names +`property_reported_total`. They are donor observations or named routing +contributions, never already-reconciled model draws. + +## Measurement bridge + +Ordinary interest uses qualified `TRDINT_VAL`; dividends use qualified +`DIV_VAL`; broad signed property receipts use qualified `RNT_VAL`. The latter +is not a pure rental-income tax leaf. Dividends need a qualified yes/positive +or no/zero receipt. A receipt-yes zero for dividends or signed property stays +unknown under the published none-or-NIU coding. + +Retirement-account interest uses the two qualified account slots. Known no +with both slots NIU and zero supplies the already-qualified zero contributions. +Known yes requires at least one declared account with a known positive amount. +Every other slot must be another known positive account or an explicitly unused +slot: readable account code 0, published amount 0, in-range amount literal and +`unreported_account_slot` source status. The latter contributes a **derived** +zero and gets its own boolean, slot count and derivation label. Its original +canonical amount stays unknown in provenance. An amount zero by itself, +a declared account with unknown/zero amount, yes without any active account, +a missing code, or contradictory receipt/account evidence does not resolve the +retirement total. The derivation applies only when the complete route resolves. + +The reported donor total is separately retained as `INT_VAL + DIV_VAL + RNT_VAL`. +The component sum uses ordinary plus retirement-account interest, dividends and +broad property receipts. Both `interest_component_discrepancy` and +`reported_minus_component_total` remain visible, with signed magnitudes. No +balancing rewrites an ASEC observation. Allocation never filters the donor fit. +Disclosure flags do not prove the cause of a discrepancy. + +Other-income clearance requires known nonreceipt with a NIU category, or a +readable reported category outside property codes 5–8 and unspecified code 19. +A reported outside category resolves its route even if its own dollar amount +is ambiguous; no other-income amount is included in this basis. Missing, +unreadable or contradictory routing stays unresolved. The dividend qualifier's +nullable survivor clearance is checked against its readable codes and retained +as `survivor_visible_routes_clear`. It covers only `SUR_SC1` and `SUR_SC2`. +The 2025 dictionary describes `SRVS_VAL` as including edited sources 1/2 plus +unedited sources 3/4 (PDF page 49, printed page 6C-28); those additional source +types are not qualified here. A positive survivor receipt therefore cannot +establish complete absence of estate/trust income even when both visible +slots are outside codes 8 and 10. + +The first bridge accepts only known survivor nonreceipt with readable NIU +slots, exposed as `survivor_full_scope_clear`. Positive receipts have a separate +`survivor_additional_sources_unresolved` exclusion and weighted coverage row. +Possible visible property overlaps and unknown visible routes retain their own +exclusion reasons. Overlapping exclusion rows must not be added together. +Neither other income nor survivor amounts enter the component or reported sum. + +`reported_total_eligible` requires age 15+, a known finite reported total, and +clear other-income routes and complete survivor scope. It is a diagnostic sample, not a second +aggregate model. `joint_component_fit_eligible` additionally requires all four +components and an exactly zero interest discrepancy. Missing or under-15 values +are never turned into analytic zeros. Negative and zero net totals remain valid: +`(100, 20, 0, -120)` has total zero and keeps both positive interest components. + +The summary reports `design_weighted_person_mass` (one mapped original household +design weight per selected person) separately from `union_household_design_mass` +(each household with at least one selected person counted once). Overlapping +selection groups can share household mass. Stable finite summation avoids +row-order-dependent loss of small weights. These are coverage diagnostics, not +calibrated population claims. Conditional donor support, bias from exclusions +and held-out model quality still need the model owner's diagnostics. + +## Source and acceptance boundaries + +The source contracts use the pinned +[2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +(SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`). +The bridge targets the broad property concept in the +[2024 ACS questionnaire](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18). +The live ACS guide follows the 2025 form; its retirement-interest wording and +the historical 2016 Census methods discussion corroborate the concept without +establishing vintage or respondent identity. ASEC calendar-year and ACS +rolling-year measurement remain different. Broad property routing and +retirement-account earnings remain distinct from taxability. + +The bounded tests use invented tables and a composition test through the real +pure source projectors using invented literals. They cover both identity axes, +permutations, missing/contradictory routes, unknown account slots, signed and +zero-net examples, separate masks, allocation preservation, weight mapping, +empty/zero-weight records and refusal of malformed/nonfinite descriptions. +They do not run source qualification, native data, the country model, fitting, +cloning or replay. The later graph owns joint draws and signed reconciliation; +the later host must preserve original ASEC observations and share survey draws +across original/PUF clone pairs before the PUF alternative takes ownership. diff --git a/docs/us-current-asec-retirement-basis.md b/docs/us-current-asec-retirement-basis.md new file mode 100644 index 000000000..890b15df8 --- /dev/null +++ b/docs/us-current-asec-retirement-basis.md @@ -0,0 +1,164 @@ +# ASEC retirement candidate ledger + +`current_asec_retirement_basis.py` describes potential retirement-bridge support +from the accepted retirement-detail and income-routing projections. It preserves +source observations, route uncertainty and published accounting differences. +It produces no fiscal inputs, fitted model, graph host or source authority. + +## API + +`build_asec_retirement_basis` accepts keyword arguments `retirement_detail`, +`income_routing`, `original_household_membership`, +`original_household_design_weights`, and `assumptions`. The two DataFrames must +have identical ordered int64 `person_id` indices, exact native person IDs, and +original interview ages. Shared amount observations and pension receipt codes +must agree. A joint permutation is accepted; a one-sided permutation refuses. +The current contract requires at least one selected person. + +Membership must cover exactly those selected person IDs in the same order. +The weight Series must have a unique int64 `household_id` index containing +exactly the households represented by that membership, with finite nonnegative +values. Extra unselected household keys refuse, including keys from a larger +original survey. The caller must select the exact original household weights +without replacing them with clone, importance, or calibrated weights. Original +weight custody remains external; a Series does not authenticate DESIGN status. + +It returns `AsecRetirementBasis` with six descriptive fields: + +- `person`: source identifiers, observed routed subtotals, candidate family + intervals, accounting differences and candidate-support flags. +- `slots`: ten records per person, retaining six pension/disability/survivor + slots and four applicable/off-route distribution slots, source codes, + published amounts, knownness, source statuses and bridge routes. +- `provenance`: detached, prefixed copies of both complete supplied projections, + including allocation and disclosure flags. +- `exclusions`: uncertainty and route diagnostic masks, including cases that + remain numerically usable. These masks are not all applied as donor filters. +- `summary`: counts and supplied original DESIGN-weight support for each + candidate tier and diagnostic mask. +- `assumptions_payload`: canonical immutable bytes identifying the choices. + +The source projection dataclasses and these outputs contain mutable descriptive +tables. Neither their type nor a digest grants source admission. A future host +must retain and requalify the actual original source owners, complete person +scope, table seals and original DESIGN weights around its relevant I/O. This +pure function does not write to either input, calibrate weights or attach clones. + +## Required choices + +Every `RetirementCandidateAssumptions` field must be supplied. No country caller +or development scenario is enabled by this change. + +| Field | Supported values | +| --- | --- | +| `pension_annuity_regularity` | `unresolved`, `assume_regular` | +| `disability_pension_eligibility` | `unresolved`, `assume_qualifying` | +| `survivor_annuity_overlap` | `unresolved` | +| `withdrawal_regularity_netting` | `unresolved` | +| `aggregate_accounting` | `exact_visible_balance_only` | + +`assume_regular` is a measurement scenario about pension/annuity regularity. +`assume_qualifying` explicitly assumes both the disability-pension scope and +regularity needed for the candidate bridge. Neither is inferred from an annual +amount, age, `DIS_HP`, or `DIS_CS`. The latter source answers remain provenance. +There are no private-pension, regular-withdrawal or taxable fractions. Changing +either supported scenario changes `to_bytes()`; unsupported choices refuse. + +## Routes and candidate amounts + +The accepted [retirement-detail source](us-current-asec-retirement-detail-source.md) +and [income-routing source](us-current-asec-income-routing-source.md) retain the +published codebooks. This ledger imports those definitions and applies a separate +bridge route classification: + +| Family | Candidate routes | Separate or unresolved routes | +| --- | --- | --- | +| Pension | Codes 1–6 | 7 Railroad; 8 unresolved | +| Disability | Codes 2–5, subject to the explicit scenario | 6 Railroad; 1/7/8/9 other compensation; 10 unresolved | +| Survivor | Codes 1–4 are visible candidate subtotals | 5 Railroad; 8 property; 9 annuity overlap; 6/7/10 unresolved | +| Distribution | Applicable account codes 1–7 retained individually | Regularity/netting remain unknown; account 4 is a type of IRA | +| Other income | 2/13 indicate additional retirement candidates | 1 Social Security overlap; 8 property overlap; 19 unspecified | + +The retirement ledger never consumes account-earnings amounts or adds Railroad +amounts to Social Security. Other-income amounts are retained separately and +never added to the candidate sum. Categories 2/13/19 leave combined candidate +scope unresolved; known other categories can remain outside that sum. These +routes do not establish any person's tax treatment. + +A source-known amount and a published literal remain separate. A declared +positive slot contributes to its observed routed subtotal. A readable unused +NIU slot can contribute an explicitly labeled structural zero to a numerical +comparison, but its source-known amount remains missing. The subtotal is only +a sum of the readable known slots; it never asserts that unknown slots are zero. +Yes-receipt dollar zeros stay ambiguous where the source domain says none/NIU. +Annuity preserves its distinct valid dollar zero and published `-1` NIU encoding. + +The four published total-minus-slot comparisons are recomputed and checked +against R1. `DBTN_VAL` compares the main distribution slots only; the applicable +known distribution total is supplied independently by the routing owner. No +comparison is allocated to another source or attributed causally to disclosure. +For age 58 and over, the main slots apply and this exact comparison must be +finite and zero before a distribution candidate interval is available. A +positive residual leaves additional scope unresolved; a negative residual is +contradictory accounting. Both retain the published DBTN total, applicable slot +total and signed difference without a cap, clipping or redistribution. The main +slot comparison does not constrain the separate applicable young slots below 58. + +For pension/disability candidate bounds, all relevant slots must be readable and +receipt-consistent, and their exact integer-dollar sum must equal the aggregate. +Positive discrepancy leaves additional scope unresolved; negative discrepancy +is contradictory accounting. Neither is clipped, used as a cap, or silently +allocated. Readable outside routes are subtracted only under this exact balance. +With unresolved regularity/scope the lower bound is zero; under the named +scenario it includes only known candidate-route amounts. Unresolved source +routes can widen the candidate interval without becoming identified labels. + +Positive survivor receipt keeps full survivor bounds unknown, even when its +published comparison balances. Visible slots do not prove the scope of sources +3/4. Only qualified no receipt with two NIU slots and zero total supplies a +zero survivor family amount. Source 9 alongside positive annuity is explicitly +flagged; no overlap priority or max/sum assumption is applied. + +A fully resolved applicable distribution composition produces a zero-to-total +candidate interval only when the applicable aggregate check also passes. +Missing/unreadable account types, ambiguous active zeros or off-route +answers/dollars prevent that interval. Off-route receipt and account literals +must be readable NIU codes, and off-route amounts must be readable zeros; +unreadable literals are uncertainty, not evidence of absence. The routing owner's +known total is still preserved when the stricter candidate test fails. + +Routed subtotals and distribution account subtotals are evidence only, not +admitted candidate amounts. In particular, account subtotals can be finite when +all account types/amounts are readable but the applicable receipt is unreadable +or the DBTN comparison fails. `distribution_account_composition_known` describes +that account evidence; it does not establish a valid receipt, aggregate balance, +regularity, netting, or an available interval. Consumers must use the candidate +interval availability and family statuses rather than infer admission from a +finite subtotal. Raw observations remain in the detached provenance regardless. + +Under-15 people retain their source evidence with outside, unresolved-outside or +contradictory-outside status and no candidate zeros. Combined bounds are available +only when every family and other-income scope passes. `candidate_basis_eligible` +means this conservative numerical support exists. `candidate_point_under_assumptions` +means the candidate bounds coincide; `point_identified` and +`fiscal_outputs_produced` remain false. No training label is created merely by +calling a quantity an interval endpoint. + +## Source and validation boundaries + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +provides the source categories and domains. The [2024 ACS subject definitions, +page 95](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf#page=95) +provide the target's regularity and disability scope. The ledger's conservative +accounting and scenario choices are separate decisions. It does not equate the +surveys' reference periods, allocate joint income, or derive source regularity +or tax treatment from those documents. + +Tests use the actual pure R1/routing projectors on invented records. They check +source routes, exact accounting, unknowns, assumption identities, interview-age +routes, native IDs above `2**53`, one-sided mutations, permutations, detached +provenance and descriptive design-weight support. Person mass uses the supplied +original household weight for each included person; household mass counts each +included household once. The Series has no independent weight-kind authority. +Allocation/disclosure flags do not filter candidates. No native data, source +capture, engine, fitting, calibration, PUF or release acceptance is involved. diff --git a/docs/us-current-asec-retirement-detail-source.md b/docs/us-current-asec-retirement-detail-source.md new file mode 100644 index 000000000..e125201e4 --- /dev/null +++ b/docs/us-current-asec-retirement-detail-source.md @@ -0,0 +1,108 @@ +# Current ASEC retirement details + +`current_asec_retirement_detail_source` qualifies pension, disability and +survivor source details from the current ASEC member owned by an authenticated +survey preparation. This is source evidence for a later ACS retirement bridge. +It produces no model, tax leaf, payment-frequency classification or new issuer. + +## Source and field ownership + +The source is the [2025 ASEC public dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`, +for calendar-year 2024 income. Page references below are physical PDF pages; +the corresponding printed pages are 6C-23 through 6C-39. + +| Amount fields | Qualification | +| --- | --- | +| `PNSN_VAL`, `ANN_VAL`, `DST_VAL1/2`, `DST_VAL1_YNG/2_YNG` | Reuse immutable printed domains from the existing income-routing owner. Compare exact retained current-money values and validity. They remain reference observations here; their canonical receipt/account interpretation stays with that owner. | +| `DIS_VAL1/2` | Explicit dictionary domains on page 44, also compared against actual retained current-money values and validity. | +| `PEN_VAL1/2`, `SUR_VAL1/2`, `DSAB_VAL`, `DBTN_VAL`, `SRVS_VAL` | Independently qualify literals from the exact authenticated source member using published width, range, universe and zero semantics. These seven fields are absent from the current retained-money roster; the qualifier never fabricates a `ready.field()` for them. | + +The live money domains must agree with the eight existing fields. A future +money roster containing one of the seven independent fields fails until the +corresponding retained comparison is reviewed. The amount-entry cache contains +immutable entries; each public call returns a detached mapping. + +The complete source join verifies native person keys, household IDs, person +line numbers and source ages. Selected original ASEC persons retain their +actual composed person IDs. No clone or PUF population is modified. The source +literal table covers the complete current source roster; the person table +covers selected ASEC persons. + +## Knownness and comparisons + +`PEN_YN`, `DIS_YN` and `SUR_YN` are separate from their two source codes and +amounts (pages 44–45, 47 and 49–50). A positive readable amount is canonical +only with a valid positive source code and yes receipt. A no receipt, NIU source +slot and zero amount yields explicitly derived known nonreceipt. A yes receipt +with a declared source and zero amount stays ambiguous. A yes receipt with a +zero source slot and amount stays unreported, not an observed analytical zero. +Missing, malformed, out-of-range and contradictory values stay distinct. + +People under 15 never acquire analytical zeros. Unreadable outside-universe +answers stay unresolved; readable nonzero outside-universe evidence is flagged +as contradictory. `DIS_CS` and `DIS_HP` are preserved source answers. The latter +includes limited work as well as prevented work and does not establish the +ACS disability-pension criterion. + +Four raw numerical comparisons retain total-minus-slot differences: + +- `PNSN_VAL - PEN_VAL1 - PEN_VAL2`; +- `DSAB_VAL - DIS_VAL1 - DIS_VAL2`; +- `SRVS_VAL - SUR_VAL1 - SUR_VAL2`; +- `DBTN_VAL - DST_VAL1 - DST_VAL2`. + +These comparisons include readable published none/NIU zeros; they are not +proof of analytical completeness. Missing or malformed components produce an +unknown comparison. No difference is allocated to a pension, taxable income +or missing source. `DBTN_VAL` is retirement distributions, not disability, and +its printed formula uses the main slots rather than young-person slots. + +`SRVS_VAL` includes edited sources 1/2 and unedited sources 3/4 (page 49). +Its visible pair is therefore never declared exhaustive. Source labels retain +Railroad Retirement, estate/trust, annuity and unspecified categories without +assigning them to an ACS aggregate. In particular, positive survivor receipt +with two visible non-property routes does not prove the absence of additional +estate/trust income. + +Allocation and top-code flags preserve their own published domains. Source +allocation flags with values 0/1/9 do not acquire all intermediate header-range +values. `I_SURVL2` retains its printed `SURV_VAL2` universe spelling. Allocation +does not invalidate an otherwise observed amount; a disclosure flag does not +explain a particular component discrepancy by itself. + +## Why the ACS bridge remains separate + +The [2024 ACS questionnaire, question 43g](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +asks about regular account withdrawals, while [ASEC Q98Ar](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf#page=205) +asks about any withdrawal/distribution. “Regular IRA” is an account label, not +payment frequency. Account type alone does not establish taxability. The +qualifier assigns neither regularity nor tax treatment and makes no ACS +retirement-component allocation. + +## API and validation boundary + +`project_retirement_detail_literals(DataFrame)` is a descriptive pure +projection. `qualify_current_asec_retirement_detail(preparation)` requires the +actual preparation and original source ownership, captures the pinned member, +checks source identity and retained money, and requalifies the parent after +the last source I/O. Its last checks include actual owner identity and a full +physical result seal. + +The retained-money comparison follows the existing owner's annuity encoding: +published `ANN_VAL=-1` matches normalized positive zero with `DECLARED_NIU`, +while the descriptive literal and published amount remain `-1`. A dollar-zero +literal cannot substitute for that NIU/status pair. Other valid dollar amounts +retain exact float64 comparison. This distinction is tested through actual +invented source preparation, as well as direct mismatched-status controls. + +`retirement_detail_values_seal(values)` covers table axes, dtypes, exact values, +nullable Float64 backing storage and masks, literals and detached evidence. +The returned frozen dataclass contains mutable descriptive tables and grants no +source admission. A consuming host retains/revalidates the actual parent and +the result at its own boundaries; a copied dataclass is not an issuer. + +Tests use only invented records and the real source-preparation path. They +cover source joins, retained versus independent fields, member-byte drift, +late parent mutation and result mutation beneath null masks. No native source +or country engine is needed, and passing tests do not certify a release. diff --git a/docs/us-current-property-income-sources.md b/docs/us-current-property-income-sources.md new file mode 100644 index 000000000..b722a1218 --- /dev/null +++ b/docs/us-current-property-income-sources.md @@ -0,0 +1,109 @@ +# Original property-income source composition + +`qualify_current_property_income_sources` composes the source branches for the +property-income model. It borrows the existing shared-predictor preparation, +qualifies the original ACS anchors and ASEC interest, income-routing and +dividend/survivor observations, and applies the reviewed pure ASEC donor basis. +It fits no model, supplies no tax split and performs no clone attachment. + +The arguments match the existing shared predictor boundary: the actual +preparation, allocated population and clone population, with optional shared +demographic conditioning and geography configuration. The allocated/clone +objects satisfy that existing ownership contract. Their weights never supply +the donor weights. All donor membership and DESIGN weights come from the +original source Frame selected by the shared qualifier. + +The result is descriptive `QualifiedPropertyIncomeSources`. Its source values, +source Frame and full origin document are retained for inspection and later +source/clone joins. The object has no issuer registry and cannot replace the +actual retained preparation. The complete descriptive seal is a mutation +check, not a credential. Downstream owners must qualify before consumption +and after their last relevant I/O before returning or exporting a successor. + +## Selected branches + +`donor_frame` is the original ASEC DESIGN Frame restricted to jointly eligible +persons, with referenced groups pruned by the existing Frame selector. +`donor_columns` contains the qualified shared features, reported property total +feature and four original components. It uses the same exact ordered person +IDs as that branch. The four component columns and reported total import their +canonical names from `property_income_constants.py`. + +`recipient_frame` selects original ACS persons with a finite, known adult INTP +anchor. `recipient_columns` contains the same shared features plus the qualified +ADJINC-adjusted anchor under the reported-total feature name. The existing +recipient matrix codec preserves their ordered identities and float64 bits. +`recipient_matrix` is the encoded matrix. Source AGEP and canonical shared +feature age remain distinguishable; no raw age is overwritten by the feature. +Shared earnings features preserve the existing qualifier's declared NIU operator; +this composition adds no new zero completion for earnings or property anchors. + +Adding these model columns to a graph population is a later visible graph +operation. The selected Frames themselves keep all original source columns, +weights, memberships, strata and metadata. The source composition does not +quietly change a Frame's tax leaves or model columns. + +The complete original selected source axis is never reduced in `origins` or +`source_frame`. `origin_document` preserves all original entity origin records; +`donor_basis` contains every original ASEC row's component knownness, provenance, +exclusions and design-weight summaries. `recipient_diagnostics` preserves every +selected ACS row's raw literals and parsed anchor metadata, plus separate +under-15 and unknown-anchor exclusions and the eligible-recipient mask. + +An empty eligible branch has an empty indexed columns table and `None` for +its Frame, and no recipient matrix when no ACS recipient is eligible. Complete +diagnostics remain available. The existing Frame selector and matrix codec +require nonempty inputs; the composition does not manufacture a record or zero +to satisfy them. A later model host must refuse fitting an empty branch or make +an explicitly reviewed completion decision. + +## Exact identity and lifetime + +The complete shared source Frame, its origins and the original preparation's +person origin records must agree in order, source channel and native identity. +Each ASEC qualifier must exactly match the selected ASEC person and native-ID +axes; the pure basis also checks their source ages. ACS anchors must match the +selected original ACS axis. Donor feature and recipient matrix axes are checked +before selection. No sorting, inner join or missing-key fill hides a mismatch. + +The donor household design weights are mapped by exact original membership. +Their person mapping must match the original Frame's resolved DESIGN weights; +importance or clone weights are never substituted. This is source construction, +not engine microsimulation aggregation. + +Each initial qualifier's complete physical description is sealed as soon as it +returns, and checked while subsequent qualifiers do I/O. After pure composition, +all five original source qualifications are repeated and compared to those +seals. Every returned value, including nullable data beneath missing masks, +is checked throughout. The actual preparation is then rechecked after the last +qualifier's I/O. The final checks are pure: retained issuer identity, nested +original source Frames, allocated/clone support, the exact optional geography +configuration payload and all complete derived seals. + +The income-routing qualifier has no standalone physical-seal API, so this +composition uses the existing full table-stamp helper over its complete person +and original-literal tables plus evidence. Existing shared/ACS/interest/dividend +seal APIs are reused directly. No source-reader implementation or new authority +framework is introduced. + +## Verification scope + +The invented fixture creates closed source bytes before issuance, then executes +the actual survey source graph and source composition. It includes a negative +ACS anchor, malformed adult anchor, under-15 blank anchor, a resolved ASEC +ordinary/retirement-interest route, and excluded ASEC overlap/unknown cases. +Controls check ordered axes, weight kind, adjusted bits, all returned seal +surfaces and copied preparation refusal. A separate injected mutation isolates +the final original-source I/O fence using previously checked descriptive values; +it does not claim another complete source replay. + +These checks run without native microdata, QRF, PUF or a country engine. Model +quality, donor conditional support, country financial integration, both-clone +fanout, replay and release acceptance remain later boundaries. + +The initial composition acceptance used donor basis `2c959096eec0441e2acd975e63522aac7f1b7016`. +A subsequent primary-source review found that survivor slots 1/2 do not establish +complete absence of estate/trust overlap: the aggregate can include sources 3/4. +The country host must adopt the separate conservative donor-basis correction +before using this branch for fitting. The initial test's eligible donor reports +no survivor income, so its observed values do not rely on a yes-survivor clearance. diff --git a/docs/us-current-survey-amount-successor.md b/docs/us-current-survey-amount-successor.md new file mode 100644 index 000000000..250bab1ad --- /dev/null +++ b/docs/us-current-survey-amount-successor.md @@ -0,0 +1,196 @@ +# Current survey amount successor + +Implementation plan and source inventory, 12 September 2026. This is a local +successor to the retained survey PUF55 run, not an amendment to the frozen d35 +native pilot. The branch starts at reviewed integration 99543c3. + +## Immediate scope + +| Native ASEC field | Canonical person input | Reporting evidence | +| --- | --- | --- | +| UC_VAL | unemployment_compensation | UC_YN receipt code; amount question universe UC_YN = 1 | +| PHIP_VAL | health_insurance_premiums_without_medicare_part_b | All persons | +| PMED_VAL | other_medical_expenses | All persons | +| POTC_VAL | over_the_counter_health_expenses | All persons | + +These four names are absent from both PUF tax-detail ownership rosters. They +already have qualified current-money domains and direct amount mappings in +`cps_carried_current.py`. The complete legacy carry helper is not reusable: +it also writes existing PUF leaves and a legacy Social Security decomposition. +The modern `survey_social_security` source owner remains the authority for +Social Security; no age-62 zero fallback or prior wages is introduced here. + +The current ASEC source is the authenticated original 2025 person member, +income year and price basis 2024. ACS observations use the already qualified +2024 source predictors, whose earnings refer to the rolling prior 12 months. +That temporal harmonization is a documented model choice, not equivalent +observation windows. No new donor dataset is needed. + +## Amount and reporting status are different + +The [2025 CPS ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +PDF page 50, defines UC_YN codes 0 (NIU), 1 (yes), and 2 (no), with the receipt +question asked of persons aged 15 or older. UC_VAL is asked of recipients; +its zero code conflates none with NIU. The maintained domain metadata pins +this dictionary to SHA256 +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The three health amounts have all-persons universes (PDF page 83). + +A small bounded UC_YN projection reads the same exact authenticated original +member. Native person and household/line coordinates must join the original +current-money owner exactly; carried columns and detached JSON do not confer +source authority. The projection retains the original amount, amount validity, +source classification, zero_origin, literal receipt code and reporting status. +Today's upstream preparation requires complete qualified current money before +issuing its handle. Literal missing-amount controls test the join's unknown +semantics; they do not claim that a missing amount passes this unchanged parent +prerequisite. Reporting unknownness can still occur with numerically valid amounts. + +UC attachment and donor policy: + +- A valid positive amount with an in-universe yes code is known receipt. +- A zero amount with an in-universe no code is known nonreceipt. +- Yes with zero is an ambiguous amount, not known nonreceipt. It remains in + the source projection but is excluded from fit and canonical attachment. +- NIU, blank/unrecognized codes, missing amounts, and contradictory amount/ + receipt pairs remain unresolved and are excluded from fit and attachment. +- Out-of-universe age does not infer a zero. ACS persons outside the receipt + question universe also retain unknown UC rather than receiving an adult draw. + +The health fields retain qualified valid all-persons amounts, including genuine +zero-dollar codes. No noanswer cell is converted to zero. Allocation and source +status remain evidence; known source values are not claimed to be unallocated. +PHIP_VAL is deliberately retained as the declared premium source, rather than +silently substituting PHIP_VAL2, whose treatment of inconsistent reported zero +premiums differs. The [Census health insurance user note](https://www.census.gov/programs-surveys/cps/technical-documentation/user-notes/health-insurance-user-notes/User-Notes-Selected-CPS-ASEC.html) +describes that distinction. Person-level conditional fitting preserves modeled +cross-component dependence; it does not certify household or insurance-unit +correlation, which still needs empirical validation. + +## One family, explicit graph operations + +The family has one parameterized source projection and attachment mechanism. +Each selected field declares its source transformation, donor eligibility, +recipient matrix, QRF fit and draw as graph nodes/artifacts. UC has its own +reporting mask. The health group uses an ordered PHIP → PMED → POTC conditional +chain, retaining observed health-component dependence through prior-target +conditioning instead of fitting the three amounts independently. This avoids +filling unknown UC to make a joint matrix finite. Fits use the existing regime-gated QRF implementation and +current source predictor qualification; donor weights remain original design +weights before allocation or cloning. No PUF value becomes an ASEC donor. + +Draws are keyed to the original ACS source person, then deliberately shared by +its original and PUF support clones. This preserves source observations and +paired support interpretation; clone labels are checked during attachment. +Source amount values and reporting status are copied to both ASEC clones. +Every existing parent column, owner, axis, membership, atomic geography, +weight, design anchor and mass ledger must remain identical. + +The parent is the exact `SurveyPuf55Run` checked by +`check_survey_puf55_run`. The original handle is retained and checked before +consumption and after source/store/export I/O. A checked JSON payload or copied +dataclass cannot issue authority. The extension copies kernel/source-codec +registries privately, preserves all prefix declarations/keys/artifacts, and +verifies actual observed populations on cold and required replay. Failed checks +revoke the original handle. The checked-output branch supplies this additive +API; the frozen native packet is left untouched. + +## Verification + +First run cheap invented controls: all UC code/amount/universe combinations, +health zero and unknown behavior, literal bounds and source-coordinate joins, +clone pairing, missing/duplicate source IDs, invalid source/clone channels, +ownership collisions and raw/status/zero-origin mutation. + +Then an actual invented authenticated-source fixture runs the family through +real graph cold and required replay, QRF train/apply, coverage, complete Frame +store/export/readback, parent retention and unchanged prefix seals. Rejection +controls mutate source projection, parent handle/population, declared graph, +artifact bytes, fit/draw lineage and output attachment independently. Acceptance +requires closed receipts and independent review. No native run, engine outcome, +held-out quality, calibration or release certification is claimed by these tests. + +## Remaining input inventory + +The accepted invented PUF baseline has 161 required names, of which 100 are +absent. Its 61 present names are not necessarily complete: some PUF leaves are +populated only on PUF clones. Counts are inventory for that fixture, not a native +coverage measurement. The next stage reduces absent names while reporting +unresolved cells separately. + +| Input family | Existing source producer or route | Integration requirement | +| --- | --- | --- | +| UC and three health amounts | authenticated current-money owner; direct `cps_carried_current` maps | This stage | +| Social Security components | `current_social_security_source`, `survey_social_security` | Preserve source reporting basis and unknown components; never legacy age split | +| Health coverage | Authenticated `current_survey_health_source` and explicit `graph_current_survey_health` fragment | This stage; seven narrow ACS concepts remain unknown where the survey cannot distinguish them | +| Disability, student status, children, veteran status | `eligibility_inputs` | Qualify literals and reporting universes before reuse | +| Parent relationships | `eligibility_inputs._own_children_in_household` resolves PEPAR1/PEPAR2 then drops links | Issue 884: retain parent_1_id/parent_2_id with household/line and clone joins; unresolved and absent differ; consumer issue 9404 is separate | +| Marital roles | `relationship_inputs` | Current source observations and clone preservation | +| Hours, work history, occupation, wage/tips | `hours_worked`, `org_wages`, `weeks_unemployed`, `sipp_tips` | Source/QRF qualification and explicit ownership; prior wages excluded | +| Child support, disability income, workers compensation | respective maintained runtime modules | Current source amount/universe qualification, then existing fit contracts | +| Housing, energy, care, education | `housing`, `energy_subsidy`, `adultcare`, education producers | Household/SPM source and clone-keyed joins; reconcile PUF tuition ownership | +| Financial wealth, vehicles, loans | SCF/SIPP producer modules | Actual donor authentication, existing fit dependencies and held-out quality | +| Retirement distributions/contributions | maintained distribution/contribution modules | Reconcile already PUF-owned taxable IRA/pension and self-employed pension leaves | +| Pregnancy, SSI disability, voluntary filing | maintained producer modules | Resolve source authority and distinct behavioral assumptions | +| Take-up | `take_up_contract` | Unsourced rates remain unresolved; no convenience constant fills | +| Mortgage details, marketplace benchmark ratio, remaining retirement details | Not yet mapped in this bounded inventory | Identify actual producer/owner or make an explicit source/model decision | + +The inventory used tracked files and `rg --no-ignore` within explicit source +directories because ordinary searches can skip the tracked build package. +Existing function names establish implementation leads; they do not prove that +those producers are integrated into the current graph or current native file. + + +## Fixed country composition + +`run_us_survey_enrichment` retains the original checked `SurveyPuf55Run` and +assembles the amount family and independently source-qualified health fragment. +Both attach to `survey_puf55.receiving`; the health CREATE and final attachment +explicitly consume `survey_amounts.attach/attachment`. Health adds nine coverage +inputs and their literal/status/knownness evidence after all four monetary leaves. +The host preserves the complete parent Frame, owners, design anchors and mass ledger, +compares every unchanged prefix version against the original parent manifest, and +requalifies both original-source projections before issuing its output. There is +one private issued-handle registry and one public checked country output, not a +family-level authority or executor. The health fragment's descriptive values do +not grant access to native sources. + +The combined invented fixture includes the exact source ACS HINS/flags and ASEC +NOW/flags before authenticating source owners. It runs the original financial and +PUF stages, both enrichment fragments, a required replay, input-coverage diagnosis +and complete Frame store/readback. No native build, model engine execution, +calibration, or release approval is implied by these checks. + + +## Invented acceptance, 12 September 2026 + +The final production source passed the 271-node combined graph: 245 validated +prefix hits followed by 26 newly executed enrichment nodes, then a required +replay with all 271 hits. Complete parent columns, owners, design anchors, +geography and the mass ledger were preserved. The full resulting Frame was +stored and read back with exact physical comparison. The fixture has 18 people; +required-name coverage improved from 61 to 74 of the 161-input profile, leaving +87 missing names. Presence does not imply complete values or statistical quality. + +All 18 people have the three health-cost values and an ESI observation. UC has +10 known and eight unresolved values. The narrow Medicaid-at-interview input has +eight known and ten unresolved values, preserving the ACS semantic gap. These +are counts from invented records, not estimates or native-data diagnostics. + +Acceptance combines five passing ordered tests from the third full attempt and +one corrected, focused final parent-revocation test. The third attempt retained +its failed result because that last test named a nonexistent generic income +column; it stopped before mutating anything. The final test uses the actual +`employment_income_before_lsr` input and verifies permanent parent and child +revocation after mutation, including refusal after restoring the original value. +It rebuilds a real parent and one child, without repeating the already-passed +required replay or readback. The focused attempt passed with all 857 Python +source pins stable, 624.872 seconds wall time and 885,669,888 bytes peak RSS. + +The separate 32-case cheap suite covers UC reporting classifications, literal +missingness, source/clone identity, ownership collisions and source-origin +serialization when `person_id` names both an index and a column. The serialization +binds the original int64 axis and its name separately from ordered column values. +Independent review covers the frozen UC source, health fragment, combined host +and this serialization correction. Preserved failed attempts remain part of the +local acceptance record; no native execution or release certificate is implied. diff --git a/docs/us-current-survey-property-graph.md b/docs/us-current-survey-property-graph.md new file mode 100644 index 000000000..61c0c4a45 --- /dev/null +++ b/docs/us-current-survey-property-graph.md @@ -0,0 +1,55 @@ +# Current survey property-income graph + +The opt-in `graph_current_survey_property.py` fragment adds 16 operations to the existing US financial graph. It puts qualified ASEC components and modeled ACS components on the complete initial clone pair. The country financial host retains the original source owners, verifies every graph observation and artifact, and issues the result. The fragment creates no separate issuance registry. + +`PropertyIncomeOptions(scales=..., atol=..., rtol=..., n_estimators=...)` requires all four settings. The options object is frozen and has a canonical `to_bytes()` representation. No numeric modeling default is supplied. The first three components are nonnegative; broad property receipts can be signed. A negative or zero anchor can coexist with positive components and an offsetting negative broad-property component. + +## Visible operations + +| Operation | Meaning | +| --- | --- | +| `survey_property.source_projection` | Bind the actual qualified property composition, the existing financial predictor projection/matrix, original host artifacts, and geography validation when present. Emit a projection and exact donor/recipient matrices. | +| `survey_property.asec_eligible_donor` | Filter the original CREATE by the exact eligible ASEC person IDs. Preserve original household DESIGN weights, including their person mapping. | +| `survey_property.asec_donor_columns` | Materialize shared predictor columns, reported total, and four qualified components. | +| `survey_property.acs_eligible_recipient` | Filter original CREATE to ACS persons whose source anchor is known and in the adult reporting universe. | +| `survey_property.acs_recipient_columns` | Materialize shared predictors and adjusted INTP as the reported-total feature. | +| `survey_property.fit.000`–`.003` | Fit the existing DESIGN-weighted chained QRF over the four components. | +| `survey_property.apply.000`–`.003` | Draw once per eligible original ACS person, preserving ordered prior-component conditioning. | +| `survey_property.draws` | Retain raw draws and their complete model/application history. | +| `survey_property.reconcile` | Apply the shared signed-total reconciliation rule with the explicit scales and tolerances. | +| `survey_property.attach` | Join original source identities to both initial clones and retain all existing rows, entities, weights, geography, metadata, and legacy financial leaves. | + +Both model branches leave CREATE before importance allocation. Their FILTER operations return person keep masks, preserving actual raw source columns. Shared predictor materialization, including the existing analytical treatment of ACS earnings NIU, happens through declared column operations. The FILTER does not substitute the separately prepared feature frame for CREATE. + +The projection binds complete descriptive physical seals, including unknown masks and excluded rows, not just the finite fitting matrices. It carries the donor exclusion summary and recipient exclusion counts. No excluded person silently becomes a zero-valued donor or recipient. Empty eligible donor or recipient branches are currently refused before fitting; an empty-branch execution policy is not implemented. + +## Attachment and knownness + +The attachment owns the four `property_*` components, their four `draw_property_*` columns, `property_reported_total`, reconciliation adjustments, nullable active-bound diagnostics, residual and objective, two source-basis discrepancy columns, and two knownness flags. `owned_columns()` is the authoritative roster. + +ASEC persons retain each individually qualified component and reported total, including NaN for an unresolved value. Donor exclusion does not erase otherwise known amounts. The source-basis discrepancies remain available, and the complete source qualification/exclusion descriptions stay bound to the projection. ASEC raw-draw and reconciliation diagnostics are absent because no draw or reconciliation is performed on these observations. + +For example, a qualified survivor estate/trust route can exclude an ASEC person from the joint four-component fit while that person's ordinary interest and dividends remain known. The attachment copies those independent amounts to both initial clones. The separate tax-leaf successor splits each known amount independently; the donor exclusion does not turn it into an unknown tax input. A focused regression exercises this path with authenticated invented survey files and the real attachment and split functions, without fitting a QRF or rerunning the complete financial host. + +Eligible ACS persons receive one ordered joint draw and its reconciled components. Their original `person_id` and pandas row index are checked separately, then the original support ID, native source ID, source label, and exact clone roles establish the two-row fan-out. Excluded ACS persons receive NaN for model components and the attached anchor, with false `property_anchor_known` and `property_components_known`. Their original source anchor status remains in the qualified source projection; under-15 values are not silently converted into a new analytical zero. + +`property_anchor_known` describes the attached anchor's finite value. `property_components_known` describes all four attached components being finite; it is distinct from eligibility for donor fitting. Reconciliation bound flags use nullable booleans so an unperformed reconciliation does not become a false observation. + +## Tax and period boundaries + +The eight existing financial tax leaves remain unchanged. Attachment reads all eight and explicitly follows the legacy financial attachment, checks them against its retained draw history, then adds property columns. The legacy capital-gains chain still conditions on legacy INT and DIV draws. The receipt records `tax_split_rebased: false` and the capital-gains limitation. `property_legacy_interest_draw_minus_reconciled_interest` measures the legacy ACS interest draw minus the reconciled ordinary plus retirement interest; it is a diagnostic, not a new tax assignment. + +Broad property receipts are not identified with rental income or a tax-law leaf. Retirement-account earnings are not retirement withdrawals. Tax-leaf rebasing and a CAP model conditioned on reconciled components require a separate declared successor. + +Periods remain those of the qualified sources: ASEC 2025 interview and calendar-2024 income; ACS 2024 rolling prior twelve months, adjusted by ADJINC to the declared price basis. No equivalence of the reporting windows is asserted. The person-level conditional chain is not a certification of household-level dependence or held-out fit quality. + +## Host integration and verification + +- `current_survey_property_nodes(qualified, clone_frame, host_pins=..., options=...)` declares the fragment. +- `register_property_kernels(...)` registers the country and shared numerical kernels over the retained source owners. +- `reconstruct_property_results(..., artifacts=..., legacy_matrix_producer_key=...)` returns eight expected non-fit results: projection, two filters, two model-column operations, draws, reconciliation, and attachment. The host patches these at their declared population versions and checks actual observations. The separate model-receipt verifier binds the eight fit/apply receipts to actual donor/recipient populations and stored model bytes. +- `verify_materialized_property_income(..., legacy_population=..., population=...)` requalifies actual source owners and compares the full final population against attachment applied to the independently checked legacy population. + +Reconstruction validates the complete ordered raw-draw history, training-model references, recipient index, person IDs and summaries. It invokes the existing deterministic reconciliation rule to derive the expected reconciliation columns; it never fits or draws another model. Store artifact type, producer-key and key validation remains a prerequisite owned by the host. The host must retain its existing final source, implementation, cache, full-population and issuance checks after relevant I/O. + +Changing only scales affects the reconciliation and attachment declarations, leaving source selection and QRF fitting declarations unchanged. The absence of property options in a host call preserves the existing strict legacy path. Graph replay validates cached output observations as well as labels; a cache hit is not source admission or a release verdict. diff --git a/docs/us-input-coverage-diagnostic.md b/docs/us-input-coverage-diagnostic.md new file mode 100644 index 000000000..1304d2fd0 --- /dev/null +++ b/docs/us-input-coverage-diagnostic.md @@ -0,0 +1,3 @@ +# US population input-coverage diagnostic + +[`diagnose_us_input_coverage`](../packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py) takes a `Population`, its `CompiledGraph` and attached `RunManifest`, and returns a typed `PopulationInputCoverage` report. Its default `USInputProfile.NATIONAL_CD` describes 161 national/congressional-district input names; [`required_us_inputs()`](../packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py) keeps the historical 163-name default. The national/CD profile omits only engine `block_geoid` and `tract_geoid`, while separately reporting assigned `census_block_geoid`; neither profile adds `employment_income_last_year`. The diagnostic checks complete declared current ownership and replayed storage, then reports actual entity grains, source-channel/clone groups, null and invalid values, typed weights, and masked-writer versus carried rows. Missing grain and applicability remain unresolved. It verifies consistency of supplied graph objects; source ancestry and source requalification remain the host's responsibility. [Twenty-five frozen controls](../experiments/us-input-coverage25-20260910.json) passed, including actual cold/required replay and late-mutation refusals, with 515 physical source/owned checks and zero model/resource inputs. This is scoped evidence, not maintained-CI validation: four eager calibration imports differ from the frozen run and retain their newer maintained versions. The report establishes neither complete input coverage, statistical signal, applicability, native build acceptance nor release eligibility. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md new file mode 100644 index 000000000..8914d6bea --- /dev/null +++ b/docs/us-launch-review.md @@ -0,0 +1,454 @@ +# US graph build: review guide + +This branch consolidates the US graph build work for review. The intended file +supports national and 119th congressional district analysis from one population. +It is still a development build: complete enrichment, fiscal calibration and +release verification have not passed together. + +## Architecture and review order + +The intended construction order is ACS + CPS ASEC → complete survey multispine +including its initial support/PUF clones → household Census block assignment → +derived larger geographies → donor enrichment → rules evaluation and calibration +→ verified full and pruned exports. Each resulting household retains its assigned +location through the later stages. The PUF +operator acts on the combined survey frame. A separate ASEC–PUF base is not the +intended architecture. + +National and local analysis share one fully constructed, enriched Frame. The +analysis branch begins after harmonization, cloning, atomic geography assignment +and imputation. Calibration changes weights; L0 controls sparsity. An export can +select a geographic scope and remove zero-weight households, but must preserve +every retained record's input values, knownness, entity relationships and assigned +geography. There is no separate national or local enrichment operation. A source +or mapping correction produces a new common parent Frame for both views. + +Release verification must bind each full or pruned export to that parent's +identity and its calibration target/weight specification, compare retained +inputs by stable entity ID, and reject changed inputs, dangling entity links or +regenerated geography. Weight and scope changes are explicit exceptions; record +values are not. This is the required contract, not a claim that all of these +checks have already passed on the release candidate. The separate legacy +ACS-local hours omission in [#765](https://github.com/PolicyEngine/microcosm/issues/765) +illustrates the drift this common construction path removes. + +Pre-calibration source quality and post-calibration export integrity are distinct +checks. Joint calibration may change an origin's contribution or set its weights +to zero. A pruned export must not be required to make every survey origin +independently resemble a national population, or to rerun imputation to satisfy +such a requirement. + +The supplied-parent export comparison now passes 36 focused checks, including +actual checkpoint write/readback, complete household selection, unchanged input +values and missingness, signed and unsigned stable IDs, and full/pruned/local +weight and scope handling. A household with zero original weight can receive +positive calibrated weight without mutating its parent or other entity weights. +The comparison binds the full ordered weight vector and specification, including +rows outside the selected scope. It deliberately reports calibration ancestry +and release eligibility as unverified: the owning build must establish those +and seal its inputs across export I/O. See the +[scoped export acceptance](../experiments/us-common-frame-export36-20260910.json). + +The earlier ten-node survey prefix assigns a block before cloning, and its +thirteen-node age-calibration extension passes cold execution and required +replay on invented originals. A twenty-node extension now carries atomic +geography through current financial imputation and required replay. Native +Delaware and complete national block support pass source normalization; +national/CD acceptance of the enriched population remains pending. The earlier joint tract/district +operator is a separate path. See +[Geography assignment](geography-assignment.md) for the country contracts, +implementation and source boundaries. + +The 10 September sequencing correction now passes 70 inventoried controls on +invented inputs, across seven separately guarded runs, plus one separate default +compatibility case. The nine-node prefix completes its initial clones before +block assignment; the nineteen-node financial extension and twelve-node age +extension retain that geography through cold execution and required replay. +Assignment uses the qualified original-source key and stable clone discriminator, +so changing row order or numeric household coordinates does not redraw the same +clone. Observed state/PUMA constraints, input knownness, entity links, complete +population ownership and weights remain checked. Different clones may draw +different blocks; an occasional shared block is valid. + +The financial donor FILTER requires the exact typed geography-validation artifact +and retains original ASEC DESIGN weights. The shared gate emits this artifact +only when requested, preserving its default API for UK consumers. Two test-only +corrections repair the CREATE source declaration and isolate the obsolete-order +budget counterexample from unrelated serialized storage differences; production +matches the independently reviewed candidate. See the +[scoped postclone acceptance](../experiments/us-postclone-geography-70-controls-20260910.json). + +The earlier ten/twenty-node native runs retain their original order. Native +postclone execution, the new-order PUF host, held-out model quality, national/local +fiscal calibration and full release acceptance remain pending. The age control +uses invented targets and does not establish that complete calibration path. + +| Area | Main source entry points | What to review | +| --- | --- | --- | +| Graph execution and storage | `packages/microcosm-graph/src/microcosm/graph/{decl,executor,manifest,store,codecs,schema}.py` | Typed inputs and outputs, ownership, artifact ancestry, cache identity and nullable round trips | +| Survey source preparation | `packages/microcosm-build/src/microcosm/build/us_runtime/{acs_population_catalogue,asec_population_catalogue,survey_population_preparation}.py` | Source identity, entity links, source universes and current income transformations | +| Combined survey and clone | `us_runtime/{graph_composed_population,graph_survey_population,graph_combined_clone}.py` | ACS and ASEC composition before the clone; native versus detail channels | +| Enrichment | `us_runtime/{full_puf_enrichment,graph_full_puf_enrichment,graph_current_survey_puf_transfer}.py` | Target ordering, conditioning, observed-value preservation and complete replay | +| Conditional models | `packages/microcosm-fit/src/microcosm/fit/{qrf_target,graph_legacy_train,graph_legacy_apply_matrix}.py` | Reusable model artifacts, deterministic draws and target regimes | +| Geography | `atomic_geography.py`; `us_runtime/{atomic_block_support,atomic_block_api_sources,survey_atomic_geography,graph_atomic_survey_population}.py` | Atomic block assignment after complete initial cloning, observed-source constraints, versioned mappings, stable clone identities and the optional typed validation prerequisite | +| Survey mass and calibration | `us_runtime/{survey_origin_budget,graph_survey_budget,graph_survey_calibration}.py`; `packages/microcosm-calibrate/src/microcosm/calibrate/{group_bounds,solve}.py` | Original survey mass, grouped bounds, fixed support and existing ungrouped solver behavior | +| Compatibility | `packages/microcosm-build/src/microcosm/build/{frame_checkpoint,us_runtime/__init__}.py` | Current-main APIs, checkpoint metadata and existing country consumers | + +Paths beginning `us_runtime/` are relative to +`packages/microcosm-build/src/microcosm/build/`. +The branch contains a substantial consolidation; review each area and its tests +before deciding how to land it. Shared model work overlaps with +[#873](https://github.com/PolicyEngine/microcosm/pull/873). Anthony considers his +runtime proposal [#885](https://github.com/PolicyEngine/microcosm/pull/885) +superseded by this direction; it is reference material, not a prerequisite to +retain or merge separately. This does not imply that its GitHub PR is closed or +that this draft has received his approval. + +## Parallel review and dependencies + +Completion of this integration draft is not a prerequisite for all UK work. +Depend on identified shared changes where necessary, with independently reviewed +PRs and compatible interfaces. Keep the US native build, UK candidate repair, +source/target reconciliation and shared runtime work moving in parallel. + +| Reviewer | Review focus in this draft | Work that can continue independently | +| --- | --- | --- | +| Anthony | Shared graph/Frame and calibration compatibility; target hierarchy and metadata flow into diagnostics/dashboard, including #855 | Target/dashboard changes follow #900; UK staging #896 proceeds independently of full UK graph conversion and native US PUF integration | +| Maria | Register each existing UK pipeline stage as a graph node using this branch's shared machinery; verify UK behavior and coverage | #900 lands before graph conversion; UK imputation, support/fit diagnosis and candidate validation continue; graph-node registration stacks on #893, with stage reordering in a later separate PR | +| US integration owner | Resolve overlap, maintain exact compatibility evidence, complete the US population and release checks, and extract shared changes agreed in review | US-specific source preparation, imputation, quality diagnostics and progressive local builds | + +Coordination updated 10 September: Maria is deliberately stacking her UK +graph-node registration on #893. That specific dependent work should pin the +parent revision and coordinate shared-interface changes here. Her source/target +repairs and Anthony's staging work do not have to wait for the whole draft. +PR #900 merged at 14:54 UTC and is incorporated here from main at +`6f7571e1ab8c516154289bd8ebf75cb2646f03b5`. The combination preserves both parent +histories and does not rerun or recertify the frozen US controls. +Anthony's hierarchy/diagnostics [#855](https://github.com/PolicyEngine/microcosm/pull/855) +and its dashboard consumer follow the target reconciliation in +[#900](https://github.com/PolicyEngine/microcosm/pull/900). Registering UK graph +nodes and changing their order are separate review steps. The US integration +work must not build a competing full UK graph; its shared atomic-area adapter +is an input to Maria's owned integration. + +Record shared dependencies as named changes with pinned revisions. Extract a +smaller shared prerequisite where useful, while preserving compatibility with +Maria's intentional stack. Proposed sequencing does not certify or merge any PR. + +The shared post-clone geography interface now has a tested additive validation +artifact contract. The default gate API remains unchanged; UK registration can +adopt the optional typed prerequisite independently of the US stage ordering. +Existing target/source repairs can proceed on their reviewed base. Passing this +draft's invented controls does not certify Maria's UK candidate, and the final +US release need not gate her independent source and calibration work. No +whole-PR merge is implied by this review map. + +## Verification and review scope + +The first native twenty-node survey, block-geography and financial cold pilot +now passes at 1/1000: 1,584 source households and 3,464 people become 3,168 +household records and 6,928 person records after cloning. Ten financial nodes ran +cold over the stored ten-node prefix, which retains the block assigned before +cloning. The composition retains location inheritance and attaches seven financial +fields while preserving the other population fields. It took 110.12 minutes and +11.45 GB peak memory. The final receipt and independent source/resource +postcheck pass. This is the original frozen 491-source implementation; newer +cache and budget changes are separate revisions. Required native cached replay +now also passes: all twenty final nodes and ten prefix nodes were cache hits, +the four non-manifest exports matched the cold bytes, and the portable manifest +retained the same content and stored-artifact identities. Replay took 74.17 +minutes and 11.31 GB peak memory. Independent postchecks verified the recorded +result and exact code/resource identities without reopening native inputs or +exported payloads. PUF augmentation, model quality and calibration remain pending. +See the [scoped replay result](../experiments/us-native-atomic-financial-required-replay-20260910.json) and +[scoped native cold result](../experiments/us-native-atomic-financial-cold-20260910.json). + +The previously published head `a85c9cbb79081a980e7d3afe8916c80153de5bf8` +passed 36 integration controls and 57 enrichment/replay controls on invented +inputs. The 36 cover eight solver, eleven snapshot/storage, nine geography and +eight facade cases. The 57 include cold execution, required replay, reconstruction +through a fresh Frame store, inherited tax-unit values and malformed ancestry +refusals. Source/control, model-declaration and resource pins matched before, +after and in external postchecks; the tax-benefit engine was not executed. + +Those receipts are identified by SHA-256 +`ad8ab6d8761db3d21343bc1fc60023c0dfa305e1367622630a5de6c68892d62a` +and `85a192f8c5c851648875d66b90bea6b8a4ee11e86e67d1875c3ca34a076ee014`. +These checks do not certify subsequent source additions or the entire repository. + +CI exposed test-helper collection failures and a pool/registry circular import. +This revision adds the test helper directory to pytest's path and moves the +registry import after the pool functions it validates. Two fresh-process import +regressions are assigned to the US-engine CI lane; their runtime result remains +pending. No import-order result is inferred from an AST check. + +The corrected checkpoint test selection passed all 58 invented controls against +the current integration checkpoint source, including nullable integer widths, +missing-value masks, deterministic round trips and malformed v4 refusals. It +took 5.36 seconds wall time and 348.5 MB peak RSS. All 45 source/control files, +three provider pins and two code resources matched in the external postcheck. +Receipt: `f8009ec6d07d076eff6a90ec97061d538756293547a21758ab2bc38fa5f7aab8`. + +Fable's earlier replay review closed seven findings. A separate review of the +published consolidation identified the checkpoint test gap and additional +grouped-bound, metadata-store and facade coverage work. The portable grouped-bound, +fixed-support and metadata-store selection now passes all 117 tests with no skips. +The fixes reject nonfinite stored JSON as `StoreCorrupt` and normalize the deprecated +`apg` alias before grouped-mode validation. Source/control, resource and provider +bytes match before, after and in an external postcheck. Receipt: +`3674af9d610417897539da6eae766726d0f582ae6dd3bc2c3e550fefa45bab68`. +The historical platform fixture and complete facade import coverage remain separate. +Environment-specific prepatch byte fixtures do not establish portable CI byte +parity. Existing v1 graph stores remain preserved; v2 execution uses a new store +rather than silently promoting old frames. + +Private runtime records and generated population artifacts stay outside this +source PR. Receipt identifiers are audit references, not reproducibility inputs. + +The composed 156-case test run against source commit `8157300` at documentation +head `78d84f9` passed 152 cases and failed four current-survey cases with +`PRODUCER_CHANGED`. Its execution guard also recorded unexpected test-bytecode +and installed-model cache-directory probes. Source/control, model-source and +resource bytes remained unchanged. This is a failed integration run, not a +replacement for the earlier passing component evidence. Its receipt is +`67ff10fef45c58d2d6ea15c72b4311a4daa901dd2aae263fbb863d38c13bc7c8`. +The successor now passes all 314 selected invented integration and identity +checks with no skips or unexpected refusals. This includes the original 156 +cases, five metadata-copy/mutation regressions and 153 encoder controls; seven +synthetic-issuance controls remain explicitly outside this selection. Shallow +copy, deepcopy and pickle previously added a class cache entry captured by +Python 3.14 annotation closures. An explicit immutable-metadata representation +avoids that mutation without weakening producer checks. Two stale ACS pins were +updated only after verifying AST equality with their accepted source revisions. +All 476 source/control, 5,983 model-source and 12 resource entries matched before, +after and in the external postcheck. The run took 253.5 seconds and 738.7 MB peak +RSS. Receipt: +`d940913f918af4455dab2b9e680d55f9efc7bb922f72ff3d8fe26b0530b070f7`. + +A separate 16-case current seven-node age-development run initially passed 13 +cases and failed three on exact diagnostic reconstruction. The checker omitted +two solver-option fields added by the consolidated solver. Reconstructing the +grouped solver's closing-state selection and empty selection receipt fixes that +mismatch while retaining exact whole-document comparison. All 16 tests now pass, +including actual seven-node execution and late target/successor mutation checks, +in 170.3 seconds at 541.9 MB peak RSS. All 469 source/control, 5,983 model-source +and 12 resource entries matched before/after/current checks. Receipt: +`5849dd42e914e4d8304a5d290ff451da45910e7ac9a61e76fff36468949ffe3a`. +Broader CI's missing helpers and stale schema, seed and runtime classification +expectations remain separate from these selected passing suites. + +## Component evidence and remaining work + +Separate component work has verified the national Census joint-geography source +artifact: 2,462 PUMAs, 87,841 joint cells, all 50 states plus DC and 436 districts +including DC. PUF source ingestion retained 207,692 ordinary records and excluded +four disclosure records. A corrected SCF wage-code interpretation passed source +preparation and model-mechanism checks. Those results belong to their respective +component revisions; they do not establish a calibrated result on this branch. + +The corrected canonical-donor codec v2 was rebuilt from the same accepted typed +PUF source. All 207,692 returns and 59 outputs passed selected-cohort replay and +byte/value serialization checks in 24.37 seconds, using 1.95 GB peak RSS. The 481 +source/control files and four resources matched their before/after/current pins. +Receipt: `6b334ca8df6c6ba721b60f84d28c43151a1e1b54e097616a3c0fa208daeb8cba`. +This run constructs the donor; it does not fit or place it onto survey recipients. + +The identity encoder passed two 40-case preflights and four 40-case +timing runs in both execution orders. The 36 tiny invented cases retain exact +identity bytes and mutation checks. Nine of the 72 case/trial comparisons were +slower, including the small Python-string Frame in both orders; larger numeric +fixtures improved substantially. These timings do not establish genuine-build +speed. It is now integrated and passes the current 314-case selection, with the +newer current-wage projection preserved byte for byte. That projection was absent +from the benchmark's accepted baseline. + +The following work remains open: + +The [US and UK release path](us-uk-release-path.md) puts these implementation +items in the complete population-quality, calibration and consumer-release +sequence. Passing component controls is one step in that sequence. + +1. Extend the accepted native twenty-node atomic and financial cold/replay + pilot to the complete survey scale. Its + invented cold/required replay, complete retained + fields and geography, default three-feature versus explicit five-feature + conditioning, and both final mutation refusals pass on invented inputs. + Two test-code mistakes in the initial four-case run were corrected and + rerun separately; the passing mutation cases were preserved. See the + [composition acceptance](../experiments/us-atomic-financial-composition-acceptance-20260909.json). + Native fit quality and admission of the financial result into subsequent + calibration remain unassessed. +2. Integrate the adopted Social Security conditioning measurement and qualify + recipient return roles before native PUF55 fitting. Thirty-one invented + controls pass the authenticated filer/joint-spouse report-sum proxy, source + preservation, missingness and refusal checks. An incomplete selected report + uses a separately declared eight-predictor route; a known total uses nine. + The helper uses the preparation's retained modeled roles, whose relationship + to current-money tax-unit construction still needs explicit qualification. + See the [measurement decision](puf55-survey-ss-measurement-decision.md) and + [scoped control result](../experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json). + Route-aware graph fitting and attachment remain pending. The genuine + 207,692-return canonical donor has already passed its 55-output projection; + that does not establish recipient matching. Complete survey + beneficiary/component modeling, and supply SCF loan inputs. Separate component runs passed + 128 profile/compatibility controls and 26 current-survey predictor controls; + see [the implementation status](current-survey-puf59-progress.md) and + [donor construction and growth](puf2015-canonical59-and-growth.md). + The genuine 207,692-return donor construction does not establish survey fit + quality, complete enrichment or a releasable population. +3. Verify every applicable model input on each source/clone channel. The proposed + national/CD profile retains 161 required model inputs, with state and county + but without requiring the engine to consume block and tract. Independently, + the build must retain one assigned Census block and derive its larger + geographies. Prior wages remain excluded. A source operator's existence is + not complete cell coverage. +4. Retain the earlier seven-node age-development result as historical evidence; + the twenty-node cold/replay pilot in item 1 is the newer native milestone. + The seven-node run passed at 1/1000: 1,584 source households, + 3,168 records after the support clone, final export/readback and owner/target + verification, 71.1 minutes and 8.26 GB peak RSS. Its source is frozen at + `a9895d8a5`, before the new Social Security, PUF55 and atomic-geography work. + The prior two-hour failure and stale-derived-attachment failure remain failed + historical runs. [The scoped acceptance record](../experiments/us-survey-age-development-20260909.json) + does not certify enrichment, full national/CD calibration or a release. +5. Run real model evaluation, national/CD calibration, holdout checks and complete + export/replay verification, followed by a dashboard bound to that exact file. + Check every pruned analysis file separately. + +These are active workstreams. No release, merge or deployment is implied by this +draft, and the source changes do not relax the outstanding acceptance checks. + +The two-route numerical finalization layer now passes 46 invented controls, +including 110 actual target fits, 220 applications and comparisons against the +maintained whole-cohort finalizer for both routes together and either route +alone. The tests also refuse changed donor row order, weights and late raw or +donor mutations. The v2 numerical receipt seals the complete finalized candidate +before return and refuses a change during receipt encoding. This verifies the +numerical layer; typed model ancestry and complete survey attachment remain +separate graph checks. +The first canonical donor CREATE attempt failed during kernel construction +because its live-code checker followed a circular function closure. The +[failed observation](../experiments/us-puf55-canonical-create-failed-20260910.json) +is preserved. The cycle correction passed its ten new marker controls, but +the [39-case follow-up](../experiments/us-puf55-canonical-cycle39-failed-20260910.json) +still refused before CREATE because the code-state snapshot changed during +initialization. The changing member was the serialized Python code object; +the corrected marker retains the actual code object and its immutable fields. +After correcting three obsolete test accesses to the public PopulationView API, +all 47 canonical CREATE/converter controls pass, including cold execution, +required replay and teardown. The corrected numerical and canonical sources are +adopted locally. These checks use invented donors; the native full PUF host +remains pending. See the +[46-control numerical evidence](../experiments/us-puf55-numerical-output-seal-46-controls-20260910.json) +and [47-control canonical evidence](../experiments/us-puf55-canonical-create-47-controls-20260910.json). + +All 149 selected calibration controls now pass on the combined US/UK solver, +including grouped bounds, fixed zero support, informed gates, budget search, +ordinary best-iterate behavior and strict diagnostic payloads. The run took +4.57 wall seconds at 394 MB peak memory; independent checks confirm the exact +34 source files, copied fixtures and all 149 unique test results. The preceding +run stopped before tests because two still-denied import probes exceeded their +reporting ceilings; the fresh invocation changed only those reporting ceilings +and run identities. No data access was added. This is invented-data solver +verification, separate from native national/CD calibration; see the +[calibration evidence](../experiments/us-calibration-consolidation-149-20260910.json). + +Recipient qualification now covers all 28 distinct original public controls: +27 passed in the original run and the five-case corrected detached-output run +passed, with four overlapping controls. The original run remains failed because +one NaN mutation test did not change its input bits; its correction flips a bit +in the actual float64 storage. This is a compound result across two source +snapshots, not a single passing 28-case run. Exact original-source issuers and +the 22-node cold/required graph are included. See the +[recipient evidence](../experiments/us-puf55-public-recipient-controls-20260910.json). + +The population-only Census API adapter passes thirty-five invented controls. +The native Delaware source control preserves all 15,317 populated blocks and +reconciles them to the independent Census state total of 989,948. It counts +4,881 zero-population blocks, checks CD119/PUMA joins and exactly reads back its +79,769-byte support artifact. The run took 10.94 seconds and 1.735 GB peak RSS. +The first attempt correctly refused an incomplete ZIP-member declaration; +a separate metadata probe established the six-member archive roster, and the +corrected check still decompresses only `NationalCD119.txt`. The +[accepted source control](../experiments/us-atomic-native-de-corrected-1-control-20260909.json) +preserves the original failed evidence. This is not national population or +release acceptance. Delaware-only support must not enter the national survey +graph. + +The subsequent population-only national acquisition and normalization now pass +for all fifty states and DC. The 104 exact sources produce 5,769,942 populated +blocks, independently reconciling every state to a total of 331,449,281 people. +Every block retains its identity and population, with complete CD119/PUMA joins. +The full support artifact is 28,862,508 bytes and passes exact serialization and +readback. Normalization took 82.97 seconds and 6.814 GB peak RSS. Its SHA-256 is +`5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960`; +see the [national source control](../experiments/us-atomic-native-national-1-control-20260909.json). +Postchecks reauthenticated code and compared the 104 native pins from bounded +receipts without reopening source bodies. National support normalization and +household assignment in the small native financial pilot now pass. Evaluating +geographic fit and calibrating the enriched survey remain separate acceptance +steps. + +The financial successor's unchanged positive control now passes with both +original fixture teardowns and final code/resource checks: 647.19 CPU seconds, +651.21 wall seconds and 570.9 MB peak RSS. This separately reviewed invocation +used a 1,200 CPU-second budget; the earlier 600-second attempt remains recorded +as incomplete. No memo or profiler was used. The [positive control evidence](../experiments/us-financial-successor-positive-20260910.json) +checks financial replay and a subsequent weight-only change. The [seven remaining refusal and callback controls](../experiments/us-financial-successor-remaining-controls-20260910.json) +now also pass with their original shared fixture and teardowns: 798.61 CPU seconds, +802.89 wall seconds and 563.6 MB peak RSS. All eight original admission controls +are accepted on the same source snapshot. A subsequent source review found an +additional detached-view callback gap in the budget and weight-only views. +The correction is adopted after four targeted controls passed in 543 wall +seconds, including nine callback branches and independent source/resource checks. +The original eight and corrected four are separate source-version checks; see +the [correction evidence](../experiments/us-budget-detached-view-correction-20260910.json). +The native pilots exclude this overlay, and native calibration remains separate. + +A separate, bounded profile of the unchanged invented cold/required financial +fixture passes its test, original teardown and all final code/resource checks: +303.90 CPU seconds, 308.26 wall seconds and 594.3 MB peak RSS. This measures the +fixture rather than the additional financial and weight-only admission work. +See the [profile evidence](../experiments/us-financial-fixture-profile-20260910.json). +Loaded ACS code verification accounts for 180.20 cumulative seconds inside the +298.15-second profiled fixture. Its nested function checker runs 2,009,108 times +and uses 91.05 self seconds; complete Frame identity uses 7.31 cumulative seconds. +Cumulative timings overlap and profiling adds overhead. The invocation-local +code comparison memo now passes all 18 focused invented controls, including +changed source, aliases, globals, closures, mutable constants and the historical +catalogue caller. Source and resource checks pass before, after and in the +external postcheck. See the [control evidence](../experiments/us-acs-loaded-code-controls-20260910.json). +The paired full-fixture run also passed correctness and final checks, but took +319.79 wall seconds versus 308.26 for the baseline. The memo added eligibility +work and demonstrated no speed improvement, so its implementation was not +adopted. The exact experimental patch and [paired comparison](../experiments/us-acs-code-memo-comparison-20260910.json) +remain recorded. A separate [one-call diagnostic](../experiments/us-acs-producer-profile-20260910.json) +now passes without constructing a fixture: the original producer takes 0.656 +profiled wall seconds. Its 157 compiler calls take 0.238 seconds, including +0.122 seconds inside 103 AST parses; those overlapping times must not be added. +A bounded bytecode compilation cache now passes 25 invented controls and final +code/resource checks, with fresh source reads and every loaded-function check +retained. No native-data speedup is established. The compilation cache has separately passed +the complete invented fixture and is adopted; the native financial cold and +required-replay pilots passed with their original frozen code. The earlier +startup refusal remains preserved and confers no data acceptance. + +Current CI separately reports stale source-attested spec/seed fingerprints. +The worker resource identity test now follows the actual lazy import closure +and includes an invented opened-JSON regression; its new CI result is pending. +Neither the component passes nor this profile establishes a green consolidation. + +## Related reviews + +- [Microcosm → Orrery exporter, #888](https://github.com/PolicyEngine/microcosm/pull/888): + already published separately. It preserves the supplied schema metadata and + exact large integers; graph visibility does not infer domain verdicts. +- [Orrery review index, #6](https://github.com/TheAxiomFoundation/orrery/pull/6) + and [search improvements, #4](https://github.com/TheAxiomFoundation/orrery/pull/4). +- [Dynamics graph integration, #420](https://github.com/PolicyEngine/microcosm-dynamics/pull/420): + owned by the separate retirement-model task and tested on synthetic inputs. +- [Candidate methods and paper updates, site #72](https://github.com/PolicyEngine/microcosm.institute/pull/72): + public component evidence and remaining release checks. + +The accepted local shared viewer remains a separate consumer. There is no package +upgrade or competing generic graph shell in this consolidation. diff --git a/docs/us-property-completion-routing.md b/docs/us-property-completion-routing.md new file mode 100644 index 000000000..debe0ee45 --- /dev/null +++ b/docs/us-property-completion-routing.md @@ -0,0 +1,82 @@ +# Property completion routing + +`current_property_completion_routing.build_property_completion_routing(qualified, clone_frame)` describes the remaining ordinary-interest and dividend work over the original ACS/ASEC source population. It assigns no amounts, fits no model and changes no Frame. Its `PropertyCompletionRouting` return value is descriptive; constructing it grants no source, complete-parent or release authority. + +The inputs are the existing `QualifiedPropertyIncomeSources` description and the complete initial clone Frame. The operation checks original source order, integer identities, source-native agreement, source ages, component knownness, the exact two clone roles and original household DESIGN weights. It preserves the complete input physical descriptions. Actual source custody remains external: a country host must retain and requalify the preparation and source issuers around relevant I/O. Neither the description's type nor a separately supplied DESIGN vector authenticates that custody. + +## Routes and retained reasons + +Each original person receives one route: + +| Route | Meaning | +| --- | --- | +| `unsupported_under15_measurement` | The source age is below15. No child amount or adult extrapolation is supplied. | +| `source_review_required` | A required source amount/receipt or ACS anchor is malformed, outside its published domain, inconsistent or has an invalid adjustment. | +| `carry_known_components` | Both independent ASEC components are known. Joint-fit exclusion does not erase them. | +| `existing_acs_anchor_decomposition` | The adult ACS aggregate anchor is observed and qualifies for the existing decomposition branch. The separate components remain unresolved at this stage. | +| `acs_anchor_completion_review` | An adult ACS anchor remains missing. This operation supplies no fallback. | +| `asec_component_completion_review` | At least one adult ASEC component remains unresolved. Any independently known other component is preserved. | + +Under15 is the first route; malformed, positive-outside-universe and contradictory observations remain visible as overlapping reasons. Adult source errors precede generic missingness. Joint-donor exclusions remain separately named reasons and never by themselves turn otherwise known O/D into unknowns. Unknown new source statuses refuse classification instead of silently entering a generic completion branch. + +The long component table preserves amount, individual knownness and the exact reporting/literal statuses. `observed` denotes a qualified observed amount; `derived` distinguishes a qualified known-nonreceipt zero; `unresolved` retains missing amounts. An observed ACS aggregate is recorded separately from its unresolved individual components. No component receives a `modeled` origin here; only a later operation consuming actual checked model outputs can assign that origin. Allocation and disclosure flags remain source provenance rather than donor filters. + +## Outputs and support + +- `person`: one row per original, with source/native identity, source age, original household, independent O/D knownness, separate ACS anchor and exclusive route. +- `components`: two rows per original for the two required component families. +- `reasons`: all overlapping required-source and joint-fit diagnostics. +- `clones`: exact original/native/source identity and clone roles mapped to the initial receiving person IDs. +- `summary`: raw original-person and unique-household counts, household-inherited DESIGN person mass and union-household DESIGN mass for each route/reason. Clones do not multiply these summaries. Zero-weight support remains visible in raw counts. +- `payload`: deterministic JSON with the tables and explicit statements that it neither assigns amounts nor authenticates a source or complete parent. Integer IDs remain integers, including those above2⁵³; unknown amounts serialize as null. + +These are support diagnostics, not prevalence estimates or calibrated totals. Overlapping reason masses must not be added as if the reasons were disjoint. + +## Integration boundary + +`property_completion_artifact_output()` supplies the typed `completion_routing` artifact declaration. This additive slice does not connect a new node or change the accepted38-node host. The next host integration should emit the payload at the existing property source-projection boundary, bind the actual retained source and clone descriptions, and recompute/compare it during requalification and reconstruction. That host change requires its own source/graph identity and observation checks. + +This operation does not create a child-income measurement model, fill an unknown adult input, assert that the PUF covers a missing source population or relax the existing completeness gate. + +## Opt-in host artifact + +`PropertyIncomeOptions(completion_routing=True, ...)` requests a fourth typed +artifact, `completion_routing`, from the existing +`survey_property.source_projection` node. The default remains disabled and +retains the previous canonical option payload. Neither setting changes the +numerical model declarations, source universes, clone attachment or tax split. +An implementation source edit does change executable cache identities. + +The private artifact contains the complete original-person, component, reason +and initial-clone tables. The source receipt contains an explicitly allowlisted +aggregate summary and its private artifact digest. The actual graph HTML and +text views expose receipts, so private identifiers and per-person values must +never enter this summary. Aggregate DESIGN support is descriptive; it is not +calibrated representation, a release gate or a disclosure certificate. + +Source execution reuses its existing qualification and builds routing once. +Independent host reconstruction checks exact diagnostic bytes and the expected +aggregate receipt on cold execution and required replay. Ordinary branch checks +retain their existing three source-artifact checks and do not recompute routing. +The host retains all declared artifact hashes and canonical options; its checked +parent/source lifetime remains the authority boundary. The pure routing result +and the artifact grant no source or complete-parent authority. + +The bounded integration controls exercise qualified invented source descriptions, +option/declaration behavior, tamper refusal and the real public serializers. +Actual 38-node host acceptance is a separate test scope; this feature does not +claim native completion, PUF readiness or a released dataset. + +The repository's existing 38-node tax-host fixture now enables completion +routing on both its cold and required calls. Its controls retain full population +and unknown-input checks, verify the private typed artifact and aggregate receipt +on replay, and refuse changes to the retained artifact bytes. The local native +harness predicates can share those fixtures through a separately pinned acceptance +wrapper; they are not a machine-specific dependency of repository tests. + +The bounded acceptance passed 27 focused controls and three actual invented +cold/required host tests at source `c06d29ea8`. The latter reuse one 38-node pair, +check the full saved Frame after final source revalidation, and reject changed +retained artifact bytes. They preserve the fixture's false tax-input completeness +verdict. Exact scope and receipt hashes are in the +[13 September integration record](../experiments/us-launch-consolidation-20260913.md). diff --git a/docs/us-property-income-graph.md b/docs/us-property-income-graph.md new file mode 100644 index 000000000..d06b4c869 --- /dev/null +++ b/docs/us-property-income-graph.md @@ -0,0 +1,58 @@ +# Property income model graph + +`microcosm.build.us_runtime.graph_property_income.property_income_nodes` +declares ten numerical nodes: four conditional QRF fits, four chained draws, +one operation that writes the raw draws as columns, and one signed +reconciliation. It uses the existing QRF and reconciliation implementations. +The fragment consumes separately supplied original donor and recipient +populations; it does not qualify native sources or attach results to clones. + +The shared columns in `property_income_constants.py` describe ordinary +interest, retirement-account interest, dividends and signed broad property +receipts. The first three are nonnegative; broad property receipts may be +negative. `property_reported_total` is a required predictor and the recipient +reconciliation anchor. The source bridge supplies the ASEC reported aggregate +for donors and the adjusted ACS INTP amount for recipients. It must preserve +their different reference periods and account for unresolved income routes. + +The training wrapper requires finite float64 predictors and components, +original DESIGN weights, and a component sum equal to the supplied donor +aggregate. It delegates actual fitting to `LegacyQRFTrainKernel`; it does not +filter, repair or reconcile donor observations. The owning source preparation +must establish original household membership, design-weight mapping, eligible +donor support and source knownness before invoking this fragment. A weight +kind label alone does not establish original source authority. + +Recipient draws preserve the complete target order and earlier raw draws as +conditioning inputs. The column operation binds every raw value to its apply +checkpoint, exact recipient index, sibling producer, and final model history. +The graph executor supplies the authenticated artifact edges. Direct calls or +copied checkpoint bytes do not authorize source or model admission. Person IDs +remain int64 independently of the pandas index used by QRF checkpoints. + +Reconciliation retains raw draws in separate `draw_` columns and exposes each +component's adjustment, active bound, residual and objective. Scales and +tolerances are required parameters. Changing them recomputes reconciliation; +changing recipient anchors recomputes recipient draws while reusing donor fits. +Negative or zero net income never forces all positive components to zero: +positive interest and a property loss can coexist at either total. + +Missing anchors and under-15 source blanks require an explicit upstream +decision; this fragment does not manufacture zeros. Original ASEC observations +are not reconciled to ACS respondents. The next country integration must join +the complete source-qualified basis, draw once per original ACS person, retain +the result on both initial clones, and derive the declared tax splits before +running the dependent PUF successor. Retirement-account earnings are not +ordinary taxable/exempt interest. Broad property receipts are not yet a +rental-income tax leaf. PUF-owned alternatives must not subsequently be forced +back onto the survey anchor. + +The initial invented test uses 44 donor persons and four recipients, including +negative/zero totals, large int64 identities, original membership, metadata and +a zero-weight household. It executes the real twelve-node graph including two +source nodes, compares draws with direct weighted QRF, compares reconciled +values with the pure projection and reopens the store in required-cache mode. +Twenty-two tests also cover delegated training-code identity, dependency reuse, incorrect weight kinds, +unknown/inconsistent donors, altered draw history and invalid declarations. +They establish numerical and graph contracts, not native source acceptance, +statistical adequacy, tax correctness or release readiness. diff --git a/docs/us-property-model-receipts.md b/docs/us-property-model-receipts.md new file mode 100644 index 000000000..1c16e9618 --- /dev/null +++ b/docs/us-property-model-receipts.md @@ -0,0 +1,65 @@ +# Property model receipt verification + +`verify_property_model_receipts(nodes, donor_population, recipient_population, +artifacts)` in `graph_property_income_receipts.py` reconstructs the existing +four fit and four apply receipt dictionaries. The caller passes the ordered +property fragment, the two original `Population` branches, and authenticated +artifact bytes keyed by `(node_id, output_name)`. Additional non-model fragment +nodes and their artifacts may be present; the country host verifies those. + +The caller must first bind the store bytes, typed producer outputs and exact +population dependencies. This helper loads the existing trusted model format, +which contains pickle. A self-consistent digest or a returned receipt does not +authenticate arbitrary model bytes or establish source authority. The owning +host retains responsibility for source qualification, complete population +identity, physical seals, borrowed inputs and final I/O checks. + +## Checks + +The helper reconstructs the shared graph declarations for the four ordered +property targets and compares their complete normative bytes. This binds input +slices, population versions, predictors, target order, seed, phase, fit options, +model/training sibling edges, prior raw-draw edges and output artifact types. +Training stays donor-only. The existing model protocol initializes its exact +chain-start state against the original design-weight donor without fitting. +Each trusted model must match the complete ordered training history, actual +consumed float64 donor bytes, donor pandas index, resolved weights, configuration +and preceding/succeeding training states. The first three property components +remain nonnegative and the four-component sum must equal the observed donor +aggregate. Both branches require finite float64 model columns and unique int64 +person IDs. + +Ordinary population apply checkpoints have a pandas-index identity but no +independent recipient-feature digest. The verifier therefore performs four +additional `apply_target` operations using the retained fitted models, current +recipient features, raw prefix and exact initial draw RNG. It compares the +resulting raw bytes and complete application states with the published +artifacts. It validates every model/raw hash in the ordered application history. +It never calls `fit_target` or a model-fitting convenience method. + +This replay establishes output consistency with the supplied feature values; +a model can produce the same output for different inputs. Exact feature and +person-ID provenance remains the caller's authenticated population binding. +The existing protocol uses pandas row labels for model state and raw draws; +the verifier does not substitute entity IDs or invent a new identity digest. +Graph owners must continue to bind the distinct int64 person axis themselves. + +## Scope and cost + +The return value uses the existing raw kernel receipt schema; the host adds and +checks the executor capability/input-writer envelope through its existing +`_states` path. It adds no status, +source issuer, receipt artifact type, model format or host framework. The +helper does not fit, reconcile, attach clones, assign tax treatment or modify +its input frames. Cost per call is one chain initialization, four trusted model +decodes and four deterministic apply replays over the recipient branch. The +owning host decides when to perform this validation; no native-scale runtime +claim follows from the small tests. + +Tests reuse the actual twelve-node property graph fixture with two trees, +44 invented donor persons, four recipient persons, non-default pandas row +labels, int64 IDs above `2**53` and zero-weight records. Original cold and +required-cache kernel receipts match the verifier. Separate controls alter +donor/recipient axes and values, design weights, graph declarations, model and +training bytes, model/raw history and draw RNG. Country engines, native sources, +the full PUF host and publication are outside these checks. diff --git a/docs/us-property-tax-leaves.md b/docs/us-property-tax-leaves.md new file mode 100644 index 000000000..afc520911 --- /dev/null +++ b/docs/us-property-tax-leaves.md @@ -0,0 +1,124 @@ +# Property tax leaf rebase + +`graph_property_tax_leaves.py` provides three deterministic graph nodes for an +already checked population with property components and incumbent tax leaves. +It does not qualify sources, fit models, attach clones, grant a checked-run +handle or wire the country/PUF host. The owning host verifies the original +component observations, clone pairing and complete frame before using it. + +## Numerical operation + +`split_property_tax_leaves(person)` returns four float64 columns indexed by exact +int64 `person_id`. It imports the maintained fractions from `cps_carried`: + +| Input component | Primary leaf | Complement leaf | +| --- | --- | --- | +| `property_ordinary_interest` (`O`) | `taxable_interest_income = O * 0.680` | `tax_exempt_interest_income = O - taxable_interest_income` | +| `property_dividends` (`D`) | `qualified_dividend_income = D * 0.448` | `non_qualified_dividend_income = D - qualified_dividend_income` | + +The complement uses subtraction after computing the primary. This is a declared +operation order, including for the dividend complement that legacy code computes +by multiplying the complementary fraction. These fractions are modelling +assumptions; they do not establish a person's observed tax treatment. + +Interest and dividend knownness are separate. A finite nonnegative input produces +its two leaves; a missing input produces two NaNs, replacing any earlier tax +predictions. Zero stays an observed numerical zero. A negative or infinite O/D +input, an infinite retirement component, incompatible dtype or ambiguous person +identity refuses. `property_retirement_interest` is retained as an auxiliary +component and never enters either tax-interest leaf. Its missing value does not +invalidate known O/D values. Broad property receipts, net anchor sign, source age, +donor-fit eligibility and earlier predictions never supply missing O/D values. +No under-15 zero completion or missing-adult prediction occurs here. + +## Three graph nodes + +Call `property_tax_leaf_nodes(frame, population=base_version, +projection=projection_input, reconciliation=reconciliation_input, atol=..., +rtol=...)`. Both tolerances must be explicit finite nonnegative Python floats. +The two `ArtifactInput` aliases must be `projection` and `reconciliation`; the +host supplies their actual producer IDs and nominal types. Register +`PropertyTaxReceivingKernel`, `PropertyTaxLeavesKernel` and +`PropertyTaxLeafGateKernel`. + +When compiling before the earlier financial/property nodes execute, the host +may pass `anticipated_outputs=tuple_of_owned_descriptors`. The factory combines +those existing declarations with the real checked Frame's entity/axis/group +shapes and dtypes. It creates no placeholder values or synthetic population. +It rejects duplicate descriptors, unknown entities, structural ID fields, +masked outputs and incompatible or unclaimed rewrites. At execution the host +re-declares the nodes from the actual complete post-property Frame and requires +the same normative declarations before reconstructing the results. The default +empty descriptor tuple still requires a complete actual input Frame. + +1. `survey_property.tax_receiving` opens a new population version using + FILTER-all. This inherits the prior owners and permits explicit replacement of + the four incumbent leaves. The executor adds a real FILTER conservation record + to the population mass ledger. It preserves the existing `Frame.mass_log`. +2. `survey_property.tax_leaves` replaces the four leaves on that version and + emits `diagnostics` of type `microcosm.us.property_tax_leaf_diagnostics/v1`. +3. `survey_property.tax_leaf_gate` recomputes the leaves and their knownness, + checks exact leaf bits, partition sums within the declared tolerances, and the + complete diagnostics payload. It emits `verification` of type + `microcosm.us.property_tax_leaf_verification/v1`. + +The gate binds ordered person IDs without floating-point conversion, current +O/D/retirement bytes, all four leaf byte digests, fractions and operation order, +individual knownness, and the typed projection/reconciliation artifact keys and +payload hashes. It includes every physical person, including dependents and +zero-weight members. `complete` is false if any person lacks O or D; a successful +numerical check with missing inputs reports `evidence_absent`. Complete numerical +inputs report `pass`. Neither outcome establishes source, model, tax-validity or +release authority. The artifact explicitly leaves full-frame verification to +the host. + +## Supported receiving shapes and host responsibilities + +`Frame.select(all)` currently refuses link tables and normalizes group indices. +The declaration factory therefore refuses links, nondefault group indices, +orphan groups, empty person frames and groups lacking a readable non-ID column. +The last condition ensures the kernel context exposes group IDs for the +receiving check. Orphans are already invalid at Frame construction; the factory +also detects a later membership mutation. The kernel rechecks exposed group +membership and logical group indices immediately before returning its keep mask. +It never silently prunes a group or switches structural operation. + +For supported frames, person rows/order/IDs, entity tables, weights and their +kinds, schema, strata, metadata and frame mass log are preserved, apart from the +four declared leaf replacements. The new population version, owners and +conservation record are intentional changes. Kernel contexts do not expose the +complete schema, metadata or population ledger, so the host must retain and +verify those independently. It must also qualify the complete supported shape +before adding these nodes, bind actual artifact producers and stored payloads, +and recheck the original owners and output after its final relevant I/O. + +For independent expected-result reconstruction, the host can call each kernel's +`run(KernelContext)` on the retained current population and authenticated typed +artifact aliases. The existing executor `_project_context` and `_apply_result` +provide the canonical context projection and result application. FILTER results +must be materialized through the validated keep mask before `population.patch`; +passing its raw `keep` result directly to that lower-level patch function omits +the executor's frame selection. Ordinary rebase/gate results use the existing +patch path. These deterministic kernels consume no random draws. Expected +artifact bytes and receipts can then be compared with the actual store and node +states, and expected full populations with the host's retained observations. +This fragment adds no separate executor, authority registry or public run issuer. + +Future PUF integration must depend on this verification artifact and require +complete tax leaves over every physical member the PUF consumer needs. The +fragment does not add that host dependency itself. Matching checked inputs give +matching outputs on paired clones; the fragment does not infer or repair clone +pairing. Capital gains retain their earlier conditioning and are not rebased by +this change. + +## Verification scope + +The 32 invented controls exercise a real four-node graph including its source, +ContentStore cold execution and required-cache replay. They cover negative, +zero-net and positive anchors with loss offsets, independent O/D unknowns, +known O/D with unknown retirement/broad components, integer IDs above `2**53`, +row permutations, empty pure input, malformed values, altered leaf bits, +diagnostics and artifact bindings, full-frame preservation, zero-weight records, +the population conservation record and unsupported receiving shapes. No native +survey, country engine, PUF execution, fitting or publication is part of this +acceptance. diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md new file mode 100644 index 000000000..e79d05090 --- /dev/null +++ b/docs/us-uk-release-path.md @@ -0,0 +1,325 @@ +# US and UK release path + +Planning snapshot, 13 September 2026. This is a release work plan, not a +certification of either country. Component tests, historical candidates and +current-candidate acceptance are distinct evidence. + +## Products and common construction + +The US product must support national and congressional-district analysis. The +UK product must support national, constituency and local-authority analysis. +Each country should have one fully constructed population from which full and +smaller analysis files are derived. Record the survey, monetary, policy and +boundary reference periods explicitly; do not infer them from a filename. + +After constructing and harmonizing the complete multispine, including its +initial clones, assign one atomic area to each resulting household and derive +larger geographies from versioned mappings. Distinct clones may receive different +areas, constrained by observed source geography and keyed by stable clone +identity. Subsequent enrichment retains the assigned location. Once construction +and enrichment are complete, +calibration changes household weights; scope selection and pruning retain the +original values, missingness and relationships for every retained household. + +For a shared set of weights and consistent measurement definitions, additive +local totals must reconcile to national totals. Separately refitted compact +views need declared approximation tolerances and their own diagnostics. A +common source population alone does not guarantee identical estimates. + +## Release sequence + +| Stage | US work | UK work | Exit evidence | +| --- | --- | --- | --- | +| Source and period coverage | ACS and ASEC survey multispine; PUF tax detail; SCF and other required auxiliary sources; declared aging | Raw FRS 2024–25; SPI/HMRC, WAS, LCFS/ETB and other required stages; declared uprating | Source register, licensed-input boundaries, source definitions, missingness, units and periods; every applicable input has a producer | +| Complete population | Harmonized relationships and tax/benefit units; geography; financial and PUF enrichment; remaining benefit, disability, housing and asset inputs | Integrated FRS stages and current relationship/claimant definitions; income, wealth, consumption and benefit inputs | Small real end-to-end build; source-channel coverage; plausible distributions and joints; no silent engine-default substitutions | +| Geographic support | Block assignment after initial multispine cloning; versioned county/PUMA/CD derivation; enough household types in each district | Shared atomic-area assignment after full spine cloning, and derivation for each nation; adequate constituency/LA support | Observed geography respected; distinct stable clone draws; subsequent location invariance; support and fit measured at advertised levels | +| Model and calibration | Locked US model; national/state/CD target matrix from identified facts | Locked UK model and companion input changes; national/local matrix with corrected UC and other target definitions | Baseline engine evaluation, calibrated population, target-level errors, support/weight diagnostics and explicit outstanding failures | +| Independent quality | Held-out distributions and geographic cells; tax/benefit totals and representative reforms; comparison with the incumbent | Current release battery, held-out areas and complete incumbent comparison; resolve remaining support and target-fit failures | Candidate-specific reports; explained regressions; thresholds agreed before choosing a candidate | +| Scale and compact views | Increase local build size after smaller checks; derive and refit the intended full/sparse family | Choose a size that meets local support and quality needs; complete compact/exact-count release evidence | Measured time, memory and cache replay; retained-input invariance; full/compact error and performance comparisons | +| Release and consumption | Exact-file dashboard, methods/papers, immutable package and real PolicyEngine loading | Exact-file dashboard and certification; immutable cut, loader/pointer integration and PolicyEngine-UK adoption | Fresh export/readback, reproducible manifest and checksums, consumer calculation smoke test, reviewed release and rollback path | + +## Current position + +The immediate US step is a small native property/tax build with an original-record +completion report. The completion diagnostic passes 29 pure controls, 27 focused +graph controls and three actual invented 38-node cold/required acceptance tests. +The latter preserve complete populations and exports while retaining unknowns; +the tax-input completeness gate remains false on that fixture. +Native PUF qualification follows resolution of the applicable unknown inputs. +Retirement candidate accounting has passed 80 controls but does not yet supply +fiscal inputs. The [13 September evidence](../experiments/us-launch-consolidation-20260913.md) +records these limits and the scoped hashing benchmark. + +UK work proceeds independently on the shared same-kind weight update and +metadata/mass-log/column-order interfaces needed by its country graph. That +source proposal is under independent review and has not run its invented +runtime checks. It establishes no new UK candidate or release acceptance. + +| Stage | US demonstrated result | UK demonstrated result | +| --- | --- | --- | +| Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; maintained two-route PUF and subsequent amount/health hosts pass scoped invented controls; native PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | +| Geography | Small native post-clone block assignment and financial extension passed on 11 September; strict required replay passed on 12 September; complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | +| Calibration | Declared national/state/CD measurement and dense grouped calibration stages pass on invented inputs; fully enriched native solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | +| Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | +| Full/compact files | Small prefix replay and 36 supplied-parent full/pruned/local export checks pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | +| Delivery | No accepted new-architecture release or default consumer adoption | Assembly implementation and historical assembled cuts exist; current certification and default promotion pending | + +The 11 September native pilot passed the corrected survey → full initial +support clone → atomic block assignment → financial-enrichment order. At a +1/1000 source sample, its 1,584 source households and 3,464 people become +3,168 households and 6,928 people after cloning. The nineteen-node graph fits +and attaches seven financial fields. Its closed receipt is tied to frozen +source `2ca11c85a`, and reports 5,362 seconds elapsed and 9.31 GB peak RSS. +This supersedes the earlier assignment-before-cloning milestone for ordering; +it does not establish full PUF enrichment or national/CD calibration. + +On 12 September, a strict required replay of that frozen source passed with a +fresh output directory and a read-only retained store. It verified the source +bytes again, reused all nine prefix and nineteen financial-graph nodes, and +reproduced the four non-manifest exports byte for byte. The portable manifest +also matched after excluding only operational timing and cache-hit fields. +The closed run took 3,981 seconds and peaked at 10.99 GB RSS; source, resource, +native-input, owned-file and thread-control checks remained unchanged, with no +unexpected access refusals. See the +[replay record](../experiments/us-financial19-required-replay-20260912.md). +Meanwhile the maintained graph's observer-mutation defect has been fixed and +independently reviewed. The shared graph consolidation merged in +[PR #913](https://github.com/PolicyEngine/microcosm/pull/913) at +`a9cc63e737fd4619a304117dc2ec7dcd86d97901`; the release branch has incorporated +that main revision without changing its existing graph implementation. Those +software changes still need their +own native candidate verification; this replay certifies only its frozen +financial stage, not the newer source or a release. + +The maintained PUF55 host is now adopted. Its 31 small controls and two actual +full-host controls pass on invented inputs: both conditioning routes, 110 fits +and 110 applications, cold execution, required replay, whole-population +preservation and numerical finalizer equality. The full-host run took 742 +seconds and 0.74 GB peak RSS. Those 64 invented donors do not establish the +runtime or quality of the full native donor. A separate current-source native +packet uses a fresh writable store and the actual retained financial run. +See [the PUF host record](../experiments/us-puf55-host-adoption-20260912.md). + +That native packet's second attempt closed with an assertion failure after +about 82 minutes, during its financial-stage acceptance and before any PUF +execution. The source and resource pins remained unchanged and no unexpected +access refusal was reported. Source review identified an invalid harness +assumption: the atomic host first materializes CREATE and ALLOCATION, so those +two nodes legitimately hit when its expanded graph runs in the same store. +Requiring every prefix node to miss rejects that valid execution. The refusal +record does not identify its precise assertion frame, so this diagnosis does +not establish completed native financial acceptance. The corrected v3 harness +passes two actual invented financial cold/replay controls and thirteen bounded +source and diagnostic controls. After independent review of its exact packet, +one fresh native run started with unchanged calculation source, native inputs +and resource limits. On 13 September, the owned process was no longer present +and neither a final receipt nor a PUF manifest existed. Its last recorded +progress was about 5.76 hours wall time, 6.41 hours process CPU and 28.29 GB peak +RSS. The exit code and stopping cause are unavailable; the CPU total exceeding +the configured budget does not establish that cause. This attempt is closed +without verified native PUF output. A successor requires a fresh reviewed +packet; the preceding source, input and failure evidence remain preserved. + +The maintained financial graph now retains an eighth field: the tax-exempt +interest remainder of its existing observed or modeled total. The split remains +a modeling assumption. Twenty-five financial/source controls and three full +invented PUF replay, finalizer and retained-output controls pass; the PUF stage +preserves the original-channel remainder. The historical native financial +receipt above still covers seven fields. See +[the interest correction](../experiments/us-survey-interest-conservation-20260912.md). +A separate reviewed source-hashing optimization preserves exact digest bytes +and passes thirty source tests. Its repeated-string benchmark improvement is +not a measured native build improvement; see +[the scoped benchmark](../experiments/us-source-string-seal-20260912.md). + +The new fiscal measurement stage passes all 66 interpreter/stage checks on the +combined integration, including an actual US-engine dividend calculation. It +uses declared target bindings, household-aligned CSR measurements and explicit +national/state/CD masks. Missing consumed predicates refuse, indicator sums +are numeric, and local calculation helpers participate in cache identity. +Specialized unsupported bindings refuse; this is not full fiscal-registry +coverage. + +The dense grouped calibration stage is now integrated. Independent review of +its numerical adapter found no actionable defects, and all 167 selected +calibration tests passed. The combined integration also passed all 42 tests +for the measurement and dense stages together. The adapter uses the existing +grouped Adam solver, keeps initially zero weights fixed, enforces group bounds +and refuses a result exceeding the declared original row caps. It returns +calibrated weights and schema-8 diagnostics. Numeric bound arrays do not prove +their source: the complete-population host must authenticate the original +sampling references and retained parent before and after use. Native fit, +full-size memory and calibration acceptance remain unestablished. + +The dense fiscal kernel now forwards an optional host-owned target-snapshot +observer to the actual grouped solver. Tiny invented graph tests verify equal +weights, artifacts, receipts and cache identity with observation on/off, plus +silent required replay and final mutation refusal after the sink. The integrated +suite has 473 passes and one explicitly recorded historical thread-configuration +skip; five final calibration-identity checks also pass. Source admission, +production cadence costs and a dashboard consumer remain separate. See +[the fiscal snapshot evidence](../experiments/fiscal-target-snapshot-host-20260912.md). + +Fiscal leaf-policy infrastructure now distinguishes required producers from +explicitly documented proposed assumptions. It passes 91 source/mock and +invented fiscal graph checks. The default remains strict all-producer; +assumption execution and inactive-leaf exemptions remain unsupported until +complete-parent admission and exhaustive dependency proof exist. No US +assumptions or new dataset inputs are enabled. See +[the policy contract and boundary](fiscal-leaf-policy.md). + +Complete-population ancestry remains an explicit requirement. The PUF host now +retains its actual checked result for downstream consumption through +`check_survey_puf55_run`. Constructing a dataclass or decoding its receipt cannot +issue that authority. Consumers must recheck the original run before use and +after their last relevant I/O. The corrected checker passes 25 controls on +actual invented cold/required execution, including extra manifest attachments, +source/store corruption and late output mutation. Independent review approved +the source and mechanical test colocation; 45 cases share one expensive fixture. +See [the retained-output evidence](../experiments/us-puf55-checked-output-20260912.md). + +The fixed post-PUF country host now assembles source-qualified unemployment, +three health-cost amounts and nine current health-coverage inputs. It retains +the checked PUF owner and attaches both fragments to the same receiving +population, preserving existing columns, geography, design anchors and mass +ledger. ASEC observations and unresolved values are retained on both clones; +modeled ACS draws are deliberately shared within each original person's clone +pair. Reporting universes remain explicit; no prior-wage or age-only zero +fallback is introduced. The narrower ACS coverage concepts remain nullable +where the source does not establish equivalence. + +The accepted invented fixture reports 74 present input names out of 161, up +from 61, leaving 87 missing. It executed 26 new enrichment nodes over a validated +245-node prefix, then required all 271 replay hits and complete Frame readback. +Its six logical controls passed across five tests in the main attempt and a +corrected focused parent-revocation follow-up, not one all-green full run. +Independent review approved that scoped evidence. After integration with the +eighth interest field and merged shared graph, 80 cheap source, attachment and +health-fragment controls pass. Declaration checks still give 271 nodes: the +eighth interest field adds an output to the existing financial attachment, +not a new fit or graph node. This integration did not repeat the full combined +host or measure fresh native coverage. See +[the combined integration record](../experiments/us-survey-enrichment-integration-20260912.md) +and [source/model decisions](us-current-survey-amount-successor.md). + +Actual graph tracing also confirms that the legacy full ASEC carry and ACS +gap-fill are not executed by this host. Remaining pension, retirement-account, +property, farm and other-income inputs therefore need source-qualified producers +and appropriate ACS completion. The ACS property-income aggregate should +condition a joint decomposition before the dependent PUF fits; separately adding +rent afterward would not conserve that observed anchor. Source measurement +bridges and overlapping income categories must be resolved explicitly before +claiming reconciliation. The broader ACS retirement total is not merely pension +plus regular-IRA income. + +Two reviewed building blocks for that completion are adopted in the release +branch. The [ACS anchor qualifier](../experiments/us-current-acs-income-anchors-20260912.md) +preserves original INTP/RETP amounts, adjustment factors, original ages and +allocation flags through the retained source owner; its 35 invented-source +controls pass. The [signed reconciliation helper](../experiments/signed-income-reconciliation-20260912.md) +projects joint component draws onto a caller-qualified total while allowing +declared signed components and preserving loss offsets; its 45 numeric controls +pass. An explicit property-income option now integrates the source bridge, +four conditional component models, signed reconciliation and clone attachment +into the existing financial host. Its actual invented 35-node construction and +required replay pass, preserving the full preceding population. The new columns +remain available beside the retained earlier population. A further explicit +three-node extension rebases four interest/dividend tax inputs and checks their +conservation and completeness. Its 38-node cold/required host passes eleven +invented-source controls; unknown inputs remain unknown and an incomplete gate +refuses PUF qualification. Retirement completion and complete PUF/native adoption +remain separate work. The focused PUF +handoff retest passes after correction of an invalid invented source record; +see the [scoped host evidence](../experiments/us-property-financial-host-20260912.md). +The [13 September integration record](../experiments/us-launch-consolidation-20260913.md) +also covers retirement source qualification, UK lineage and the CI repairs. + +The remaining critical sequence is complete PUF/input integration, a small +real full build, model and calibration evaluation, then progressive scale +and release verification. The existing age-development runner is a separate +prefix demonstration, not calibration of the nineteen-node financial result. See +[the US review guide](us-launch-review.md) for scoped acceptance records. + +UK work already includes raw-source enrichment, national/local calibration, +size experiments and release assembly. Its recorded P50/P95b 55,000-household +candidates fail area-support and target-fit checks; the skipped holdout does +not establish an incumbent-comparison pass. Historical dense assembly also +does not satisfy today's release battery. See +[the size evidence](../experiments/355-uk-dataset-size-receipts.md) and +[the current dense release runbook](uk-dense-release-assembly-runbook-762.md). + +The current UK route clones for geographic support and then uses the existing +OA ladder. It has not adopted the new shared atomic geography graph keyed to +stable post-clone household identity. Its existing placement after cloning is +consistent with the user's 10 September ordering correction; the source, +identity and shared-operator migration remain separate work. Maria's +[PR #900](https://github.com/PolicyEngine/microcosm/pull/900) updates Chronicle +household targets and the Northern Ireland lookup; that source work does not +by itself migrate the assignment owner. Anthony's +[PR #896](https://github.com/PolicyEngine/microcosm/pull/896) provides staging +telemetry and synthetic smoke evidence, not a calibrated release. + +The shared UK adapter now has accepted supplied-array tests for England/Wales +Output Areas, Scottish Output Areas and Northern Ireland Data Zones. These +checks cover source conventions, observed-region constraints, stable post-clone +identity and row-order/subset invariance. Ordinary UK imports also preserve the +public export contract without eagerly loading unrelated stages or country +engines. All 52 checks passed with unchanged source bytes and no unexpected +access attempts; see the [scoped acceptance record](../experiments/uk-atomic-area-lazy52-20260910.json). +No native UK area source, whole-spine build or calibration result is certified +by those checks. Maria can use these shared primitives in the existing UK graph +integration rather than starting a second country runtime. + +UK release coverage must be re-established on the chosen integrated build and +locked model. The declared 145-input contract's older candidate evidence must +not certify the current stage set. The +[migration epic](https://github.com/PolicyEngine/microcosm/issues/665) tracks +source and calibration work; +[exact-count release parity](https://github.com/PolicyEngine/microcosm/issues/898) +and [default promotion](https://github.com/PolicyEngine/microcosm/issues/823) +remain separate consumer-release requirements. Existing open issues are leads +for candidate-specific verification, not proof every historical defect remains. + +## Team meeting decisions + +This document is the shared US/UK release-path reference. The country review +guides and run receipts provide supporting detail; update the sequence and +ownership here when they change. + +The 10 September coordination sequence is: + +- Maria's #900 source/target reconciliation merged on 10 September at 14:54 UTC; + Anthony's #855 target hierarchy/diagnostics and corresponding dashboard + consumer follow it. Its merge clears that dependency, not all release gates. +- Maria owns registering every existing UK stage as a graph node, stacking on + #893. Changes to UK stage order follow in a separate PR. The shared atomic-area + adapter work supports that integration and does not create a second UK graph. +- Anthony's UK staging #896 can proceed independently. His earlier runtime + proposal #885 is superseded in intent, not another required implementation. +- US construction and verification continue alongside these changes. Complete + graph coverage in both countries is required for the broader advertised + launch; passing tests for individual stages does not establish that coverage. + +The latest UK full rebuild and evaluation were reported as underway at the +meeting. Its new report must replace historical quality evidence only after the +actual candidate and results are available; the approximate four-hour runtime +reported for that build is not a US or release ETA. + +1. Confirm the release years, advertised geographies and full/compact products. +2. Assign one accountable owner to each country candidate and one owner to each + shared dependency: runtime, calibration, diagnostics and release consumers. +3. Review every row above for the latest actual candidate. Name the missing + artifact and next measurable result, rather than reporting code completion + as population acceptance. +4. Agree calibration, holdout, incumbent and representative-reform criteria, + including which differences need explanation and which block release. +5. Set dates for the next complete small candidate, the full quality review and + the production-sized candidate using measured runs rather than linear timing + extrapolation. + +Country data work, target/source reconciliation, independent validation, and +dashboard/consumer integration can run in parallel. Each final dashboard and +release verdict must refer to the same candidate bytes. Cross-sectional release +does not require retirement dynamics, every possible forecast year, or a +complete Chronicle/Orrery redesign. Keep those workstreams active without +making them prerequisites for the first accepted cross-section. diff --git a/docs/validation-cost-and-borrow-boundaries.md b/docs/validation-cost-and-borrow-boundaries.md new file mode 100644 index 000000000..f88c2cf4c --- /dev/null +++ b/docs/validation-cost-and-borrow-boundaries.md @@ -0,0 +1,43 @@ +# Validation cost and borrow boundaries + +The US graph must detect changed sources, substituted artifacts and changes to retained Populations without repeating the same expensive work at every nested call. This note records the current decision and the proposals still under review. It does not change dataset release requirements. + +## Adopted: reuse source compilation + +The ACS native coverage owner now keeps a bounded process-local cache of compiled code. Its key includes exact source bytes, filename, mode, flags, inheritance setting, effective optimization level and compiler identity. Every source read, AST check and loaded-function, global, alias and closure check remains fresh. Cached code is neither source authority nor a portable receipt. + +The change passes 25 focused controls and the original invented financial fixture, including cold execution, required replay and final checks. One paired profile took 216.82 wall seconds versus 308.26 seconds before the change, a 29.66% reduction. CPU time fell from 303.90 to 214.69 seconds. These instrumented observations establish an improvement for that fixture; they do not establish native-data performance. See [the experiment](../experiments/us-acs-compilation-cache-adoption-20260910.json) for exact source, test and verification identities. + +The native pilot already in progress retains its original frozen source snapshot. + +## Next: remove duplicate work within a check + +A source review confirmed nine direct Frame storage traversals in a healthy `verify_acs_native_coverage` call: three calls to `_verify_frame`, each performing three storage hashes. Some repetitions occur across supplemental axis and resolved-weight checks, so collapsing all nine into three is not automatically equivalent under the existing mutation checks. + +The next candidate will share the first storage calculation with the supplemental fingerprint calculation, while preserving the independent final prepared-Frame storage pass and all three outer verification points. That would reduce nine direct traversals to six. It needs a regression that mutates a cell during the supplemental check, unchanged receipt and fingerprint comparisons, and a paired measurement before adoption. + +## Keep separate: code, files and retained objects + +Three different facts currently appear together in nested validation: + +- Loaded-code identity: the interpreter's functions, code, globals, aliases and closures. +- Agreement with files: source bytes, archive snapshots and stored graph artifacts still match their recorded identities. +- Retained-object identity: the actual Frame, Population, weights and ancestry still match their issued state. + +An independent Fable 5.1 review recommended making these checks explicit and performing expensive file verification primarily at public entry and emission boundaries. That is a proposed contract change, not an adopted optimization. Moving file checks would change when an altered archive is detected. It requires a documented public-entry contract and tests at every relevant callback, file borrow and emission boundary before implementation. + +Similarly, an AST cache would hold mutable objects, unlike compiled code. It needs its own ownership design and evidence; the compilation-cache result does not establish its safety or benefit. + +## Composition review + +The same review raised kernel-registry sharing, registration order and recipient ordering. These require owner-level checks before being treated as bugs. For example, each financial run currently constructs a fresh survey prefix internally; it does not accept an existing prefix for a second extension. The PUF raw-route merger separately derives and verifies the complete receiving tax-unit axis before restoring row order. Neither observation alone proves every composition boundary correct. + +The budget returned-view correction is now adopted. Detached documents and grouped bounds are constructed after the last borrowed I/O, then checked against the issued payload and retained owners. Four targeted controls passed, including nine callback branches; the post-run verification rechecked 501 source/owned files, 5,983 model-code files and 12 resources. The earlier eight financial-successor controls remain evidence for their original source revision. These are separate component runs, not twelve tests of one integrated revision. See [the correction experiment](../experiments/us-budget-detached-view-correction-20260910.json). + +A second Fable source review found no confirmed correctness or ownership bug in the public PUF recipient interface. It recommended explicit cold-cache reuse assertions, exact comparisons with the original issuers, and clearer documentation of the original-source reads required by the recipient kernels. The frozen two-positive pilot remains unchanged; those additions belong to subsequent controls. + +The immediate implementation priorities remain actual public PUF recipient and graph checks, authenticated donor/model binding, one whole-cohort PUF finalization and attachment, then native calibration and release verification. Performance work proceeds alongside those tasks. + +## Review scope + +Fable 5.1 completed a read-only source review on September 10, 2026, using the saved Hivesight login. Its requested scratch-cache read was denied and that limitation was disclosed in its result. It supplied architectural advice, not runtime approval or a release verdict. The recommendations above are distinguished from independently verified source findings and adopted changes. diff --git a/experiments/893-reconciliation-amendments-19-20-20260912.md b/experiments/893-reconciliation-amendments-19-20-20260912.md new file mode 100644 index 000000000..f1627eb2b --- /dev/null +++ b/experiments/893-reconciliation-amendments-19-20-20260912.md @@ -0,0 +1,722 @@ +# #893 reconciliation to main's amended graph interface — lane report + +> **12 September Codex continuation:** this is the historical layer proposal +> at `80fef2217`, not an approved extraction plan. The final battery +> `bt0nx8fcc` was killed during its first pytest group: CI-group verification +> and coverage check exited zero, but no final consumer result was produced. +> Independent review found that G4 exposes mutable admitted populations to +> observers and that the layer order omits hard packaged-resource dependencies. +> Fix and review both before extracting or landing layers. PR912 is also held +> for fixes from Astra's executed review; the branch-head assumption below is +> superseded until its corrected head lands. +> +> Resource repair `a4b6fe88c` restores all three omitted ASEC JSON contracts +> from their exact historical blobs, matching unchanged loader pins. Eight +> targeted tests and wheel-content verification pass. These three resources, +> `acs_2024_housing_universe.json`, and current-money domains/consumers/price-basis +> resources must accompany their first PR-1 loaders or an earlier prerequisite; +> Python import analysis alone cannot establish extraction order. No engine +> defaults were admitted and no source/engine identity was refreshed. + +> **Later 12 September correction:** `6f66fa545` closes the demonstrated G4 +> mutation defect by handing observers detached snapshots, including nested +> table values, metadata, schema, weights and ledgers. Independent review is +> clean; 524 graph tests, five focused cases and all four existing synthetic +> consumers passed, including financial and calibration teardown checks. An +> earlier source-edit-during-test failure remains preserved and is excluded. +> Entry 7.4 below is historical: the current observer need not be trusted to +> leave its snapshot unmodified. Retaining snapshots still costs memory. +> +> `cb27bdb0b` makes the offline explanation display executor-owned unreached +> and gate-exception states independently from cache hits. Sixteen graph +> explanation/actual-gate cases and two free-form diagnostic controls passed. +> Its new `explain → availability` import is recorded in the implementation +> inventory; availability was already bound as a whole module in all ten +> using scopes. No scope, dependency, resource, or unbound-use exemption was +> added. All eight prepared-resource/full-manifest tests passed afterward. +> +> Corrected PR912 head `5e33d6971` is undergoing exact-head CI and Fable review. +> Graph-only extraction is being prepared against that revision, incorporating +> G1–G4 and numbered charter entries. Neither that extraction nor the remaining +> country layers are declared accepted by this historical report. + +> **Subsequent 12 September integration checks:** corrected PR912 head +> `5e33d6971` received Fable approval; its merge remains conditional on CI. +> The local integration merged that exact revision in `9e61d0d04` and +> recomputed the combined source/seed identities. All 734 graph and fitting +> tests passed. The six-file spec/build sweep collected 248 tests: 247 passed +> and one failed because its invented PUMA fixture still used schema 1 and +> omitted the required joint tract/CD population support. The fixture now +> supplies consistent schema-2 support and explicit invented provenance; that +> single test passed when rerun. The original failed sweep and intermediate +> incomplete fixture repair are preserved. This is separate-run evidence, +> not a claim that the complete 248-test sweep was rerun successfully. +> +> Shared graph consolidation is now public as PR913, temporarily based on +> PR912, with independent Fable review in progress. Its runtime matches this +> integration's graph implementation. Native strict replay, PUF55 host +> acceptance and complete national/CD release acceptance remain separate +> work and are not established by these software checks. + + +Date: 2026-09-12. Worktree `~/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910`, +branch `microcosm-us-launch-verified-lanes-20260910` (PR #893's branch is +`microcosm-us-launch-integration-20260909`). Started at `069d5ed9a`. No push, no +new branch, no stash, `uv.lock` untouched. Every commit below is on the +checked-out branch. + +## 1. Outcome + +- `origin/main` (`0c3f4f651`, amendment 19) and + `origin/amend-keyed-seed-and-uniform-draws` (`23ba24770`, amendment 20 / PR + #912) are merged. **#912 was merged from its branch head**, so once it lands + the dispatcher should re-run `git merge origin/main`; for the graph and fit + packages, `docs/graph-interface.lock` and `docs/graph-acceptance.md` that is + expected to be a no-op (this tree already carries `23ba24770`'s bytes for all + of them, verified with `git diff --quiet 23ba24770 HEAD -- `). +- Every frozen or amended file equals `23ba24770` byte for byte: `decl.py`, + `kernel.py`, `keys.py`, `serialize.py`, `view.py`, `artifact_edges.py`, + `randomness.py`, `population.py`, `fit/qrf.py`, the lock, the charter, + `test_graph_kernel_contract.py`, and every `test_acceptance_*` file. The + interface-lock test passes (it is part of the 518-test graph run below). +- Four not-yet-amended graph pieces the US runtime consumes were re-applied + as separate named commits on top of main's files; four pieces with no + consumer were dropped (section 3). +- The five graph tests the brief named are green; the whole graph package is + green (518 passed); the fit package is green (191 passed); the spec-engine + check, `ci_test_groups --verify` and ruff are clean. Build-side results are + in section 5. +- The layer map (section 6) partitions the branch's remaining 407-file diff + against `origin/main` into the eleven staged layers, the later closure and + post-staging work, and the graph pieces, with per-layer file lists, test + files, import dependencies, and a proposed landing order. Section 7 has a + draft charter amendment entry per kept graph piece. Section 8 lists what + needs Max. + +Lane commits (newest first): + +``` +07566eb17 Re-pin the US implementation inventory for the reconciled graph package +c704a1be6 Pin that unreached propagates through a structural node and its version +2e1857a4b Bring the #893 reconciliation journal up to the piece commits +a16eeacf8 Re-apply the private population observer on run_graph +d1019762b Re-apply executor execution states: gate exceptions leave their consumers unreached +db1b7821a Re-apply complete Frame metadata storage and non-finite JSON refusal in the store +752ab840f Re-apply the raw-byte source codec on main's graph package +cff8fbf32 Re-pin the calibrate and simulate H1 parity cases the branch's kernels move +dc621c14c Re-pin the seed protocol and compiled seed map digests on the merged tree +051357909 Merge origin/amend-keyed-seed-and-uniform-draws (23ba24770, amendment 20) +3010b7788 Merge origin/main (0c3f4f651, amendment 19) into the #893 integration branch +3ad1b1aae Open the #893 reconciliation lane journal +``` + +## 2. The two merges and the per-file resolution + +Baseline before any change (`uv run --no-sync pytest packages/microcosm-graph/tests`): +5 failed, 362 passed, exit 1 — exactly the five the brief names +(`test_acceptance_b_ownership::test_b2_executor_enforces_ownership`, +`test_acceptance_h_parity::test_h1_kernel_parity`, +`test_graph_executor::test_fit_qrf_tolerance_source_hash_pin_is_current`, +`test_graph_kernel_contract::test_context_numerics_default_empty_and_carry_scopes`, +`test_graph_serialize::test_generated_parity_graphs_bind_real_kernels_and_direct_bytes`). + +### Merge 1: `git merge origin/main` (commit `3010b7788`) + +Git reported 8 content conflicts plus one in `PROGRESS.md`; `store.py`, +`__init__.py` and `PROGRESS.md` auto-merged. Resolution: + +| Path | Resolution | +| --- | --- | +| `packages/microcosm-graph/src/microcosm/graph/{artifact_edges,decl,executor,kernel,keys,manifest,serialize,view}.py` (conflicts) | main's bytes (`git checkout origin/main -- packages/microcosm-graph`) | +| `packages/microcosm-graph/src/microcosm/graph/{store,__init__}.py` (auto-merged) | main's bytes, same checkout; the branch's hunks return as pieces A and B | +| `packages/microcosm-graph/src/microcosm/graph/{attachments,availability,schema}.py` (branch-only) | removed in the merge (`git rm`); `availability.py` returns as piece C, the other two are dropped | +| `packages/microcosm-graph/src/microcosm/graph/randomness.py` (branch-only until amendment 20) | kept as the branch had it; replaced by `23ba24770`'s in merge 2 | +| `packages/microcosm-graph/tests/*` | main's bytes; the branch's copies were strict subsets of main's (numstat showed only removals), so this also dropped nothing of the branch's; `test_frame_metadata_store.py` and `test_lazy_snapshot_metadata_integration.py` (branch-only) removed here, the first returns with piece B, the second is dropped | +| `packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json` | main's (consistent with fit/qrf.py being main's in this merge) | +| `packages/microcosm-fit/src/microcosm/fit/qrf.py` (auto-merged to the branch's) | main's bytes; the branch's `predict_from_uniforms` returns with amendment 20 in merge 2 (the branch's qrf.py was byte-identical to `23ba24770`'s) | +| `packages/microcosm-fit/src/microcosm/fit/{_graph_legacy_apply,_graph_legacy_qrf,graph_legacy_apply,graph_legacy_apply_matrix,graph_legacy_qrf,graph_legacy_train,model_input,qrf_target}.py` (branch-only) | kept: not amended content, consumed by the US runtime (layer 01) | +| `docs/graph-interface.lock`, `docs/graph-acceptance.md` | main's bytes | +| `PROGRESS.md` | both sides kept: this lane's section, then main's Amendment 19 and #907 sections, then the branch's history | +| everything else | ordinary auto-merge | + +### Merge 2: `git merge 23ba24770` (commit `051357909`) + +Six conflicts: + +| Path | Resolution | +| --- | --- | +| `packages/microcosm-graph/src/microcosm/graph/randomness.py` (add/add) | `23ba24770`'s (adds the signed-zero normalisation the branch lacked) | +| `packages/microcosm-fit/tests/test_qrf_stateless.py` (add/add) | `23ba24770`'s (182 lines; the branch's 159-line copy was a strict subset) | +| `docs/evidence/spec-engine/us-f0-coverage.json` | `23ba24770`'s, then recomputed (section 4) | +| `packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py` (seed digests) | `23ba24770`'s, then recomputed (section 4) | +| `packages/microcosm-build/tests/test_spec_engine_country_bundles.py` (AM/BE/UK digests) | `23ba24770`'s; see section 5 for the merged-tree check | +| `packages/microcosm-build/tests/test_us_multispine_pool_tool.py` (pool tool `spec_sha256`) | `23ba24770`'s; see section 5 | + +After merge 2 the graph package, fit sources, lock and charter were +byte-identical to `23ba24770` except the eight branch-only fit modules above; +`docs/graph-interface.lock` matched `shasum -a 256` of the two frozen files. + +## 3. The not-yet-amended graph pieces + +Consumers were found by an AST scan of every `.py` under +`packages/microcosm-{build,calibrate,fit,frame,data}` and `tools/` for names +imported from `microcosm.graph.*` that `23ba24770`'s graph package does not +define, plus greps for the keyword-level uses (`_population_observer=`, +`population_retention=`, `emit_validation_artifact=`). + +### Kept (each re-applied on main's file as its own commit, with tests) + +| Piece | Commit | Consumers outside the graph package | Graph-level tests | +| --- | --- | --- | --- | +| A. `codecs.py` raw-byte mode: `SourceBytesCodec`, `register_bytes`/`load_bytes`, `raw-bytes-v1`, `RAW_BYTES_MAX_BYTES`, `load_source_bytes` (+ two `__init__` exports) | `752ab840f` | 13 modules: `graph_atomic_geography`, `graph_geography`, `graph_sources`, `graph_atomic_survey_population`, `graph_puf55_canonical_donor`, `puf_raw_source`, `puf_monetary_source`, `puf_monetary_agi_projection`, `puf_full_source_graph`, `atomic_block_sources`, `atomic_block_api_sources`, `survey_atomic_geography`, `puf55_survey_recipients`; 8 US test files | the branch had none; 4 added to `test_graph_codecs.py` (8 passed) | +| B. `store.py` Frame-metadata storage (`microcosm-graph-frame-v2`, `_encode/_decode_frame_metadata`, `frame_metadata_sha256`, `_put(validate_existing=…)`) and the non-finite JSON decode hooks | `db1b7821a` | `survey_population_replay.same_replayed_frame`, `population_input_coverage`, `graph_full_puf_enrichment` (all call `_encode_frame_metadata`), and every US stage relying on `Frame.metadata` surviving a store round trip | `test_frame_metadata_store.py` (the branch's file, 19 collected tests) | +| C. `availability.py` + executor/manifest execution states: `gate_exception`, `unreached`, `blocked_by`, cache-record schema 3, manifest schema 4, refusal of kernel-authored execution metadata; replaces amendment 19's refusal of a gate declaring a typed output | `d1019762b` | `graph_atomic_survey_clone` (`atomic_geography_nodes(..., emit_validation_artifact=True)` makes the geography gate declare a typed output, which main's executor refuses outright), its consumers `survey_age_calibration`, `graph_current_survey_predictors`, `survey_origin_budget`; `population_input_coverage` (`execution_state` on every producer receipt); build test `test_atomic_geography.py` parametrises the declaration | the branch had none; main's `test_a_gate_kernel_may_not_declare_a_typed_artifact_output` replaced by 6 tests in `test_graph_executor.py` (five in `d1019762b`, the structural-propagation one in `c704a1be6`) | +| D. `run_graph(_population_observer=…)` | `a16eeacf8` | `survey_age_calibration`, `graph_atomic_survey_financial`, `graph_survey_population`, `graph_atomic_survey_population`; 8 US test files | the branch had none; 1 added | + +What was deliberately not re-applied inside those files: the branch's +docstring deletions on the amended surface; its `_opaque_artifact_key` +inline hash (main's `graph_keys.opaque_artifact_key` is the same +derivation); its `KernelContext` argument order (frozen file); its eager +artifact payload reads on cache hits (main authenticates edges without +reading, `9018c4420`); its `StoreMiss`-on-hit for a missing declared artifact +(main's `5c4a8efde` decision, `StoreCorrupt`); the removal of main's #907 +datetime64/timedelta64 object-leaf refusal in `store.py`; and main's +`_validate_typed_ancestry` gate-ancestry shape check and the +`NodeRejectedError`→`ValueError` conversion (`402d9a631`, `7d45c32e5`), which +the branch's older copy lacked. + +### Dropped (no consumer left) + +| Piece | Why dropped | +| --- | --- | +| `attachments.py`, `_PopulationRetention`, `_LazyPopulations`, `run_graph(population_retention="lazy")`, `PopulationView.__slots__ = ("__weakref__",)` (staged as layer 08 GRAPH-ATTACHMENT-METADATA) | no call site outside the graph package passes `population_retention`; its only consumer was the branch's own `test_lazy_snapshot_metadata_integration.py`, dropped with it. It is a memory optimisation for long graphs (the native pilot's 11.5 GB peak is in the PR body); if it is wanted it is a self-contained later PR on top of piece B, which it imports. | +| `schema.py` (`graph_schema`, 316 lines) | imported by nothing in the repository (the `graph_schema_version` hits in the spec bundle are unrelated). | +| `keys.py` `_stream_file` chunked source hashing | no consumer and no test; it changes no identity (same digest sequence, it only bounds memory and refuses a file that changes while hashed). Main's `keys.py` is kept byte-identical. Candidate for a small standalone PR if large-source memory matters. | +| the `_write_node` per-coordinate memory refactor and the `del series` | no consumer; behaviour-preserving; main's amended `_write_node` kept. | + +## 4. Identities re-pinned on the merged tree, and why + +- **Seed protocol and compiled seed map digests** (`inventory_coverage.py` + `EXPECTED_HASHES`, `docs/evidence/spec-engine/us-f0-coverage.json`; + commit `dc621c14c`). Both hash the exact source bytes of the attested kernel + modules in `spec_engine/seeds.py` (no `microcosm.graph` module is on that + list, so the graph pieces do not move them). The merged tree carries the + branch's `acs_transfer` and `housing_inputs` changes and, through amendment + 20, the same `fit.qrf` bytes the branch already carried, so the actual + digests are the values the branch pinned in `1734b9e90` + (`a775cccd…` / `c15bff65…`), not `23ba24770`'s (computed on a tree without + the two US kernel changes). Computed directly (`spec.seed_protocol.implementation_sha256`, + `sha256_json(compiled.seed_stream_map.to_wire())`), then the report was + regenerated with `tools/spec_engine_coverage.py` and `--check` exits 0. +- **H1 parity pins for `calibrate` and `simulate`** (commit `cff8fbf32`). + Six graph tests were red on the merged tree, all on these two cases. + `CalibrateKernel.implementation_hash` binds `microcosm.calibrate`'s solve and + diagnostics modules, which the branch's grouped-bound solver path changes + (layer 05); `SimulateRulesKernel.implementation_hash` binds + `microcosm.frame.bundle`, to which the branch adds `Frame.__reduce__` (layer + 01). Re-recorded with `tools/graph_parity_repin.py calibrate` and + `… simulate`, which keep every pinned platform (both cases pin only + `arm64/darwin/py3.14`) and leave `direct.csv` untouched. The `fit.qrf` pin + came from `23ba24770` and needed no move. +- **The US implementation inventory** (`us_runtime/graph_implementation_inventory.json`; + commit `07566eb17`). This reviewed dependency fence is checked by every US + source stage before it runs (`graph_implementation.implementation_manifest`), + and its refusal surfaces as `PREPARATION_ISSUANCE_REFUSED` (the preparation + wraps every exception; the true cause was recovered from the exception's + `__context__`). Three entries no longer held: the `microcosm.graph` roster + listed the dropped `attachments.py` and `schema.py` (39 survey tests red on + that alone); the contracts for `executor.py`, `manifest.py`, `keys.py` and + `population.py` reflected the branch's versions rather than main's plus the + kept pieces, and `attachments.py` was a listed stage module in ten stages; + and `congressional_district_vintage.py`'s contract predated main's + `05ff48f05` (merged into the branch on 2026-09-11), which added the + `microcosm.calibrate.geography_constants` import — an inherited drift the + branch already carried at `069d5ed9a`, verified by recomputing the + pre-merge tree's contracts against its own inventory. Recomputed with the + module's own `_dependency_contract` / `_covered_imports`; the import + classification set is the exact union again. Seven of the ten stages now + build their manifest; three cannot (section 8.5). +- **Country-bundle digests and the pool tool's `spec_sha256`**: resolved to + `23ba24770`'s values in merge 2; their merged-tree check is in section 5. + +## 5. Commands, exit codes, counts + +All from the worktree root with the locked engine environment +(`uv sync --all-packages --locked --extra us --extra uk` → exit 0 before any +change). Tests ran with `uv run --no-sync pytest … -p no:cacheprovider`. + +| Stage | Command | Result | +| --- | --- | --- | +| baseline (`069d5ed9a`) | `pytest packages/microcosm-graph/tests` | 5 failed, 362 passed, exit 1 | +| baseline | `python tools/ci_test_groups.py --verify` | exit 0 | +| after both merges (`051357909`) | `pytest packages/microcosm-graph/tests` | 6 failed, 485 passed, exit 1 (calibrate/simulate parity pins; section 4) | +| after both merges | `pytest packages/microcosm-fit/tests` | 191 passed, exit 0 | +| after both merges | `python tools/spec_engine_coverage.py --check` | exit 1 (`seed protocol content digest differs`, `compiled seed map digest differs`) | +| after both merges | `python tools/ci_test_groups.py --verify` | `verification=ok`, exit 0 | +| after both merges | `ruff check .` | All checks passed, exit 0 | +| seed re-pin (`dc621c14c`) | `python tools/spec_engine_coverage.py` then `--check` | exit 0 / exit 0 | +| parity re-pin (`cff8fbf32`) | `python tools/graph_parity_repin.py calibrate`; `… simulate` | exit 0 / exit 0 | +| parity re-pin | `pytest test_acceptance_h_parity.py test_graph_parity_pins.py test_graph_parity_repin.py test_graph_serialize.py` | 27 passed, exit 0 | +| piece A (`752ab840f`) | `pytest packages/microcosm-graph/tests/test_graph_codecs.py` | 8 passed, exit 0 | +| piece B (`db1b7821a`) | `pytest test_frame_metadata_store.py test_graph_store.py test_graph_population.py test_graph_executor.py test_graph_manifest.py` | 274 passed, exit 0 | +| piece C (`d1019762b`) | `pytest packages/microcosm-graph/tests` (whole package, includes the interface-lock test) | **518 passed, exit 0** | +| piece C | `ruff check .`; `ruff format --check packages/microcosm-graph` | exit 0 / exit 0 | +| piece D (`a16eeacf8`) | `pytest test_graph_executor.py test_acceptance_b_ownership.py test_acceptance_f_gates.py` | 94 passed, exit 0 | +| structural test (`c704a1be6`) | `pytest packages/microcosm-graph/tests/test_graph_executor.py` | 82 passed, exit 0 | +| inventory re-pin (`07566eb17`) | `pytest test_us_survey_origin_budget.py::test_actual_initial_budget_and_zero test_us_survey_age_calibration_run.py::test_actual_seven_node_calibration_preserves_full_population` (two of the 39 formerly refused) | 2 passed, exit 0 | +| final tree | `python tools/ci_test_groups.py --verify` | exit 0 | +| final tree | `python tools/spec_engine_coverage.py --check` | exit 0 | + +Build-side and shared consumers, final tree (`07566eb17`). A first battery at `a16eeacf8` found 39 survey tests refused (`PREPARATION_ISSUANCE_REFUSED`) — all traced to the implementation inventory above, re-pinned in `07566eb17` — and one non-existent file name in my own raw-bytes group list (`test_us_puf_raw_source.py`, removed); the battery was then rerun in full: + +_(Historical draft marker resolved: task `bt0nx8fcc` was killed during availability consumers. No final battery counts or exit code exist; later groups did not start.)_ + +## 6. Layer map: landing #893 as sequential PRs + +Basis: `git diff --name-status origin/main HEAD` = 407 files (340 added, 67 +modified, 0 deleted), each attributed to the first branch commit that touched +it (`git log --name-only --no-merges 3094bfe84..HEAD`), with the eleven +`Stage integration layer NN` commits mapped to their layer names, the later +staging commits to layers 12–14, the sixty post-staging commits of 10–12 +September grouped by subject (P1–P6), and the graph pieces of section 3 as +their own layer. Import dependencies come from an AST scan of every changed +`.py` file resolved against that attribution (`microcosm.*` imports only). +The scripts and their JSON output are in this lane's scratchpad +(`layers.py`, `layers2.py`, `layers.json`, `per-file-deps.json`). + +Two corrections to the staged layer names as they sit in git history: + +- Layer 08 GRAPH-ATTACHMENT-METADATA is empty in the map: its only file, + `attachments.py`, is dropped (section 3). +- Layer 09 F-JOINT-GEOGRAPHY-GATE has no file of its own: its 27-line change + lives in `graph_geography.py`, which layer 01 created, so it lands with + layer 01. +- Nineteen files differ from `origin/main` only because `origin/main` does not + yet carry amendment 20 (`kernel.py`, `randomness.py`, `fit/qrf.py`, + `test_graph_kernel_contract.py`, `test_graph_parity_*`, + `test_graph_randomness.py`, `tools/graph_parity_repin.py`, the three + `pins.json`, the lock, the charter, the amendment-20 journal/receipts/ + changelog fragment) or were re-pinned by this lane. They vanish from the diff + when #912 lands and are not #893 content; they are listed under + "already on main's way" below and belong to no PR. + +### Dependency facts the order rests on + +Three kinds of link were measured, strongest first: + +1. **Hard**: a file imports a module that does not exist on `origin/main` + (a file the branch adds). These force order. +2. **Name-hard**: a file imports, from a module main already has, a *name* + that main's version of that module does not define (checked by parsing + `git show origin/main:`). Only three survive after discarding + submodule imports that merely look like names: `GroupedUpperBounds` + (layer 03's two-line re-export in `calibrate/__init__.py` of layer 01's + `group_bounds.py`; imported by layer 12's and 13's tests), + `decode_us_puma_ladder` (layer 06's `puma_ladder.py`; imported by layer + 13's `test_us_geography_integration.py`), and the graph pieces' names + (`load_source_bytes`, `load_raw_bytes`, `RAW_BYTES_MAX_BYTES`, + `_encode_frame_metadata`, `execution_state`). +3. **Soft**: a file imports a module main already has that the branch also + modifies (`frame_checkpoint`, `asec_checkpoint`, `operator_boundary`, + `us_runtime/__init__`, `calibrate/__init__`, `solve.py`, `bundle.py`, + `variable_labels.py`, and amendment 20's `qrf.py`/`kernel.py`). These do + not force order by themselves; each PR's own test run is what proves the + modified behaviour is or is not needed. + +Hard links between layers (source and tests): + +| Layer | hard on | of which tests only | +| --- | --- | --- | +| 01 SAFE-ADDITIVE | nothing | — | +| 03 ACCEPTED-SHARED-RESTORE | 01 (`group_bounds`, `reported_coverage_source`, `_person_signal_summary`, `operator_column_contracts`, `table_identity`), P3 (`asec_raw_stage_v4`) | P3 link is `test_us_asec_checkpoint.py` only | +| 04, 06 | nothing hard (soft on 01/03) | — | +| 05 SOLVE-MERGE-PROPOSAL | 01 (`group_bounds`) | — | +| 07 F-CATALOGUE-OPTIMIZATION | 01 (`survey_population_preparation`) | all three files are tests | +| 10 SOURCE-CLOSURE | G1 (soft) | — | +| 11 PLACEMENT-ADDITIONS | 01 (the eight fit modules, `full_puf_enrichment`, `graph_sources`, `survey_population_replay`), G1, G2 | — | +| 12 ORDINARY-CLOSURE | 01 (fit modules), 03 (`GroupedUpperBounds`) | all files are tests/fixtures | +| 13 INTEGRATION-REGRESSIONS | 01 (`graph_geography`), 03 (`GroupedUpperBounds`), 06 (`decode_us_puma_ladder`) | all three are tests | +| P1 POST-CLONE-ATOMIC-GEOGRAPHY | 01 (`randomness` = amendment 20, fit modules, `graph_combined_clone`, `graph_sources`, `graph_survey_population`, `survey_*`), G1, G3 (the gate's declared artifact), P2 | P2 link is two tests (`test_us_current_survey_predictor_demographics.py`, `test_us_survey_origin_budget_predictor_compatibility.py`) | +| P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR | 01 (`model_input`, `_graph_legacy_qrf`, `graph_legacy_*`, and eleven test-side modules), 11 (`graph_full_puf_enrichment`, two tests), P1 (`graph_atomic_survey_financial`, one test), G1 | 11 and P1 links are tests only | +| P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET | 01 (`reported_coverage_source` from `asec_raw_stage_v4`; six test-side modules), G2, G3 (`population_input_coverage`), P2 (`asec_demographic_source`) | P2 link is `test_us_asec_demographic_source.py` only | +| P4 | nothing hard (soft on 03) | — | +| P5 | nothing hard (soft on 01/03/04/06) | — | +| P6 UK-ADAPTER-DOCS-AND-RECORDS | P1 (`atomic_geography`) | — | + +Two cycles exist and both are test-only: P1↔P2 (three tests) and 03↔P3 +(one test). Every source-level dependency is acyclic. + +### Landing order + +1. **PR-G1 raw-byte codec** — piece A (`codecs.py`, `__init__.py`, + `test_graph_codecs.py`). No dependency. Charter entry 7.1. +2. **PR-G2 Frame metadata storage** — piece B (`store.py`, + `test_frame_metadata_store.py`, + `changelog.d/20260906-frame-metadata-storage.fixed.md`). No dependency. + Charter entry 7.2. +3. **PR-G3 execution states** — piece C (`availability.py`, `executor.py`, + `manifest.py`, `test_graph_executor.py`). No code dependency, but it + supersedes an amendment-19 ruling: **needs Max's decision (section 8) + before it is opened.** Charter entry 7.3. +4. **PR-G4 population observer** — piece D (`executor.py`, + `test_graph_executor.py`). Stacks on G3 only because both edit + `executor.py`; G3+G4 can be one PR. Charter entry 7.4. +5. **PR-1 SAFE-ADDITIVE** — all of layer 01: 101 non-test files (100 modules + plus `graph_implementation_inventory.json`; `randomness.py` is excluded, it + is amendment 20) and its 26 test files, plus layer 03's two-line + `calibrate/__init__.py` re-export of `GroupedUpperBounds` (it belongs with + `group_bounds.py`) and layer 09's 27 lines (already inside + `graph_geography.py`), and `graph_implementation_inventory.json`, which + pins the `microcosm.graph` roster and module contracts and therefore has + to be re-pinned against whatever main's graph package is when PR-1 opens + (`07566eb17` shows the procedure). Hard on G1, G2 and G4 only (`_population_observer` + in `survey_age_calibration` and `graph_survey_population`). Its 21 soft + links are listed in the scratchpad `pr1-refined.json`; the PR's gate is + its own 26 test files on main + G1–G4 + PR-1. +6. **PR-2 US runtime restore** — layers 03 (without the `__init__` re-export + moved to PR-1, and with `asec_raw_stage_v4.py` moved in from P3 so + `test_us_asec_checkpoint.py` runs), 04 (`puf_support.py`), 05 + (`calibrate/solve.py`), 06 (`us_runtime/__init__.py`, `puma_ladder*.py`, + `tools/build_us_puma_ladder_artifact.py`, six tests), 12's calibrate + tests and fixture, 13's three regressions and `docs/us-launch-review.md`, + 14 (`dfa7f872c`), and the two re-pins that these files move: the + calibrate H1 pin (`solve.py`) and the seed digests (`acs_transfer.py`, + `housing_inputs.py`). 03/04/05/06 touch disjoint files and could be four + PRs (05 first, since only it is hard on 01); they are one here because + their tests import across all four. +7. **PR-3 catalogue, source closure, placement** — layers 07 (three digest + tests + fragment), 10 (`graph_acs_housing_universe.py`, eight JSON source + definitions), 11 (`graph_full_puf_enrichment.py` + test), and 12's fit + tests (`test_graph_legacy_*`, `test_qrf_target.py`, + `legacy-slice-apply-v1.json`). Hard on PR-1 and G1/G2. +8. **PR-4 calibration diagnostics** — P4 (`variable_labels.py`, its test, + fragments, experiments). Soft on PR-2. +9. **PR-5 survey population, catalogue and budget** — P3 minus + `asec_raw_stage_v4.py`: `common_frame_export_contract`, + `input_coverage_profile`, `population_input_coverage`, + `frame/bundle.py`'s `__reduce__` (carries the simulate H1 re-pin), ten + tests including `test_us_spine_blindness.py`; + `test_us_asec_demographic_source.py` rides with PR-7 instead. Hard on + PR-1, G2, G3. +10. **PR-6 post-clone atomic geography** — P1 (`atomic_geography.py`, + `graph_atomic_geography.py`, `atomic_block_*`, `graph_atomic_survey_*`, + `survey_atomic_geography.py`, `current_survey_geography.py`, + `tools/ci_test_groups.py`, fourteen tests); its two P2-importing tests + ride with PR-7. Hard on PR-1, G1, G3. +11. **PR-7 PUF55 two-route and canonical donor** — P2 (nineteen modules, + twenty-one tests plus the three deferred from PR-5/PR-6, PUF docs and + growth evidence, `pyproject.toml`'s `pythonpath` line). Hard on PR-1, + PR-3 (11), PR-5, PR-6. +12. **PR-8 CI, seed-identity diagnostics and spine blindness** — P5 + (`.github/workflows/test.yml`, `tools/spec_seed_identity_diagnostics.py`, + the seven `test_spec_seed_identity_*` files, `CLAUDE.md`, + `test_spec_engine_loader.py`, `test_us_puf_support.py`, + `test_us_stacked_spine.py`, the AM/BE/UK bundle digests). Soft on PR-2 + and PR-5; last because the diagnostic's roster names modules from every + earlier PR. +13. **PR-9 UK adapter, docs and records** — P6 (`uk_runtime/atomic_area_support.py`, + `atomic_household_identity.py`, two tests, `.gitattributes`, records). + Hard on PR-6. + +Every `experiments/` record and `changelog.d/` fragment rides with the PR +whose code it describes (listed per layer below). The "already on main's +way" files (amendment 20, listed at the top of this section) belong to no PR. + +### Per-layer file lists + +#### 00 JOURNAL (1 non-test files, 0 test files) + +- `./`: `PROGRESS.md` + +#### 01 SAFE-ADDITIVE (101 non-test files, 26 test files) + +- `packages/microcosm-build/src/microcosm/build/`: `survey_allocation.py`, `survey_domain_sample.py`, `table_identity.py` +- `packages/microcosm-build/src/microcosm/build/cd_benchmark/`: `__init__.py`, `canonical.py`, `origin.py`, `protocol.py`, `reasons.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `_asec_current_money_codec.py`, `_person_signal_summary.py`, `acs_housing_universe.py`, `acs_housing_universe_source.py`, `acs_native_coverage_binding.py`, `acs_person_coverage_authentication.py`, `acs_person_coverage_columns.py`, `acs_population_catalogue.py`, `asec_2024_native_population.py`, `asec_coverage_authentication.py`, `asec_current_money.py`, `asec_current_money_graph_resources.py`, `asec_current_money_resources.py`, `asec_current_money_selection.py`, `asec_current_money_source.py`, `asec_current_money_units.py`, `asec_engine_evaluation.py`, `asec_household_coverage_fields.py`, `asec_household_observations.py`, `asec_housing_status.py`, `asec_housing_status_source.py`, `asec_housing_universe.py`, `asec_housing_universe_source.py`, `asec_income_observations.py`, `asec_original_household_weights.py`, `asec_person_coverage_source.py`, `asec_person_income_source.py`, `asec_population_catalogue.py`, `asec_prepared_source.py`, `asec_student_controls.py`, `cd_reference.py`, `cd_reference_sources.py`, `cps_carried_current.py`, `demographic_calibration_graph.py`, `full_puf_enrichment.py`, `graph_asec_income.py`, `graph_asec_prepared.py`, `graph_combined_clone.py`, `graph_composed_asec_binding.py`, `graph_composed_asec_measures.py`, `graph_composed_contracts.py`, `graph_composed_population.py`, `graph_context.py`, `graph_current_survey_puf_transfer.py`, `graph_geography.py`, `graph_housing_universe.py`, `graph_implementation.py`, `graph_implementation_inventory.json`, `graph_national_age_counts.py`, `graph_native_household_origin.py`, `graph_native_origin_implementation.py`, `graph_puf_detail_transfer.py`, `graph_puf_diagnostic_consumer.py`, `graph_sources.py`, `graph_survey_age_artifact.py`, `graph_survey_budget.py`, `graph_survey_calibration.py`, `graph_survey_population.py`, `national_age_activation.py`, `native_household_origin.py`, `operator_column_contracts.py`, `puf_detail_transfer.py`, `puf_diagnostic_consumer.py`, `puf_growth.py`, `puf_growth_graph.py`, `puf_monetary_agi_projection.py`, `puf_monetary_source.py`, `puf_price_baseline.py`, `puf_raw_source.py`, `reported_coverage_source.py`, `source_csv_builtin.py`, `survey_age_activation.py`, `survey_age_calibration.py`, `survey_age_sources.py`, `survey_calibration_diagnostics.py`, `survey_catalogue_selection.py`, `survey_observed_age.py`, `survey_origin_budget.py`, `survey_population_domains.py`, `survey_population_preparation.py`, `survey_population_replay.py` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `group_bounds.py` +- `packages/microcosm-fit/src/microcosm/fit/`: `_graph_legacy_apply.py`, `_graph_legacy_qrf.py`, `graph_legacy_apply.py`, `graph_legacy_apply_matrix.py`, `graph_legacy_qrf.py`, `graph_legacy_train.py`, `model_input.py`, `qrf_target.py` +- `packages/microcosm-frame/src/microcosm/frame/adapters/`: `_policyengine_us_source_index.py`, `policyengine_us.py` +- `packages/microcosm-graph/src/microcosm/graph/`: `randomness.py` +- tests: `test_us_acs_housing_source.py`, `test_us_acs_person_coverage_authentication.py`, `test_us_acs_person_coverage_columns.py`, `test_us_acs_population_catalogue.py`, `test_us_asec_2024_native_population.py`, `test_us_asec_coverage_authentication.py`, `test_us_asec_current_money_source.py`, `test_us_asec_person_income_source.py`, `test_us_current_survey_puf_host.py`, `test_us_current_survey_puf_transfer.py`, `test_us_full_puf_enrichment.py`, `test_us_graph_survey_population.py`, `test_us_national_age_counts.py`, `test_us_puf_detail_transfer.py`, `test_us_puf_price_baseline.py`, `test_us_survey_age_activation.py`, `test_us_survey_age_artifact.py`, `test_us_survey_age_calibration_run.py`, `test_us_survey_age_development.py`, `test_us_survey_age_sources.py`, `test_us_survey_calibration.py`, `test_us_survey_observed_age.py`, `test_us_survey_origin_budget.py`, `test_us_survey_population_preparation.py`, `test_us_survey_population_replay.py`, `test_policyengine_us_ownership_index.py` +- source imports from: 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece), P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET, P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS +- tests import from: 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) + +#### 02 GRAPH-RESTORE (1 non-test files, 0 test files) + +- `packages/microcosm-graph/src/microcosm/graph/`: `kernel.py` + +#### 03 ACCEPTED-SHARED-RESTORE (20 non-test files, 2 test files) + +- `packages/microcosm-build/src/microcosm/build/`: `frame_checkpoint.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `acs_inputs.py`, `acs_pums.py`, `acs_transfer.py`, `asec_checkpoint.py`, `congressional_district_vintage.py`, `cps_carried.py`, `eligibility_inputs.py`, `hours_worked.py`, `housing_inputs.py`, `multispine_pool.py`, `operator_boundary.py`, `prior_year_income.py`, `puf_capital_gains_tail.py`, `qbi_inputs.py`, `relationship_inputs.py`, `spine_assembly.py`, `stacked_spine.py` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `__init__.py` +- `packages/microcosm-fit/src/microcosm/fit/`: `qrf.py` +- tests: `test_us_acs_pums.py`, `test_us_asec_checkpoint.py` +- source imports from: 01 SAFE-ADDITIVE, 04 PUF-SUPPORT-MERGE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS +- tests import from: 01 SAFE-ADDITIVE, 06 J-GRAPH-COMPATIBILITY, P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET + +#### 04 PUF-SUPPORT-MERGE (1 non-test files, 0 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `puf_support.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE + +#### 05 SOLVE-MERGE-PROPOSAL (1 non-test files, 0 test files) + +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `solve.py` +- source imports from: 01 SAFE-ADDITIVE + +#### 06 J-GRAPH-COMPATIBILITY (4 non-test files, 6 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `__init__.py`, `puma_ladder.py`, `puma_ladder_sources.py` +- `tools/`: `build_us_puma_ladder_artifact.py` +- tests: `test_us_acs_multispine.py`, `test_us_acs_multispine_legacy_builder.py`, `test_us_base_pool.py`, `test_us_multispine_pool_tool.py`, `test_us_puma_ladder.py`, `test_us_puma_ladder_sources.py` +- tests import from: 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE + +#### 07 F-CATALOGUE-OPTIMIZATION (1 non-test files, 3 test files) + +- `changelog.d/`: `us-survey-catalogue-digest.changed.md` +- tests: `test_us_survey_catalogue_digest.py`, `test_us_survey_catalogue_digest_authority.py`, `test_us_survey_catalogue_digest_fast_path.py` +- tests import from: 01 SAFE-ADDITIVE, 06 J-GRAPH-COMPATIBILITY + +#### 10 SOURCE-CLOSURE (9 non-test files, 0 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `acs_2024_housing_universe.json`, `asec_current_money_consumers_v1.json`, `asec_current_money_domains_v1.json`, `asec_current_money_price_basis_v1.json`, `graph_acs_housing_universe.py`, `native_origin_graph_inventory.json`, `puf_2015_monetary_agi_source_projection.json`, `puf_2015_monetary_source_projection.json`, `puf_2015_raw_source_definition.json` +- source imports from: G RAW-BYTES-CODEC (graph piece) + +#### 11 PLACEMENT-ADDITIONS (1 non-test files, 1 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `graph_full_puf_enrichment.py` +- tests: `test_us_graph_full_puf_enrichment.py` +- source imports from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### 12 ORDINARY-CLOSURE (0 non-test files, 8 test files) + +- tests: `legacy_prepatch.json`, `test_group_bounds.py`, `test_grouped_fixed_support.py`, `legacy-slice-apply-v1.json`, `test_graph_legacy_matrix.py`, `test_graph_legacy_qrf.py`, `test_qrf_stateless.py`, `test_qrf_target.py` +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, G RAW-BYTES-CODEC (graph piece) + +#### 13 INTEGRATION-REGRESSIONS (2 non-test files, 3 test files) + +- `changelog.d/`: `us-launch-integration.fixed.md` +- `docs/`: `us-launch-review.md` +- tests: `test_us_geography_integration.py`, `test_us_runtime_facade_union.py`, `test_solve_us_launch_integration.py` +- tests import from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### Already on main's way (amendment 20 / this lane's re-pins; not #893 content) (10 non-test files, 9 test files) + +- `./`: `PROGRESS-amendment-20-keyed-draws.md` +- `changelog.d/`: `amend-keyed-seed-and-uniform-draws.added.md` +- `docs/`: `graph-acceptance.md`, `graph-interface.lock` +- `experiments/`: `amendment-20-keyed-draws-receipts.md`, `us-native-atomic-financial-required-replay-20260910.json`, `us-puf-donor-boundaries-20260910.json`, `us-puf55-public-recipient-controls-20260910.json`, `us-puf55-two-route-numerical-20260910.json` +- `tools/`: `graph_parity_repin.py` +- tests: `test_us_puf55_model_boundaries.py`, `test_kernels.py`, `pins.json`, `pins.json`, `pins.json`, `test_graph_kernel_contract.py`, `test_graph_parity_pins.py`, `test_graph_parity_repin.py`, `test_graph_randomness.py` +- source imports from: G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### G RAW-BYTES-CODEC (graph piece) (2 non-test files, 1 test files) + +- `packages/microcosm-graph/src/microcosm/graph/`: `__init__.py`, `codecs.py` +- tests: `test_graph_codecs.py` +- tests import from: G STORE-FRAME-METADATA (graph piece) + +#### G STORE-FRAME-METADATA (graph piece) (2 non-test files, 1 test files) + +- `changelog.d/`: `20260906-frame-metadata-storage.fixed.md` +- `packages/microcosm-graph/src/microcosm/graph/`: `store.py` +- tests: `test_frame_metadata_store.py` + +#### P1 POST-CLONE-ATOMIC-GEOGRAPHY (38 non-test files, 16 test files) + +- `changelog.d/`: `893-population-only-block-source.added.md`, `893-source-demographic-conditioning.added.md`, `893-survey-atomic-prefix.added.md`, `shared-atomic-geography.added.md`, `us-atomic-financial-composition.added.md`, `us-national-atomic-support.added.md` +- `experiments/`: `us-atomic-age-v2-1-controls-20260909.json`, `us-atomic-block-adapter-20260909.json`, `us-atomic-block-api-sources-35-controls-20260909.json`, `us-atomic-block-sources-36-controls-20260909.json`, `us-atomic-budget-semantic-8-controls-20260909.json`, `us-atomic-clone-graph-20260909.json`, `us-atomic-financial-4-controls-20260909.json`, `us-atomic-financial-composition-acceptance-20260909.json`, `us-atomic-financial-corrected-2-controls-20260909.json`, `us-atomic-national-acquisition-20260909.json`, `us-atomic-native-de-1-control-20260909.json`, `us-atomic-native-de-corrected-1-control-20260909.json`, `us-atomic-native-national-1-control-20260909.json`, `us-atomic-survey-population-controls-20260909.json`, `us-budget-predictor-compatibility-1-controls-20260909.json`, `us-financial-demographics-3-controls-20260909.json`, `us-postclone-geography-70-controls-20260910.json`, `us-puf55-native-donor-20260909.json`, `us-survey-geography-graph-controls-20260909.json`, `us-survey-geography-source-controls-20260909.json` +- `packages/microcosm-build/src/microcosm/build/`: `atomic_geography.py`, `graph_atomic_geography.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `atomic_block_api_sources.py`, `atomic_block_sources.py`, `atomic_block_support.py`, `current_survey_geography.py`, `graph_atomic_survey_clone.py`, `graph_atomic_survey_financial.py`, `graph_atomic_survey_population.py`, `graph_current_survey_geography.py`, `survey_atomic_geography.py` +- `tools/`: `ci_test_groups.py` +- tests: `test_atomic_geography.py`, `test_us_atomic_block_api_sources.py`, `test_us_atomic_block_api_sources_native_de.py`, `test_us_atomic_block_api_sources_native_national.py`, `test_us_atomic_block_sources.py`, `test_us_atomic_block_support.py`, `test_us_atomic_survey_clone_graph.py`, `test_us_current_survey_geography.py`, `test_us_current_survey_predictor_demographics.py`, `test_us_graph_atomic_survey_financial.py`, `test_us_graph_atomic_survey_population.py`, `test_us_graph_current_survey_geography.py`, `test_us_postclone_geography_identity.py`, `test_us_survey_age_calibration_atomic_geography.py`, `test_us_survey_origin_budget_atomic_geography.py`, `test_us_survey_origin_budget_predictor_compatibility.py` +- source imports from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR + +#### P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR (51 non-test files, 21 test files) + +- `./`: `pyproject.toml` +- `changelog.d/`: `893-age-count-fixture.internal.md`, `893-diagnostic-refusal-context.internal.md`, `893-historical-zero-provenance.internal.md`, `puf55-canonical-donor.added.md`, `puf55-survey-social-security.added.md`, `survey-social-security-source.added.md`, `us-puf55-canonical-and-output-seals.added.md`, `us-puf55-survey-ss-measurement.added.md` +- `docs/`: `current-survey-puf59-progress.md`, `geography-assignment.md`, `puf2015-canonical59-and-growth.md`, `puf55-survey-ss-measurement-decision.md`, `survey-social-security-source.md`, `us-uk-release-path.md` +- `docs/evidence/puf2015-target2024/`: `GROWTH-RECIPE.json`, `INDEX-VALUES.json`, `NATIONAL-GROWTH-EXTRACT.json`, `PUBLIC-WORKBOOK-SOURCES.json` +- `experiments/`: `us-acs-compilation-cache-adoption-20260910.json`, `us-budget-detached-view-correction-20260910.json`, `us-financial-fixture-profile-20260910.json`, `us-financial-successor-positive-1-limit-stop-20260909.json`, `us-financial-successor-positive-20260910.json`, `us-financial-successor-remaining-controls-20260910.json`, `us-puf55-canonical-create-47-controls-20260910.json`, `us-puf55-numerical-output-seal-46-controls-20260910.json`, `us-puf55-population-controls-20260909.json`, `us-puf55-public-recipient-positives-20260910.json`, `us-puf55-survey-ss-measurement-31-controls-20260909.json`, `us-puf55-two-route-values-controls-20260910.json`, `us-survey-age-development-20260909.json` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `asec_demographic_source.py`, `current_asec_demographics.py`, `current_social_security_source.py`, `current_survey_predictors.py`, `graph_current_survey_predictors.py`, `graph_puf55_canonical_donor.py`, `graph_puf55_survey_recipients.py`, `puf55_canonical_donor.py`, `puf55_route_finalization.py`, `puf55_survey_recipients.py`, `puf55_survey_ss_measurement.py`, `puf59_canonical.py`, `puf59_canonical_artifact.py`, `puf_full_source.py`, `puf_full_source_graph.py`, `puf_qbi_model.py`, `puf_target2024_growth.py`, `survey_financial_successor.py`, `survey_social_security.py` +- tests: `test_frame_checkpoint.py`, `test_spec_seed_diagnostic_refusal_context.py`, `test_us_acs_source_compile_cache.py`, `test_us_current_asec_demographics.py`, `test_us_current_survey_predictors.py`, `test_us_full_puf_output_profiles.py`, `test_us_graph_puf55_canonical_donor.py`, `test_us_graph_puf55_survey_recipients.py`, `test_us_graph_puf55_survey_ss.py`, `test_us_plan.py`, `test_us_puf55_route_finalization.py`, `test_us_puf55_route_numerical_finalization.py`, `test_us_puf55_survey_recipients.py`, `test_us_puf55_survey_ss_measurement.py`, `test_us_puf55_survey_ss_profile.py`, `test_us_puf59_canonical.py`, `test_us_puf_target2024_growth.py`, `test_us_runtime_import_order.py`, `test_us_survey_financial_successor.py`, `test_us_survey_origin_budget_detached_views.py`, `test_us_survey_social_security.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, 11 PLACEMENT-ADDITIONS, G RAW-BYTES-CODEC (graph piece), P1 POST-CLONE-ATOMIC-GEOGRAPHY, P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET + +#### P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET (16 non-test files, 10 test files) + +- `changelog.d/`: `common-frame-export-contract.added.md`, `us-survey-copy-and-identity.fixed.md` +- `docs/`: `us-input-coverage-diagnostic.md` +- `experiments/`: `us-acs-code-memo-comparison-20260910.json`, `us-acs-code-memo-not-adopted-20260910.patch`, `us-acs-loaded-code-controls-20260910.json`, `us-common-frame-export36-20260910.json`, `us-input-coverage25-20260910.json`, `us-postclone-budget-replay-fence4-20260910.json`, `us-survey-catalogue-memo20-20260910.json`, `us-survey-numeric-diagnostics27-20260910.json` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `asec_raw_stage_v4.py`, `common_frame_export_contract.py`, `input_coverage_profile.py`, `population_input_coverage.py` +- `packages/microcosm-frame/src/microcosm/frame/`: `bundle.py` +- tests: `test_us_acs_transfer.py`, `test_us_asec_catalogue_records_memo.py`, `test_us_asec_demographic_source.py`, `test_us_asec_household_coverage_fields.py`, `test_us_common_frame_export_contract.py`, `test_us_population_input_coverage.py`, `test_us_spine_blindness.py`, `test_us_survey_catalogue_immutable_memo.py`, `test_us_survey_frame_identity_encoding.py`, `test_us_survey_origin_budget_replay_producer.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR + +#### P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS (6 non-test files, 1 test files) + +- `changelog.d/`: `us-launch-store-grouped-validation.fixed.md`, `us-survey-diagnostic-options.fixed.md` +- `experiments/`: `us-calibration-consolidation-149-20260910.json`, `us-puf55-canonical-create-failed-20260910.json`, `us-puf55-canonical-cycle39-failed-20260910.json` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `variable_labels.py` +- tests: `test_variable_labels.py` +- tests import from: 03 ACCEPTED-SHARED-RESTORE + +#### P5 CI-SEED-IDENTITY-AND-SPINE-BLINDNESS (10 non-test files, 10 test files) + +- `./`: `CLAUDE.md` +- `.github/workflows/`: `test.yml` +- `changelog.d/`: `893-diagnostic-current-lock.fixed.md`, `893-spec-identity-diagnostics.internal.md`, `893-worker-resource-trace.fixed.md` +- `docs/evidence/spec-engine/`: `us-f0-coverage.json` +- `experiments/`: `spec-seed-current-lock-20260910.json`, `us-acs-producer-profile-20260910.json` +- `packages/microcosm-build/src/microcosm/build/spec_engine/`: `inventory_coverage.py` +- `tools/`: `spec_seed_identity_diagnostics.py` +- tests: `test_spec_engine_country_bundles.py`, `test_spec_engine_loader.py`, `test_spec_seed_identity_cpu_bootstrap.py`, `test_spec_seed_identity_engine_parameter_files.py`, `test_spec_seed_identity_outer_failure.py`, `test_spec_seed_identity_owned_temp.py`, `test_spec_seed_identity_per_code_context.py`, `test_spec_seed_identity_system_metadata.py`, `test_us_puf_support.py`, `test_us_stacked_spine.py` +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY + +#### P6 UK-ADAPTER-DOCS-AND-RECORDS (10 non-test files, 2 test files) + +- `./`: `.gitattributes` +- `changelog.d/`: `893-uk-atomic-area-and-lazy-exports.added.md` +- `docs/`: `validation-cost-and-borrow-boundaries.md` +- `experiments/`: `uk-atomic-area-lazy52-20260910.json`, `us-financial-default-2-controls-20260909.json`, `us-native-atomic-financial-cold-20260910.json`, `us-native-postclone-financial-incomplete-20260910.json` +- `packages/microcosm-build/src/microcosm/build/uk_runtime/`: `__init__.py`, `atomic_area_support.py`, `atomic_household_identity.py` +- tests: `test_uk_atomic_area_support.py`, `test_uk_runtime_lazy_exports.py` +- source imports from: P1 POST-CLONE-ATOMIC-GEOGRAPHY +- tests import from: P1 POST-CLONE-ATOMIC-GEOGRAPHY + +**Layer-01 modules with no cross-layer import at all (53; `randomness.py` among them is amendment 20, not #893 content):** `__init__.py`, `canonical.py`, `origin.py`, `protocol.py`, `reasons.py`, `survey_allocation.py`, `survey_domain_sample.py`, `table_identity.py`, `_asec_current_money_codec.py`, `_person_signal_summary.py`, `acs_housing_universe.py`, `acs_native_coverage_binding.py`, `acs_person_coverage_authentication.py`, `acs_person_coverage_columns.py`, `acs_population_catalogue.py`, `asec_coverage_authentication.py`, `asec_current_money.py`, `asec_current_money_graph_resources.py`, `asec_current_money_resources.py`, `asec_current_money_units.py`, `asec_household_coverage_fields.py`, `asec_housing_status.py`, `asec_housing_status_source.py`, `asec_housing_universe.py`, `asec_housing_universe_source.py`, `asec_income_observations.py`, `asec_original_household_weights.py`, `asec_person_coverage_source.py`, `asec_population_catalogue.py`, `asec_prepared_source.py`, `asec_student_controls.py`, `cd_reference.py`, `cd_reference_sources.py`, `cps_carried_current.py`, `graph_implementation.py`, `graph_implementation_inventory.json`, `graph_native_origin_implementation.py`, `native_household_origin.py`, `operator_column_contracts.py`, `puf_detail_transfer.py`, `puf_diagnostic_consumer.py`, `puf_growth.py`, `puf_price_baseline.py`, `reported_coverage_source.py`, `source_csv_builtin.py`, `survey_age_activation.py`, `survey_age_sources.py`, `survey_catalogue_selection.py`, `survey_observed_age.py`, `survey_population_domains.py`, `group_bounds.py`, `_policyengine_us_source_index.py`, `randomness.py` + + +**Layer-01 modules with a cross-layer import (48; 27 of them only on the graph pieces G1/G2, the other 21 soft on modified modules of 03/05/06/P3/P4 — see `pr1-refined.json`):** `acs_housing_universe_source.py`, `asec_2024_native_population.py`, `asec_current_money_selection.py`, `asec_current_money_source.py`, `asec_engine_evaluation.py`, `asec_household_observations.py`, `asec_person_income_source.py`, `demographic_calibration_graph.py`, `full_puf_enrichment.py`, `graph_asec_income.py`, `graph_asec_prepared.py`, `graph_combined_clone.py`, `graph_composed_asec_binding.py`, `graph_composed_asec_measures.py`, `graph_composed_contracts.py`, `graph_composed_population.py`, `graph_context.py`, `graph_current_survey_puf_transfer.py`, `graph_geography.py`, `graph_housing_universe.py`, `graph_national_age_counts.py`, `graph_native_household_origin.py`, `graph_puf_detail_transfer.py`, `graph_puf_diagnostic_consumer.py`, `graph_sources.py`, `graph_survey_age_artifact.py`, `graph_survey_budget.py`, `graph_survey_calibration.py`, `graph_survey_population.py`, `national_age_activation.py`, `puf_growth_graph.py`, `puf_monetary_agi_projection.py`, `puf_monetary_source.py`, `puf_raw_source.py`, `survey_age_calibration.py`, `survey_calibration_diagnostics.py`, `survey_origin_budget.py`, `survey_population_preparation.py`, `survey_population_replay.py`, `_graph_legacy_apply.py`, `_graph_legacy_qrf.py`, `graph_legacy_apply.py`, `graph_legacy_apply_matrix.py`, `graph_legacy_qrf.py`, `graph_legacy_train.py`, `model_input.py`, `qrf_target.py`, `policyengine_us.py` + +**Layer-01 tests (all ride with PR-1; 26):** `test_us_acs_housing_source.py`, `test_us_acs_person_coverage_authentication.py`, `test_us_acs_person_coverage_columns.py`, `test_us_acs_population_catalogue.py`, `test_us_asec_2024_native_population.py`, `test_us_asec_coverage_authentication.py`, `test_us_asec_current_money_source.py`, `test_us_asec_person_income_source.py`, `test_us_current_survey_puf_host.py`, `test_us_current_survey_puf_transfer.py`, `test_us_full_puf_enrichment.py`, `test_us_graph_survey_population.py`, `test_us_national_age_counts.py`, `test_us_puf_detail_transfer.py`, `test_us_puf_price_baseline.py`, `test_us_survey_age_activation.py`, `test_us_survey_age_artifact.py`, `test_us_survey_age_calibration_run.py`, `test_us_survey_age_development.py`, `test_us_survey_age_sources.py`, `test_us_survey_calibration.py`, `test_us_survey_observed_age.py`, `test_us_survey_origin_budget.py`, `test_us_survey_population_preparation.py`, `test_us_survey_population_replay.py`, `test_policyengine_us_ownership_index.py` + +## 7. Draft charter amendment entries (one per kept graph piece) + +Written in the style of `docs/graph-acceptance.md` "Interface freeze". None of +the four touches `decl.py` or `kernel.py`, so none re-records the lock; they +are offered as numbered entries because each changes what a receipt, a store +object or an executor outcome means, which the charter records. + +**7.1 Raw-byte sources.** A source codec is registered in exactly one of two +modes and a name belongs to at most one: *Frame mode* (`SourceCodec`, +`SourceCodecRegistry.register` / `load`) decodes a source into a population +`Frame`, and *raw-byte mode* (`SourceBytesCodec`, `register_bytes` / +`load_bytes`, `load_source_bytes`) decodes it into immutable `bytes` that the +consuming kernel alone interprets. Neither mode can be loaded through the +other: the mismatch is a `TypeError` naming the mode the codec has, so no +caller receives bytes where it declared a Frame. The shipped `raw-bytes-v1` +codec reads one regular file, bounded at `RAW_BYTES_MAX_BYTES` (64 MiB) and +opened non-blocking so a directory, FIFO, device or socket is refused from +the descriptor's own mode. Identity, the pre-run content key and the post-run +mutation check stay in the executor (E1, E3 unchanged; C3's "declared +predecessors only" is untouched because a source is not a node). Why: a +lookup NPZ or a crosswalk CSV that an import kernel turns into a typed +`ArtifactOutput` (amendment 19) is not a population, and the only way to read +it before this was a Frame codec registered as a pretence. Raised by the US +launch integration branch (thirteen consumers); no key moves, because +`SourceRef.codec` was already normative and `raw-bytes-v1` is a new name. + +**7.2 Frames keep their metadata across the store.** A frame object is +written as `microcosm-graph-frame-v2`: `Frame.metadata` is encoded +losslessly (mappings, tuples, frozensets, `float64` bit patterns, scalars) +into the frame manifest, its SHA-256 is recorded in the object header, and +`_read_frame` refuses a mismatch as `StoreCorruptError`. A v1 object is +`StoreUnavailableError` on load rather than a Frame with its metadata silently +dropped, and writing a frame under a key that already holds one with different +metadata is `StoreCorruptError` (E1: a key identifies exactly one content). +The JSON decode boundary refuses `NaN`/`Infinity` and overflowing numerals, +because every store JSON is written with `allow_nan=False`, so a corrupt +manifest is `StoreCorruptError` at the boundary rather than a `TypeError` +from the canonical re-encoder (E2). Why: the US survey stages carry source +provenance in `Frame.metadata` and replay verifiers compare it +(`same_replayed_frame`); until now a population that went through the store +came back without it. No node key moves: the frame key is the node's, and +metadata was never part of a key. Raised by the US launch integration +branch (`8d9f62c40`); acceptance items E1 and E2. + +**7.3 Execution states: a gate exception leaves its consumers unreached.** +Amendment 19 refused a gate kernel that declares a typed artifact output +because it had no regime for an output a node was unable to produce. This +entry supplies the regime and lifts the refusal. The executor owns an +execution state (`"microcosm.graph.execution.v1"`) that a kernel may not +author: a gate that declares typed outputs and raises still becomes a `fail` +verdict and the run continues (amendment 7, now literally true for every +node shape), and its receipt records `gate_exception` with the outputs it +could not produce; a node whose predecessor is `unreached`, or whose declared +byte input its producer recorded as unavailable, is itself recorded +`unreached` with its blockers named by node key — it runs no kernel and +invents no column, frame, weight or byte (F1: nothing is fabricated); a +release behind such an edge derives its tier from the same ancestry and stays +`evidence` (F2). Cache records that carry a state use schema 3; a cached +unreached record is a hit only while the same inputs are unavailable for the +same reason and is `StoreCorruptError` once they exist (E3); `require` +preflight derives blockers from the cached parents. A run manifest carrying +any state serializes at schema 4 and authenticates every blocker on load +(F5, E4). No node key moves: states live in receipts and records, never in a +key. Why: the US post-clone geography gate emits a typed validation artifact +that the financial stages consume, so a geography failure must leave those +stages unreached rather than abort the run or launder the gate into a compute +node. Raised by the US launch integration branch; supersedes the interim +ruling recorded in amendment 19. + +**7.4 A private population observer.** `run_graph` accepts +`_population_observer`, a callable handed each node's admitted population +(design anchors included) after the result is applied and before the node is +persisted, on cold execution and on cache hits alike; it must not mutate the +population, an exception it raises refuses the run, and an unreached node +has no population to observe. It is an integration seam for verifiers (the +US survey stages assert design-weight provenance through it), not a kernel +capability: it enters no key, receipt or record (A1, B4 unchanged). Raised by +the US launch integration branch. + +## 8. Decisions needed from Max + +1. **Piece C supersedes an amendment-19 ruling.** The Amendment 19 lane + recorded "the gate-artifact-output refusal is the one interim ruling this + lane made on his behalf". This branch's US runtime needs the opposite (the + post-clone geography gate declares its validation artifact and three + stages consume it), so piece C re-applies the branch's execution-state + regime and replaces main's test that pins the refusal with five tests of + the new regime. Both the frozen files and every `test_acceptance_*` file + are untouched, but the executor's behaviour on that one node shape now + differs from what the charter's amendment 19 text says. Landing PR-G3 + requires adopting entry 7.3 (or an edited form) in the charter; until then + this branch's executor and the charter disagree on that sentence. +2. **Lazy population retention** (`attachments.py`, layer 08) is dropped for + lack of a consumer. If the native pilot's memory ceiling is a reason to + keep it, it is a self-contained PR on top of PR-G2. +3. **`_stream_file` and the `_write_node` refactor** are dropped for lack of a + consumer. Both are memory-only; say if either should come back as its own + small PR. +4. **The report's home.** `out.md` at the repo root is a tracked file holding + the F1 Sol gate round-1 report (2026-09-04). This report is written there + for the dispatcher but is **not committed** there; the committed copy is + `experiments/893-reconciliation-amendments-19-20-20260912.md`. +5. **Three US stages name a resource that does not exist anywhere.** + `asec_prepared_v3`, `composed_asec_binding_v1` and `composed_population_v1` + list `us_runtime/asec_current_money_engine_defaults_v1.json` in their + resources; the file is on neither this branch (at `069d5ed9a` or now) nor + `origin/main` (it appears only in two commits of another branch, + `b0e7f54b2` / `b6cfc84b0`). `implementation_manifest` for those stages has + therefore been failing since before this lane. Not invented here; the + owner of layer 10 SOURCE-CLOSURE has to admit the file or drop the entry, + and the inventory's "scope review" rule applies to `07566eb17` as well. +6. **Two build tests fail locally for reasons outside this lane** if they are + run: `test_release_target_parity.py`'s two `_feed_or_skip` tests need a 131 + MB feed outside the repository (already recorded by the Amendment 19 lane). + They were not part of this lane's runs. + +## 9. Where things are + +- Journal: `PROGRESS.md` (top section, cumulative file). +- This report: `out.md` (uncommitted, the `-o` path) and + `experiments/893-reconciliation-amendments-19-20-20260912.md` (committed). +- Scratch evidence (session-local, not committed): the AST consumer scan, + the layer scripts and JSON, and every gate log named in section 5. diff --git a/experiments/908-review-findings-recheck.json b/experiments/908-review-findings-recheck.json new file mode 100644 index 000000000..44ea5058a --- /dev/null +++ b/experiments/908-review-findings-recheck.json @@ -0,0 +1,33 @@ +{ + "code_1_aggregate_only_metadata": { + "candidate_mapping_with_denylisted_person_id_refused": true, + "candidate_mapping_with_record_vectors_refused": true, + "closed": true, + "context_record_vectors_refused": true, + "nested_context_refused": true + }, + "code_2_nested_alias": { + "caller_context_unchanged": true, + "closed": true, + "next_snapshot_unchanged": true + }, + "code_3_history_publication": { + "a_complete_write_still_publishes": true, + "closed": true, + "no_leftover_temporary": true, + "reader_sees_nothing_after_failure": true, + "retained_after_failure": 0, + "write_failed": true + }, + "code_4_identity_validation": { + "closed": true, + "invalid_cases_now_refused": [ + "epoch_exceeds_epochs", + "invalid_timestamp", + "invalid_best_metadata", + "candidate_id_not_string" + ] + }, + "reviewed_head": "bae1887ffc5f6e32edefa9774ec6428568286247", + "scope": "The review's four counterexamples replayed against this branch. Invented data only; no optimizer, engine, network or native input." +} diff --git a/experiments/908-target-snapshot-bench-receipt.json b/experiments/908-target-snapshot-bench-receipt.json new file mode 100644 index 000000000..f57535753 --- /dev/null +++ b/experiments/908-target-snapshot-bench-receipt.json @@ -0,0 +1,84 @@ +{ + "label": "SYNTHETIC \u2014 invented matrices and frames, not US or UK calibration", + "dimension_points": { + "codec_target_counts": [ + 500, + 2000, + 5000, + 10000 + ], + "solver_records": 10000, + "solver_targets": 100, + "solver_epochs": 300 + }, + "bounds": { + "numeric_threads": 1, + "cpu_seconds_limit_applied": 120, + "memory_bytes_limit_applied": null, + "peak_rss_bytes": 289800192, + "native_inputs_read": 0 + }, + "codec_costs": [ + { + "n_targets": 500, + "build_and_validate_ms": 2.099, + "serialize_ms": 0.603, + "store_write_ms": 3.913, + "serialized_bytes": 88625, + "bytes_per_target": 177.2 + }, + { + "n_targets": 2000, + "build_and_validate_ms": 9.087, + "serialize_ms": 2.321, + "store_write_ms": 13.491, + "serialized_bytes": 353895, + "bytes_per_target": 176.9 + }, + { + "n_targets": 5000, + "build_and_validate_ms": 18.943, + "serialize_ms": 5.723, + "store_write_ms": 29.589, + "serialized_bytes": 885599, + "bytes_per_target": 177.1 + }, + { + "n_targets": 10000, + "build_and_validate_ms": 38.339, + "serialize_ms": 11.357, + "store_write_ms": 60.114, + "serialized_bytes": 1771933, + "bytes_per_target": 177.2 + } + ], + "solver_overhead": [ + { + "cadence": "observer_off", + "best_of_3_seconds": 0.036, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": null, + "chunks_retained_on_disk": 0, + "retained_bytes_on_disk": 0 + }, + { + "cadence": "bounded_every_25", + "best_of_3_seconds": 0.0597, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": 14, + "chunks_retained_on_disk": 14, + "retained_bytes_on_disk": 231241 + }, + { + "cadence": "every_epoch", + "best_of_3_seconds": 0.594, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": 301, + "chunks_retained_on_disk": 256, + "retained_bytes_on_disk": 4220712 + } + ] +} diff --git a/experiments/908-target-snapshot-bench-receipts.md b/experiments/908-target-snapshot-bench-receipts.md new file mode 100644 index 000000000..faf4d8816 --- /dev/null +++ b/experiments/908-target-snapshot-bench-receipts.md @@ -0,0 +1,63 @@ +# microcosm#908 — per-target snapshot cost, SYNTHETIC measurement + +**Everything below is synthetic.** The numbers come from invented target +matrices and an invented seeded frame produced by +[`908_target_snapshot_bench.py`](908_target_snapshot_bench.py). No native +microdata was read, no country engine ran, and nothing was uploaded. These are +**not** US or UK calibration runtimes and **not** upload times. The dimension +points were chosen to bracket an order of magnitude; they are not measured from +the real US or UK target registries. + +Run 2026-09-12 on macOS 26.6.2 / arm64, one numeric thread, CPU rlimit 120 s, +peak RSS 0.29 GiB (re-measured after the adversarial-review fixes) (well inside the 2 GiB cap; macOS refused `RLIMIT_AS`, so the +script records the limit it actually applied and the peak RSS it reached). +Machine-readable copy: [`908-target-snapshot-bench-receipt.json`](908-target-snapshot-bench-receipt.json). + +## Codec and local-store cost per snapshot, by target count + +Build+validate is the full emission cost (`snapshot()` validates what it +builds). Store write is one immutable history chunk plus the atomic `latest.json` +replacement plus the index rewrite. + +| targets | build+validate (ms) | serialize (ms) | store write (ms) | serialized bytes | bytes/target | +|--------:|--------------------:|---------------:|-----------------:|-----------------:|-------------:| +| 500 | 2.10 | 0.60 | 3.91 | 88,625 | 177.2 | +| 2,000 | 9.09 | 2.32 | 13.49 | 353,895 | 176.9 | +| 5,000 | 18.94 | 5.72 | 29.59 | 885,599 | 177.1 | +| 10,000 | 38.34 | 11.36 | 60.11 | 1,771,933 | 177.2 | + +Both cost and size are linear in target count at ~177 bytes per target row +(indent=1, sorted keys). A single 10,000-target snapshot is ~1.7 MB. + +## Solver overhead at three cadences + +Invented frame: 10,000 households, 100 targets, 300 epochs, seed 0. One +discarded warm-up run, then best of three timed runs per cadence. + +| cadence | best-of-3 (s) | snapshots emitted | chunks retained | retained bytes | weights bitwise equal to observer-off | +|---|---:|---:|---:|---:|---| +| observer off | 0.0360 | — | 0 | 0 | yes (reference) | +| bounded, every 25 | 0.0597 | 14 | 14 | 231,241 | yes | +| every epoch | 0.5940 | 301 | 256 | 4,220,712 | yes | + +The closing loss is identical to all printed digits across all three +(`2.7968459435242565e-07`), and the returned weight vectors are bitwise equal. + +## What this does and does not license + +It does establish that emission is linear, that ~177 bytes/target is the wire +cost, and that turning the observer on does not change the optimizer's result. + +It does **not** license a production cadence default. This solve costs 36 ms +total, so per-snapshot cost dominates it by construction and the 16x +every-epoch ratio above is an artifact of a trivially cheap solve — a real +calibration's epoch is far more expensive, so the same absolute per-snapshot +cost would be a much smaller fraction of it. Choosing the production default +needs the native UK and US measurements #908 asks for, which this host is not +permitted to run. The shipped default is therefore *off*; a caller that opts in +picks its own cadence. + +The 5 MiB per-remote-file ceiling in the version-2 staging content policy is a +separate, unmeasured constraint here: at ~177 bytes/target a single snapshot +stays well inside it, but an accumulated history does not, which is why the +store chunks history rather than growing one document. diff --git a/experiments/908_review_findings_recheck.py b/experiments/908_review_findings_recheck.py new file mode 100644 index 000000000..01b72e10b --- /dev/null +++ b/experiments/908_review_findings_recheck.py @@ -0,0 +1,180 @@ +"""Re-run the #908 independent review's four counterexamples against the fix. + +The review's probe (head ``bae1887ff``) reproduced four defects with invented +data and recorded them in ``reproductions.json``. This script replays the same +four counterexamples, unchanged, against this branch's source and asserts each +one is now closed. It is passive: invented aggregates and temporary files only, +no engine, no optimizer, no network, no native input. + +Run with the lane's isolated interpreter, e.g.:: + + python -I -B -S experiments/908_review_findings_recheck.py +""" + +from __future__ import annotations + +import copy +import json +import pathlib +import sys +import tempfile +from datetime import UTC, datetime + +import numpy as np + +from microcosm.calibrate.target_snapshots import ( + TargetSnapshotError, + TargetSnapshotObserver, + TargetSnapshotWriter, + iter_history, + validate_target_snapshot, +) + +#: The review's own record-level vectors, copied verbatim from its probe. +VECTORS = { + "household_weights": [10.0, 20.0], + "tax_unit_id": [101, 102], + "spm_unit_id": [201, 202], +} + + +def _observer(sink=lambda payload: None, context=None): + return TargetSnapshotObserver( + sink=sink, + run_id="invented", + context={} if context is None else context, + clock=lambda: datetime(2026, 9, 12, tzinfo=UTC), + ).bind(names=("aggregate",), targets=[30.0]) + + +def _payload(): + return _observer().snapshot(np.array([30.0]), epoch=1, epochs=2, iterate="current") + + +def _accepted(payload) -> bool: + try: + validate_target_snapshot(payload) + except TargetSnapshotError: + return False + return True + + +def _refused(call) -> bool: + try: + call() + except TargetSnapshotError: + return True + return False + + +def finding_1_aggregate_only_metadata() -> dict[str, object]: + """Record vectors and mapping identifiers no longer have a shape to ride in.""" + candidate = _payload() + candidate["candidate_id"] = copy.deepcopy(VECTORS) + person = _payload() + person["candidate_id"] = {"person_id": [1, 2]} + return { + "context_record_vectors_refused": _refused(lambda: _observer(context=VECTORS)), + "candidate_mapping_with_record_vectors_refused": not _accepted(candidate), + "candidate_mapping_with_denylisted_person_id_refused": not _accepted(person), + "nested_context_refused": _refused( + lambda: _observer(context={"display": {"unit": "USD"}}) + ), + "closed": True, + } + + +def finding_2_nested_alias() -> dict[str, object]: + """A sink cannot reach the caller's metadata or the next snapshot.""" + context = {"display_unit": "USD"} + + def mutate(payload): + payload["context"]["display_unit"] = "changed-by-sink" + + bound = _observer(sink=mutate, context=context) + bound.emit(np.array([30.0]), epoch=1, epochs=2, iterate="current") + following = bound.snapshot(np.array([30.0]), epoch=2, epochs=2, iterate="current") + return { + "caller_context_unchanged": context == {"display_unit": "USD"}, + "next_snapshot_unchanged": following["context"] == {"display_unit": "USD"}, + "closed": True, + } + + +def finding_3_history_publication() -> dict[str, object]: + """A partial chunk is never visible, and a failed write leaves nothing.""" + import os + + observed: dict[str, object] = {} + with tempfile.TemporaryDirectory(prefix="microcosm-908-fix-recheck-") as tmp: + writer = TargetSnapshotWriter(pathlib.Path(tmp)) + real_fsync = os.fsync + calls: list[int] = [] + + def failing_fsync(descriptor): + calls.append(descriptor) + if len(calls) == 1: + # The review interrupted the chunk write after ten bytes; this + # interrupts it after all of them, which is strictly harder. + raise OSError("invented disk failure before publication") + return real_fsync(descriptor) + + os.fsync = failing_fsync + try: + try: + writer(_payload()) + except OSError: + observed["write_failed"] = True + finally: + os.fsync = real_fsync + observed["retained_after_failure"] = len(writer.retained()) + observed["reader_sees_nothing_after_failure"] = list(iter_history(tmp)) == [] + observed["no_leftover_temporary"] = ( + sorted(p.name for p in (pathlib.Path(tmp) / "history").iterdir()) == [] + ) + writer(_payload()) + observed["a_complete_write_still_publishes"] = len(list(iter_history(tmp))) == 1 + return {**observed, "closed": True} + + +def finding_4_identity_validation() -> dict[str, object]: + """The codec refuses the four impossible payloads it used to admit.""" + cases = { + "epoch_exceeds_epochs": {"epoch": 99, "epochs": 1}, + "invalid_timestamp": {"created_at": "not-a-time"}, + "invalid_best_metadata": { + "iterate": "best_retained", + "best_retained": {"available": True, "epoch": -7, "loss": "not-a-number"}, + }, + "candidate_id_not_string": {"candidate_id": {"unexpected": [1, 2]}}, + } + refused = [] + for name, changes in cases.items(): + payload = _payload() + payload.update(changes) + if not _accepted(payload): + refused.append(name) + return {"invalid_cases_now_refused": refused, "closed": len(refused) == len(cases)} + + +def main() -> int: + report = { + "scope": ( + "The review's four counterexamples replayed against this branch. " + "Invented data only; no optimizer, engine, network or native input." + ), + "reviewed_head": "bae1887ffc5f6e32edefa9774ec6428568286247", + "code_1_aggregate_only_metadata": finding_1_aggregate_only_metadata(), + "code_2_nested_alias": finding_2_nested_alias(), + "code_3_history_publication": finding_3_history_publication(), + "code_4_identity_validation": finding_4_identity_validation(), + } + for key, value in report.items(): + if isinstance(value, dict): + assert value["closed"] is True, f"{key} is not closed: {value}" + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/908_target_snapshot_bench.py b/experiments/908_target_snapshot_bench.py new file mode 100644 index 000000000..c48d02785 --- /dev/null +++ b/experiments/908_target_snapshot_bench.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""SYNTHETIC benchmark for the #908 per-target snapshot path. + +Every number this script produces comes from **invented** target matrices and +an **invented** frame built from a seeded RNG. Nothing here reads native +microdata, runs a country engine, or touches a network. The dimension points +below were chosen to bracket an order of magnitude; they are NOT measured from +the real US or UK calibration target registries, and the timings are NOT US or +UK calibration times or upload times. Read them as "what does the codec and the +local store cost per target row and per epoch", nothing more. + +Run (from the repo root, inside the isolated lane interpreter):: + + python experiments/908_target_snapshot_bench.py + +Bounded on purpose: one numeric thread, a 2 GiB address-space cap and a 120 s +CPU-time cap are set before torch loads, so the script cannot quietly grow into +a machine-sized job. +""" + +from __future__ import annotations + +import json +import os +import resource +import sys +import tempfile +import time +from pathlib import Path + +_MEMORY_BYTES = 2 * 1024**3 + + +def _bound_resource(which: int, value: int) -> int | None: + """Apply a soft rlimit where the platform allows it; report what stuck.""" + soft, hard = resource.getrlimit(which) + if hard != resource.RLIM_INFINITY and value > hard: + return None + try: + resource.setrlimit(which, (value, hard)) + except (ValueError, OSError): + return None + return value + + +_CPU_LIMIT = _bound_resource(resource.RLIMIT_CPU, 120) +_MEMORY_LIMIT = _bound_resource(resource.RLIMIT_AS, _MEMORY_BYTES) +if _MEMORY_LIMIT is None: + # macOS refuses RLIMIT_AS here; RLIMIT_DATA is the portable fallback. + _MEMORY_LIMIT = _bound_resource(resource.RLIMIT_DATA, _MEMORY_BYTES) +for _variable in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS"): + os.environ.setdefault(_variable, "1") + +import numpy as np # noqa: E402 - after the resource and thread bounds +import pandas as pd # noqa: E402 - after the resource and thread bounds +import torch # noqa: E402 - after the resource and thread bounds + +from microcosm.calibrate import ( # noqa: E402 - after the resource bounds + EVERY_EPOCH, + Target, + TargetSet, + TargetSnapshotCadence, + TargetSnapshotObserver, + TargetSnapshotWriter, + calibrate, +) +from microcosm.frame import ( # noqa: E402 - after the resource bounds + EntitySchema, + Frame, + WeightKind, + Weights, +) + +torch.set_num_threads(1) + +#: Invented target-count dimension points for the codec measurement. +TARGET_COUNTS = (500, 2_000, 5_000, 10_000) + +#: Invented solver dimension point: households x targets x epochs. +SOLVER_RECORDS = 10_000 +SOLVER_TARGETS = 100 +SOLVER_EPOCHS = 300 + +#: Timed repeats per cadence; the table reports the best (least noisy) run. +REPEATS = 3 + +_SCHEMA = EntitySchema(group_entities=("household",)) + + +def _bound(n_targets: int): + """A bound observer over ``n_targets`` invented targets, sinking to nowhere.""" + rng = np.random.default_rng(0) + names = tuple(f"synthetic_target_{index:06d}@2024" for index in range(n_targets)) + targets = rng.lognormal(12.0, 1.0, n_targets) + observer = TargetSnapshotObserver(sink=lambda payload: None, run_id="synthetic") + return observer.bind(names=names, targets=targets), rng.lognormal( + 12.0, 1.0, n_targets + ) + + +def codec_costs() -> list[dict[str, object]]: + """Build/validate/serialize/write cost per snapshot, by target count.""" + rows: list[dict[str, object]] = [] + for n_targets in TARGET_COUNTS: + bound, estimates = _bound(n_targets) + repeats = max(3, 30_000 // n_targets) + start = time.perf_counter() + for _ in range(repeats): + # snapshot() validates the payload it builds, so this is the full + # build + validate cost a real emission pays. + payload = bound.snapshot( + estimates, epoch=1, epochs=1, iterate="current", loss=0.1 + ) + build_seconds = (time.perf_counter() - start) / repeats + + start = time.perf_counter() + for _ in range(repeats): + serialized = json.dumps(payload, indent=1, sort_keys=True, allow_nan=False) + serialize_seconds = (time.perf_counter() - start) / repeats + + with tempfile.TemporaryDirectory() as directory: + writer = TargetSnapshotWriter(Path(directory), history_limit=8) + start = time.perf_counter() + for index in range(repeats): + copy = dict(payload) + copy["sequence"] = index + 1 + writer(copy) + write_seconds = (time.perf_counter() - start) / repeats + + rows.append( + { + "n_targets": n_targets, + "build_and_validate_ms": round(build_seconds * 1e3, 3), + "serialize_ms": round(serialize_seconds * 1e3, 3), + "store_write_ms": round(write_seconds * 1e3, 3), + "serialized_bytes": len(serialized.encode("utf-8")), + "bytes_per_target": round( + len(serialized.encode("utf-8")) / n_targets, 1 + ), + } + ) + return rows + + +def _synthetic_frame_and_targets() -> tuple[Frame, TargetSet]: + rng = np.random.default_rng(1) + household_ids = np.arange(SOLVER_RECORDS, dtype="int64") + columns = { + f"measure_{index:03d}": rng.lognormal(3.0, 0.5, SOLVER_RECORDS) + for index in range(SOLVER_TARGETS) + } + weights = np.full(SOLVER_RECORDS, 100.0) + household = pd.DataFrame({"household_id": household_ids, **columns}) + person = pd.DataFrame( + {"person_id": household_ids, "person_household_id": household_ids} + ) + frame = Frame( + {"person": person, "household": household}, + _SCHEMA, + {"household": Weights(values=weights, kind=WeightKind.DESIGN)}, + ) + targets = TargetSet( + [ + Target( + name=name, + period=2024, + entity="household", + measure=name, + # An invented 8% miss, so the optimizer has somewhere to go. + value=float((values * weights).sum()) * 1.08, + ) + for name, values in columns.items() + ] + ) + return frame, targets + + +def solver_overhead() -> list[dict[str, object]]: + """Added wall-clock from snapshot emission, at three cadences. + + One discarded warm-up run first (the first calibrate of a process pays + matrix-build and torch warm-up that would otherwise be charged to whichever + cadence happened to run first), then the best of ``REPEATS`` timed runs per + cadence. Each run's returned weights are compared bitwise against the + observer-off run, so the table also evidences that emission changed nothing. + """ + frame, targets = _synthetic_frame_and_targets() + cadences = ( + ("observer_off", None), + ("bounded_every_25", TargetSnapshotCadence(every=25)), + ("every_epoch", TargetSnapshotCadence(every=EVERY_EPOCH)), + ) + calibrate(frame, targets, epochs=SOLVER_EPOCHS, seed=0) # warm-up, discarded + + rows: list[dict[str, object]] = [] + reference: np.ndarray | None = None + for label, cadence in cadences: + best_seconds = float("inf") + emitted = retained_chunks = retained_bytes = 0 + for _ in range(REPEATS): + with tempfile.TemporaryDirectory() as directory: + observer = ( + None + if cadence is None + else TargetSnapshotObserver( + sink=TargetSnapshotWriter(Path(directory)), + run_id="synthetic", + cadence=cadence, + ) + ) + start = time.perf_counter() + result = calibrate( + frame, + targets, + epochs=SOLVER_EPOCHS, + seed=0, + target_snapshots=observer, + ) + best_seconds = min(best_seconds, time.perf_counter() - start) + history = Path(directory) / "history" + if observer is not None: + retained_chunks = len(list(history.glob("*.json"))) + retained_bytes = sum( + path.stat().st_size for path in history.glob("*.json") + ) + emitted = int( + json.loads( + (Path(directory) / "history_index.json").read_text( + encoding="utf-8" + ) + )["last_sequence"] + ) + if reference is None: + reference = result.weights.copy() + identical = True + else: + identical = bool(np.array_equal(reference, result.weights)) + rows.append( + { + "cadence": label, + f"best_of_{REPEATS}_seconds": round(best_seconds, 4), + "closing_loss": float(result.closing_loss), + "weights_bitwise_equal_to_observer_off": identical, + "snapshots_emitted": emitted or None, + "chunks_retained_on_disk": retained_chunks, + "retained_bytes_on_disk": retained_bytes, + } + ) + return rows + + +def main() -> int: + report = { + "label": "SYNTHETIC — invented matrices and frames, not US or UK calibration", + "dimension_points": { + "codec_target_counts": list(TARGET_COUNTS), + "solver_records": SOLVER_RECORDS, + "solver_targets": SOLVER_TARGETS, + "solver_epochs": SOLVER_EPOCHS, + }, + "bounds": { + "numeric_threads": torch.get_num_threads(), + "cpu_seconds_limit_applied": _CPU_LIMIT, + "memory_bytes_limit_applied": _MEMORY_LIMIT, + "peak_rss_bytes": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + "native_inputs_read": 0, + }, + "codec_costs": codec_costs(), + "solver_overhead": solver_overhead(), + } + json.dump(report, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/fiscal-target-snapshot-host-20260912.md b/experiments/fiscal-target-snapshot-host-20260912.md new file mode 100644 index 000000000..824fe94e2 --- /dev/null +++ b/experiments/fiscal-target-snapshot-host-20260912.md @@ -0,0 +1,113 @@ +# Target snapshots through the US fiscal graph + +The actual `FiscalDenseCalibrationKernel` now accepts an optional host-owned +`TargetSnapshotObserver` and forwards it to its existing grouped `calibrate` +call. Default construction and explicit `None` preserve the unobserved path. +Observer configuration stays on the kernel instance; it is absent from graph +parameters, implementation identity inputs, cache keys, artifacts and receipts. +The [usage documentation](../docs/calibration-target-snapshots.md#us-fiscal-host) +shows how a caller supplies the sink. + +This integration combines grouped core `2f62bb8448010bcc9b9e1419ea24aafcb85917d1` +with PR #914 head `16fc6cb501ccb58ceb158e9064bf4f39ee23479c`, including reviewed +repair `fbb109987e30f09ccc87fc402c47190d7c3ecfd2`. The grouped loop now reports +completed optimizer updates, starting at zero, while cadence continues to use +the one-based evaluation ordinal. Its selected snapshot remains the final +closing state at `epoch == epochs`, with no retained best. Ordinary Adam, +proximal and winning budget-probe labels preserve their reviewed repairs. +No numeric update, accepted projection, optimizer selection or RNG operation +was changed by the added fiscal observer boundary. + +## Integrated execution evidence + +The full calibration test directory, existing dense fiscal tests and nine new +fiscal snapshot controls completed with **473 passed, one skipped and no +failures or errors**. The run took 12.051 seconds wall time, 12.375 CPU seconds +and 491,569,152 bytes peak RSS. All 478 Python source files under shard source +directories retained their hashes; Torch threads were 1/1. + +The one skip is the saved prepatch exact-byte fixture. It requires its recorded +Torch interop setting of 18, while this bounded run used one. The current +on/off numerical comparisons and all new fiscal controls executed; the skipped +historical comparison is not claimed as a pass. The earlier grouped lane's +443-test evidence remains separate from this integrated run. + +The new controls exercise the real solver and tiny invented fiscal graph: + +- Default, explicit `None`, and observers with different labels/cadences produce + exactly equal weights, graph artifacts and receipts. Kernel implementation + hashes and declared node identities are equal across observer configurations. +- Snapshots use the actual measurement's ordered target names and values. The + emitted in-loop estimates equal the existing float32 loss tensors, and final + estimates equal the final float64 diagnostics. A six-update run at cadence + three emits completed-update labels 0, 2, 5 and selected 6. +- Every delivered dictionary is detached. A sink can damage its own aggregate + metadata and target rows without changing later snapshots or returned results. +- Exceptions from current and selected sinks propagate. Selected sinks that + corrupt live measurement values, household IDs or accepted weights are + refused by the existing final checks after callback completion. +- Actual graph executions with observation off/on have identical node keys, + weights and artifact bytes. Required replay with a fresh failing observer and + forbidden optimizer hook hits every cached node and invokes neither hook. + A cache replay therefore generates no new optimizer snapshots. + +The first attempt is preserved as failed test evidence. Its final-weight +corruption control tried normal assignment to an already read-only array and +stopped before reaching the intended final guard. The corrected test explicitly +injects a corrupted live weight container; final acceptance then refuses it. +This was a test-only correction; production solver and host bytes were unchanged. + +| Local evidence | SHA256 | +| --- | --- | +| Integrated execution receipt | `b4fb9b48a1d1ca4739e6d22d78968eae14a4855789548c9805068b7ffac2d38f` | +| Integrated JUnit result | `68b110709ade099efc3084af2e0adae0944d6a251172bce8496a3c265e20ed00` | +| Fiscal host source | `c26a9484b596015b4ffaf6a6a1e68e45f0ab4e81e63b5b612149777b100d56b9` | +| New fiscal snapshot test source | `f60d71aa9a4ac6bcf897dd937fe787eb21b0866cdd04223dba513af684849fcb` | +| Reconciled solver source | `8f22f325ed06cde31c4c25467efa39a43ba8ccef9598b14e468701f3263c1010` | + +These bounded wrappers apply runtime/thread limits and source hashes. They are +not the strict native pilot audit guard. The reviewed execution path uses +invented sources and no country simulation; flags or table counts alone do not +establish native/source admission. + +## Source-derived identities + +The merge had five derived-identity conflicts. Final values were recomputed +from the combined checkout using the existing spec loader/compiler, complete +coverage verifier and selective calibration parity tool. No parent branch's +hash was accepted as the final value. Fit and simulation parity pins were left +byte-identical. The parity tool also checks that the local direct calibration +output remains equal to its recorded bytes before rewriting its key. + +| Identity | Final value | +| --- | --- | +| Seed protocol | `5f93b3ec98ada30338b06ad978a6e1b47d0f9f916af89418500cea435235ad06` | +| Seed map | `da07a54ab2bc4e297ac7d0693a8559b359de230787757e4b6e26dcb619c8e5a0` | +| US resolved spec | `54aa5d96b062a66207dfe49d4a03817e9658bd0ac11acb37546b1bf2a01f533d` | +| Minimal loader golden | `a866bfe36a9eeb3b9a9888466b4b906faf4d8da57daf380d9bbc8ccf22e1e048` | +| Fiscal kernel implementation | `3d68e0f9b95a6d07f7a61b93f2044157ba4ab925e09a0a9600beebc16b60b8b7` | +| Calibration parity node | `f85d2af2276acc617779567e7ce8c97878febda0062acd7d2ddf326f4629d520` | + +The first identity-helper invocation correctly refused two advertised source +roots from the shared editable environment. Its failure is preserved. The +successful invocation disabled automatic site initialization and supplied the +owned shard roots plus dependency site-packages explicitly. No source-identity +check was relaxed. Its report SHA256 is +`69666f687e025d9a27d9d76b2cc87fefc2622734a56fc4530674b9de019ff23d`. + +Five focused checks of the final minimal/country spec goldens and calibration +parity identities then passed without failures or skips. They took 58.312 +seconds wall time with all 478 source pins unchanged. The receipt SHA256 is +`ee662e419c17acc27b76761b32f2bdbb096f2dc993e1efeb437d3f01006573e8`; +the JUnit result SHA256 is +`a60a4ffcc11b4afde6fe64a332c2bcfe54b8ac298f9aa523d9a4279089ccd116`. +No UK or US country-simulation test was selected. + +## Remaining acceptance + +This is solver and fiscal-host instrumentation, not a dataset release or +dashboard deployment. Complete-population/source authority remains external. +Native calibration quality, observer overhead at production target counts, +cadence choice, staging upload and a dashboard consumer need their own evidence. +Retain prior histories with their actual run identities; do not label cached +replay as new optimizer progress. diff --git a/experiments/grouped-target-snapshot-integration-20260912.json b/experiments/grouped-target-snapshot-integration-20260912.json new file mode 100644 index 000000000..5f7c0e72b --- /dev/null +++ b/experiments/grouped-target-snapshot-integration-20260912.json @@ -0,0 +1,90 @@ +{ + "lane": "grouped-target-snapshot-integration-20260912", + "date": "2026-09-12", + "scope": "shared calibration solver seam only; no host wiring, no native build, no release action, nothing pushed", + "base": { + "origin_main": "116d46ee9dc2aafdc68259b7c06e4c3462522e8b", + "moved_since_pin": false, + "note": "origin/main was fetched and is byte-identical to the shared base both reviewed heads were cut from, so no main drift had to be preserved" + }, + "integrated_heads": { + "grouped_us_solver": "536f1ceefcdafda3cc619c14b4da18e012a7be57", + "target_snapshots_pr914": "b43369dc49e175803f62020cc1e72fa53926aed8", + "merge_base_of_the_two": "116d46ee9dc2aafdc68259b7c06e4c3462522e8b" + }, + "pinned_source_hashes_verified_before_merge": { + "grouped/solve.py": "925a1b6bae1c714de93df3ee44e26afeed7f3912377b93942f3a5173281d851a", + "grouped/calibrate__init__.py": "69ac527f4732cdb67c4807bcb8ee9faec56174621f663ec491140dfc324a705f", + "snapshots/solve.py": "f43be0efecdd21b3f5caa9789edc8463a127423c665523ac7f0c8bc48f4b4eed", + "snapshots/calibrate__init__.py": "6b66552827852dd16a36512bdef76c28e47842d9a6b0b59d176a4a18a71f48ae", + "all_four_matched_the_pin_file": true + }, + "commits": [ + {"sha": "fff20fca56f41060db55742b884a6df2693e139f", "what": "merge of the two reviewed heads; neither solver replaced"}, + {"sha": "e64d5a8850ab301b7e819a6b972f235c51688568", "what": "grouped Adam snapshot emission and retain-best separation"}, + {"sha": "7ac2b864a93977ffa98fa86674c09e04ea6b20b5", "what": "source-derived identity recomputation"}, + {"sha": "2f62bb8448010bcc9b9e1419ea24aafcb85917d1", "what": "grouped x snapshot cross-product controls"} + ], + "final_source_hashes": { + "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py": "e031ab5823d0f7bff43642e6ebb160ad867ec387e88658534ccd1e6ef62b273d", + "packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py": "1841264fe0c09a96ccf6aacf467cfa201de92ee67185b00cd378456fc7631a78", + "packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py": "a86958e3e94ca2454d997567883e2223e832f7edb5d994f689f0a64ddad1a88a", + "packages/microcosm-calibrate/src/microcosm/calibrate/group_bounds.py": "19a440a31d164832f61c84b2c1277a7a8798681f6830700bf03511dce7524557" + }, + "identity_recomputation": { + "rule": "recomputed against the merged checkout and the isolated interpreter; no value equals either branch's, so none is an ours/theirs choice", + "values": [ + {"artifact": "spec_engine/inventory_coverage.py EXPECTED_HASHES.seed_protocol", "grouped": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47", "snapshots": "d052fd875d014f6fa28d6f5b052735912f464d4efb2c98ecf373346b8cc454d3", "recomputed": "57e21b6522cfccc9b2d5e725503d9d39255abe5d5c9b391563d08ee664dea988"}, + {"artifact": "spec_engine/inventory_coverage.py EXPECTED_HASHES.seed_map", "grouped": "be2d24ad74bdc6339dfa073d01469074baee4e3a691eaa350451d36fa17d13dc", "snapshots": "d5a9694aac3e351f987b4f08ef8add0c25fcd3390d6bb2a851246da03adde516", "recomputed": "4ae2a0634e3a92061beba5bf5059db7c30b4ff6f1898e639697d246cb7e73878"}, + {"artifact": "us resolved spec_sha256 (us-f0-coverage.json and test_us_multispine_pool_tool.py)", "grouped": "f03e4d08d0ba4e15da53aab86bd2fbf50286e19e248657ac859b760103af2405", "snapshots": "ff2c97039853e18bfc0eb5388eb08c688cae33068b742f232d4c06a18b8efd88", "recomputed": "15e61194b4077de58272b14d83af95f7f54427aeab251429ada3cda01d93cbc8"}, + {"artifact": "test_spec_engine_loader.py minimal-loader golden", "grouped": "77f043a13853bbc109b6aa8cb42635ab1b2ebea69a1997b5bdb4c78a3726aeb3", "snapshots": "8a24089831b3a8f0cfe1be094769af7a30b7b74dbc7d4009e26e9f56444cba4c", "recomputed": "3868d14d5df04b5ab10ed4f0a56c2d9652c15c219840ed88d437f847fa481de7"}, + {"artifact": "parity kernels/calibrate/pins.json implementation_hash", "grouped": "59ca046b7c26bd872e56bd4cf3dd6170748fbe04e5a5da7b711e94aea8d3cc74", "snapshots": "46b52a20a7a5fac27d354c735a09ae5ba171d62bd9010fbc3079b06614cc2a32", "recomputed": "f2dd92eaaebba782e100cb92d90524d3ae6e2bab9ffedbacbe8df0257d5cae3d"}, + {"artifact": "parity kernels/calibrate/pins.json node_key", "grouped": "faaccb273d0aedeb1cca2e1b52c3e27678867512493f3c0379f596890c258cd1", "snapshots": "970720b71cb2579a5e2f714ca47deabc28c9249a83007ebfda83e2e7446d9311", "recomputed": "ae92142a3bf0934ac6b6751aca904c1847cdff3ca8533af83141d2cbf1ccf102"}, + {"artifact": "test_spec_engine_country_bundles.py am", "grouped": "c15bd12508731eb11fe061df810d9047b4c62c2887645cb19d3c54c7a104aa1c", "snapshots": "untouched-by-S", "recomputed": "258b940476e9f611d5e12d471dead7427479f36267eb40ed459fcd2f3dbd18ae"}, + {"artifact": "test_spec_engine_country_bundles.py be", "grouped": "5285b530974a9b8f7c21497fb140fdbd6060c3f1cc983987f35e9f244e71ee62", "snapshots": "untouched-by-S", "recomputed": "a61b65f7a1424c090081632fb1ddacdb85dae1b85920faff5dd7cb144cb1b5a1"}, + {"artifact": "test_spec_engine_country_bundles.py uk", "grouped": "1ac642230eb3166dd9ea2a113536985e35bf2898f914e289f5564ac8fde6e0b4", "snapshots": "untouched-by-S", "recomputed": "66837fd0fad4956566655f558d10f8d39e83523ad873b17e9f37ce0cb5012724"} + ], + "deliberately_not_repinned": [ + "packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json (G's value preserved; it hashes microcosm/frame/bundle.py, not calibrate.solve)", + "packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/ (neither head changed microcosm/fit/qrf.py)", + "parity direct.csv outputs (the calibrate kernel's outputs did not move, only its implementation hash)" + ], + "tooling": { + "us-f0-coverage.json": "tools/spec_engine_coverage.py (then --check exits 0)", + "calibrate parity pin": "tools/graph_parity_repin.py calibrate", + "loader golden and bundle digests": "taken from the observed value the failing identity assertion reports" + } + }, + "diff_summary": { + "vs_grouped_head": {"files": 25, "insertions": 4287, "deletions": 20, "note": "every deletion is an identity pin line, an extended import, an extended docstring, or a line S replaced; nothing from the grouped head was lost"}, + "vs_snapshot_head": {"files": 432, "insertions": 148099, "deletions": 3251, "note": "the grouped branch's whole delta"} + }, + "controls": { + "microcosm-calibrate shard": {"passed": 443, "failed": 0}, + "microcosm-graph shard + test_us_multispine_pool_tool + 4 spec-engine identity files": {"passed": 775, "failed": 0}, + "per_file": { + "test_target_snapshots.py": {"cases": 52, "added": 0}, + "test_grouped_fixed_support.py": {"cases": 42, "added": 18}, + "test_group_bounds.py": {"cases": 82, "added": 3}, + "test_solve_us_launch_integration.py": {"cases": 12, "added": 4} + }, + "tools/spec_engine_coverage.py --check": "exit 0", + "tools/ci_test_groups.py --verify": "verification=ok; no test file changed lane and none is [defaulted]", + "ruff": "format clean, check clean on packages/microcosm-calibrate" + }, + "environment": { + "interpreter": "borrowed root .venv/bin/python 3.14.4 run with -I -B -S and an explicit sys.path", + "sys_path_order": "this worktree's packages/*/src first, then the repo root, then the borrowed site-packages", + "imports_asserted_in_worktree": ["microcosm.frame", "microcosm.calibrate", "microcosm.build", "microcosm.fit", "microcosm.graph"], + "threads": "OMP/OPENBLAS/MKL/NUMEXPR/VECLIB all 1", + "pytest": "PYTEST_DISABLE_PLUGIN_AUTOLOAD=1, -p no:randomly", + "locked_versions": {"numpy": "2.4.6", "pandas": "3.0.3", "scipy": "1.17.1", "torch": "2.12.0", "platform": "arm64/darwin/py3.14"}, + "nothing_installed": true + }, + "not_done_on_purpose": [ + "host observer wiring in us_runtime/graph_fiscal_dense_calibration.py", + "any global observer registry, Node.params callback or filesystem path in graph params", + "any claim that a required replay reruns the optimizer", + "native build, PUF fixture, country build, dependency install, push, PR, publication or merge" + ] +} diff --git a/experiments/grouped-target-snapshot-integration-20260912.md b/experiments/grouped-target-snapshot-integration-20260912.md new file mode 100644 index 000000000..5e317e0bf --- /dev/null +++ b/experiments/grouped-target-snapshot-integration-20260912.md @@ -0,0 +1,130 @@ +# Grouped solver x calibration target snapshot integration — 2026-09-12 + +Historical scope note, 12 September 2026: this records grouped core +`2f62bb8448010bcc9b9e1419ea24aafcb85917d1` and its earlier test run. The later +[fiscal-host integration](fiscal-target-snapshot-host-20260912.md) adds the +host observer and PR #914's completed-update convention. The 443-test result +and host-wiring gaps below remain evidence of this earlier revision. + +Scope, evidence and residual risks for branch +`grouped-target-snapshot-integration-20260912`. Source-only; no native build, +no PUF fixture, no country build, no release action, nothing pushed. + +## What this is + +The bounded integration of the reviewed calibration target snapshot observer +(microcosm#908, head `b43369dc49e175803f62020cc1e72fa53926aed8`) with the +reviewed US grouped/fixed-zero Adam solver (head +`536f1ceefcdafda3cc619c14b4da18e012a7be57`), on a fresh `origin/main` +(`116d46ee9dc2aafdc68259b7c06e4c3462522e8b`). Main had not moved from the +shared base, so no main drift had to be preserved. + +Executed against the read-only design checklist +`grouped-snapshot-integration-review.md` and its pins. All four pinned source +hashes on both heads matched the pin file byte for byte before the merge. + +## What it is not + +- **Not** host wiring. `us_runtime/graph_fiscal_dense_calibration.py` still + calls `calibrate` without an observer, so this branch produces no snapshot + file for the actual US fiscal path. Wiring it is a separate, explicitly + host-owned step. No global observer registry, no callback or filesystem path + in graph `Node.params`, and no claim that any replay reruns the optimizer. +- **Not** a dashboard, a staging upload, or a cadence/performance acceptance. +- **Not** a release, a publication or a promotion of any artifact. + +## The one real code change + +A clean textual merge leaves grouped runs uninstrumented: `_optimize` returns +early into `_optimize_grouped` before reaching any snapshot hook S added to the +ordinary Adam loop, so a caller passing `target_snapshots` to a grouped +`calibrate` received exactly one closing snapshot and no trajectory. + +1. `_optimize` forwards its bound observer into `_optimize_grouped` + (`snapshots=snapshots`), and `_optimize_grouped` takes a default-`None` + `snapshots` argument. +2. `_optimize_grouped` emits inside the epoch loop, after the existing progress + callback and before `backward()`, from the exact float32 `estimate` tensor + the epoch's loss was computed from: `iterate=current`, `precision=float32`, + `loss=trajectory[epoch]`, `best_retained={available: False, epoch: None, + loss: None}`. +3. `calibrate` reads retain-best off the solver actually used rather than + inferring it from an empty `iterate_selection_receipt`, and stamps the + grouped closing snapshot at `epoch == epochs`. The receipt itself stays + empty, exactly as before. +4. Grouped snapshots carry three bounded aggregate scalars in `selection`: + `rule: "closing_state"`, `constraint_mode: "grouped_upper_bounds"`, + `grouped_preserve_zeros: `. + +Nothing else in either solver moved. No optimizer or matrix dtype, active mask, +zero template, L2 denominator, log/exp operation, optimizer construction, seed, +projection call, accepted-buffer assignment or `log_w` overwrite changed; there +is still exactly one `log_w` overwrite per accepted projection, the final no-op +projection and accepted-byte equality are unchanged, and the stored-result +admission (weight bytes, fixed-zero mask, bounds, ordered household IDs) still +runs after the last sink. + +### Why the `selection` labels + +Grouped Adam is a closing-state algorithm; it never runs the retain-best rule. +Without a label, its `best_retained.available: False` is indistinguishable from +an ordinary run whose retain-best rule happened to be off, and a consumer would +read the two the same way. Checklist section 2 sanctions exactly this — +"Optional grouped labels must be bounded aggregate scalars, e.g. constraint +mode and whether fixed-zero support is enabled". The labels mirror the run's +own `options["iterate_selection"]` and reuse the receipt's `rule` vocabulary; +they are three JSON scalars carrying no record-level content. This is the one +place the integration goes beyond the checklist's literal field list for +in-loop snapshots, and it is a single expression to remove if root disagrees. + +## Evidence + +- Grouped runs are bit-identical with the observer on and off across + dense x CSR, positive x fixed-zero support, and plain / warm-started / L2: + returned weights, loss trajectory, options, closing loss, stored frame + weights, ordered IDs, fixed-zero bytes, and the full sequence of private + post-projection proof payloads. +- Exactly 20 `_apply_constraint` and 3 `problem.estimates` calls with the + observer on and off on the 20-epoch fixture: cadence buys observation, not + evaluation. Torch and numpy RNG state are identical after each pair of runs. +- Each in-loop row carries the exact float32 tensor the epoch's loss was + computed from, verified against a wrapped `_apply_constraint`, and differs on + all 20 epochs from a recomputation on the accepted vector the private + observer reports next. +- The closing row equals the reused float64 final diagnostics estimates. +- A sink that ruins every delivered payload changes neither the weights nor the + next payload. A sink that mutates live frame IDs at the closing emission + still trips the ordered-ID admission guard. +- Zero epochs through the internal grouped seam emits nothing and evaluates + nothing. +- The deliberately oscillating example still selects grouped closing output + while ordinary Adam selects an earlier best, and each solver's closing + snapshot now says which rule produced it. + +Identity recomputation. Editing `solve.py` — an attested kernel module — moves +the seed protocol digest, every resolved-spec digest that folds it in, and the +calibrate kernel's implementation hash. Each value below was recomputed against +this checkout with the isolated Python 3.14 interpreter; none equals either +branch's value, so none is an ours/theirs choice. The simulate parity pin and +the fit.qrf pin were deliberately left alone: neither hashes +`microcosm.calibrate.solve`, and both are byte-unchanged from the grouped head. + +## Residual risks + +1. **Cadence cost on a real US run is unmeasured here.** The only overhead + evidence in the tree is #908's synthetic benchmark; this branch adds no + native measurement. A country-scale cadence choice is still unowned. +2. **The `selection` labels are an integration judgement**, not a checklist + requirement, on the in-loop rows specifically. See above. +3. **Host wiring remains the gap between this and any dashboard.** Green tests + here say the solver seam is correct; they say nothing about whether any real + US calibration emits anything, because none does yet. +4. **The identity recomputation is only as good as its interpreter.** Digests + fold locked dependency versions; a lane on different locked versions will + compute different parity keys. The values here were taken under + numpy 2.4.6 / pandas 3.0.3 / scipy 1.17.1 / torch 2.12.0 on + arm64/darwin/py3.14, which is what the calibrate parity pin records. +5. **The engine-gated identity tests were exercised with the engines present** + in the borrowed environment. A lane without the US extra will skip + `test_spec_engine_loader.py`'s golden and the country-bundle proofs rather + than check them. diff --git a/experiments/signed-income-reconciliation-20260912.md b/experiments/signed-income-reconciliation-20260912.md new file mode 100644 index 000000000..4d5b6453e --- /dev/null +++ b/experiments/signed-income-reconciliation-20260912.md @@ -0,0 +1,85 @@ +# Deterministic reconciliation to a signed income total + +`microcosm.fit.signed_reconciliation.reconcile_signed_total` projects a joint +draw onto a caller-qualified total. It is a separate numeric operation for a +future graph node. No existing model, source qualifier, financial attachment, +PUF host or native candidate invokes it in this change. + +For draw `q`, anchor `A`, strictly positive scales `s`, and a declared set `B` of +nonnegative components, it solves the strictly convex problem + +```text +minimize sum_j ((z_j - q_j) / s_j)^2 +subject to sum_j z_j = A + z_j >= 0 for j in B +``` + +At least one component must remain unrestricted, making every finite signed +anchor feasible in exact arithmetic. Component names, the bound mask and scales +are explicit parameters. Ordinary interest and retirement-account interest can +therefore be separate components; this operator assigns neither a source meaning +nor tax treatment to either. It introduces no nonzero-presence restriction. + +Define `w_j = (s_j / max(s))^2`. Uniform scale normalization preserves the +minimizer. Given an active bound set `C`, the equality solution for the remaining +free coordinates `F` is + +```text +t = (A - sum_{j in F} q_j) / sum_{j in F} w_j +z_j = q_j + t*w_j for j in F +z_j = 0 for j in C +``` + +Start with every coordinate free. Fix any bounded coordinate with a negative +candidate to zero, then solve again. Removing negative candidates can only lower +`t`, so a fixed coordinate cannot need releasing later. An unrestricted +coordinate always remains free. The algorithm therefore terminates after at +most `len(B) + 1` solves, without a general optimizer or random draws. The final +free-coordinate stationarity and active-bound dual feasibility are checked. + +The signed anchor is never clamped or used as a divisor. A zero total does not +force its components to zero. For example, `[8, 4, -3]` with unit scales and a +zero anchor becomes `[5, 1, -6]`. Positive income and offsetting property loss +remain possible for negative, zero or positive totals. An exactly feasible draw +is preserved, including its float64 signed-zero bits. + +Draws have shape `(..., k)` and anchors have exactly the batch shape `(...)`. +Scales and the boolean bound mask each have shape `(k,)`; broadcasting an anchor +across records is deliberately not implicit. Empty batches are supported. +Inputs are finite real numeric values converted into detached float64 arrays. +The result preserves the raw draw, anchor, component roster and scales, together +with projected values, adjustment vectors, signed sum residuals, active bounds, +normalized shifts, objectives and KKT diagnostics. These arrays are descriptive, +mutable values, not an authority or immutable execution receipt. + +Callers must supply finite nonnegative `atol` and `rtol`. The sum check uses +`abs(residual) <= atol + rtol*abs(A)`. Component stationarity and dual checks use +the same form with the maximum absolute draw, result and proposed adjustment. +The sum uses `math.fsum`; no residual is patched into a final component and no +display rounding changes the optimum. Nonfinite input, intermediate or objective +overflow, a normalized scale weight lost to underflow, or failure of the stated +tolerances causes refusal. Floating-point refusal is possible even when an +exact-arithmetic solution exists. + +The 45-case guarded suite passed with zero failures/errors/skips in 4.470 seconds +wall time, 2.899 CPU seconds and 405,143,552 bytes peak RSS. This includes 160 +randomized small problems compared against an independent oracle that enumerates +bound faces, solves dense block KKT systems and selects the minimum objective. +It also verifies negative and zero-net totals, loss offsets, unequal scales, +scale-rescaling invariance, active bounds, exact feasible identity, detached +storage, alternate component rosters, multidimensional and empty batches, invalid +inputs and unstable arithmetic. All 990 source-plus-owned hashes and 15 allowed +resource hashes were unchanged; no child or unexpected refusal occurred. The +normal guard-induced dateutil zoneinfo warning occurred. + +Production source SHA256: +`d56c21959121cc7500f06cd747c74bea51561deba537ce2c58266d5966ae84b2`. +Test SHA256: `b5ee23c6ff4f633434c96a9e17c8b82b254435757b6eb24061b1835f57bdaa77`. +The local `codex-signed-reconciliation-20260912/numeric-v2/numeric-v2.json` receipt +SHA256 is `2f071f5116489c1febb4f594c97781c9b8d88d68cf125e94aec446d80e7c3321`. +The earlier guard preflight failure is preserved; it executed no numeric tests. + +This numerical acceptance does not establish ACS/ASEC measurement equivalence, +resolve the property or retirement donor bridge, approve scale choices, fit a +model, or certify a release. Source qualification, donor definitions, a named +reconciliation node and dependent model replay remain separate work. diff --git a/experiments/spec-seed-current-lock-20260910.json b/experiments/spec-seed-current-lock-20260910.json new file mode 100644 index 000000000..0005829fa --- /dev/null +++ b/experiments/spec-seed-current-lock-20260910.json @@ -0,0 +1,22 @@ +{ + "approved_lock_sha256": "751d5ef5d25406bbae1798667f0e29890d4aad933d323c12912c7d45d8809bb9", + "before_lock_sha256": "4ef1ef6eb39b65ebc47c2c00bafa44e7c872544b1b0146dd8493bcfe45b2da1b", + "ci_job": "https://github.com/PolicyEngine/microcosm/actions/runs/34506624007/job/102970618835", + "data_or_release_acceptance": false, + "diagnostic_rerun": "pending CI on corrected source", + "diagnostic_sha256": "18468f5319324169896942230ca3584188c9840fff03e913aaf16e8360adf4ef", + "local_subject_execution": false, + "matches_existing_worker_approval": true, + "observed_refusal": { + "code": "LOCK", + "completed": false, + "coverage_pass": false, + "phase": "environment" + }, + "only_lock_delta": "Declare existing openpyxl package in microcosm-build[uk] dependency and requires-dist; no new resolved package or version change", + "package_count_before_after": 125, + "resolved_package_versions_unchanged": true, + "scope": "Source-only audit of current-lock diagnostic correction", + "source_change": "One literal, retaining exact lock equality; no allowance for arbitrary locks", + "upstream_worker_approval_commit": "0d978f20bf4d1d899b707fe7d33ab084c153ee4f" +} diff --git a/experiments/uk-atomic-area-lazy52-20260910.json b/experiments/uk-atomic-area-lazy52-20260910.json new file mode 100644 index 000000000..076f3a32f --- /dev/null +++ b/experiments/uk-atomic-area-lazy52-20260910.json @@ -0,0 +1,37 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "country_distribution_paths_dereferenced_by_postchecker": false, + "cpu_seconds": 2.335378, + "guard_policy_passed": true, + "model_resources": 0, + "model_source_files": 0, + "native_payloads_read": 0, + "physical_postcheck": "pass", + "physical_postcheck_sha256": "dc9ca4ad0c4cd77f40a4a0affb12487975d88a6e9d0e7ff2a8e41011e4430071", + "receipt_sha256": "b6abf03c647e79415f593a7f8a2cfc933b6ac1a47cf92ba552f2e0cf38d3a838", + "release_acceptance": false, + "resources": 0, + "reviewer": "root", + "scope": "Invented three-system UK atomic-area adapter and ordinary lazy-package behavior; no native UK population or calibration acceptance", + "source_count": 64, + "source_files": { + "packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py": "accb5208bfc22f0d2905dad12ad390b416f424506e61c417d169215fe4bad78d", + "packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py": "b2006451f4afcbc2c5192eb421dd7c94d11262889c0a10aa2d1040f1fa28ec8e", + "packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py": "8d2164ae90eb521ee838936c6ca17e1ce08f43650889ea14afc9bcb82301a325", + "packages/microcosm-build/src/microcosm/build/uk_runtime/content_identity.py": "589002ba67e1c669f40b13968f16ee439937bfbd7e36d6ad4ac9ee195af2b3e5", + "packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_geography.py": "5b7287cd228fa7f5385cc19e6ce73184749b3c8b655953440d7273f86f3eabb3", + "packages/microcosm-build/tests/test_uk_atomic_area_support.py": "2109e447bf9446b6c4047bc742f0d9c1ed93f89b926fb9be335ae9567292759a", + "packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py": "207f5ba4b21737902e42ed8f72f3251425f1aa68959309d3f2ff0345f0f4296b" + }, + "source_map_sha256": "e7c59f91af92643378cefffb2a59be700d1940d3b3dc5aadfb4a4bf0df8e9eff", + "source_plus_owned": 73, + "suite_accepted": true, + "supplied_geography_conventions": [ + "EW OA 2021", + "Scotland OA 2022", + "Northern Ireland Data Zone 2021" + ], + "tests_passed": 52, + "wall_seconds": 3.199442916957196, + "xml_sha256": "34a5afffba8a3f7cc8933e306da19e76c8b597c9d31cffbadfeda902a1edbf8d" +} diff --git a/experiments/us-acs-code-memo-comparison-20260910.json b/experiments/us-acs-code-memo-comparison-20260910.json new file mode 100644 index 000000000..1cdec2fc8 --- /dev/null +++ b/experiments/us-acs-code-memo-comparison-20260910.json @@ -0,0 +1,217 @@ +{ + "baseline": { + "cpu_seconds": 303.897467, + "peak_rss_bytes": 594296832, + "wall_seconds": 308.2577062920318 + }, + "baseline_receipt_sha256": "784d46a3a5ed3994f1018f6d05900122abdc6f63a0f18e318002fedfe7059af9", + "correctness_controls": 18, + "cpu_ratio": 1.0405705290067455, + "date": "2026-09-10", + "decision": "not_adopted", + "experiment_patch_sha256": "bc2b245b712256578ad6587c7a242af38e4244eaeacc268b589b2ed0c38ae557", + "guard_sha256": "8a2cf87bd0a00c1de1d8e4263764394551efed8e1ecb3d3297eeb5b1ab0eaf5e", + "implementation_restored_to_sha256": "db7aaae06961cc1ed9ce13b28dac74d8fd1041b9cd4c32294d2b11105fbef7af", + "native_speedup_established": false, + "optimized": { + "cpu_seconds": 316.226748, + "peak_rss_bytes": 601653248, + "wall_seconds": 319.78662233403884 + }, + "optimized_receipt_sha256": "49016ede5317b1e025e6d476364398b8233e3c589b7b80ef78781d154ab9f0d0", + "optimized_source_sha256": "4dc64558ed453f1c06557ef005e42193278496a9c31b374475417c407c037573", + "profile_and_original_teardown_passed": true, + "reason": "No measured speed improvement in the same instrumented fixture; added eligibility work exceeded the comparison savings.", + "release_acceptance": false, + "scope": "one paired invented cold/required fixture run, identical cProfile policy and dimensions", + "successor_controls_executed": false, + "timing_limitations": [ + "cProfile overhead changes elapsed time and can trigger the unchanged limits", + "partial snapshots omit unfinished active-call time until profiler flush", + "only exact frozen-source coordinates are reported; external time remains within cumulative callers", + "cumulative function times overlap and must not be summed" + ], + "top_optimized_records": [ + { + "calls": 1, + "cumulative_seconds": 309.704728666, + "file": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "function": "known_financial_run", + "line": 19, + "recursive_calls": 0, + "self_seconds": 0.0052826240000000005 + }, + { + "calls": 2, + "cumulative_seconds": 308.7401165, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py", + "function": "run_atomic_survey_financial", + "line": 538, + "recursive_calls": 0, + "self_seconds": 0.005962359 + }, + { + "calls": 110, + "cumulative_seconds": 224.98760203900002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_validate", + "line": 1071, + "recursive_calls": 0, + "self_seconds": 0.00429048 + }, + { + "calls": 108, + "cumulative_seconds": 220.871511795, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_checked", + "line": 1091, + "recursive_calls": 0, + "self_seconds": 0.000492867 + }, + { + "calls": 652, + "cumulative_seconds": 195.60535725600002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_producer", + "line": 223, + "recursive_calls": 0, + "self_seconds": 4.97942924 + }, + { + "calls": 14, + "cumulative_seconds": 169.824184456, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py", + "function": "reconstruct_atomic_survey_geography", + "line": 307, + "recursive_calls": 0, + "self_seconds": 0.006830769 + }, + { + "calls": 38464, + "cumulative_seconds": 168.34809192, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_live_code", + "line": 137, + "recursive_calls": 0, + "self_seconds": 8.731628122 + }, + { + "calls": 8, + "cumulative_seconds": 139.864694334, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py", + "function": "qualify_current_survey_predictors", + "line": 304, + "recursive_calls": 0, + "self_seconds": 0.0066878350000000005 + }, + { + "calls": 2, + "cumulative_seconds": 119.01038833400001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py", + "function": "run_atomic_survey_population", + "line": 213, + "recursive_calls": 0, + "self_seconds": 0.001211868 + }, + { + "calls": 6, + "cumulative_seconds": 116.761678708, + "file": "packages/microcosm-graph/src/microcosm/graph/executor.py", + "function": "run_graph", + "line": 2158, + "recursive_calls": 0, + "self_seconds": 0.007467228 + }, + { + "calls": 15, + "cumulative_seconds": 115.40236200000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py", + "function": "qualify_current_survey_geography", + "line": 220, + "recursive_calls": 0, + "self_seconds": 0.0036097990000000003 + }, + { + "calls": 2010412, + "cumulative_seconds": 114.621920672, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "check", + "line": 176, + "recursive_calls": 51772, + "self_seconds": 86.630530445 + }, + { + "calls": 348, + "cumulative_seconds": 111.58561966500001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_producer", + "line": 102, + "recursive_calls": 0, + "self_seconds": 0.135475169 + }, + { + "calls": 150, + "cumulative_seconds": 98.955415883, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "verify_acs_native_coverage", + "line": 490, + "recursive_calls": 0, + "self_seconds": 0.010436341 + }, + { + "calls": 8, + "cumulative_seconds": 98.24177225000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py", + "function": "_initial", + "line": 355, + "recursive_calls": 0, + "self_seconds": 0.0019372500000000002 + }, + { + "calls": 116, + "cumulative_seconds": 75.394407754, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_checked", + "line": 406, + "recursive_calls": 0, + "self_seconds": 0.003720891 + }, + { + "calls": 110, + "cumulative_seconds": 71.55709778800001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "verify_acs_source_catalogue", + "line": 430, + "recursive_calls": 0, + "self_seconds": 0.000156618 + }, + { + "calls": 4, + "cumulative_seconds": 70.568924458, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py", + "function": "_qualified", + "line": 269, + "recursive_calls": 0, + "self_seconds": 0.0008037900000000001 + }, + { + "calls": 23, + "cumulative_seconds": 52.046634627, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py", + "function": "qualify_current_asec_demographics", + "line": 194, + "recursive_calls": 0, + "self_seconds": 0.003941233000000001 + }, + { + "calls": 2, + "cumulative_seconds": 42.917792458, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py", + "function": "run_authenticated_survey_population", + "line": 815, + "recursive_calls": 0, + "self_seconds": 0.0007081860000000001 + } + ], + "wall_ratio": 1.037400252472796 +} diff --git a/experiments/us-acs-code-memo-not-adopted-20260910.patch b/experiments/us-acs-code-memo-not-adopted-20260910.patch new file mode 100644 index 000000000..246a86e19 --- /dev/null +++ b/experiments/us-acs-code-memo-not-adopted-20260910.patch @@ -0,0 +1,497 @@ +--- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py ++++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py +@@ -48,13 +48,103 @@ + raise ACSNativeCoverageBindingError(code) + + +-def _live_code(module, compiled): ++def _immutable_code_constants(value, immutable_codes, depth=0): ++ """Recognize exact immutable compiler values without invoking custom equality. ++ ++ Runtime-created CodeType constants need not be compiler values. Unfamiliar ++ values and deeply nested trees take the original uncached equality path; ++ eligibility is never a new source-acceptance requirement. Positive code ++ entries retain the object itself so integer identity keys cannot be reused. ++ """ ++ if depth > 64: ++ return False ++ kind = type(value) ++ if any( ++ kind is scalar ++ for scalar in ( ++ type(None), ++ bool, ++ int, ++ float, ++ complex, ++ str, ++ bytes, ++ type(Ellipsis), ++ ) ++ ): ++ return True ++ if kind is tuple or kind is frozenset: ++ return all( ++ _immutable_code_constants(item, immutable_codes, depth + 1) ++ for item in value ++ ) ++ if kind is not CodeType: ++ return False ++ if immutable_codes.get(id(value)) is value: ++ return True ++ # Check all stored public CodeType fields, not merely co_consts: runtime ++ # constructors may admit scalar subclasses with custom comparison methods. ++ if not all( ++ type(getattr(value, name)) is int ++ for name in ( ++ "co_argcount", ++ "co_posonlyargcount", ++ "co_kwonlyargcount", ++ "co_nlocals", ++ "co_stacksize", ++ "co_flags", ++ "co_firstlineno", ++ ) ++ ) or not all( ++ type(getattr(value, name)) is str ++ for name in ("co_filename", "co_name", "co_qualname") ++ ): ++ return False ++ if not all( ++ type(getattr(value, name)) is bytes ++ for name in ("co_code", "co_linetable", "co_exceptiontable") ++ ): ++ return False ++ for name in ("co_names", "co_varnames", "co_freevars", "co_cellvars"): ++ items = getattr(value, name) ++ if type(items) is not tuple or any(type(item) is not str for item in items): ++ return False ++ if type(value.co_consts) is not tuple or not _immutable_code_constants( ++ value.co_consts, immutable_codes, depth + 1 ++ ): ++ return False ++ immutable_codes[id(value)] = value ++ return True ++ ++ ++def _same_loaded_code(live, reference, code_pairs, immutable_codes): ++ """Memoize equality only for strongly retained, immutable code pairs.""" ++ if type(live) is not CodeType or type(reference) is not CodeType: ++ return live == reference ++ key = (id(live), id(reference)) ++ retained = code_pairs.get(key) ++ if retained is not None and retained[0] is live and retained[1] is reference: ++ return True ++ eligible = _immutable_code_constants( ++ live, immutable_codes ++ ) and _immutable_code_constants(reference, immutable_codes) ++ matches = live == reference ++ if matches and eligible: ++ code_pairs[key] = (live, reference) ++ return matches ++ ++ ++def _live_code(module, compiled, code_pairs=None, immutable_codes=None): + """Check loaded Python implementations against their current source bytes. + + Covers module functions, imported function aliases and source class methods; + generated dataclass methods have no source code object. This is a drift + check in a trusted process, not a sandbox against arbitrary Python execution. + """ ++ if code_pairs is None: ++ code_pairs = {} ++ if immutable_codes is None: ++ immutable_codes = {} + path = getattr(module, "__file__", None) + if path is None: + return +@@ -102,15 +192,19 @@ + + visit(compile(Path(path).read_bytes(), path, "exec", dont_inherit=True)) + compiled[path] = codes ++ live = function.__code__ ++ _require(function.__globals__ is vars(origin), "LOADED_PRODUCER") + _require( +- function.__globals__ is vars(origin) +- and function.__code__ == compiled[path].get(function.__code__.co_qualname), ++ _same_loaded_code( ++ live, compiled[path].get(live.co_qualname), code_pairs, immutable_codes ++ ), + "LOADED_PRODUCER", + ) + + for cell in function.__closure__ or (): + if isinstance(cell.cell_contents, FunctionType): + check(cell.cell_contents) ++ _require(function.__code__ is live, "LOADED_PRODUCER") + + for value in vars(module).values(): + if isinstance(value, FunctionType): +@@ -147,10 +241,12 @@ + name = package + "." + relative.removesuffix(".py").replace("/", ".") + names.add(name.removesuffix(".__init__")) + compiled = {} ++ code_pairs = {} ++ immutable_codes = {} + for name in sorted(names): + module = sys.modules.get(name) + if module is not None: +- _live_code(module, compiled) ++ _live_code(module, compiled, code_pairs, immutable_codes) + return { + "preparation": manifest, + "coverage": coverage._producer(), +--- /dev/null ++++ b/packages/microcosm-build/tests/test_us_acs_loaded_code_validation.py +@@ -0,0 +1,351 @@ ++"""Invented Python source controls for invocation-local ACS code equality memo.""" ++ ++from __future__ import annotations ++ ++import sys ++from pathlib import Path ++from types import CodeType, FunctionType, ModuleType ++ ++import pytest ++ ++from microcosm.build.us_runtime import acs_native_coverage_binding as owner ++ ++ ++def _module(tmp_path, monkeypatch, name, source): ++ path = tmp_path / (name + ".py") ++ path.write_text(source, encoding="utf-8") ++ module = ModuleType(name) ++ module.__file__ = str(path) ++ module.__package__ = "" ++ monkeypatch.setitem(sys.modules, name, module) ++ exec(compile(source, str(path), "exec", dont_inherit=True), vars(module)) ++ return module ++ ++ ++def _code(source="def f():\n return 7\n"): ++ compiled = compile(source, "invented-code.py", "exec", dont_inherit=True) ++ return next(value for value in compiled.co_consts if type(value) is CodeType) ++ ++ ++def _state(): ++ return {}, {}, {} ++ ++ ++def _refuses(): ++ return pytest.raises(owner.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$") ++ ++ ++def _cell(value): ++ return (lambda: value).__closure__[0] ++ ++ ++def test_repeated_exact_pair_skips_only_second_deep_comparison(): ++ live, reference = _code(), _code() ++ pairs, immutable = {}, {} ++ eligibility_calls = [] ++ eligibility_code = owner._immutable_code_constants.__code__ ++ ++ def profile(frame, event, _arg): ++ if event == "call" and frame.f_code is eligibility_code: ++ eligibility_calls.append(1) ++ ++ previous = sys.getprofile() ++ try: ++ sys.setprofile(profile) ++ assert owner._same_loaded_code(live, reference, pairs, immutable) ++ first_calls = len(eligibility_calls) ++ assert first_calls > 0 ++ assert owner._same_loaded_code(live, reference, pairs, immutable) ++ assert len(eligibility_calls) == first_calls ++ finally: ++ sys.setprofile(previous) ++ assert list(pairs) == [(id(live), id(reference))] ++ assert pairs[(id(live), id(reference))][0] is live ++ assert pairs[(id(live), id(reference))][1] is reference ++ assert immutable[id(live)] is live ++ assert immutable[id(reference)] is reference ++ ++ ++def test_wrong_strong_reference_does_not_authorize_an_identity_key(): ++ live, reference, changed = _code(), _code(), _code("def f():\n return 8\n") ++ pairs = {(id(changed), id(reference)): (live, reference)} ++ assert not owner._same_loaded_code(changed, reference, pairs, {}) ++ ++ ++def test_historical_two_argument_caller_keeps_fresh_validation(tmp_path, monkeypatch): ++ module = _module( ++ tmp_path, monkeypatch, "memo_legacy_call", "def f():\n return 7\n" ++ ) ++ compiled = {} ++ owner._live_code(module, compiled) ++ owner._live_code(module, compiled) ++ monkeypatch.setattr(module.f, "__code__", _code("def f():\n return 8\n")) ++ with _refuses(): ++ owner._live_code(module, compiled) ++ ++ ++def test_import_alias_is_rechecked_after_pair_hit(tmp_path, monkeypatch): ++ origin = _module(tmp_path, monkeypatch, "memo_origin", "def f():\n return 7\n") ++ alias = _module(tmp_path, monkeypatch, "memo_alias", "from memo_origin import f\n") ++ state = _state() ++ owner._live_code(origin, *state) ++ owner._live_code(alias, *state) ++ alias.f = FunctionType(origin.f.__code__, vars(origin)) ++ with _refuses(): ++ owner._live_code(alias, *state) ++ ++ ++def test_equal_code_with_non_origin_globals_refuses_after_pair_hit( ++ tmp_path, monkeypatch ++): ++ module = _module( ++ tmp_path, monkeypatch, "memo_globals", "def f():\n return 7\nalias = f\n" ++ ) ++ state = _state() ++ owner._live_code(module, *state) ++ module.alias = FunctionType(module.f.__code__, {"__name__": module.__name__}) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++def test_replaced_code_refuses_after_pair_hit(tmp_path, monkeypatch): ++ module = _module(tmp_path, monkeypatch, "memo_replaced", "def f():\n return 7\n") ++ state = _state() ++ owner._live_code(module, *state) ++ monkeypatch.setattr(module.f, "__code__", _code("def f():\n return 8\n")) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++_CLOSURE_SOURCE = """def factory(): ++ def inner(): ++ return 7 ++ def wrapped(): ++ return inner() ++ return wrapped ++wrapped = factory() ++""" ++ ++ ++def test_replaced_closure_function_is_rechecked_after_pair_hit(tmp_path, monkeypatch): ++ module = _module(tmp_path, monkeypatch, "memo_closure", _CLOSURE_SOURCE) ++ state = _state() ++ owner._live_code(module, *state) ++ cell = module.wrapped.__closure__[0] ++ original = cell.cell_contents ++ bad = FunctionType(original.__code__, {"__name__": module.__name__}) ++ monkeypatch.setattr(cell, "cell_contents", bad) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++def test_distinct_closure_with_same_outer_code_is_not_memoized_as_valid( ++ tmp_path, monkeypatch ++): ++ module = _module(tmp_path, monkeypatch, "memo_second_closure", _CLOSURE_SOURCE) ++ state = _state() ++ owner._live_code(module, *state) ++ original = module.wrapped.__closure__[0].cell_contents ++ bad = FunctionType(original.__code__, {"__name__": module.__name__}) ++ module.second = FunctionType( ++ module.wrapped.__code__, vars(module), closure=(_cell(bad),) ++ ) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++def test_source_read_and_replacement_remain_fresh_within_shared_invocation_state( ++ tmp_path, monkeypatch ++): ++ source = "def f():\n return 7\n" ++ module = _module(tmp_path, monkeypatch, "memo_source", source) ++ path = Path(module.__file__) ++ state = _state() ++ reads = [] ++ reader = Path.read_bytes.__code__ ++ ++ def profile(frame, event, _arg): ++ if ( ++ event == "call" ++ and frame.f_code is reader ++ and frame.f_locals["self"] == path ++ ): ++ reads.append(1) ++ ++ previous = sys.getprofile() ++ try: ++ sys.setprofile(profile) ++ owner._live_code(module, *state) ++ owner._live_code(module, *state) ++ finally: ++ sys.setprofile(previous) ++ # One AST read on each scan, plus the existing first code-compilation read. ++ assert len(reads) == 3 ++ changed = "def f():\n return 8\n" ++ fired = [] ++ ++ def change_after_scan(frame, event, _arg): ++ if ( ++ event == "return" ++ and frame.f_code is owner._live_code.__code__ ++ and frame.f_locals["module"] is module ++ and not fired ++ ): ++ fired.append(1) ++ path.write_text(changed, encoding="utf-8") ++ exec(compile(changed, str(path), "exec", dont_inherit=True), vars(module)) ++ ++ try: ++ sys.setprofile(change_after_scan) ++ owner._live_code(module, *state) ++ with _refuses(): ++ owner._live_code(module, *state) ++ finally: ++ sys.setprofile(previous) ++ assert fired == [1] ++ # A distinct producer invocation recompiles the new current source bytes. ++ owner._live_code(module, *_state()) ++ ++ ++def test_code_swap_on_pair_return_refuses_before_function_check_returns( ++ tmp_path, monkeypatch ++): ++ module = _module(tmp_path, monkeypatch, "memo_return", "def f():\n return 7\n") ++ state = _state() ++ owner._live_code(module, *state) ++ original = module.f.__code__ ++ changed = _code("def f():\n return 8\n") ++ target = owner._same_loaded_code.__code__ ++ fired = [] ++ ++ def profile(frame, event, _arg): ++ if ( ++ event == "return" ++ and frame.f_code is target ++ and frame.f_locals["live"] is original ++ and not fired ++ ): ++ fired.append(1) ++ module.f.__code__ = changed ++ ++ previous = sys.getprofile() ++ try: ++ sys.setprofile(profile) ++ with _refuses(): ++ owner._live_code(module, *state) ++ finally: ++ sys.setprofile(previous) ++ module.f.__code__ = original ++ assert fired == [1] ++ ++ ++@pytest.mark.parametrize( ++ "container", [lambda x: x, lambda x: (x,), lambda x: frozenset((x,))] ++) ++def test_mutable_custom_constants_never_gain_pair_or_eligibility_authority(container): ++ class MutableEqual: ++ def __init__(self): ++ self.equal = True ++ ++ def __eq__(self, _other): ++ return self.equal ++ ++ __hash__ = object.__hash__ ++ ++ value = MutableEqual() ++ reference = _code() ++ live = reference.replace(co_consts=(None, container(value))) ++ pairs, immutable = {}, {} ++ for equal in (True, False, True): ++ value.equal = equal ++ # CPython may reject custom constant types before calling __eq__. ++ # Preserve the original decision; do not assume initial acceptance. ++ expected = live == reference ++ assert owner._same_loaded_code(live, reference, pairs, immutable) is expected ++ assert not pairs ++ assert id(live) not in immutable ++ ++ ++def test_nested_code_with_mutable_constant_remains_uncached(): ++ mutable = [] ++ inner = _code().replace(co_consts=(None, mutable)) ++ outer = _code().replace(co_consts=(None, inner)) ++ reference = _code().replace(co_consts=(None, _code())) ++ pairs, immutable = {}, {} ++ for content in ((), (7,), (8,)): ++ mutable[:] = content ++ assert owner._same_loaded_code(outer, reference, pairs, immutable) is ( ++ outer == reference ++ ) ++ assert not pairs ++ assert id(inner) not in immutable ++ assert id(outer) not in immutable ++ ++ ++def test_constant_eligibility_never_calls_custom_metaclass_equality(): ++ class CustomType(type): ++ def __eq__(cls, _other): ++ raise AssertionError("constant eligibility called custom type equality") ++ ++ class CustomValue(metaclass=CustomType): ++ pass ++ ++ live = _code().replace(co_consts=(None, CustomValue())) ++ immutable = {} ++ assert owner._immutable_code_constants(live, immutable) is False ++ assert id(live) not in immutable ++ ++ ++def test_scalar_subclasses_and_deep_values_fall_back_without_new_refusal(): ++ class CustomInt(int): ++ pass ++ ++ deep = 7 ++ for _ in range(70): ++ deep = (deep,) ++ for value in (CustomInt(7), deep): ++ live = _code().replace(co_consts=(None, value)) ++ reference = live.replace() ++ pairs, immutable = {}, {} ++ assert owner._same_loaded_code(live, reference, pairs, immutable) is ( ++ live == reference ++ ) ++ assert not pairs ++ ++ ++def test_compiler_constant_shapes_and_nested_functions_are_cacheable(): ++ source = """def f(x): ++ def inner(): ++ return (None, True, 7, 1.25, 2j, 'text', b'bytes', ...) ++ return x in {1, 2, 3}, inner ++""" ++ live, reference = _code(source), _code(source) ++ pairs, immutable = {}, {} ++ assert owner._same_loaded_code(live, reference, pairs, immutable) ++ assert pairs[(id(live), id(reference))] == (live, reference) ++ assert all(type(code) is CodeType for code in immutable.values()) ++ ++ ++def test_source_class_methods_properties_and_dataclass_escape_keep_existing_behavior( ++ tmp_path, monkeypatch ++): ++ source = """class Example: ++ @classmethod ++ def named(cls): ++ return cls.__name__ ++ @staticmethod ++ def value(): ++ return 7 ++ @property ++ def amount(self): ++ return 7 ++""" ++ module = _module(tmp_path, monkeypatch, "memo_methods", source) ++ state = _state() ++ owner._live_code(module, *state) ++ owner._live_code(module, *state) ++ # Match only the pre-existing generated-code escape; it is not broadened. ++ module.generated = FunctionType( ++ _code().replace(co_filename=""), vars(module) ++ ) ++ owner._live_code(module, *state) diff --git a/experiments/us-acs-compilation-cache-adoption-20260910.json b/experiments/us-acs-compilation-cache-adoption-20260910.json new file mode 100644 index 000000000..157e4596a --- /dev/null +++ b/experiments/us-acs-compilation-cache-adoption-20260910.json @@ -0,0 +1,75 @@ +{ + "control_postcheck_sha256": "58f10a538ad6db8efa30a4029a6eefaa4b0451c1bbb325e21d13a4bd2cc39cd7", + "control_receipt_sha256": "53b578332356b5e5700e2d1a6c40b2b1ccab47440aa886e53db9f19746c8c074", + "control_tests": 25, + "date": "2026-09-10", + "decision": "Adopt bounded process-local reuse of compiled bytecode keyed by complete compilation inputs. Retain every fresh source read, AST check, and loaded function, alias, global and closure check.", + "limitations": [ + "One paired cProfile observation, not a generalized benchmark or a native-data speedup.", + "Cache bounds retained source keys, not total process memory.", + "Compiler warning and audit events occur on cache misses; fresh AST parsing still occurs on every loaded-code check.", + "Cache shares the existing trusted-process boundary; it does not authenticate coherent forgery of private process state.", + "The already running native pilot retains its frozen original implementation." + ], + "native_acceptance": false, + "paired_fixture": { + "comparison": { + "baseline_cpu_seconds": 303.897467, + "baseline_postcheck_sha256": "146aea7545077b933f921fc53a15340b5818f478408199c12d101c0c920477de", + "baseline_profile_sha256": "badec8594bb1421483e0090f273f79f7560adc0b13b68434f089442c7c8d4bbb", + "baseline_wall_seconds": 308.2577062920318, + "cpu_reduction_fraction": 0.2935343617062791, + "same_original_fixture_scope": true, + "scope": "one_paired_profile_observation_not_generalized_benchmark", + "sole_production_delta_owner_sha256": "534042fdd4f2af5a238b556882d3b75c5495d9909e240bb07201ec48bd2bb4da", + "wall_reduction_fraction": 0.2966248653600234 + }, + "cpu_seconds": 214.693118, + "declared_invented_dimensions": { + "atomic_support_blocks": 4, + "clone_households": 12, + "clone_persons": 18, + "counts_are_declared_from_frozen_source_not_native_measurements": true, + "demographic_predictor_count": 5, + "financial_output_count": 7, + "fixture_function": "known_financial_run", + "fixture_source": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "graph_nodes_per_run": 20, + "n_estimators": 2, + "origin": "invented_original_source_fixture", + "runs": [ + "cold_auto", + "required_replay" + ], + "source_households": 6, + "source_persons": 9, + "state_fips_count": 2 + }, + "guard_sha256": "a665ab886e7feeebadffc630175734434a0667cd6b5a0fe812c4cd2cb4f42ca2", + "junit_sha256": "3b17eabe18fb9d8e6d87cb7f84b5eae00cc331086ab4d90239456c00263f2639", + "peak_rss_bytes": 581271552, + "profile_sha256": "0348d21a7ed9edc1dd2152595b13fcc91d51e6347619d80b78b76b4c1a96e5a9", + "receipt_sha256": "ed856c409f4923e49379044a003f652ec3fcdfa29789be52dd8fee33802f01d2", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 500 + }, + "wall_seconds": 216.82080566696823 + }, + "paired_postcheck_sha256": "fc2c232eee9904077e578ead9ea6e68f4d9d44c7ac9930d44cb1f08c6b67eba8", + "postcheck_correction": "The first checker incorrectly excluded the inherited root pyproject.toml source entry. The preserved original refused before physical verification; the corrected checker accepts that exact entry and passed. No fixture rerun was needed.", + "protocol": "microcosm.acs-source-compilation-cache-adoption.v1", + "release_acceptance": false, + "source_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py": "534042fdd4f2af5a238b556882d3b75c5495d9909e240bb07201ec48bd2bb4da", + "packages/microcosm-build/tests/test_us_acs_source_compile_cache.py": "3ee1adffa8a1420d6b95f0f6b8cc966f5193c6221420aed9b8dbf83b543d9dd6" + }, + "source_peer_review_sha256": "e5de7a750a3e33a985009a7e330f182a97c7c833bbc0a09e204125028203c244" +} diff --git a/experiments/us-acs-loaded-code-controls-20260910.json b/experiments/us-acs-loaded-code-controls-20260910.json new file mode 100644 index 000000000..5fec133ec --- /dev/null +++ b/experiments/us-acs-loaded-code-controls-20260910.json @@ -0,0 +1,33 @@ +{ + "accepted": true, + "acs_source_sha256": "4dc64558ed453f1c06557ef005e42193278496a9c31b374475417c407c037573", + "comparison_policy": "Memo only successful immutable CodeType identity pairs within one producer invocation; retain source, alias, globals and closure checks. Historical two-argument catalogue caller preserved.", + "cpu_seconds": 3.121793, + "date": "2026-09-10", + "financial_successor_acceptance": false, + "guard_sha256": "66d02d798933d7e2a677775c4037df9e16e4f5e6f45cd42363b78234daf56448", + "implementation_adopted": false, + "native_acceptance": false, + "peak_rss_bytes": 353026048, + "performance_gain_established": false, + "receipt_sha256": "da8cde54cb9c5b3ed909ed88b7ab12d6ab824148462593be798dfc86048b4bf1", + "release_acceptance": false, + "runtime": "Python 3.14.4; one numerical thread; no country engine, network, child process or native data access.", + "scope": "invented_acs_code_equality_controls", + "source_map_sha256": "3e5545835bf269a165738a6f59cdd9235171044120113bdd5568a030e59cdc66", + "subsequent_comparison": "us-acs-code-memo-comparison-20260910.json", + "test_sha256": "6d2e43acc5aea4533a99d12adfb94db00629c0f01f25b725d427bd47c7f1aa1f", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 18 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 500 + }, + "wall_seconds": 4.655691749998368 +} diff --git a/experiments/us-acs-producer-profile-20260910.json b/experiments/us-acs-producer-profile-20260910.json new file mode 100644 index 000000000..8c76e9386 --- /dev/null +++ b/experiments/us-acs-producer-profile-20260910.json @@ -0,0 +1,354 @@ +{ + "accepted_diagnostic": true, + "cprofile_options": { + "builtins": true, + "subcalls": false + }, + "current_reverified_files": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 500 + }, + "date": "2026-09-10", + "decision": "Source-only experiment in bounded bytecode compilation reuse; preserve fresh reads and all loaded-function checks. No optimization adopted or native speedup established.", + "fixture_constructed": false, + "graph_executed": false, + "guard_sha256": "426cd0370e3e94d4d4e022fdcf954b89b373af7992da88e2d21a37e8670ec2a4", + "native_inputs_used": false, + "postcheck_sha256": "3426339877891cc0659118f788487fe54e7659094dcd454e193284cf6a6e60e7", + "producer_completed": true, + "producer_invocations_requested": 1, + "profile_sha256": "6b4508e6e96226746381a9862997c22eebb9880a328d50b24b9c19f9fd0e78a1", + "profiled_elapsed_cpu_seconds": 0.6406650000000003, + "profiled_elapsed_wall_seconds": 0.6561586250318214, + "protocol": "microcosm.acs-original-producer-profile.v1", + "receipt_sha256": "a956298f6bbe11438256ac5ad7183098c89ed889c457666602c68c32d4bc5e12", + "records": [ + { + "calls": 1, + "cumulative_seconds": 0.6560935, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_producer", + "kind": "frozen_source", + "line": 129, + "recursive_calls": 0, + "self_seconds": 0.005839213 + }, + { + "calls": 1, + "cumulative_seconds": 0.42354929100000005, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "implementation_manifest", + "kind": "frozen_source", + "line": 411, + "recursive_calls": 0, + "self_seconds": 0.000314576 + }, + { + "calls": 59, + "cumulative_seconds": 0.36231329, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_dependency_contract", + "kind": "frozen_source", + "line": 303, + "recursive_calls": 0, + "self_seconds": 0.005696675 + }, + { + "calls": 59, + "cumulative_seconds": 0.355838413, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_dependency_details", + "kind": "frozen_source", + "line": 202, + "recursive_calls": 0, + "self_seconds": 0.023903589000000003 + }, + { + "calls": 44, + "cumulative_seconds": 0.212273085, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_live_code", + "kind": "frozen_source", + "line": 51, + "recursive_calls": 0, + "self_seconds": 0.021575394 + }, + { + "calls": 2586, + "cumulative_seconds": 0.128561793, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "check", + "kind": "frozen_source", + "line": 86, + "recursive_calls": 66, + "self_seconds": 0.0032278470000000003 + }, + { + "calls": 979, + "cumulative_seconds": 0.103954326, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "visit_FunctionDef", + "kind": "frozen_source", + "line": 243, + "recursive_calls": 344, + "self_seconds": 0.000322374 + }, + { + "calls": 9482, + "cumulative_seconds": 0.052384176000000005, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "visit_Call", + "kind": "frozen_source", + "line": 260, + "recursive_calls": 2364, + "self_seconds": 0.002310004 + }, + { + "calls": 3, + "cumulative_seconds": 0.016132043000000002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_validate_package_roster", + "kind": "frozen_source", + "line": 332, + "recursive_calls": 0, + "self_seconds": 7.2842e-05 + }, + { + "calls": 1, + "cumulative_seconds": 0.014024958, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py", + "function": "_producer", + "kind": "frozen_source", + "line": 124, + "recursive_calls": 0, + "self_seconds": 4.2163e-05 + }, + { + "calls": 36445, + "cumulative_seconds": 0.008270251000000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "visit_Name", + "kind": "frozen_source", + "line": 252, + "recursive_calls": 0, + "self_seconds": 0.005725035000000001 + }, + { + "calls": 16, + "cumulative_seconds": 0.005612417000000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py", + "function": "_sha", + "kind": "frozen_source", + "line": 65, + "recursive_calls": 0, + "self_seconds": 8.211e-06 + }, + { + "calls": 1, + "cumulative_seconds": 0.004030375, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_package_roots", + "kind": "frozen_source", + "line": 180, + "recursive_calls": 0, + "self_seconds": 9.082e-06 + }, + { + "calls": 59, + "cumulative_seconds": 0.0036707920000000004, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_covered_imports", + "kind": "frozen_source", + "line": 286, + "recursive_calls": 0, + "self_seconds": 0.001535637 + }, + { + "calls": 3277, + "cumulative_seconds": 0.002018209, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "visit", + "kind": "frozen_source", + "line": 97, + "recursive_calls": 3223, + "self_seconds": 0.0012540350000000001 + }, + { + "calls": 1451, + "cumulative_seconds": 0.001515904, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 229, + "recursive_calls": 0, + "self_seconds": 0.000852642 + }, + { + "calls": 181, + "cumulative_seconds": 0.000619031, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_digest", + "kind": "frozen_source", + "line": 176, + "recursive_calls": 0, + "self_seconds": 6.2668e-05 + }, + { + "calls": 118, + "cumulative_seconds": 0.000576835, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_canonical", + "kind": "frozen_source", + "line": 170, + "recursive_calls": 0, + "self_seconds": 7.5121e-05 + }, + { + "calls": 555, + "cumulative_seconds": 0.00047688000000000006, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 289, + "recursive_calls": 0, + "self_seconds": 0.00047688000000000006 + }, + { + "calls": 1, + "cumulative_seconds": 0.000265083, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_inventory", + "kind": "frozen_source", + "line": 312, + "recursive_calls": 0, + "self_seconds": 4.0999000000000005e-05 + }, + { + "calls": 1, + "cumulative_seconds": 0.000205333, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_projections", + "kind": "frozen_source", + "line": 352, + "recursive_calls": 0, + "self_seconds": 4.0830000000000005e-06 + }, + { + "calls": 76, + "cumulative_seconds": 0.00019908300000000002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "columns", + "kind": "frozen_source", + "line": 355, + "recursive_calls": 74, + "self_seconds": 4.3078e-05 + }, + { + "calls": 3811, + "cumulative_seconds": 0.00012806000000000002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_require", + "kind": "frozen_source", + "line": 46, + "recursive_calls": 0, + "self_seconds": 0.00012806000000000002 + }, + { + "calls": 555, + "cumulative_seconds": 8.0924e-05, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 290, + "recursive_calls": 0, + "self_seconds": 8.0924e-05 + }, + { + "calls": 303, + "cumulative_seconds": 4.1642000000000006e-05, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 358, + "recursive_calls": 0, + "self_seconds": 3.0757000000000004e-05 + }, + { + "calls": 240, + "cumulative_seconds": 3.2792e-05, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 296, + "recursive_calls": 0, + "self_seconds": 3.2792e-05 + }, + { + "calls": 1, + "cumulative_seconds": 8.400000000000001e-08, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py", + "function": "_require", + "kind": "frozen_source", + "line": 60, + "recursive_calls": 0, + "self_seconds": 8.400000000000001e-08 + }, + { + "calls": 157, + "cumulative_seconds": 0.237641084, + "function": "builtins.compile", + "kind": "builtin", + "recursive_calls": 0, + "self_seconds": 0.237641084 + }, + { + "calls": 1, + "cumulative_seconds": 2.92e-07, + "function": "time.perf_counter", + "kind": "builtin", + "recursive_calls": 0, + "self_seconds": 2.92e-07 + }, + { + "calls": 1, + "cumulative_seconds": 3e-06, + "function": "time.process_time", + "kind": "builtin", + "recursive_calls": 0, + "self_seconds": 3e-06 + }, + { + "calls": 103, + "cumulative_seconds": 0.12155241700000001, + "function": "ast.parse", + "kind": "stdlib", + "recursive_calls": 0, + "self_seconds": 4.1292000000000004e-05 + } + ], + "scope": "one_original_producer_invocation_after_normal_helper_imports", + "source_or_population_authority_issued": false, + "successor_or_release_acceptance": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "timing_limitations": [ + "normal imports are outside profiling and no fixture is requested", + "one producer call has no within-process repetition or warmup", + "cumulative times overlap and must not be summed", + "ast.parse wraps compile and their cumulative times overlap", + "cProfile adds overhead; this diagnostic cannot establish a speedup", + "source rows cover only the three declared owner/manifest modules", + "other external time remains within cumulative callers", + "compiled-reference visitation and deep code equality remain unmodified" + ], + "whole_guard": { + "cpu_seconds": 4.779278, + "peak_rss_bytes": 481591296, + "wall_seconds": 6.85519720899174 + } +} diff --git a/experiments/us-atomic-age-v2-1-controls-20260909.json b/experiments/us-atomic-age-v2-1-controls-20260909.json new file mode 100644 index 000000000..c7fb9cfa7 --- /dev/null +++ b/experiments/us-atomic-age-v2-1-controls-20260909.json @@ -0,0 +1,34 @@ +{ + "code_resources": 12, + "configured_resource_limits": { + "cpu_seconds": 600, + "wall_seconds": 900 + }, + "cpu_seconds": 601.979524, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 506183680, + "publisher_provenance_established": false, + "receipt_sha256": "fc77f8ead3a5b8f2f611aa8d1d20d4c19ab6b315322be17241c0b9ec5c12bbdf", + "release_eligible": false, + "resource_observation": "The configured limits were unchanged. Reported cumulative CPU was 601.979524 seconds; the guarded process returned zero after all checks.", + "result": 0, + "scope": "Actual thirteen-node cold and required replay age calibration over invented original sources and block support. Invented data only; no native or publisher admission, no release.", + "source_and_owned_files": 492, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "a10b9f3552c84563d845675c772130073b925b5835d6730ffb9afc0e93170777", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "c0667ad329af2f58901b0fd2f333489ad2366b83e2bac8ded95b51088d9e5d5e", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "25a414517974d345541bcec6152b61151b09ccc476a5350a36015c4b33dcd70e", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "d130335f5fb8d66e3e032984c326e1077b0a889162da8d97324f85866095905d", + "packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py": "d81a82b9aedecd26d62831e93355c233093dc8fc3adec1e702d8c7f138fd8bbd" + }, + "unexpected_refusals": {}, + "wall_seconds": 613.9018378330511 +} diff --git a/experiments/us-atomic-block-adapter-20260909.json b/experiments/us-atomic-block-adapter-20260909.json new file mode 100644 index 000000000..1980d44db --- /dev/null +++ b/experiments/us-atomic-block-adapter-20260909.json @@ -0,0 +1,25 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 3.078341, + "district_relation": "official_tabulation", + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 353140736, + "receipt_sha256": "404deb7666b3ac9b8a654181228c545ab95c131f30a1fbe65a2e15d5a08a4c4c", + "release_eligible": false, + "result": 0, + "sampling_basis": "2020_census_persons", + "scope": "Invented US block normalization and declaration through shared assignment, derivation and validation; native support and graph integration remain pending.", + "source_and_owned_files": 479, + "source_sha256": "8d776a60e5ace340b0de745461b89679653522bb672b95ab43bdfabda0f602ee", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 13 + }, + "unexpected_refusals": {}, + "wall_seconds": 4.607667459000368 +} diff --git a/experiments/us-atomic-block-api-sources-35-controls-20260909.json b/experiments/us-atomic-block-api-sources-35-controls-20260909.json new file mode 100644 index 000000000..6053026d2 --- /dev/null +++ b/experiments/us-atomic-block-api-sources-35-controls-20260909.json @@ -0,0 +1,26 @@ +{ + "code_resources": 12, + "cpu_seconds": 2.923694, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 349110272, + "publisher_provenance_established": false, + "receipt_sha256": "e76a4373281f38f98d794c89265cedbaf50744e63c60f5256dbbdb7905a3a586", + "release_eligible": false, + "result": 0, + "scope": "35 invented population-only Census API source controls through actual CD/PUMA parsers and atomic normalization. Exact response header, row/code types, source hashes, state/block totals, retained byte snapshots and mapping completeness; no queries, credentials, native inputs, publisher admission or release.", + "source_and_owned_files": 496, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 35 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_api_sources.py": "4bb9f58364e75846d4cbb19c3e0254468089481e28caacb67ae17eb87d802235", + "packages/microcosm-build/tests/test_us_atomic_block_api_sources.py": "69821ca1b81c47647d4a6a7faaf0bf9279889ef409368ed20b84779613ab5241" + }, + "unexpected_refusals": {}, + "wall_seconds": 4.598902583005838 +} diff --git a/experiments/us-atomic-block-sources-36-controls-20260909.json b/experiments/us-atomic-block-sources-36-controls-20260909.json new file mode 100644 index 000000000..52c25620e --- /dev/null +++ b/experiments/us-atomic-block-sources-36-controls-20260909.json @@ -0,0 +1,26 @@ +{ + "code_resources": 12, + "cpu_seconds": 2.88845, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 350126080, + "publisher_provenance_established": false, + "receipt_sha256": "a64003994a3f3823d06a22cbf7376ae8bcb3714d1a25110782cc72e83bfcf1cd", + "release_eligible": false, + "result": 0, + "scope": "36 invented source-byte adapter controls through actual PL/CD/PUMA parsers and atomic normalization: exact source/ZIP/CRC/bounds/roster/population/mapping checks, retained byte snapshots, output codec cap. Whole native PL acquisition remains excluded; no native/publisher/release acceptance.", + "source_and_owned_files": 494, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 36 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py": "b5be0b06085f0037206836a16625bb4be5dfe6dc6a12659f207e871aea53340b", + "packages/microcosm-build/tests/test_us_atomic_block_sources.py": "5b9a9fc74b14e354ec63b237b33736434a7b66f89135e798d222c32cc7ad788c" + }, + "unexpected_refusals": {}, + "wall_seconds": 3.594141000008676 +} diff --git a/experiments/us-atomic-budget-semantic-8-controls-20260909.json b/experiments/us-atomic-budget-semantic-8-controls-20260909.json new file mode 100644 index 000000000..0d13a0f9e --- /dev/null +++ b/experiments/us-atomic-budget-semantic-8-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "code_resources": 12, + "cpu_seconds": 313.261743, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 447545344, + "publisher_provenance_established": false, + "receipt_sha256": "76a132f01c33bd71af65eb0c17bb2187d9b15dfefc6e7e61ad5f935ea14f59bb", + "release_eligible": false, + "result": 0, + "scope": "Corrected semantic budget replay identity and FIFO/owner/mutation refusals. Invented data only; no native or publisher admission, no release.", + "source_and_owned_files": 492, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 8 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "a10b9f3552c84563d845675c772130073b925b5835d6730ffb9afc0e93170777", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "c0667ad329af2f58901b0fd2f333489ad2366b83e2bac8ded95b51088d9e5d5e", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "25a414517974d345541bcec6152b61151b09ccc476a5350a36015c4b33dcd70e", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "d130335f5fb8d66e3e032984c326e1077b0a889162da8d97324f85866095905d", + "packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py": "d81a82b9aedecd26d62831e93355c233093dc8fc3adec1e702d8c7f138fd8bbd" + }, + "unexpected_refusals": {}, + "wall_seconds": 321.5954028330161 +} diff --git a/experiments/us-atomic-clone-graph-20260909.json b/experiments/us-atomic-clone-graph-20260909.json new file mode 100644 index 000000000..a30a95cfe --- /dev/null +++ b/experiments/us-atomic-clone-graph-20260909.json @@ -0,0 +1,33 @@ +{ + "all_location_columns_inherited": "passed", + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 4.555835, + "household_weight_halves": "passed", + "households_after": 8, + "households_before": 4, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 364150784, + "persons_after": 12, + "persons_before": 6, + "preclone_gate_order": "passed", + "production_source_sha256": "ef9b562b355abc450f3d2934ab9be04e49387d03de4b39e37fa09b1c0c1905c8", + "receipt_sha256": "1bb75a5ee53edcbdd199a6d0163cbe75c32e54c80513cb3eb8d13021c3c2b1c3", + "release_eligible": false, + "required_replay": "passed", + "result": 0, + "scope": "Invented combined-survey actual graph: atomic assignment and derivation before support clone, inherited mapping gate and required replay. Native source and runner integration remain pending.", + "source_and_owned_files": 481, + "source_only_peer_review": "Codex peer found no actionable findings in adapter, composition and tests; no independent runtime or native verification.", + "subset_mapping_check": "passed", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 6.214872374985134 +} diff --git a/experiments/us-atomic-financial-4-controls-20260909.json b/experiments/us-atomic-financial-4-controls-20260909.json new file mode 100644 index 000000000..508aeb133 --- /dev/null +++ b/experiments/us-atomic-financial-4-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "code_resources": 12, + "control_accepted": false, + "cpu_seconds": 467.05899700000003, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 565133312, + "publisher_provenance_established": false, + "receipt_sha256": "ae0397762505db7059865abf28e739b534026416cd39b690ca50dc27310cb34b", + "release_eligible": false, + "result": 1, + "scope": "Four invented atomic-plus-financial controls: twenty-node cold/required replay with five qualified features, default three-feature compatibility, detached money mutation refusal at final support return, and final geography mutation refusal. No fit quality, native admission, PUF recipient admission, budget successor, or release acceptance.", + "source_and_owned_files": 497, + "test_counts": { + "errors": 0, + "failures": 2, + "skipped": 0, + "tests": 4 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "0e32704bd82e32382cd03936b8cb6faada32afc10e9256c78c9a4b2f9b3372c8", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "af365e9135f2fffa9cdcaed643be9a082cb2d382255c9d93fdd4e7cd4cfe8a77", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "fd643fcaca956ea4830e2022ad737c7321dbc3e59057230c56c31f303e8f5c27", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py": "8d0fc394f7f59bf64f387cb137819c1789bfdf310f1573a4b41d4bd3ea3cfcdb" + }, + "unexpected_refusals": {}, + "wall_seconds": 472.7063618339598 +} diff --git a/experiments/us-atomic-financial-composition-acceptance-20260909.json b/experiments/us-atomic-financial-composition-acceptance-20260909.json new file mode 100644 index 000000000..d6f21bf54 --- /dev/null +++ b/experiments/us-atomic-financial-composition-acceptance-20260909.json @@ -0,0 +1,42 @@ +{ + "calibration_successor_accepted": false, + "completed_controls": [ + "cold and required replay across all20 actual graph nodes", + "complete retained fields, metadata, IDs, weights, owners and clone inheritance", + "explicit five-feature conditioning and default three-feature compatibility", + "refuse detached money mutation during final support return", + "refuse materialized geography mutation during final owner return" + ], + "fit_quality_accepted": false, + "graph_nodes": 20, + "native_payloads_read": 0, + "native_survey_replay_accepted": false, + "negative_controls_repeated": false, + "population_scope": "invented original survey inputs; native survey replay pending", + "puf_attachment_accepted": false, + "release_eligible": false, + "runs": [ + { + "experiment": "us-atomic-financial-4-controls-20260909.json", + "failed_test_code_defects": [ + "nonexistent assertion helper name", + "incorrect expected refusal string" + ], + "receipt_sha256": "ae0397762505db7059865abf28e739b534026416cd39b690ca50dc27310cb34b", + "result": "two negative mutation controls passed; two test-code defects failed" + }, + { + "experiment": "us-atomic-financial-corrected-2-controls-20260909.json", + "receipt_sha256": "adc535f414c62ab01240f98d0a50f498f4b345531157f1b67c6be23c6deaa0b5", + "result": "two corrected tests passed with the same implementation and full cold/required fixture" + } + ], + "status": "invented_composition_controls_passed", + "test_only_corrections": true, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "0e32704bd82e32382cd03936b8cb6faada32afc10e9256c78c9a4b2f9b3372c8", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "af365e9135f2fffa9cdcaed643be9a082cb2d382255c9d93fdd4e7cd4cfe8a77", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "fd643fcaca956ea4830e2022ad737c7321dbc3e59057230c56c31f303e8f5c27", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py": "1b441049e527f24f339e2c2fed289ae8ff434015d7bc0b025cebda87f32d52d0" + } +} diff --git a/experiments/us-atomic-financial-corrected-2-controls-20260909.json b/experiments/us-atomic-financial-corrected-2-controls-20260909.json new file mode 100644 index 000000000..042978bd3 --- /dev/null +++ b/experiments/us-atomic-financial-corrected-2-controls-20260909.json @@ -0,0 +1,24 @@ +{ + "code_resources": 12, + "control_accepted": true, + "cpu_seconds": 248.723104, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 552976384, + "publisher_provenance_established": false, + "receipt_sha256": "adc535f414c62ab01240f98d0a50f498f4b345531157f1b67c6be23c6deaa0b5", + "release_eligible": false, + "result": 0, + "scope": "Identical unexecuted-v2 source closure; select only the two previously failing corrected tests, retaining their actual cold/required replay fixture. Passed original negative controls are not rerun. No ongoing successor implementation, fit quality, native admission, or release acceptance.", + "source_and_owned_files": 497, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "tested_source_changes": {}, + "unexpected_refusals": {}, + "wall_seconds": 252.2364875409985 +} diff --git a/experiments/us-atomic-national-acquisition-20260909.json b/experiments/us-atomic-national-acquisition-20260909.json new file mode 100644 index 000000000..42485f90e --- /dev/null +++ b/experiments/us-atomic-national-acquisition-20260909.json @@ -0,0 +1,598 @@ +{ + "acquired_at_utc": "2026-09-09T23:45:56.994286+00:00", + "acquisition_receipt_sha256": "6e36758e59aa8536d35d25881caed122111da59e200a634c5a4b8d7baa6485d5", + "automatic_retry": false, + "cpu_seconds": 16.783967, + "de_metadata_sha256": "6fd38afe7e3b33995b13e14bc112827936a7871fd58b66a61e5b9b825350bb5d", + "guard_sha256": "1c9bf5810b787f19043cb744f1155d823b0f80703038c2a0f861fdcdf8276a18", + "native_inputs_reopened_for_postcheck": false, + "native_normalization": false, + "peak_rss_acceptance_bytes": 4294967296, + "peak_rss_bytes": 390889472, + "project_imports": false, + "release_eligible": false, + "result": 0, + "rss_hard_os_enforcement": false, + "scope": "104 once-only verified-HTTPS requests: P1_001N block populations and independent state totals for50states+DC, CD119 mapping archive and2020tract-to-PUMA mapping. Acquisition only; no normalization, survey construction or calibration.", + "script_and_roster_before_after_equal": true, + "script_sha256": "eccecc8f8927dd52b2f704cb1cc22c3c3fb10a59f7c1ae01f301edee240c8894", + "source_admission_issued": false, + "source_count": 104, + "sources": [ + { + "id": "pl2020_p1_state01_blocks", + "sha256": "2abb5d0be868baaa5d0b6c05490d832941084da51f1457090a10d9ff9657606d", + "size_bytes": 6427012 + }, + { + "id": "pl2020_p1_state01_total", + "sha256": "3a6d19e9e904b80afffb901a71bdda9cb5101bca1889dfc8efc50c8c39688ef5", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state02_blocks", + "sha256": "8534edc8dd05f33c878f33b7eb50ec359a3aff95198dbb91f95dc854fac10574", + "size_bytes": 982680 + }, + { + "id": "pl2020_p1_state02_total", + "sha256": "ba9387573141f95e32d1440192e5a14efe431ab1e54ae7bbf0fe2433adf61d43", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state04_blocks", + "sha256": "81ea2d71faef8aaebb95bf986a87c905b09bfd30036ca5d096da67e91fd7c8cf", + "size_bytes": 5395609 + }, + { + "id": "pl2020_p1_state04_total", + "sha256": "b7ba8c426c875025aa324794719d978c3167e3a49e332e4af9b8fdf83b7a231c", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state05_blocks", + "sha256": "627457173db87dd14a2cfdb5464ac69941c306229423431962f5e2a23b4e522e", + "size_bytes": 4704516 + }, + { + "id": "pl2020_p1_state05_total", + "sha256": "b67fed8d924bca67ea48d091b4677828288b5d4b76cf57e1ce5777e7b55af6d4", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state06_blocks", + "sha256": "af233c9cf00396a0e61735430268f09df3301695d4790ec577c58b3df2e30957", + "size_bytes": 18139193 + }, + { + "id": "pl2020_p1_state06_total", + "sha256": "6bcb55b367bdde18380c87a526c18ebc7bd929c6100951942c7df517ddba02c2", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state08_blocks", + "sha256": "148ad5135f22c8410ee3baadc40c9f22cef8960423a05d4a4d9f940e44969ae9", + "size_bytes": 4866694 + }, + { + "id": "pl2020_p1_state08_total", + "sha256": "d730fcb0310c63c7ab6d6cd8c59474790be28953a3e791e1086f5d77feca9764", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state09_blocks", + "sha256": "ef40acb552ad9447cba791ccc0a24aed8f572891eb81ae4561ee0d61de836c54", + "size_bytes": 1747484 + }, + { + "id": "pl2020_p1_state09_total", + "sha256": "96a3d6fd5abfd6b32b44653adbcce8f1fe1e133a5cd474718d0d7aad7ec305fd", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state10_blocks", + "sha256": "1a7a314a84aa0049adb0865d0bfd7466d808b36460f67eef6f3143a331fe6408", + "size_bytes": 702646 + }, + { + "id": "pl2020_p1_state10_total", + "sha256": "ac40e616243c2ea14060dd460009e0a7f15755324a17387adf8768e35787fb79", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state11_blocks", + "sha256": "d23c7ae418bc35270dc6eec54c014c92cd370aab7ff92de8168ff5c182db84a1", + "size_bytes": 210965 + }, + { + "id": "pl2020_p1_state11_total", + "sha256": "fd791728a6b47cb82daf1fdaa156cf3a166815510f8057341e44bc8b9c36f61b", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state12_blocks", + "sha256": "b8c5633d903a040396c7f4d386aacb72e7c698253851fde6a115720bc90ed5e0", + "size_bytes": 13566697 + }, + { + "id": "pl2020_p1_state12_total", + "sha256": "41b9bcced940cfe83c8d27aea2c3ff69e0c48b3e7dbd88ded2fb6c1cd5b8bff2", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state13_blocks", + "sha256": "d2ee481c091a4c316fbef61bc62d069a884fbd52d658cd8a43de8fd4a1c2a90f", + "size_bytes": 8068936 + }, + { + "id": "pl2020_p1_state13_total", + "sha256": "a6bf2b3f49f8393f3ff7d1762b99d927d949d4c18e65c2995488ee88e4b7b8f5", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state15_blocks", + "sha256": "12360bccbca022409819393314c4f22d960d5d7f8edc302f3b9feefc73a011b4", + "size_bytes": 514556 + }, + { + "id": "pl2020_p1_state15_total", + "sha256": "7ed5585fbe97d5dab88b0ab10de66f30e90fa4a14b0e8989c02d32c2c4d631fd", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state16_blocks", + "sha256": "74ee9f18bddd2d67b90d93d808944edac3d410cc95a761a34980e3ce55f151a9", + "size_bytes": 2821287 + }, + { + "id": "pl2020_p1_state16_total", + "sha256": "8f1db7ed1b345ba86cdce141ab9d645c2cff91b88e0191fe557089fe0ceb4fbe", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state17_blocks", + "sha256": "6b0eed0c5ec1bcc326cffecc00c98d77a389fd3aa01b2163a99ca174088861ca", + "size_bytes": 12820271 + }, + { + "id": "pl2020_p1_state17_total", + "sha256": "c02690f4ec95f7f8a9f3bd3a994d5dee9877ad3c2d2e07de9eeb2ebcb9bd0757", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state18_blocks", + "sha256": "3c91aeb1afc98189affebfd1a5d6884cc03755e413e46b906a4b0d47240c9255", + "size_bytes": 7095958 + }, + { + "id": "pl2020_p1_state18_total", + "sha256": "572d8013dcc0fb51697478648a8c1e9ddf0c8ef0c2aa583a4eb44e5c81c2ddcc", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state19_blocks", + "sha256": "98f4efd90409774596b5bc623078d75f579d781ca3bc392b586a24fa67e638a1", + "size_bytes": 6033554 + }, + { + "id": "pl2020_p1_state19_total", + "sha256": "5cd722954a80d81e07f29fe65056cd0f40967c163935eb450320d4e73a5d4169", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state20_blocks", + "sha256": "35e21c30e14e0c4bfdb3645076cdc7a89ef4708c17373047526875be9cb0ab41", + "size_bytes": 5931398 + }, + { + "id": "pl2020_p1_state20_total", + "sha256": "aaaa97cb26fbc4db8318b32549da0a4f64a3fc4ca316ef9fcc5b2da6bb3b1c77", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state21_blocks", + "sha256": "975c9821955f9472f2222f85e089ce6ce9caff44a18bed9edd797ca13bda49c8", + "size_bytes": 4591236 + }, + { + "id": "pl2020_p1_state21_total", + "sha256": "c65f5deb2c5dacbcd8c1e84e4caf7a41458399f3a388425fbc5a7af69dba0bb9", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state22_blocks", + "sha256": "9bdacd2c2ccfa5bac93f5df47ec1e63fe39643c15633e7d33f464ad7a171d869", + "size_bytes": 4942488 + }, + { + "id": "pl2020_p1_state22_total", + "sha256": "db924ac57fd4177be3508d795ce912f3b55454f8eb6331fcc297252c2bb20a19", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state23_blocks", + "sha256": "eb26cdc0e114b07154096f592447df295d7fcda6dd890ebfe6e012d7d7bb40ca", + "size_bytes": 1629179 + }, + { + "id": "pl2020_p1_state23_total", + "sha256": "2a6bc352f6304d0ceb8850333a47378fa2a324a1163850d56931b424b8281089", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state24_blocks", + "sha256": "2d181f3e6bf3c96c8c2f458024a699be8e4c8b2bb80b6433ee95c69efe655b29", + "size_bytes": 2925422 + }, + { + "id": "pl2020_p1_state24_total", + "sha256": "b9de28b8819798a841c36ac13d35cdc75573c8e47ec232d195817239ab0bff55", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state25_blocks", + "sha256": "ca73c4d84aaf50e22ca9b25c4fdc54ca314167473a5ba1c9ee31fac1881396e3", + "size_bytes": 3749394 + }, + { + "id": "pl2020_p1_state25_total", + "sha256": "db5b65758be7c88e4bcafd4413f2813fa32de96307aa50a450f28bdc14e3bb39", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state26_blocks", + "sha256": "732595037f8d58aee0d01a3b68e71a18bc8a7144c81433d1dfc805a1a83cfe8d", + "size_bytes": 8845352 + }, + { + "id": "pl2020_p1_state26_total", + "sha256": "29a349d67a3c6219b9675e6cc5e5bd9a0bf3511e4b48005a6c9a9215be033069", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state27_blocks", + "sha256": "8910dd3e06035629bbef028441acb5d29f29ea555381df98076a05289bf29803", + "size_bytes": 6863252 + }, + { + "id": "pl2020_p1_state27_total", + "sha256": "13b45f3487425184637b01256644d5f86599d6a445b1a4f1d9fb86ef3790cef9", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state28_blocks", + "sha256": "33350d9a37d539dd4724dc7f4b969de0fd1dbfcb5207cae093b8ed73e0a67d68", + "size_bytes": 3878667 + }, + { + "id": "pl2020_p1_state28_total", + "sha256": "5cabdefebb5363751a244a9b12720ff43e9ac3a6fef7ba8c1ee7f79e7adaf745", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state29_blocks", + "sha256": "0e9b1a932c3cfb283a80a0e831192482ee9db6f01d7aad83de67a16e5c4075d5", + "size_bytes": 8749056 + }, + { + "id": "pl2020_p1_state29_total", + "sha256": "b75c227a323978f36e25e2efab8ec11b20579b06d6393da203cb36d533743a6d", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state30_blocks", + "sha256": "b9a81e34cc8c228fef203728f4134f96b7a8acfc1b14395e2de251d7ab0e3964", + "size_bytes": 3033388 + }, + { + "id": "pl2020_p1_state30_total", + "sha256": "caa4444d14b67700b3db4a5653412bd633c995df0d279473441b885a40102968", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state31_blocks", + "sha256": "bebe7ca15c979b170cfc3e6f49e27cdc50f5639de2f9a1955e60187849404e56", + "size_bytes": 4095921 + }, + { + "id": "pl2020_p1_state31_total", + "sha256": "30c8aae0e54d78eeb7857f0954be04c95c3a2278a030e78317f2a361fe40c683", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state32_blocks", + "sha256": "f34926bc3ada9c07a7ace640a787495814b9893c8351b34a9a0fe9bcf91acd75", + "size_bytes": 1991100 + }, + { + "id": "pl2020_p1_state32_total", + "sha256": "3f3957d218a7eb94ae360e03f6b5924d95ab0e8cb80b685d0fc74ff3de3349a7", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state33_blocks", + "sha256": "11e83119f4924fdbc1c0dcf6f96113c84ba4173c507290578835dfd2c7f97dfa", + "size_bytes": 1110353 + }, + { + "id": "pl2020_p1_state33_total", + "sha256": "b2c3dae17fcbdccaa8eddf6f3e5fdb742d4b0ec017084610958d5182c0f70ce7", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state34_blocks", + "sha256": "280a8f28f13545bbff7d432e2356cdaeaedd9e0ea026f7f05f4d9e83dd28886e", + "size_bytes": 4823492 + }, + { + "id": "pl2020_p1_state34_total", + "sha256": "517cdd27711beea019fa1efc18c6017690cd1d44617aa446a52677e94281d0cc", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state35_blocks", + "sha256": "a012da889bc274fb518c2a258c9b1394f43c24e6d58fb3661151565cc5ce8757", + "size_bytes": 3690294 + }, + { + "id": "pl2020_p1_state35_total", + "sha256": "9632c8400a3052e17d23dd71427ccf66e8c062b15cb58df2cb949ecb7a287e9e", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state36_blocks", + "sha256": "7a47a92501f986ebca42a27a180fe897b4afb2c37cd463360179a0b32500ab2d", + "size_bytes": 10067666 + }, + { + "id": "pl2020_p1_state36_total", + "sha256": "e34eac8cee690ba81ef7ad607cee5e06697834056165979faf2148627d5aeb5f", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state37_blocks", + "sha256": "3a721726b3ee7a8426411e5ba5267b0c0f88d4123916e8ed40d71e0a601eb797", + "size_bytes": 8216489 + }, + { + "id": "pl2020_p1_state37_total", + "sha256": "bf0b0c6a90da12c51a71942982ec8a10b8e1dece22601462dd0391c2dc758ba2", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state38_blocks", + "sha256": "6f8985d46a8d8e97fce2f563f20442f7c6e73a0503cddf58ff59746e829e2b68", + "size_bytes": 2892476 + }, + { + "id": "pl2020_p1_state38_total", + "sha256": "3616ffbe27406d4e881587912b59c4494d2a0aa71132a25a6f8108bae23ccbb5", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state39_blocks", + "sha256": "aeb3eff352866ece982d0a31cfaf3ad63877bd8e2598bede9d883a0a168b450c", + "size_bytes": 9606700 + }, + { + "id": "pl2020_p1_state39_total", + "sha256": "3a93ca174df19c6f3377abf023177def2d8ff585b2182d56a9abd75b2e3454e3", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state40_blocks", + "sha256": "2078a38b5338f474099f7e1fe2ba0dba3cab95b1644db5abd6c52ca4a8a57703", + "size_bytes": 6212394 + }, + { + "id": "pl2020_p1_state40_total", + "sha256": "bd8b724c531ca5d6605aa1752c70c00a2666755070a18ef73ac0a3423d2812a5", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state41_blocks", + "sha256": "3cafb0c96a666767ea005ac1f08ca3171379934352c4d2ef285fd6ce44196ccd", + "size_bytes": 4521307 + }, + { + "id": "pl2020_p1_state41_total", + "sha256": "fb8d53cee89c2a14485cf42cf49b4723576407db7d550897609c1f5038b20e67", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state42_blocks", + "sha256": "34840439707bedac6bf76a35863287aa24263000ff2e1dca21ecdf0030b262f6", + "size_bytes": 11713716 + }, + { + "id": "pl2020_p1_state42_total", + "sha256": "b07244d4d81570c0c8bf49dc3653c382e8e3e74a2d798fb32c85e5808fd789a7", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state44_blocks", + "sha256": "8f8eb7ee08020dd7e3a49dab4d95f461acd5d29572fe8bf5b55f200ff84e7231", + "size_bytes": 894186 + }, + { + "id": "pl2020_p1_state44_total", + "sha256": "ec6f0565a6ebceccf1bf1a665f46a8105852b352762416b350f9630f802a8223", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state45_blocks", + "sha256": "a3daf2da71c547c02d8c260b009be331cd218c84896d602fafcdbc12acd59c3f", + "size_bytes": 5086274 + }, + { + "id": "pl2020_p1_state45_total", + "sha256": "3d15445808b86c1f785d6a1668f69eae773fbc75a462d3ead77c99bcc769ba9c", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state46_blocks", + "sha256": "1700e07604a6fb79d5cfb34fcb6bfe3d810429ecbca4e88d3012b8b2070a516d", + "size_bytes": 2447858 + }, + { + "id": "pl2020_p1_state46_total", + "sha256": "e5b885739fae3a34f28cc11f739435e8cd39926ebeb0a2ef1a18ae3ff035becd", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state47_blocks", + "sha256": "d8522733ea6f5971d5f340d86f71f7174b54f502ddd84a79d703fbd72d343eb2", + "size_bytes": 6232909 + }, + { + "id": "pl2020_p1_state47_total", + "sha256": "32de25caad8e71a6bb74f5639f5e9eae1c95836f61743275c60b76f716d0c790", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state48_blocks", + "sha256": "2e997c530f51c17b595226374c50eb1ed249d2d6945e37dde9a1ceed8a161c6d", + "size_bytes": 23175159 + }, + { + "id": "pl2020_p1_state48_total", + "sha256": "469d7f5efd70e8918a3c9744d7c7742e190e54b52d5b331faf648907f5882b0b", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state49_blocks", + "sha256": "73207e9605fda4b336c404455c9a369ed64e39e7ecebd623b633835a9664e495", + "size_bytes": 2471351 + }, + { + "id": "pl2020_p1_state49_total", + "sha256": "825dab7318dcc582b32eb2723473be42d111a00d32f47d88ae93beade83e2fae", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state50_blocks", + "sha256": "7e34e58509cec16138d786dda2e34a45370fc1386108591182313f306b252daa", + "size_bytes": 850760 + }, + { + "id": "pl2020_p1_state50_total", + "sha256": "c5ccb3d18ec85abae8ca3958cd4cca738812021cee35dd34be09150c07aa3c3d", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state51_blocks", + "sha256": "0d979951968f230cd5b7235f703e9acfabcafbeed99c4c018e34c0f88ba0ca52", + "size_bytes": 5678471 + }, + { + "id": "pl2020_p1_state51_total", + "sha256": "87ce18c978a4361494de558f74c03ce956079f6cfa8fb0401c8e77e5bf234c97", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state53_blocks", + "sha256": "0e3ec5282f8d894a504625cc9a366ada08430b60317fb5d529a982b0e2787a64", + "size_bytes": 5490219 + }, + { + "id": "pl2020_p1_state53_total", + "sha256": "ef5ad0f000c5505f923e4c91a5516b02138669c8b08b7a26c547cc856a7891e4", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state54_blocks", + "sha256": "28f1b09668784b1d8a064bd1433a1446e9697d8da2988c611bbc267ebee4456f", + "size_bytes": 2507211 + }, + { + "id": "pl2020_p1_state54_total", + "sha256": "b1cefc08915ac5c9bd05f47636d5ce6307b8ecb43c3c32502de2ec6d8b144c13", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state55_blocks", + "sha256": "fc491abf3fcd1a1fd3807ecde1d95b68bc2cab4e0a235a1b51f095ab5a2afcfa", + "size_bytes": 7025682 + }, + { + "id": "pl2020_p1_state55_total", + "sha256": "1c7bbd06d50dfe00b2e4461d2b92e98fac585f4b7b8c2425782bc822984fc5e9", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state56_blocks", + "sha256": "55aef5b66c91b47bda087b490e44a12067eabe7634b444351172e59ae126e66a", + "size_bytes": 1843303 + }, + { + "id": "pl2020_p1_state56_total", + "sha256": "0684454d90b7e6430f6f55991791113091593c8dab5577ff8d6dcc2d1fa2e10d", + "size_bytes": 38 + }, + { + "id": "census_cd119_bef", + "sha256": "1433feb5178dc7b4188ee30f5f7f715851f4400740b8fe1ce606a876c6294bd6", + "size_bytes": 22959130 + }, + { + "id": "census_tract2020_puma2020", + "sha256": "5262f460b2c8dbe86b00549e916317b1791fdc595c9af72ab18f6f6a67040531", + "size_bytes": 1709076 + } + ], + "state_fips": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56" + ], + "transferred_bytes": 306552379, + "wall_seconds": 328.79918500001077 +} diff --git a/experiments/us-atomic-native-de-1-control-20260909.json b/experiments/us-atomic-native-de-1-control-20260909.json new file mode 100644 index 000000000..84d9bb62e --- /dev/null +++ b/experiments/us-atomic-native-de-1-control-20260909.json @@ -0,0 +1,42 @@ +{ + "archive_members": [ + "01_AL_CD119.txt", + "13_GA_CD119.txt", + "22_LA_CD119.txt", + "36_NY_CD119.txt", + "37_NC_CD119.txt", + "NationalCD119.txt" + ], + "automatic_retry": false, + "code_resources_rechecked": 12, + "corrected_control_executed": false, + "cpu_seconds": 3.002104, + "diagnosis": "The acquired CD archive contains six members; the test incorrectly declared only NationalCD119.txt. The exact-roster validation correctly refused it.", + "diagnostic_receipt_sha256": "54405497257274709b458baad3d0a8b0f4fd2727406d794d42f962d12cd851ca", + "failure_class": "ValueError", + "frozen_before_after_equal": true, + "frozen_code_current_equal": true, + "metadata_probe_member_contents_opened": 0, + "model_source_files": 5983, + "native_inputs_reopened_in_postcheck": false, + "native_resource_pins_receipt_only": 4, + "peak_rss_bytes": 372555776, + "receipt_sha256": "063d36af8b4a8352bb573396b39a921c21c8703d484e32542f24680702428df0", + "release_eligible": false, + "reserved_housing_targets_read": false, + "result": 1, + "scope": "One native DE original P1 population/state-total plus CD119/PUMA source normalization control. Exact four input pins, block preservation and state reconciliation, full support readback. No national coverage, survey Frame, source admission or release acceptance.", + "selected_member": "NationalCD119.txt", + "selected_member_size_bytes": 163499110, + "source_and_owned_files": 497, + "source_control_accepted": false, + "status": "failed_pending_diagnosis", + "test_counts": { + "errors": 0, + "failures": 1, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 4.633959750004578 +} diff --git a/experiments/us-atomic-native-de-corrected-1-control-20260909.json b/experiments/us-atomic-native-de-corrected-1-control-20260909.json new file mode 100644 index 000000000..1fea35f10 --- /dev/null +++ b/experiments/us-atomic-native-de-corrected-1-control-20260909.json @@ -0,0 +1,175 @@ +{ + "code_resources_rechecked": 12, + "cpu_seconds": 9.85925, + "frozen_before_after_equal": true, + "frozen_code_current_equal": true, + "model_source_files": 5983, + "national_coverage": false, + "native_inputs_reopened_in_postcheck": false, + "native_resource_pins_receipt_only": 4, + "normalization": { + "exact_support_readback": true, + "national_coverage": false, + "protocol": "microcosm.us.native-de-atomic-api-control.v1", + "release_eligible": false, + "source_admission_issued": false, + "source_receipt": { + "exact_input_hashes_verified": true, + "limits": { + "max_line_bytes": 65536, + "max_member_bytes": 536870912, + "max_response_rows": 250000, + "max_source_bytes": 67108864, + "max_total_geography_bytes": 805306368, + "max_zip_members": 6 + }, + "protocol": "microcosm.us.atomic-block-api-sources.v1", + "publisher_provenance_established": false, + "reconciliation": { + "output_state_population": { + "10": 989948 + }, + "populated_blocks": 15317, + "population_total": 989948, + "selected_geography_bytes": 165910870, + "state_population": { + "10": 989948 + } + }, + "release_eligible": false, + "request_origin_verified": false, + "source_admission_issued": false, + "source_ids": { + "district": "census-2025-cd119-NationalCD119.txt", + "population": "census-2020-dec-pl-api-P1_001N", + "puma": "census-2020-Census-Tract-to-2020-PUMA" + }, + "sources": { + "district": { + "assigned_blocks": 8174799, + "crc_checked_members": [ + "NationalCD119.txt" + ], + "delegate_normalization": "98_to_00", + "delegate_records": 47999, + "district_relation": "official_tabulation", + "selected_member": "NationalCD119.txt", + "selected_member_sha256": "3c5afab5b61e2d7151206f977981702815692df6f3680e7624a6c3f93808fddf", + "selected_member_size_bytes": 163499110, + "sha256": "1433feb5178dc7b4188ee30f5f7f715851f4400740b8fe1ce606a876c6294bd6", + "size_bytes": 22959130, + "source_id": "census-2025-cd119-NationalCD119.txt", + "source_records": 8174955, + "unassigned_blocks": 156, + "unselected_member_contents_read": false, + "zip_members": [ + "01_AL_CD119.txt", + "13_GA_CD119.txt", + "22_LA_CD119.txt", + "36_NY_CD119.txt", + "37_NC_CD119.txt", + "NationalCD119.txt" + ] + }, + "population": [ + { + "block_rows": 20198, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:10" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "1a7a314a84aa0049adb0865d0bfd7466d808b36460f67eef6f3143a331fe6408", + "size_bytes": 702646 + }, + "populated_blocks": 15317, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "10", + "state_population": 989948, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:10" + ] + ] + }, + "sha256": "ac40e616243c2ea14060dd460009e0a7f15755324a17387adf8768e35787fb79", + "size_bytes": 38 + }, + "zero_population_blocks": 4881 + } + ], + "puma": { + "selected_state_tract_mappings": 262, + "sha256": "5262f460b2c8dbe86b00549e916317b1791fdc595c9af72ab18f6f6a67040531", + "size_bytes": 1709076, + "source_id": "census-2020-Census-Tract-to-2020-PUMA" + } + }, + "state_fips": [ + "10" + ], + "support_sha256": "a0ef3f86be7c76306527ec2cecc70455a9d5b51af0a1c441d30fc532057db84d" + }, + "source_receipt_sha256": "b0783c16860e1333b271876a48f0b82d16c543813c93904c3eb460aecfc1111b", + "state_fips": [ + "10" + ], + "status": "source_control_passed", + "support_filename": "de-atomic-support.npz", + "support_sha256": "a0ef3f86be7c76306527ec2cecc70455a9d5b51af0a1c441d30fc532057db84d", + "support_size_bytes": 79769, + "survey_frame_created": false + }, + "normalization_metadata_sha256": "6fd38afe7e3b33995b13e14bc112827936a7871fd58b66a61e5b9b825350bb5d", + "peak_rss_bytes": 1734787072, + "receipt_sha256": "ea260a22f8701839aa0ac9beab98d82e17738cf9359570a0317e26338697d26e", + "release_eligible": false, + "result": 0, + "scope": "Correct only the native DE test archive roster to the six metadata-observed names; selected decompression/CRC remains solely NationalCD119.txt. No adapter, input, other source or guard behavior change. Failed original run preserved.", + "source_and_owned_files": 497, + "source_control_accepted": true, + "status": "DE_source_control_passed", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "tested_source_changes": { + "packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_de.py": { + "after": "4b918e4d28044045faad6618dd931077b3b07086929c34aa31cff049fdf096a2", + "before": "e22a550f4e95592a6bd596f32ffd772d6f43117f284ce0eebe992b0de42ed7ab" + } + }, + "unexpected_refusals": {}, + "wall_seconds": 10.939260708983056 +} diff --git a/experiments/us-atomic-native-national-1-control-20260909.json b/experiments/us-atomic-native-national-1-control-20260909.json new file mode 100644 index 000000000..822ea0e7c --- /dev/null +++ b/experiments/us-atomic-native-national-1-control-20260909.json @@ -0,0 +1,3434 @@ +{ + "code_resources_rechecked": 12, + "cpu_seconds": 81.174622, + "frozen_before_after_equal": true, + "frozen_code_current_equal": true, + "model_source_files": 5983, + "native_inputs_reopened_in_postcheck": false, + "native_resource_pins_receipt_only": 104, + "normalization": { + "exact_support_readback": true, + "mapping_oracle": "separate maintained-parser pass; complete selected joins", + "national_coverage": true, + "population_oracle": { + "01": { + "block_rows": 185976, + "ordered_positive_block_population_sha256": "db629f189d2658bec941146c499464bec1d7278d83da0b07ac8ed338d5d993ab", + "populated_blocks": 128366, + "state_population": 5024279, + "zero_population_blocks": 57610 + }, + "02": { + "block_rows": 28568, + "ordered_positive_block_population_sha256": "c82ab4e84dfee99b16fc0397e3bf57d0ef5c87e1e37e5fc766879c690d0f4caf", + "populated_blocks": 11765, + "state_population": 733391, + "zero_population_blocks": 16803 + }, + "04": { + "block_rows": 155444, + "ordered_positive_block_population_sha256": "57059741545abff19005434c387543790c8cb047bd8453c17aada74bda370c46", + "populated_blocks": 107938, + "state_population": 7151502, + "zero_population_blocks": 47506 + }, + "05": { + "block_rows": 136422, + "ordered_positive_block_population_sha256": "af7158b0ab1d4e4baf46766a0953bfe74f52036272af13c4cfe6b65ef572c5ed", + "populated_blocks": 88121, + "state_population": 3011524, + "zero_population_blocks": 48301 + }, + "06": { + "block_rows": 519723, + "ordered_positive_block_population_sha256": "fc244613d2faab0aa55e0064fff67ee2df2cda32d8e1cfef0f934a6e80e41a68", + "populated_blocks": 377591, + "state_population": 39538223, + "zero_population_blocks": 142132 + }, + "08": { + "block_rows": 140345, + "ordered_positive_block_population_sha256": "d69d69546b402b60bd4eb98fbb4003417ddc3d26c442e6051437628dff7708a1", + "populated_blocks": 99899, + "state_population": 5773714, + "zero_population_blocks": 40446 + }, + "09": { + "block_rows": 49926, + "ordered_positive_block_population_sha256": "5be4f0b5fba158152132db2671dbb607f0c8233c1dfc532d5beb99d51965ee0d", + "populated_blocks": 42008, + "state_population": 3605944, + "zero_population_blocks": 7918 + }, + "10": { + "block_rows": 20198, + "ordered_positive_block_population_sha256": "f32d84817caa2c3f3ea776da426dbee30f049c82b9335af41699ec4bee00e235", + "populated_blocks": 15317, + "state_population": 989948, + "zero_population_blocks": 4881 + }, + "11": { + "block_rows": 6012, + "ordered_positive_block_population_sha256": "c0d37648dff6f3ba16d524e2e375417669565acab25027d64ec03b05c89c603d", + "populated_blocks": 4500, + "state_population": 689545, + "zero_population_blocks": 1512 + }, + "12": { + "block_rows": 390066, + "ordered_positive_block_population_sha256": "683fb2c4854b4c7ede8815bfc5620c3cf26a5d9149eab4bbd25fd4eedb5eb00e", + "populated_blocks": 289652, + "state_population": 21538187, + "zero_population_blocks": 100414 + }, + "13": { + "block_rows": 232717, + "ordered_positive_block_population_sha256": "dbc9e610217cd9e5de12fb3825addb9508ae368c1dc4a88b98801960ff91cdb0", + "populated_blocks": 165333, + "state_population": 10711908, + "zero_population_blocks": 67384 + }, + "15": { + "block_rows": 14732, + "ordered_positive_block_population_sha256": "245ccc4831fdcb75b1d2d5202f0d38c2ac9c1fab91e4916a61191729d4fbc88f", + "populated_blocks": 10169, + "state_population": 1455271, + "zero_population_blocks": 4563 + }, + "16": { + "block_rows": 81879, + "ordered_positive_block_population_sha256": "47cc73fe9b89c0db1c66621484f12b2a426f7332c289c28a0c30915ab7135dc8", + "populated_blocks": 46134, + "state_population": 1839106, + "zero_population_blocks": 35745 + }, + "17": { + "block_rows": 369978, + "ordered_positive_block_population_sha256": "bf7e305eae384c4338298d4b2dad71cfa48a5a0dc2858dda558b92bc26cbceaa", + "populated_blocks": 278166, + "state_population": 12812508, + "zero_population_blocks": 91812 + }, + "18": { + "block_rows": 204568, + "ordered_positive_block_population_sha256": "a85fae03b48ad4703a5ed7f39bb81eca4974df835d7c9e99590486fbaf11e163", + "populated_blocks": 162739, + "state_population": 6785528, + "zero_population_blocks": 41829 + }, + "19": { + "block_rows": 175199, + "ordered_positive_block_population_sha256": "f89589b4fd957d0643af40784431bd793d8ebfa05ceed97b3921689f424ee938", + "populated_blocks": 124175, + "state_population": 3190369, + "zero_population_blocks": 51024 + }, + "20": { + "block_rows": 172529, + "ordered_positive_block_population_sha256": "75877b15ff41b689be4774c7ddd9294701b97b1762fd4a935d10fce1cfb9c2e4", + "populated_blocks": 109149, + "state_population": 2937880, + "zero_population_blocks": 63380 + }, + "21": { + "block_rows": 132662, + "ordered_positive_block_population_sha256": "c7b86e0bef4455178c4da1b99b16e615454602062db58de9b9e8bf5a389c9954", + "populated_blocks": 91572, + "state_population": 4505836, + "zero_population_blocks": 41090 + }, + "22": { + "block_rows": 142874, + "ordered_positive_block_population_sha256": "fa3122a5aee3bfb4f78dba6ad64d582c1515ee045298e8f2359870ec8d884457", + "populated_blocks": 92180, + "state_population": 4657757, + "zero_population_blocks": 50694 + }, + "23": { + "block_rows": 47138, + "ordered_positive_block_population_sha256": "c870139aa6e05735c7f1ff1e05399995dc54a722dc2df99198059e7f89324f85", + "populated_blocks": 30548, + "state_population": 1362359, + "zero_population_blocks": 16590 + }, + "24": { + "block_rows": 83827, + "ordered_positive_block_population_sha256": "519b855f8fa876aed559a5c4b57458301e9aa1323bd7ae5575b916055abe00f4", + "populated_blocks": 65274, + "state_population": 6177224, + "zero_population_blocks": 18553 + }, + "25": { + "block_rows": 107278, + "ordered_positive_block_population_sha256": "aa920cf2bee71bf21a2ef68d76b04eb417c2c72fbcf468523fd032956917ac41", + "populated_blocks": 88207, + "state_population": 7029917, + "zero_population_blocks": 19071 + }, + "26": { + "block_rows": 254730, + "ordered_positive_block_population_sha256": "8e57ac1ca0ab6cafc9385f23b658a0eafd4f4e01f55468d42da327355d95a445", + "populated_blocks": 198244, + "state_population": 10077331, + "zero_population_blocks": 56486 + }, + "27": { + "block_rows": 198705, + "ordered_positive_block_population_sha256": "e616047478baea1a265c120762ebc1ad234531a1c1459e8a97263dfa5645b225", + "populated_blocks": 137689, + "state_population": 5706494, + "zero_population_blocks": 61016 + }, + "28": { + "block_rows": 112241, + "ordered_positive_block_population_sha256": "ab34034369ef3795f7a2a8112de09a48789c5921102f64681e8f772fe5f24fe8", + "populated_blocks": 77426, + "state_population": 2961279, + "zero_population_blocks": 34815 + }, + "29": { + "block_rows": 253632, + "ordered_positive_block_population_sha256": "ec441928aa5f5ef297b96c37db8f81132a0ac50399eada71edff432e0d922b03", + "populated_blocks": 169004, + "state_population": 6154913, + "zero_population_blocks": 84628 + }, + "30": { + "block_rows": 88417, + "ordered_positive_block_population_sha256": "b09a76edbea2619f38d2ad70a31108726f24abd23b17099a0ff010f07b48b43d", + "populated_blocks": 43717, + "state_population": 1084225, + "zero_population_blocks": 44700 + }, + "31": { + "block_rows": 119103, + "ordered_positive_block_population_sha256": "ab1afedaab5dd2392593270ce5fd7536d54bfe4cf99b34d53ce7a66aad96830f", + "populated_blocks": 79779, + "state_population": 1961504, + "zero_population_blocks": 39324 + }, + "32": { + "block_rows": 57409, + "ordered_positive_block_population_sha256": "760c7b9e590052d373316c9800545a20a71495b42f0d4b7a20b55b2fbf6f12f9", + "populated_blocks": 35249, + "state_population": 3104614, + "zero_population_blocks": 22160 + }, + "33": { + "block_rows": 31948, + "ordered_positive_block_population_sha256": "d88a28f2f35cf1b8ef64fd5d42600e416aad959de76e06de9ac76dd4e86a21a8", + "populated_blocks": 25317, + "state_population": 1377529, + "zero_population_blocks": 6631 + }, + "34": { + "block_rows": 137972, + "ordered_positive_block_population_sha256": "d924c85d9ae80d624ed2902e925a9c7dabb4a996680a081fbe237fcd9a1746d3", + "populated_blocks": 113212, + "state_population": 9288994, + "zero_population_blocks": 24760 + }, + "35": { + "block_rows": 107215, + "ordered_positive_block_population_sha256": "3905ad0b0d16d2c86502cfa4797024fd6d2f895efe0280379bb7963836ccab82", + "populated_blocks": 56605, + "state_population": 2117522, + "zero_population_blocks": 50610 + }, + "36": { + "block_rows": 288819, + "ordered_positive_block_population_sha256": "65a2be271d6a207e1f3e7cabedd6778d19d909ba0e37f5232bb010ad8a30b2f7", + "populated_blocks": 230339, + "state_population": 20201249, + "zero_population_blocks": 58480 + }, + "37": { + "block_rows": 236638, + "ordered_positive_block_population_sha256": "0c135c4919f7b904a298fbb183620e452e02ba9fca6644e27c2e8cf65e40ed50", + "populated_blocks": 174988, + "state_population": 10439388, + "zero_population_blocks": 61650 + }, + "38": { + "block_rows": 84566, + "ordered_positive_block_population_sha256": "fb4b3241fa5f1d81ed332353f90b0130b2383dd17e840a11badc1c911d155597", + "populated_blocks": 40198, + "state_population": 779094, + "zero_population_blocks": 44368 + }, + "39": { + "block_rows": 276428, + "ordered_positive_block_population_sha256": "a3e40aaf54aaaa347573881c39a06eed40a1e7d02b1e57a4a70fdfdaafb6b7fb", + "populated_blocks": 219669, + "state_population": 11799448, + "zero_population_blocks": 56759 + }, + "40": { + "block_rows": 180154, + "ordered_positive_block_population_sha256": "10240ff44682b047e63e5da0e4e8a5c9f6750d80b9a7f176ad397500f06492c1", + "populated_blocks": 121826, + "state_population": 3959353, + "zero_population_blocks": 58328 + }, + "41": { + "block_rows": 130807, + "ordered_positive_block_population_sha256": "01b6c50dbc1320f96741aa159354a3e42b318ee7e3bda21dac3d5f122e6799b0", + "populated_blocks": 79081, + "state_population": 4237256, + "zero_population_blocks": 51726 + }, + "42": { + "block_rows": 336985, + "ordered_positive_block_population_sha256": "70bb990a03502d1c2359e19a5f469727933661dcabfdcb9082c7c9f5ba34ee07", + "populated_blocks": 275023, + "state_population": 13002700, + "zero_population_blocks": 61962 + }, + "44": { + "block_rows": 25649, + "ordered_positive_block_population_sha256": "2777acda8ed305d3fcad00ec03bcc9d5b16d295a82fc8250caa73a0acf4184cd", + "populated_blocks": 21382, + "state_population": 1097379, + "zero_population_blocks": 4267 + }, + "45": { + "block_rows": 146844, + "ordered_positive_block_population_sha256": "d1a102de62516378b036f30c3642a17d3f2a4040b8a76e380b2f01f8b33d8a4f", + "populated_blocks": 105469, + "state_population": 5118425, + "zero_population_blocks": 41375 + }, + "46": { + "block_rows": 71383, + "ordered_positive_block_population_sha256": "8c372d4e82aff59d8701ecd211e24224885561645bedb7de74db55ca0c60205c", + "populated_blocks": 40766, + "state_population": 886667, + "zero_population_blocks": 30617 + }, + "47": { + "block_rows": 179717, + "ordered_positive_block_population_sha256": "f3804b33c433afd24a7b433c6524b0d124754d289cc2d589798fde5987794357", + "populated_blocks": 133846, + "state_population": 6910840, + "zero_population_blocks": 45871 + }, + "48": { + "block_rows": 668757, + "ordered_positive_block_population_sha256": "83903c280fd7fb16e9cc5a5ce7266753a0445118c8a3151bfadc86e27b891b3d", + "populated_blocks": 449085, + "state_population": 29145505, + "zero_population_blocks": 219672 + }, + "49": { + "block_rows": 71207, + "ordered_positive_block_population_sha256": "d1ea8f0e3bfa6225218b0f3bd0c022453fa3038a196ebef8a17dadb1be0ca575", + "populated_blocks": 46792, + "state_population": 3271616, + "zero_population_blocks": 24415 + }, + "50": { + "block_rows": 24611, + "ordered_positive_block_population_sha256": "3df326b12b20dbf6af14229fae0dc2e5733b4eabbe7a142515bebf0455881a93", + "populated_blocks": 17976, + "state_population": 643077, + "zero_population_blocks": 6635 + }, + "51": { + "block_rows": 163491, + "ordered_positive_block_population_sha256": "756a961291abef27225b26548e1fc67a58725285de9bbf4d93b04a618d7731bb", + "populated_blocks": 117361, + "state_population": 8631393, + "zero_population_blocks": 46130 + }, + "53": { + "block_rows": 158093, + "ordered_positive_block_population_sha256": "11bbff165b4c5004c526e0420dff07601a637a2d9b145288c86ffc476dcd6f5c", + "populated_blocks": 111441, + "state_population": 7705281, + "zero_population_blocks": 46652 + }, + "54": { + "block_rows": 72558, + "ordered_positive_block_population_sha256": "3fac3d7a2c520bcfb0e3ddf04776e2e147d157d58b8b8dd04cebb91ec89a8a6e", + "populated_blocks": 51850, + "state_population": 1793716, + "zero_population_blocks": 20708 + }, + "55": { + "block_rows": 203059, + "ordered_positive_block_population_sha256": "16f7d47ca52eced522acd712209a41215fd99ddb43f5a2232974b5ef6bc510de", + "populated_blocks": 144948, + "state_population": 5893718, + "zero_population_blocks": 58111 + }, + "56": { + "block_rows": 53769, + "ordered_positive_block_population_sha256": "33eac054e9aaf0d266460d4c4df67192a5ac7c993658ece720b398f868f45648", + "populated_blocks": 22857, + "state_population": 576851, + "zero_population_blocks": 30912 + } + }, + "protocol": "microcosm.us.native-national-atomic-api-control.v1", + "release_eligible": false, + "source_admission_issued": false, + "source_count": 104, + "source_read_limits": { + "mapping_verification_passes": 1, + "selected_cd_member_bytes_per_pass": 163499110, + "selected_geography_bytes_per_adapter_pass": 1237240934, + "unique_acquired_bytes": 1073741824 + }, + "source_receipt": { + "exact_input_hashes_verified": true, + "limits": { + "max_line_bytes": 65536, + "max_member_bytes": 163499110, + "max_response_rows": 2000000, + "max_source_bytes": 67108864, + "max_total_geography_bytes": 1237240934, + "max_zip_members": 6 + }, + "protocol": "microcosm.us.atomic-block-api-sources.v1", + "publisher_provenance_established": false, + "reconciliation": { + "output_state_population": { + "01": 5024279, + "02": 733391, + "04": 7151502, + "05": 3011524, + "06": 39538223, + "08": 5773714, + "09": 3605944, + "10": 989948, + "11": 689545, + "12": 21538187, + "13": 10711908, + "15": 1455271, + "16": 1839106, + "17": 12812508, + "18": 6785528, + "19": 3190369, + "20": 2937880, + "21": 4505836, + "22": 4657757, + "23": 1362359, + "24": 6177224, + "25": 7029917, + "26": 10077331, + "27": 5706494, + "28": 2961279, + "29": 6154913, + "30": 1084225, + "31": 1961504, + "32": 3104614, + "33": 1377529, + "34": 9288994, + "35": 2117522, + "36": 20201249, + "37": 10439388, + "38": 779094, + "39": 11799448, + "40": 3959353, + "41": 4237256, + "42": 13002700, + "44": 1097379, + "45": 5118425, + "46": 886667, + "47": 6910840, + "48": 29145505, + "49": 3271616, + "50": 643077, + "51": 8631393, + "53": 7705281, + "54": 1793716, + "55": 5893718, + "56": 576851 + }, + "populated_blocks": 5769942, + "population_total": 331449281, + "selected_geography_bytes": 447092359, + "state_population": { + "01": 5024279, + "02": 733391, + "04": 7151502, + "05": 3011524, + "06": 39538223, + "08": 5773714, + "09": 3605944, + "10": 989948, + "11": 689545, + "12": 21538187, + "13": 10711908, + "15": 1455271, + "16": 1839106, + "17": 12812508, + "18": 6785528, + "19": 3190369, + "20": 2937880, + "21": 4505836, + "22": 4657757, + "23": 1362359, + "24": 6177224, + "25": 7029917, + "26": 10077331, + "27": 5706494, + "28": 2961279, + "29": 6154913, + "30": 1084225, + "31": 1961504, + "32": 3104614, + "33": 1377529, + "34": 9288994, + "35": 2117522, + "36": 20201249, + "37": 10439388, + "38": 779094, + "39": 11799448, + "40": 3959353, + "41": 4237256, + "42": 13002700, + "44": 1097379, + "45": 5118425, + "46": 886667, + "47": 6910840, + "48": 29145505, + "49": 3271616, + "50": 643077, + "51": 8631393, + "53": 7705281, + "54": 1793716, + "55": 5893718, + "56": 576851 + } + }, + "release_eligible": false, + "request_origin_verified": false, + "source_admission_issued": false, + "source_ids": { + "district": "census-2025-cd119-NationalCD119.txt", + "population": "census-2020-dec-pl-api-P1_001N", + "puma": "census-2020-Census-Tract-to-2020-PUMA" + }, + "sources": { + "district": { + "assigned_blocks": 8174799, + "crc_checked_members": [ + "NationalCD119.txt" + ], + "delegate_normalization": "98_to_00", + "delegate_records": 47999, + "district_relation": "official_tabulation", + "selected_member": "NationalCD119.txt", + "selected_member_sha256": "3c5afab5b61e2d7151206f977981702815692df6f3680e7624a6c3f93808fddf", + "selected_member_size_bytes": 163499110, + "sha256": "1433feb5178dc7b4188ee30f5f7f715851f4400740b8fe1ce606a876c6294bd6", + "size_bytes": 22959130, + "source_id": "census-2025-cd119-NationalCD119.txt", + "source_records": 8174955, + "unassigned_blocks": 156, + "unselected_member_contents_read": false, + "zip_members": [ + "01_AL_CD119.txt", + "13_GA_CD119.txt", + "22_LA_CD119.txt", + "36_NY_CD119.txt", + "37_NC_CD119.txt", + "NationalCD119.txt" + ] + }, + "population": [ + { + "block_rows": 185976, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:01" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2abb5d0be868baaa5d0b6c05490d832941084da51f1457090a10d9ff9657606d", + "size_bytes": 6427012 + }, + "populated_blocks": 128366, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "01", + "state_population": 5024279, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:01" + ] + ] + }, + "sha256": "3a6d19e9e904b80afffb901a71bdda9cb5101bca1889dfc8efc50c8c39688ef5", + "size_bytes": 39 + }, + "zero_population_blocks": 57610 + }, + { + "block_rows": 28568, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:02" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "8534edc8dd05f33c878f33b7eb50ec359a3aff95198dbb91f95dc854fac10574", + "size_bytes": 982680 + }, + "populated_blocks": 11765, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "02", + "state_population": 733391, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:02" + ] + ] + }, + "sha256": "ba9387573141f95e32d1440192e5a14efe431ab1e54ae7bbf0fe2433adf61d43", + "size_bytes": 38 + }, + "zero_population_blocks": 16803 + }, + { + "block_rows": 155444, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:04" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "81ea2d71faef8aaebb95bf986a87c905b09bfd30036ca5d096da67e91fd7c8cf", + "size_bytes": 5395609 + }, + "populated_blocks": 107938, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "04", + "state_population": 7151502, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:04" + ] + ] + }, + "sha256": "b7ba8c426c875025aa324794719d978c3167e3a49e332e4af9b8fdf83b7a231c", + "size_bytes": 39 + }, + "zero_population_blocks": 47506 + }, + { + "block_rows": 136422, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:05" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "627457173db87dd14a2cfdb5464ac69941c306229423431962f5e2a23b4e522e", + "size_bytes": 4704516 + }, + "populated_blocks": 88121, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "05", + "state_population": 3011524, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:05" + ] + ] + }, + "sha256": "b67fed8d924bca67ea48d091b4677828288b5d4b76cf57e1ce5777e7b55af6d4", + "size_bytes": 39 + }, + "zero_population_blocks": 48301 + }, + { + "block_rows": 519723, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:06" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "af233c9cf00396a0e61735430268f09df3301695d4790ec577c58b3df2e30957", + "size_bytes": 18139193 + }, + "populated_blocks": 377591, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "06", + "state_population": 39538223, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:06" + ] + ] + }, + "sha256": "6bcb55b367bdde18380c87a526c18ebc7bd929c6100951942c7df517ddba02c2", + "size_bytes": 40 + }, + "zero_population_blocks": 142132 + }, + { + "block_rows": 140345, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:08" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "148ad5135f22c8410ee3baadc40c9f22cef8960423a05d4a4d9f940e44969ae9", + "size_bytes": 4866694 + }, + "populated_blocks": 99899, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "08", + "state_population": 5773714, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:08" + ] + ] + }, + "sha256": "d730fcb0310c63c7ab6d6cd8c59474790be28953a3e791e1086f5d77feca9764", + "size_bytes": 39 + }, + "zero_population_blocks": 40446 + }, + { + "block_rows": 49926, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:09" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "ef40acb552ad9447cba791ccc0a24aed8f572891eb81ae4561ee0d61de836c54", + "size_bytes": 1747484 + }, + "populated_blocks": 42008, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "09", + "state_population": 3605944, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:09" + ] + ] + }, + "sha256": "96a3d6fd5abfd6b32b44653adbcce8f1fe1e133a5cd474718d0d7aad7ec305fd", + "size_bytes": 39 + }, + "zero_population_blocks": 7918 + }, + { + "block_rows": 20198, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:10" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "1a7a314a84aa0049adb0865d0bfd7466d808b36460f67eef6f3143a331fe6408", + "size_bytes": 702646 + }, + "populated_blocks": 15317, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "10", + "state_population": 989948, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:10" + ] + ] + }, + "sha256": "ac40e616243c2ea14060dd460009e0a7f15755324a17387adf8768e35787fb79", + "size_bytes": 38 + }, + "zero_population_blocks": 4881 + }, + { + "block_rows": 6012, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:11" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "d23c7ae418bc35270dc6eec54c014c92cd370aab7ff92de8168ff5c182db84a1", + "size_bytes": 210965 + }, + "populated_blocks": 4500, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "11", + "state_population": 689545, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:11" + ] + ] + }, + "sha256": "fd791728a6b47cb82daf1fdaa156cf3a166815510f8057341e44bc8b9c36f61b", + "size_bytes": 38 + }, + "zero_population_blocks": 1512 + }, + { + "block_rows": 390066, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:12" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "b8c5633d903a040396c7f4d386aacb72e7c698253851fde6a115720bc90ed5e0", + "size_bytes": 13566697 + }, + "populated_blocks": 289652, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "12", + "state_population": 21538187, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:12" + ] + ] + }, + "sha256": "41b9bcced940cfe83c8d27aea2c3ff69e0c48b3e7dbd88ded2fb6c1cd5b8bff2", + "size_bytes": 40 + }, + "zero_population_blocks": 100414 + }, + { + "block_rows": 232717, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:13" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "d2ee481c091a4c316fbef61bc62d069a884fbd52d658cd8a43de8fd4a1c2a90f", + "size_bytes": 8068936 + }, + "populated_blocks": 165333, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "13", + "state_population": 10711908, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:13" + ] + ] + }, + "sha256": "a6bf2b3f49f8393f3ff7d1762b99d927d949d4c18e65c2995488ee88e4b7b8f5", + "size_bytes": 40 + }, + "zero_population_blocks": 67384 + }, + { + "block_rows": 14732, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:15" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "12360bccbca022409819393314c4f22d960d5d7f8edc302f3b9feefc73a011b4", + "size_bytes": 514556 + }, + "populated_blocks": 10169, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "15", + "state_population": 1455271, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:15" + ] + ] + }, + "sha256": "7ed5585fbe97d5dab88b0ab10de66f30e90fa4a14b0e8989c02d32c2c4d631fd", + "size_bytes": 39 + }, + "zero_population_blocks": 4563 + }, + { + "block_rows": 81879, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:16" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "74ee9f18bddd2d67b90d93d808944edac3d410cc95a761a34980e3ce55f151a9", + "size_bytes": 2821287 + }, + "populated_blocks": 46134, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "16", + "state_population": 1839106, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:16" + ] + ] + }, + "sha256": "8f1db7ed1b345ba86cdce141ab9d645c2cff91b88e0191fe557089fe0ceb4fbe", + "size_bytes": 39 + }, + "zero_population_blocks": 35745 + }, + { + "block_rows": 369978, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:17" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "6b0eed0c5ec1bcc326cffecc00c98d77a389fd3aa01b2163a99ca174088861ca", + "size_bytes": 12820271 + }, + "populated_blocks": 278166, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "17", + "state_population": 12812508, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:17" + ] + ] + }, + "sha256": "c02690f4ec95f7f8a9f3bd3a994d5dee9877ad3c2d2e07de9eeb2ebcb9bd0757", + "size_bytes": 40 + }, + "zero_population_blocks": 91812 + }, + { + "block_rows": 204568, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:18" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "3c91aeb1afc98189affebfd1a5d6884cc03755e413e46b906a4b0d47240c9255", + "size_bytes": 7095958 + }, + "populated_blocks": 162739, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "18", + "state_population": 6785528, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:18" + ] + ] + }, + "sha256": "572d8013dcc0fb51697478648a8c1e9ddf0c8ef0c2aa583a4eb44e5c81c2ddcc", + "size_bytes": 39 + }, + "zero_population_blocks": 41829 + }, + { + "block_rows": 175199, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:19" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "98f4efd90409774596b5bc623078d75f579d781ca3bc392b586a24fa67e638a1", + "size_bytes": 6033554 + }, + "populated_blocks": 124175, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "19", + "state_population": 3190369, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:19" + ] + ] + }, + "sha256": "5cd722954a80d81e07f29fe65056cd0f40967c163935eb450320d4e73a5d4169", + "size_bytes": 39 + }, + "zero_population_blocks": 51024 + }, + { + "block_rows": 172529, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:20" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "35e21c30e14e0c4bfdb3645076cdc7a89ef4708c17373047526875be9cb0ab41", + "size_bytes": 5931398 + }, + "populated_blocks": 109149, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "20", + "state_population": 2937880, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:20" + ] + ] + }, + "sha256": "aaaa97cb26fbc4db8318b32549da0a4f64a3fc4ca316ef9fcc5b2da6bb3b1c77", + "size_bytes": 39 + }, + "zero_population_blocks": 63380 + }, + { + "block_rows": 132662, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:21" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "975c9821955f9472f2222f85e089ce6ce9caff44a18bed9edd797ca13bda49c8", + "size_bytes": 4591236 + }, + "populated_blocks": 91572, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "21", + "state_population": 4505836, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:21" + ] + ] + }, + "sha256": "c65f5deb2c5dacbcd8c1e84e4caf7a41458399f3a388425fbc5a7af69dba0bb9", + "size_bytes": 39 + }, + "zero_population_blocks": 41090 + }, + { + "block_rows": 142874, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:22" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "9bdacd2c2ccfa5bac93f5df47ec1e63fe39643c15633e7d33f464ad7a171d869", + "size_bytes": 4942488 + }, + "populated_blocks": 92180, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "22", + "state_population": 4657757, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:22" + ] + ] + }, + "sha256": "db924ac57fd4177be3508d795ce912f3b55454f8eb6331fcc297252c2bb20a19", + "size_bytes": 39 + }, + "zero_population_blocks": 50694 + }, + { + "block_rows": 47138, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:23" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "eb26cdc0e114b07154096f592447df295d7fcda6dd890ebfe6e012d7d7bb40ca", + "size_bytes": 1629179 + }, + "populated_blocks": 30548, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "23", + "state_population": 1362359, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:23" + ] + ] + }, + "sha256": "2a6bc352f6304d0ceb8850333a47378fa2a324a1163850d56931b424b8281089", + "size_bytes": 39 + }, + "zero_population_blocks": 16590 + }, + { + "block_rows": 83827, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:24" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2d181f3e6bf3c96c8c2f458024a699be8e4c8b2bb80b6433ee95c69efe655b29", + "size_bytes": 2925422 + }, + "populated_blocks": 65274, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "24", + "state_population": 6177224, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:24" + ] + ] + }, + "sha256": "b9de28b8819798a841c36ac13d35cdc75573c8e47ec232d195817239ab0bff55", + "size_bytes": 39 + }, + "zero_population_blocks": 18553 + }, + { + "block_rows": 107278, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:25" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "ca73c4d84aaf50e22ca9b25c4fdc54ca314167473a5ba1c9ee31fac1881396e3", + "size_bytes": 3749394 + }, + "populated_blocks": 88207, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "25", + "state_population": 7029917, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:25" + ] + ] + }, + "sha256": "db5b65758be7c88e4bcafd4413f2813fa32de96307aa50a450f28bdc14e3bb39", + "size_bytes": 39 + }, + "zero_population_blocks": 19071 + }, + { + "block_rows": 254730, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:26" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "732595037f8d58aee0d01a3b68e71a18bc8a7144c81433d1dfc805a1a83cfe8d", + "size_bytes": 8845352 + }, + "populated_blocks": 198244, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "26", + "state_population": 10077331, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:26" + ] + ] + }, + "sha256": "29a349d67a3c6219b9675e6cc5e5bd9a0bf3511e4b48005a6c9a9215be033069", + "size_bytes": 40 + }, + "zero_population_blocks": 56486 + }, + { + "block_rows": 198705, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:27" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "8910dd3e06035629bbef028441acb5d29f29ea555381df98076a05289bf29803", + "size_bytes": 6863252 + }, + "populated_blocks": 137689, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "27", + "state_population": 5706494, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:27" + ] + ] + }, + "sha256": "13b45f3487425184637b01256644d5f86599d6a445b1a4f1d9fb86ef3790cef9", + "size_bytes": 39 + }, + "zero_population_blocks": 61016 + }, + { + "block_rows": 112241, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:28" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "33350d9a37d539dd4724dc7f4b969de0fd1dbfcb5207cae093b8ed73e0a67d68", + "size_bytes": 3878667 + }, + "populated_blocks": 77426, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "28", + "state_population": 2961279, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:28" + ] + ] + }, + "sha256": "5cabdefebb5363751a244a9b12720ff43e9ac3a6fef7ba8c1ee7f79e7adaf745", + "size_bytes": 39 + }, + "zero_population_blocks": 34815 + }, + { + "block_rows": 253632, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:29" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "0e9b1a932c3cfb283a80a0e831192482ee9db6f01d7aad83de67a16e5c4075d5", + "size_bytes": 8749056 + }, + "populated_blocks": 169004, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "29", + "state_population": 6154913, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:29" + ] + ] + }, + "sha256": "b75c227a323978f36e25e2efab8ec11b20579b06d6393da203cb36d533743a6d", + "size_bytes": 39 + }, + "zero_population_blocks": 84628 + }, + { + "block_rows": 88417, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:30" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "b9a81e34cc8c228fef203728f4134f96b7a8acfc1b14395e2de251d7ab0e3964", + "size_bytes": 3033388 + }, + "populated_blocks": 43717, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "30", + "state_population": 1084225, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:30" + ] + ] + }, + "sha256": "caa4444d14b67700b3db4a5653412bd633c995df0d279473441b885a40102968", + "size_bytes": 39 + }, + "zero_population_blocks": 44700 + }, + { + "block_rows": 119103, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:31" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "bebe7ca15c979b170cfc3e6f49e27cdc50f5639de2f9a1955e60187849404e56", + "size_bytes": 4095921 + }, + "populated_blocks": 79779, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "31", + "state_population": 1961504, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:31" + ] + ] + }, + "sha256": "30c8aae0e54d78eeb7857f0954be04c95c3a2278a030e78317f2a361fe40c683", + "size_bytes": 39 + }, + "zero_population_blocks": 39324 + }, + { + "block_rows": 57409, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:32" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "f34926bc3ada9c07a7ace640a787495814b9893c8351b34a9a0fe9bcf91acd75", + "size_bytes": 1991100 + }, + "populated_blocks": 35249, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "32", + "state_population": 3104614, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:32" + ] + ] + }, + "sha256": "3f3957d218a7eb94ae360e03f6b5924d95ab0e8cb80b685d0fc74ff3de3349a7", + "size_bytes": 39 + }, + "zero_population_blocks": 22160 + }, + { + "block_rows": 31948, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:33" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "11e83119f4924fdbc1c0dcf6f96113c84ba4173c507290578835dfd2c7f97dfa", + "size_bytes": 1110353 + }, + "populated_blocks": 25317, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "33", + "state_population": 1377529, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:33" + ] + ] + }, + "sha256": "b2c3dae17fcbdccaa8eddf6f3e5fdb742d4b0ec017084610958d5182c0f70ce7", + "size_bytes": 39 + }, + "zero_population_blocks": 6631 + }, + { + "block_rows": 137972, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:34" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "280a8f28f13545bbff7d432e2356cdaeaedd9e0ea026f7f05f4d9e83dd28886e", + "size_bytes": 4823492 + }, + "populated_blocks": 113212, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "34", + "state_population": 9288994, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:34" + ] + ] + }, + "sha256": "517cdd27711beea019fa1efc18c6017690cd1d44617aa446a52677e94281d0cc", + "size_bytes": 39 + }, + "zero_population_blocks": 24760 + }, + { + "block_rows": 107215, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:35" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "a012da889bc274fb518c2a258c9b1394f43c24e6d58fb3661151565cc5ce8757", + "size_bytes": 3690294 + }, + "populated_blocks": 56605, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "35", + "state_population": 2117522, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:35" + ] + ] + }, + "sha256": "9632c8400a3052e17d23dd71427ccf66e8c062b15cb58df2cb949ecb7a287e9e", + "size_bytes": 39 + }, + "zero_population_blocks": 50610 + }, + { + "block_rows": 288819, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:36" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "7a47a92501f986ebca42a27a180fe897b4afb2c37cd463360179a0b32500ab2d", + "size_bytes": 10067666 + }, + "populated_blocks": 230339, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "36", + "state_population": 20201249, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:36" + ] + ] + }, + "sha256": "e34eac8cee690ba81ef7ad607cee5e06697834056165979faf2148627d5aeb5f", + "size_bytes": 40 + }, + "zero_population_blocks": 58480 + }, + { + "block_rows": 236638, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:37" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "3a721726b3ee7a8426411e5ba5267b0c0f88d4123916e8ed40d71e0a601eb797", + "size_bytes": 8216489 + }, + "populated_blocks": 174988, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "37", + "state_population": 10439388, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:37" + ] + ] + }, + "sha256": "bf0b0c6a90da12c51a71942982ec8a10b8e1dece22601462dd0391c2dc758ba2", + "size_bytes": 40 + }, + "zero_population_blocks": 61650 + }, + { + "block_rows": 84566, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:38" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "6f8985d46a8d8e97fce2f563f20442f7c6e73a0503cddf58ff59746e829e2b68", + "size_bytes": 2892476 + }, + "populated_blocks": 40198, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "38", + "state_population": 779094, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:38" + ] + ] + }, + "sha256": "3616ffbe27406d4e881587912b59c4494d2a0aa71132a25a6f8108bae23ccbb5", + "size_bytes": 38 + }, + "zero_population_blocks": 44368 + }, + { + "block_rows": 276428, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:39" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "aeb3eff352866ece982d0a31cfaf3ad63877bd8e2598bede9d883a0a168b450c", + "size_bytes": 9606700 + }, + "populated_blocks": 219669, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "39", + "state_population": 11799448, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:39" + ] + ] + }, + "sha256": "3a93ca174df19c6f3377abf023177def2d8ff585b2182d56a9abd75b2e3454e3", + "size_bytes": 40 + }, + "zero_population_blocks": 56759 + }, + { + "block_rows": 180154, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:40" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2078a38b5338f474099f7e1fe2ba0dba3cab95b1644db5abd6c52ca4a8a57703", + "size_bytes": 6212394 + }, + "populated_blocks": 121826, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "40", + "state_population": 3959353, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:40" + ] + ] + }, + "sha256": "bd8b724c531ca5d6605aa1752c70c00a2666755070a18ef73ac0a3423d2812a5", + "size_bytes": 39 + }, + "zero_population_blocks": 58328 + }, + { + "block_rows": 130807, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:41" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "3cafb0c96a666767ea005ac1f08ca3171379934352c4d2ef285fd6ce44196ccd", + "size_bytes": 4521307 + }, + "populated_blocks": 79081, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "41", + "state_population": 4237256, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:41" + ] + ] + }, + "sha256": "fb8d53cee89c2a14485cf42cf49b4723576407db7d550897609c1f5038b20e67", + "size_bytes": 39 + }, + "zero_population_blocks": 51726 + }, + { + "block_rows": 336985, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:42" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "34840439707bedac6bf76a35863287aa24263000ff2e1dca21ecdf0030b262f6", + "size_bytes": 11713716 + }, + "populated_blocks": 275023, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "42", + "state_population": 13002700, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:42" + ] + ] + }, + "sha256": "b07244d4d81570c0c8bf49dc3653c382e8e3e74a2d798fb32c85e5808fd789a7", + "size_bytes": 40 + }, + "zero_population_blocks": 61962 + }, + { + "block_rows": 25649, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:44" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "8f8eb7ee08020dd7e3a49dab4d95f461acd5d29572fe8bf5b55f200ff84e7231", + "size_bytes": 894186 + }, + "populated_blocks": 21382, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "44", + "state_population": 1097379, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:44" + ] + ] + }, + "sha256": "ec6f0565a6ebceccf1bf1a665f46a8105852b352762416b350f9630f802a8223", + "size_bytes": 39 + }, + "zero_population_blocks": 4267 + }, + { + "block_rows": 146844, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:45" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "a3daf2da71c547c02d8c260b009be331cd218c84896d602fafcdbc12acd59c3f", + "size_bytes": 5086274 + }, + "populated_blocks": 105469, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "45", + "state_population": 5118425, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:45" + ] + ] + }, + "sha256": "3d15445808b86c1f785d6a1668f69eae773fbc75a462d3ead77c99bcc769ba9c", + "size_bytes": 39 + }, + "zero_population_blocks": 41375 + }, + { + "block_rows": 71383, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:46" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "1700e07604a6fb79d5cfb34fcb6bfe3d810429ecbca4e88d3012b8b2070a516d", + "size_bytes": 2447858 + }, + "populated_blocks": 40766, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "46", + "state_population": 886667, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:46" + ] + ] + }, + "sha256": "e5b885739fae3a34f28cc11f739435e8cd39926ebeb0a2ef1a18ae3ff035becd", + "size_bytes": 38 + }, + "zero_population_blocks": 30617 + }, + { + "block_rows": 179717, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:47" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "d8522733ea6f5971d5f340d86f71f7174b54f502ddd84a79d703fbd72d343eb2", + "size_bytes": 6232909 + }, + "populated_blocks": 133846, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "47", + "state_population": 6910840, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:47" + ] + ] + }, + "sha256": "32de25caad8e71a6bb74f5639f5e9eae1c95836f61743275c60b76f716d0c790", + "size_bytes": 39 + }, + "zero_population_blocks": 45871 + }, + { + "block_rows": 668757, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:48" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2e997c530f51c17b595226374c50eb1ed249d2d6945e37dde9a1ceed8a161c6d", + "size_bytes": 23175159 + }, + "populated_blocks": 449085, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "48", + "state_population": 29145505, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:48" + ] + ] + }, + "sha256": "469d7f5efd70e8918a3c9744d7c7742e190e54b52d5b331faf648907f5882b0b", + "size_bytes": 40 + }, + "zero_population_blocks": 219672 + }, + { + "block_rows": 71207, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:49" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "73207e9605fda4b336c404455c9a369ed64e39e7ecebd623b633835a9664e495", + "size_bytes": 2471351 + }, + "populated_blocks": 46792, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "49", + "state_population": 3271616, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:49" + ] + ] + }, + "sha256": "825dab7318dcc582b32eb2723473be42d111a00d32f47d88ae93beade83e2fae", + "size_bytes": 39 + }, + "zero_population_blocks": 24415 + }, + { + "block_rows": 24611, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:50" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "7e34e58509cec16138d786dda2e34a45370fc1386108591182313f306b252daa", + "size_bytes": 850760 + }, + "populated_blocks": 17976, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "50", + "state_population": 643077, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:50" + ] + ] + }, + "sha256": "c5ccb3d18ec85abae8ca3958cd4cca738812021cee35dd34be09150c07aa3c3d", + "size_bytes": 38 + }, + "zero_population_blocks": 6635 + }, + { + "block_rows": 163491, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:51" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "0d979951968f230cd5b7235f703e9acfabcafbeed99c4c018e34c0f88ba0ca52", + "size_bytes": 5678471 + }, + "populated_blocks": 117361, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "51", + "state_population": 8631393, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:51" + ] + ] + }, + "sha256": "87ce18c978a4361494de558f74c03ce956079f6cfa8fb0401c8e77e5bf234c97", + "size_bytes": 39 + }, + "zero_population_blocks": 46130 + }, + { + "block_rows": 158093, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:53" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "0e3ec5282f8d894a504625cc9a366ada08430b60317fb5d529a982b0e2787a64", + "size_bytes": 5490219 + }, + "populated_blocks": 111441, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "53", + "state_population": 7705281, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:53" + ] + ] + }, + "sha256": "ef5ad0f000c5505f923e4c91a5516b02138669c8b08b7a26c547cc856a7891e4", + "size_bytes": 39 + }, + "zero_population_blocks": 46652 + }, + { + "block_rows": 72558, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:54" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "28f1b09668784b1d8a064bd1433a1446e9697d8da2988c611bbc267ebee4456f", + "size_bytes": 2507211 + }, + "populated_blocks": 51850, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "54", + "state_population": 1793716, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:54" + ] + ] + }, + "sha256": "b1cefc08915ac5c9bd05f47636d5ce6307b8ecb43c3c32502de2ec6d8b144c13", + "size_bytes": 39 + }, + "zero_population_blocks": 20708 + }, + { + "block_rows": 203059, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:55" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "fc491abf3fcd1a1fd3807ecde1d95b68bc2cab4e0a235a1b51f095ab5a2afcfa", + "size_bytes": 7025682 + }, + "populated_blocks": 144948, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "55", + "state_population": 5893718, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:55" + ] + ] + }, + "sha256": "1c7bbd06d50dfe00b2e4461d2b92e98fac585f4b7b8c2425782bc822984fc5e9", + "size_bytes": 39 + }, + "zero_population_blocks": 58111 + }, + { + "block_rows": 53769, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:56" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "55aef5b66c91b47bda087b490e44a12067eabe7634b444351172e59ae126e66a", + "size_bytes": 1843303 + }, + "populated_blocks": 22857, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "56", + "state_population": 576851, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:56" + ] + ] + }, + "sha256": "0684454d90b7e6430f6f55991791113091593c8dab5577ff8d6dcc2d1fa2e10d", + "size_bytes": 38 + }, + "zero_population_blocks": 30912 + } + ], + "puma": { + "selected_state_tract_mappings": 84414, + "sha256": "5262f460b2c8dbe86b00549e916317b1791fdc595c9af72ab18f6f6a67040531", + "size_bytes": 1709076, + "source_id": "census-2020-Census-Tract-to-2020-PUMA" + } + }, + "state_fips": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56" + ], + "support_sha256": "5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960" + }, + "source_receipt_sha256": "9adaa4365fc4624dfcb80d456783d5a54806e672384e17b4e84ad08d10bde74b", + "state_fips": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56" + ], + "status": "source_control_passed", + "support_filename": "national-atomic-support.npz", + "support_sha256": "5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960", + "support_size_bytes": 28862508, + "survey_frame_created": false + }, + "normalization_report_sha256": "1da2a391b58956de6541ac8e541d1b67f5d560b601da86caca3baca1859b8c45", + "peak_rss_bytes": 6813728768, + "receipt_sha256": "6c65dee6d9317e522a2076f690341889f867a28652ba18ddd6120bc6c2a80ad0", + "release_eligible": false, + "result": 0, + "scope": "One ordinary-acceptance national source-normalization case after separate root review and approval;104 population-only/geography pins, exact51 states, no survey Frame/source admission/release eligibility. Existing failed and accepted DE runs preserved.", + "source_and_owned_files": 498, + "source_control_accepted": true, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 82.97476216696668 +} diff --git a/experiments/us-atomic-survey-population-controls-20260909.json b/experiments/us-atomic-survey-population-controls-20260909.json new file mode 100644 index 000000000..9a162052a --- /dev/null +++ b/experiments/us-atomic-survey-population-controls-20260909.json @@ -0,0 +1,31 @@ +{ + "before_after_current_frozen_source_equal": true, + "code_resources": 12, + "cpu_seconds": 306.843367, + "model_source_files": 5983, + "native_geography_acceptance": false, + "native_payloads_read": 0, + "peak_rss_bytes": 440320000, + "pending_followups": [ + "Nonblocking support-file open correction and its FIFO control are tested separately.", + "Geography-aware calibration budget and age-runner integration are tested separately.", + "Publisher-qualified native block support and a native calibrated build remain pending." + ], + "publisher_provenance_established": false, + "receipt_sha256": "025bc482224c63c06020acd537c603235b0c612dc805d3b86deccc683624993e", + "release_eligible": false, + "result": 0, + "scope": "Complete survey atomic-geography wrapper on invented original source files: cold and required replay, raw/enriched allocation separation, geography-before-clone ordering, full Frame/owner/weight/design inheritance and late mutation refusals.", + "source_and_owned_files": 489, + "test_source_sha256": "762c386d5333514ef1cd12c1573c2e97970573ae519f6faedff8f9e736f094b9", + "tested_reconstruction_source_sha256": "6c707c55e9c6a1b67241f86807700856a6438e98428b8f5704f22ae175e40f89", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 5 + }, + "unexpected_refusals": {}, + "wall_seconds": 315.1505964579992, + "wrapper_source_sha256": "794789a80298b9d239d707ba4cae605b61265846ad6dbc39330d0e1a28c2d1fc" +} diff --git a/experiments/us-budget-detached-view-correction-20260910.json b/experiments/us-budget-detached-view-correction-20260910.json new file mode 100644 index 000000000..894069e3b --- /dev/null +++ b/experiments/us-budget-detached-view-correction-20260910.json @@ -0,0 +1,39 @@ +{ + "adopted": true, + "behavior": "Move detached checked-view construction after last borrowed I/O; seal fresh document and complete maintained GroupedUpperBounds state, including stable-ID buckets, canonical digest and immutable storage, against issued payload and pure owner state.", + "budget_owner_sha256": "e1faffa1b133878b3607e888fe0c1fe0503f23d46be52eeddc054cf6c28a1c2c", + "callback_branches": { + "new_detached_values": 7, + "old_population_mutations": 2 + }, + "cpu_seconds": 540.759706, + "guard_sha256": "8c0b29efd7dc6c272b94463c996a9a6d35bb81c16857046bc75f95d11fb69087", + "historical_original_eight": { + "budget_owner_sha256": "80418528db45266e7322a8984cc63f9414f6fa3c084119cd9d0f558f4a0e0260", + "positive_postcheck_sha256": "8698b43174b17faa1bcc4123ae1b7c7c1ea65bba94f5d41f207c259dc3c951e8", + "remaining_postcheck_sha256": "423a6cd3ae1537bfe372b7d735ad3a907e6fcc6837257e6c65d538db18e13486" + }, + "native_acceptance": false, + "new_test_sha256": "4a8078e93d95cd6c807218fe021128614ce0567490ef216b7c83555a25b65a0e", + "peak_rss_bytes": 452624384, + "performance_claim": false, + "public_recipient_pilot_uses_original_frozen_budget": true, + "release_acceptance": false, + "root_postcheck_sha256": "2f94929c903c86633c199d3570625dbfda29fe57a84dab9b52502b25fe1236ed", + "scope": "Original eight financial-successor controls and four corrected budget-view controls are separate versioned runs; this is not whole-tree or native certification.", + "source_candidate_peer_sha256": "5f4ef9c2ecb9b19803b3439ec26b0bfdd92b19acc09e1579a6cc216240d35801", + "source_map_sha256": "9f5858b3bd7aceb9e9ee5ca0a2b5b4033ab9356a845364e9b07a11373de75b5f", + "status": "adopted_after_exact_component_verification", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 4 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 501 + }, + "wall_seconds": 543.4429369170102 +} diff --git a/experiments/us-budget-predictor-compatibility-1-controls-20260909.json b/experiments/us-budget-predictor-compatibility-1-controls-20260909.json new file mode 100644 index 000000000..2c4af43e8 --- /dev/null +++ b/experiments/us-budget-predictor-compatibility-1-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "code_resources": 12, + "cpu_seconds": 35.759673, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 519110656, + "publisher_provenance_established": false, + "receipt_sha256": "de102101abef2cb6238092df3cf2d8c87e885efec91349d130031919a5eabd4b", + "release_eligible": false, + "result": 0, + "scope": "Historical pair-return compatibility through both actual existing current-survey financial qualifiers; no model fitting. Invented data only; no native or publisher admission, no release.", + "source_and_owned_files": 492, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "a10b9f3552c84563d845675c772130073b925b5835d6730ffb9afc0e93170777", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "c0667ad329af2f58901b0fd2f333489ad2366b83e2bac8ded95b51088d9e5d5e", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "25a414517974d345541bcec6152b61151b09ccc476a5350a36015c4b33dcd70e", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "d130335f5fb8d66e3e032984c326e1077b0a889162da8d97324f85866095905d", + "packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py": "d81a82b9aedecd26d62831e93355c233093dc8fc3adec1e702d8c7f138fd8bbd" + }, + "unexpected_refusals": {}, + "wall_seconds": 38.938349917007145 +} diff --git a/experiments/us-calibration-consolidation-149-20260910.json b/experiments/us-calibration-consolidation-149-20260910.json new file mode 100644 index 000000000..166e89640 --- /dev/null +++ b/experiments/us-calibration-consolidation-149-20260910.json @@ -0,0 +1,50 @@ +{ + "cases": 149, + "checks": { + "copied_test_helper_fixture_files": 10, + "exact_case_and_junit_rosters": true, + "junit_selector_order": true, + "junit_unique_names": 149, + "owned_before_after_current_match": true, + "owned_files": 20, + "source_before_after_current_match": true, + "source_files": 34, + "terminal_pins_final_match": true + }, + "coverage": [ + "US grouped upper bounds, fixed zero support and unsupported combinations", + "UK informed gates and budget search", + "ordinary best iterate plus same-runtime frozen optimizer controls", + "dense/CSR combined behavior and strict diagnostic payload/rebuild" + ], + "cpu_seconds": 3.9523219999999997, + "current_34_source_subset_matches": true, + "date": "2026-09-10", + "excluded_control": "cross-runtime saved golden requiring18 interop threads; same-runtime frozen optimizer comparisons still executed", + "failed_predecessor_receipt_sha256": "240b17fdb30b8dfa0cddb3f58343fc4a1ba0bc14e44b852cd188cb1bd7a6703b", + "full_checkout_certified": false, + "guard_sha256": "c4218acbd403c1ff4b05f3cbb4d47654e1a1b51cbcf7f6b8b045c9252248aa80", + "junit_sha256": "57ba56887978b71aa86152f36f83484471a216829f4b7cca9b20c15ea1af9bee", + "native_calibration_accepted": false, + "peak_rss_bytes": 394067968, + "postchecker_sha256": "ee5bd078009c7842d7f9c19adf5452bc2c1c8c97a44c4a890c6fd4e7ee924bff", + "predecessor_failure": "collection stopped before tests: two still-denied stdlib ZIP probe counts exceeded their reporting ceilings", + "receipt_sha256": "242fa8cb76d5e6f1fa4589319bf6764c4f63620b3cda2e9e2a15f69549f362c3", + "release_acceptance": false, + "root_postcheck_sha256": "dca84da504acf53c65d68f7076ebe1ecf12a8d64a6479c40255c164717f3d10d", + "schema_version": "microcosm.scoped-calibration-control.v1", + "scope": "exact invented calibration149 source/runtime evidence only; no native, release or unrelated consolidated-checkout acceptance", + "selectors": 68, + "source_files": 34, + "source_map_sha256": "7de4ee1007343d72a3618cd95850700cb28bf87cd89405d956a0f92d37634850", + "status": "accepted_invented_calibration149", + "successor_change": "fresh once-only run; two reporting ceilings changed2 to3; exact same source, selectors, permissions and600CPU/900wall/one-thread limits", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 149 + }, + "unexpected_refusals": {}, + "wall_seconds": 4.565558874979615 +} diff --git a/experiments/us-common-frame-export36-20260910.json b/experiments/us-common-frame-export36-20260910.json new file mode 100644 index 000000000..10c567b06 --- /dev/null +++ b/experiments/us-common-frame-export36-20260910.json @@ -0,0 +1,67 @@ +{ + "adoption_formatting": { + "full_ast_unchanged": true, + "test_source_after_sha256": "d6502d76a36796081e8c937c9b80caf74806558ed89f5466c6699f89e7b46c3c", + "test_source_before_sha256": "7d5184303390508ff745f3d6f4f7d8bdd4cbb10f2e50a574adda38e5bf71c9c0", + "tool": "ruff format" + }, + "component": "US supplied-parent full/pruned/local export comparison", + "fable_source_review": { + "blocking_findings": 0, + "model": "Fable 5.1", + "record_note": "files identifies adopted bytes; tested_files identifies exact executed bytes. Full AST equality after Ruff formatting is recorded above.", + "reviewed_commit": "b591280674db6d606c7ab356cabffbb6ef582e2d", + "scope": "Six-file source review; no runtime or independent hash recomputation by the peer", + "sha256": "361eb8abccc7b227c7c2aef2e1704e807076b6481ce73e501839adf358143416", + "verdict": "clean" + }, + "files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py": "206159abfad714563fa3434209092a05f8e847d9ffef99a913373331f2ecbb8c", + "packages/microcosm-build/tests/test_us_common_frame_export_contract.py": "d6502d76a36796081e8c937c9b80caf74806558ed89f5466c6699f89e7b46c3c" + }, + "guard_sha256": "f8d8184e582a443ccae781342df2b8c0accf4c87c552ccaed843f1bad0586bd1", + "independent_review": { + "correctness_defects": 0, + "coverage_note": "Both new prune-flag cases select the same retained rows; existing full/pruned/local cases exercise differing selections.", + "sha256": "8ef45f1d1fd5b3790e9d15d42d4dcb98651c1f66e5badb059134df623c91d17b" + }, + "limits": { + "cpu_seconds": 600, + "numeric_threads": 1, + "wall_seconds": 900 + }, + "physical_postcheck_sha256": "486eb981a71ca13806ed2524cc7a7b9469751ff47d2496e85b566044b7afc0a8", + "prior_attempts": "Preserved original all-zero fixture failure, unexecuted34 packet, and two36 wrapper failures. Final36v3 corrected invocation serialization and passed all checks.", + "scope": "Invented frames and real frame-checkpoint write/readback. No native candidate, actual calibration ancestry or release is certified.", + "source_closure_parity": { + "new_files": 2, + "pyproject_difference": "Comment describing UK NISRA openpyxl use; dependency configuration unchanged", + "pyproject_toml_semantics_equal": true, + "unchanged_python_files": 57 + }, + "source_map_sha256": "d6bb7e1404569ec27c5231b37d92397399fca7f63b85744ff393dc630454215e", + "source_revision_before_adoption": "162869d4d350b06c263b2fbdbeea93b7f1b05050", + "tested_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py": "206159abfad714563fa3434209092a05f8e847d9ffef99a913373331f2ecbb8c", + "packages/microcosm-build/tests/test_us_common_frame_export_contract.py": "7d5184303390508ff745f3d6f4f7d8bdd4cbb10f2e50a574adda38e5bf71c9c0" + }, + "validation": { + "actual_graph_calibration_ancestry_verified": false, + "cpu_seconds": 1.769343, + "model_sources": 0, + "owned_files": 11, + "peak_rss_bytes": 324517888, + "physical_postcheck": "pass", + "receipt_sha256": "2b6de385041b6bb5d59ff34a2672d215de85bf0d058e6853f05a7afaf05b84f9", + "release_eligible": false, + "resources": 0, + "reviewer": "root", + "source_files": 60, + "suite_accepted": true, + "tests_failed": 0, + "tests_passed": 36, + "unexpected_refusals": {}, + "wall_seconds": 2.5767143329721875, + "xml_sha256": "3ab7cc10509aad988dd767ce39425138e9c8a67b171fcb433387057c2c1d28cb" + } +} diff --git a/experiments/us-current-acs-income-anchors-20260912.md b/experiments/us-current-acs-income-anchors-20260912.md new file mode 100644 index 000000000..17fc09e1d --- /dev/null +++ b/experiments/us-current-acs-income-anchors-20260912.md @@ -0,0 +1,67 @@ +# Qualifying observed ACS income anchors + +`qualify_current_acs_income_anchors` reads the original ACS person archive through +the retained survey preparation owner. It returns the selected ACS people's +original `INTP`, `RETP`, `ADJINC`, `AGEP`, `FINTP`, and `FRETP` literals, joined by +the original `SERIALNO` and normalized `SPORDER`. Household serial numbers are +checked through the native person's household link. The selected native person +axis and the preparation's stacked person axis remain explicit. + +These are broad survey anchors. They are not separately observed tax inputs. + +| Anchor | Source meaning | Qualification | +| --- | --- | --- | +| `INTP` | Interest, dividends, net rental, royalty, estate and trust income over the preceding 12 months | Preserve signed values, published zero, blank and malformed literals separately. | +| `RETP` | Broad retirement, survivor and disability income over the preceding 12 months, excluding Social Security | Preserve the nonnegative source amount without choosing a pension or retirement-account decomposition. | +| `ADJINC` | Factor for expressing income in the release's dollar year | Match the mapper's multiplication and division order and every retained nonmissing float64 bit. This adjustment does not convert the rolling reference period into a calendar year. | +| `AGEP` | Original source age | Use the original age for the age-15 income universe, before any downstream top-code mapping. | +| `FINTP`, `FRETP` | Allocation flags | Keep allocation separate from amount validity. A valid allocated amount remains known and explicitly labeled. | + +The [2024 ACS questionnaire, question 43](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf), +[2024 PUMS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +(printed pages 38, 43, 127 and 129), and +[2024 subject definitions](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf) +establish these meanings. The released INTP domain contains zero, -10,000 through +-4, and 4 through 999,999; RETP contains zero and 4 through 999,999. Those disclosure +bounds describe released records, not bounds on latent income. + +The output distinguishes observed, missing, malformed, outside-domain, invalid +adjustment, and outside-universe records. It supplies no analytical zero for an +under-15 person or for an adult with a blank amount. The original literal survives +every classification, including missing or unrecognized allocation flags. Numeric +coercion reproduces the existing source mapper only for the retained-storage +comparison; it never supplies observation knownness. + +The qualifier exhausts a privately captured archive, checks its recorded hash and +complete source row count, and compares selected raw fields and adjusted anchors +to the retained native frame. After capture cleanup, it rechecks the actual +preparation and catalogue owners. A final physical seal also detects mutations +that table JSON's decimal precision cannot represent. Returned values, projection +bytes and evidence are descriptive: they cannot replace the original preparation +or grant source authority. A consuming graph host must retain that owner and the +physical value seal, check them immediately before consumption, and requalify +after its last relevant I/O before returning a successor. + +The 35-case invented-source suite passed with no failures, errors or skips in +56.261 seconds wall time (55.239 CPU seconds; 487,407,616 bytes peak RSS). It covers +the real preparation issuer, household and group-quarter records, selected-row +reordering and ID changes, signed and missing amounts, allocation states, exact +adjustment bits, a copied preparation, source mutation after capture I/O, and a +returned amount mutation hidden by JSON precision. The guarded run used 982 +source files and 15 allowed resources, with all 990 source-plus-owned hashes and +all resource hashes unchanged, no children, no unexpected refusals, and Torch +threads fixed at 1/1. The normal guard-induced dateutil zoneinfo warning occurred. + +The source SHA256 is +`94cb447fe209e8b801cd6080151a93c80bfb06bb736b068affb44d9d595c50a2`; +the test SHA256 is +`9bffafce6cdf04d63940e15052306037e0bee31049996ec561d46d8cb9e1a2bc`. +The local `codex-acs-income-anchor-20260912/source-v4/source-v4.json` receipt SHA256 +is `9154b624cf98def81bcf84fb814faf72d4a866d29243eacbeef12a6607c737b6`. +Earlier failed and superseded fixture evidence remains preserved. + +This change does not fit or reconcile components, add an enrichment host, choose +between ASEC interest definitions, or resolve the retirement donor bridge. It +does not execute native microdata, a country engine or a model. Those steps need +their own source and graph acceptance before this qualification can support a +release candidate. diff --git a/experiments/us-current-health-coverage-20260912.md b/experiments/us-current-health-coverage-20260912.md new file mode 100644 index 000000000..586f8a8af --- /dev/null +++ b/experiments/us-current-health-coverage-20260912.md @@ -0,0 +1,108 @@ +# Current health coverage on the survey multispine + +This lane adds source-qualified interview-date coverage to the common ACS+ASEC +frame and its PUF clones. It does not estimate eligibility, annual receipt, +insurance costs, or a common observation date. The current ASEC arm is income +year 2024 / survey year 2025; ACS coverage was observed during 2024. Older ASEC +income-year vintages are outside this first attachment. + +## Source definitions and mapping decisions + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf) +defines these current recodes for all persons, with 1=yes and 2=no. Corresponding +`I_NOW_*` flags distinguish reported, hot-deck, logical, and whole-unit imputation. +The [2024 ACS questionnaire, item 16](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf) +asks all respondents about current coverage. The [ACS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +defines HINS1–7 (printed pages 37–38) and allocation/coverage-edit flags (126). + +| NATIONAL_CD input | ASEC source | ACS exact source in this first stage | +| --- | --- | --- | +| `has_esi` | `NOW_GRP` | `HINS1` | +| `has_marketplace_health_coverage_at_interview` | `NOW_MRK` | Gap: `HINS2` combines direct-purchase types | +| `has_non_marketplace_direct_purchase_health_coverage_at_interview` | `NOW_NONM` | Same gap | +| `has_medicaid_health_coverage_at_interview` | `NOW_CAID` | Gap: `HINS4` includes CHIP/other means-tested coverage | +| `has_other_means_tested_health_coverage_at_interview` | `NOW_OTHMT` | Same gap | +| `has_tricare_health_coverage_at_interview` | `NOW_MIL` | Gap: `HINS5` includes other military coverage | +| `has_champva_health_coverage_at_interview` | `NOW_CHAMPVA` | No separate ACS item | +| `has_va_health_coverage_at_interview` | `NOW_VACARE` | Keep `HINS6` distinct pending CHAMPVA recode reconciliation | +| `has_indian_health_service_coverage_at_interview` | `NOW_IHSFLG` | `HINS7` | + +The existing `cps_carried._fill_health_coverage_inputs` supplies the field roster +and most ASEC mappings, but its absent-column-to-zero helper is unsuitable here. +Its Medicaid mapping uses `NOW_MCAID`, the broader Medicaid/CHIP/other-means-tested +aggregate, rather than `NOW_CAID`. Both names and their distinction also appear +in the [2023](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar23.pdf) and +[2024](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf) dictionaries. +The old pipeline remains untouched; this stage refuses ownership collisions. + +This narrower mapping is supported by the pinned consumer source, rather than +inferred from the input name alone. PolicyEngine-US 1.819.0 declares the Medicaid +input as person-level current Medicaid coverage, declares other means-tested +coverage separately, and uses the Medicaid input alongside `receives_medicaid` +for California Medi-Cal continuity. The checked files are +`variables/household/expense/health/has_medicaid_health_coverage_at_interview.py` +(SHA256 `0500c4e86120b540e6e2f09fc108123758d8f9863c0e4e4bf94b178dd50e09b8`) +and `variables/gov/states/ca/chhs/is_ca_medicaid_immigration_status_eligible.py` +(`6902d01792b8d0902c2693edb78f5ba2554a34f95d4073bd4e85e4ad69ce41a5`). +Only source text was inspected; this is not an engine execution or a blanket +reinterpretation of earlier datasets. Both `NOW_CAID` and `NOW_MCAID`, with their +allocation flags, remain in the graph. A later CMS comparison must declare +whether its target covers Medicaid alone or Medicaid plus CHIP. + +The [ACS subject definitions, health insurance](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf) +describe source editing and the 2019 change to current VA enrollment wording. +The [Census programming guide](https://www.census.gov/topics/health/health-insurance/guidance/programming-code/acs-estimates.html) +still groups CHAMPVA with HINS6. Consequently this stage does not assume that +HINS6 is interchangeable with the narrower ASEC VACARE recode. Raw HINS2–6 stay +inspectable; no missing concept becomes a false flag. `HIMRKS` measures subsidized +direct purchase, not all Marketplace enrollment, and cannot repair that split. + +## Graph and authority + +Borrow the original checked `SurveyPuf55Run` and its retained survey preparation. +Capture the exact original ASEC PERSON member and ACS person archive under the +existing native owners' pins. Match source household/person coordinates to the +preparation's original person ledger; never join by row position. Record the +source codes, allocation literals, status, knownness, source year and survey year. + +The fragment exposes a source projection artifact, a CREATE source population, +literal code columns, and nine named per-field recodes. CREATE requires the +existing `survey_population_source` reference from the original source registry; +it introduces no additional source file or issuer. A post-PUF attachment follows the retained +original-person link to both clones, verifies native channel/ID and clone roster, +and adds nullable booleans plus provenance. This stage has no fitted model. +Future disaggregation models need separate declarations and assessment. + +One fixed country enrichment host assembles the amount and health fragments over +the original checked PUF handle. The health fragment receives the explicit +receiving version and the amount attachment's typed artifact edge. Its ordinary +attachment preserves all incoming columns, including amount unknowns. It does +not return a separate whole-population branch or issue a public health handle. + +Portable values do not issue authority. The owning consumer must requalify the +retained parent immediately before consumption and after its last relevant I/O, +then seal the complete successor before returning it. The fragment checks its +retained physical values around each kernel callback; callback success itself +grants no source authority. The host owns source and kernel registries, typed +artifact/store bindings, replay checks, and complete successor verification. + +## Bounded verification + +The 31-case suite passes over invented records. It exercises the real source +catalogues, preparation issuers, original-member qualifier and requalification; +literal yes/no/allocation and unknown handling; exact source-coordinate joins; +clone identity and collision refusal; all prior amount-column preservation; +and a 13-node actual graph cold run plus required replay. Independent complete +population reconstruction matches every observed health-node output. The graph +fixture's qualified value is explicitly descriptive; it does not claim a new +source admission. A separate test obtains the qualified value through the real +issuer path using privately pinned invented source bytes. + +The source-v3 guarded run closed with result 0: 20.61 seconds wall, 19.95 seconds +CPU and 490.9 MB peak RSS. Earlier failed fixtures remain preserved, including +the origin-coordinate type mismatch exposed by the real qualification test. +No native microdata, country engine, public upload or model benchmark was used. +This verifies the fragment, not a launchable population. Actual-source replay, +the combined host's final retained-parent fence and release coverage checks +remain its consumer's responsibilities; the seven unresolved ACS concepts stay +explicit gaps. diff --git a/experiments/us-financial-default-2-controls-20260909.json b/experiments/us-financial-default-2-controls-20260909.json new file mode 100644 index 000000000..1d3d92fef --- /dev/null +++ b/experiments/us-financial-default-2-controls-20260909.json @@ -0,0 +1,27 @@ +{ + "code_resources": 12, + "cpu_seconds": 93.988173, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 526712832, + "publisher_provenance_established": false, + "receipt_sha256": "d2775bd9e1c1e14dabaa9b7162070310fdd00adc8ea996084932692bce8f9dc9", + "release_eligible": false, + "result": 0, + "scope": "Two existing default financial predictor cold/required replay cases, with the accepted optional demographic implementation left disabled. Invented original sources only; no native/quality/release acceptance.", + "source_and_owned_files": 495, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "91c1a167dc6f188721e226623f3f92d3b0c57a4bf55af5f880f146cbe7c0596a", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "1c49e18612176337ab95170bb862578262aa702926148647db34e88154b24e15", + "packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py": "99d8062331165a72ee3dbad4812676cb240080de066102ccde7a2512f570e302" + }, + "unexpected_refusals": {}, + "wall_seconds": 98.86093041597633 +} diff --git a/experiments/us-financial-demographics-3-controls-20260909.json b/experiments/us-financial-demographics-3-controls-20260909.json new file mode 100644 index 000000000..fdfd17581 --- /dev/null +++ b/experiments/us-financial-demographics-3-controls-20260909.json @@ -0,0 +1,27 @@ +{ + "code_resources": 12, + "cpu_seconds": 111.759989, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 528891904, + "publisher_provenance_established": false, + "receipt_sha256": "37b9c1ceeefe38028772d8e6de64d10728c5fbcc5c6339549bb3b07c1513a6c0", + "release_eligible": false, + "result": 0, + "scope": "Three invented financial-demographic qualification, graph cold/required replay, unknown/default compatibility, and late detached-projection mutation controls. No fit quality, native admission, or release acceptance.", + "source_and_owned_files": 495, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 3 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "91c1a167dc6f188721e226623f3f92d3b0c57a4bf55af5f880f146cbe7c0596a", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "1c49e18612176337ab95170bb862578262aa702926148647db34e88154b24e15", + "packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py": "99d8062331165a72ee3dbad4812676cb240080de066102ccde7a2512f570e302" + }, + "unexpected_refusals": {}, + "wall_seconds": 114.70848075003596 +} diff --git a/experiments/us-financial-fixture-profile-20260910.json b/experiments/us-financial-fixture-profile-20260910.json new file mode 100644 index 000000000..964ba4c2e --- /dev/null +++ b/experiments/us-financial-fixture-profile-20260910.json @@ -0,0 +1,248 @@ +{ + "accepted": true, + "approval_sha256": "6e53647ed6bb1af53c120b2c8f105f3517d6264476bb767547ae2dda1cf69bc7", + "cpu_seconds": 303.897467, + "declared_invented_dimensions": { + "atomic_support_blocks": 4, + "clone_households": 12, + "clone_persons": 18, + "counts_are_declared_from_frozen_source_not_native_measurements": true, + "demographic_predictor_count": 5, + "financial_output_count": 7, + "fixture_function": "known_financial_run", + "fixture_source": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "graph_nodes_per_run": 20, + "n_estimators": 2, + "origin": "invented_original_source_fixture", + "runs": [ + "cold_auto", + "required_replay" + ], + "source_households": 6, + "source_persons": 9, + "state_fips_count": 2 + }, + "fixture_acquisition_completed": true, + "fixture_original_teardown_passed": true, + "guard_sha256": "e7972505720ee6ede8d3859e28e2c9171d4f568d477d90670371ca17a36167d1", + "native_acceptance": false, + "native_reads_stats_hashes": false, + "next_step": "Review and test per-producer code-comparison memoization without removing live globals, aliases, closure checks, source reads or final I/O seals. No optimization or larger run accepted by this diagnostic.", + "partial_snapshot_needed": false, + "peak_rss_bytes": 594296832, + "peer_source_review_sha256": "6b4614e182df7fa519a1079a6c5d99ded55bbdc45f021b46908e041c8895f4af", + "profile_scope": "fixture_acquisition_only", + "profile_sha256": "badec8594bb1421483e0090f273f79f7560adc0b13b68434f089442c7c8d4bbb", + "profiled_implementation_sha256": { + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py": "db7aaae06961cc1ed9ce13b28dac74d8fd1041b9cd4c32294d2b11105fbef7af", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "06776801268115604cd3e544381a3e4ce81588876fd10bb9a8374bab03ba7f53" + }, + "profiled_seconds": 298.15339916600004, + "protocol": "microcosm.us.financial-fixture-profile-evidence.v1", + "receipt_sha256": "784d46a3a5ed3994f1018f6d05900122abdc6f63a0f18e318002fedfe7059af9", + "release_acceptance": false, + "runtime_result": 0, + "scope": "invented_financial_fixture_profile", + "successor_acceptance": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "timing_limitations": [ + "cProfile overhead changes elapsed time and can trigger the unchanged limits", + "partial snapshots omit unfinished active-call time until profiler flush", + "only exact frozen-source coordinates are reported; external time remains within cumulative callers", + "cumulative function times overlap and must not be summed" + ], + "top_functions": [ + { + "calls": 1, + "cumulative_seconds": 298.15339916600004, + "file": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "function": "known_financial_run", + "line": 19, + "recursive_calls": 0, + "self_seconds": 0.004364749 + }, + { + "calls": 2, + "cumulative_seconds": 297.206106625, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py", + "function": "run_atomic_survey_financial", + "line": 538, + "recursive_calls": 0, + "self_seconds": 0.0062629930000000006 + }, + { + "calls": 110, + "cumulative_seconds": 212.956586461, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_validate", + "line": 1071, + "recursive_calls": 0, + "self_seconds": 0.004810231 + }, + { + "calls": 108, + "cumulative_seconds": 209.131314091, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_checked", + "line": 1091, + "recursive_calls": 0, + "self_seconds": 0.0005602070000000001 + }, + { + "calls": 652, + "cumulative_seconds": 180.19579690400002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_producer", + "line": 129, + "recursive_calls": 0, + "self_seconds": 5.073519417 + }, + { + "calls": 14, + "cumulative_seconds": 161.371121044, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py", + "function": "reconstruct_atomic_survey_geography", + "line": 307, + "recursive_calls": 0, + "self_seconds": 0.007554145000000001 + }, + { + "calls": 38464, + "cumulative_seconds": 151.50668402300002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_live_code", + "line": 51, + "recursive_calls": 0, + "self_seconds": 9.209266879000001 + }, + { + "calls": 8, + "cumulative_seconds": 133.111351877, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py", + "function": "qualify_current_survey_predictors", + "line": 304, + "recursive_calls": 0, + "self_seconds": 0.006935434000000001 + }, + { + "calls": 2, + "cumulative_seconds": 115.295601876, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py", + "function": "run_atomic_survey_population", + "line": 213, + "recursive_calls": 0, + "self_seconds": 0.001364704 + }, + { + "calls": 6, + "cumulative_seconds": 112.51926329300001, + "file": "packages/microcosm-graph/src/microcosm/graph/executor.py", + "function": "run_graph", + "line": 2158, + "recursive_calls": 0, + "self_seconds": 0.008178516 + }, + { + "calls": 15, + "cumulative_seconds": 109.250808543, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py", + "function": "qualify_current_survey_geography", + "line": 220, + "recursive_calls": 0, + "self_seconds": 0.0037998010000000002 + }, + { + "calls": 348, + "cumulative_seconds": 103.018557294, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_producer", + "line": 102, + "recursive_calls": 0, + "self_seconds": 0.14776667400000001 + }, + { + "calls": 2009108, + "cumulative_seconds": 95.553654135, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "check", + "line": 86, + "recursive_calls": 51772, + "self_seconds": 91.045168105 + }, + { + "calls": 8, + "cumulative_seconds": 93.21181225100001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py", + "function": "_initial", + "line": 355, + "recursive_calls": 0, + "self_seconds": 0.0020371490000000003 + }, + { + "calls": 150, + "cumulative_seconds": 92.18578382800001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "verify_acs_native_coverage", + "line": 394, + "recursive_calls": 0, + "self_seconds": 0.011203179 + }, + { + "calls": 116, + "cumulative_seconds": 69.695211884, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_checked", + "line": 406, + "recursive_calls": 0, + "self_seconds": 0.0037891110000000004 + }, + { + "calls": 4, + "cumulative_seconds": 66.760753874, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py", + "function": "_qualified", + "line": 269, + "recursive_calls": 0, + "self_seconds": 0.000840784 + }, + { + "calls": 110, + "cumulative_seconds": 66.121036298, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "verify_acs_source_catalogue", + "line": 430, + "recursive_calls": 0, + "self_seconds": 0.00016753900000000002 + }, + { + "calls": 23, + "cumulative_seconds": 49.949269957000006, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py", + "function": "qualify_current_asec_demographics", + "line": 194, + "recursive_calls": 0, + "self_seconds": 0.0047751980000000005 + }, + { + "calls": 2, + "cumulative_seconds": 41.477013541000005, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py", + "function": "run_authenticated_survey_population", + "line": 815, + "recursive_calls": 0, + "self_seconds": 0.0007954430000000001 + } + ], + "unexpected_refusals": {}, + "verified_after_run": { + "code_resource_files": 12, + "model_source_files": 5983, + "source_and_owned_files": 500 + }, + "wall_seconds": 308.2577062920318 +} diff --git a/experiments/us-financial-successor-positive-1-limit-stop-20260909.json b/experiments/us-financial-successor-positive-1-limit-stop-20260909.json new file mode 100644 index 000000000..c7b05b458 --- /dev/null +++ b/experiments/us-financial-successor-positive-1-limit-stop-20260909.json @@ -0,0 +1,34 @@ +{ + "all_expected_source_code_and_model_pins_match_after_stop": true, + "automatic_retry": false, + "code_resources": 12, + "completed_test_count": null, + "configured_cpu_seconds": 600, + "configured_wall_seconds": 900, + "control_accepted": false, + "expected_test_count": 1, + "final_runtime_receipt_written": false, + "guard_sha256": "d65bd3f82a64791a0720dab30a2938456a90757b6c9aa6cfa4a30662edab1b6b", + "in_process_final_verification_completed": false, + "model_source_files": 5983, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_inputs_read": false, + "native_threads": 1, + "observed_cpu_seconds": null, + "observed_peak_rss_bytes": null, + "other_selections_executed": false, + "process_exit_code": 152, + "release_eligible": false, + "resource_map_sha256": "c6a557ab9d9231457a7c9f0d1c4a19d2ea354bcdeb7063812ddc88a3b3cf6ef4", + "result": "incomplete_resource_limit_stop", + "scope": "Positive invented financial successor cold/required graph, original budget, admitted financial ancestry, weight-only transition and unchanged default path. CPU-limit stop before final receipt; no pass or new successor/native acceptance. Next action is separately reviewed bounded profiling, not a budget increase.", + "source_files": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "termination_signal": "SIGXCPU", + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "06776801268115604cd3e544381a3e4ce81588876fd10bb9a8374bab03ba7f53", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py": "613bf949a73ed4d0a6c9f92f01fb48a64b426db2825d719ccf023e938c5c18d7", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "80418528db45266e7322a8984cc63f9414f6fa3c084119cd9d0f558f4a0e0260", + "packages/microcosm-build/tests/test_us_survey_financial_successor.py": "a42e7dadf3b77f6cf976db9c904243e787f8704bbc6d1a82bd820246f3c2e980" + } +} diff --git a/experiments/us-financial-successor-positive-20260910.json b/experiments/us-financial-successor-positive-20260910.json new file mode 100644 index 000000000..fa1a8d651 --- /dev/null +++ b/experiments/us-financial-successor-positive-20260910.json @@ -0,0 +1,41 @@ +{ + "all_successor_controls_passed": false, + "approval_sha256": "91efa2d737f38d6d6b11890241a1a9b5dbb1c6840da3a49f58966101e3f967ef", + "cpu_seconds": 647.185773, + "fixture_original_teardowns_passed": true, + "guard_sha256": "a9566eb1783d71dd1fc3c0468e2f1af763fbd5ed218f0a87d2cbca3ae1d4011b", + "interpretation": "The original financial replay and financial-to-weight-only successor behavior passes; remaining refusal/callback cases and native calibration admission are pending.", + "junit_sha256": "2af08e54f972192d299281c992dd4306dde99399699c4684037ad6bf60804d9a", + "limits": { + "cpu_seconds": 1200, + "numerical_threads": 1, + "wall_seconds": 1800 + }, + "memo_or_profiler": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "original_acs_owner_sha256": "db7aaae06961cc1ed9ce13b28dac74d8fd1041b9cd4c32294d2b11105fbef7af", + "peak_rss_bytes": 570884096, + "postcheck_sha256": "8698b43174b17faa1bcc4123ae1b7c7c1ea65bba94f5d41f207c259dc3c951e8", + "receipt_sha256": "96819afd6805e5cbcd6dfb2271ec1d8c4c3b72775135904a9efe1d97cbb79791", + "release_acceptance": false, + "remaining_controls": 7, + "resource_map_sha256": "088a55cd36f83e4b466947a216b0315a983d5888c603103e575f0cc7ae2ece38", + "schema_version": 1, + "scope": "one_invented_financial_successor_positive_control", + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "one_positive_control_accepted", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 499 + }, + "wall_seconds": 651.2133005840005 +} diff --git a/experiments/us-financial-successor-remaining-controls-20260910.json b/experiments/us-financial-successor-remaining-controls-20260910.json new file mode 100644 index 000000000..3180e591b --- /dev/null +++ b/experiments/us-financial-successor-remaining-controls-20260910.json @@ -0,0 +1,61 @@ +{ + "all_successor_controls_passed": true, + "approval_sha256": "9e4ed7c7924b130212a4580fa4ee4298bf7ea33d2ae3e5f2e9798226fb07baee", + "cpu_seconds": 798.6069409999999, + "guard_sha256": "51b3a5adad3c14e11a2f69b8566702abe5aecfffbc1c315dd6f08c412c5e671e", + "interpretation": "Together with the accepted positive control, all eight original financial successor cases pass. Native artifact replay, calibration admission in the completed graph and release verification remain separate.", + "junit_sha256": "6cffe5cac9baca685a7c2c0b032580edf83059bd653d4695b08aab75172d7a8c", + "limits": { + "cpu_seconds": 2400, + "native_threads": 1, + "qrf_fit_n_jobs": "1", + "qrf_predict_workers": "1", + "rss_monitor": false, + "rss_reporting": "final peak only", + "wall_seconds": 3600 + }, + "memo_or_profiler": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "original_fixture_and_teardowns": true, + "peak_rss_bytes": 563593216, + "peer_source_review_sha256": "12a541b3bcf620f0d931815f40f6b9fffac36e59064222c071c8128b9bdbab7d", + "positive_evidence": "us-financial-successor-positive-20260910.json", + "postcheck_sha256": "423a6cd3ae1537bfe372b7d735ad3a907e6fcc6837257e6c65d538db18e13486", + "receipt_sha256": "e394516086ca0f4c773764027ad2d7270a8a9d82d038873f0640cd49e3db483d", + "release_acceptance": false, + "resource_map_sha256": "63de09c280f733c91d07e674bf753c3017b7a375901623a2200a8538f835a4a2", + "root_source_review_sha256": "db0d7f3424cd85bcbc58c690d375d0ef58c557e8d4fefad3dddc666101143c12", + "schema_version": 1, + "scope": "seven_unchanged_invented_financial_successor_refusal_and_callback_controls", + "selected_test_cases": [ + "test_us_survey_financial_successor.py::test_unissued_copy_and_wrong_original_budget_refuse", + "test_us_survey_financial_successor.py::test_arbitrary_nonweight_changes_still_refuse", + "test_us_survey_financial_successor.py::test_rehashed_graph_artifact_cannot_replace_issued_draw", + "test_us_survey_financial_successor.py::test_final_budget_support_borrow_cannot_mutate_financial_run", + "test_us_survey_financial_successor.py::test_final_support_borrow_cannot_forge_returned_view", + "test_us_survey_financial_successor.py::test_financial_borrow_cannot_relax_detached_budget_bounds", + "test_us_survey_financial_successor.py::test_run_borrow_cannot_forge_original_budget_digest" + ], + "source_count": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "seven_remaining_controls_accepted", + "subsequent_source_review": { + "covered_by_these_eight_controls": false, + "finding": "Budget and weight-only checked views were constructed before the last support I/O, exposing returned detached values to callbacks.", + "status": "additional_gap_under_correction" + }, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 7 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 499 + }, + "wall_seconds": 802.8923444160027 +} diff --git a/experiments/us-financial19-required-replay-20260912.md b/experiments/us-financial19-required-replay-20260912.md new file mode 100644 index 000000000..f0865222b --- /dev/null +++ b/experiments/us-financial19-required-replay-20260912.md @@ -0,0 +1,60 @@ +# Native financial graph required replay — 12 September 2026 + +The frozen nineteen-node financial graph passed a separate required-cache +replay on real ACS/ASEC inputs. This accepts replay of the earlier financial +pilot, not PUF enrichment, a calibration, a current-source population or release. + +## Exact scope + +- Source: `2ca11c85a0f710b8fa495e363243b940f7eabae2`. +- One collected test passed; outer guard result `0`. +- Request: source fraction `1/1000`, survey/geography seed `20260908`, two + trees, demographic conditioning enabled, `resume="require"`. +- All nine prefix nodes and all nineteen financial-graph nodes were cache hits. +- The graph, preparation, projection and recipient-matrix exports matched the + cold run byte for byte. Portable manifest content matched after removing only + start/end times and per-node timing/cache-hit fields. The complete artifact + report also matched, excluding its cache-hit and manifest-file identity fields. +- Source, resources, native inputs, owned files, baseline exports and thread + controls remained unchanged. There were no unexpected access refusals. +- Elapsed: `3981.09802937496` seconds; CPU: `3944.4073700000004` seconds; + peak RSS: `10988191744` bytes. Declared limits were 7,200 CPU seconds, + 10,800 elapsed seconds and 32 GiB peak RSS, with one numerical worker. + +The source population remains 1,584 households / 3,464 people, expanded to +3,168 households / 6,928 people before block assignment. The final stage carries +seven financial fields. The same frozen source produced the 11 September cold +pilot; no broader coverage follows from replay. + +## Evidence identities + +| Evidence | SHA256 | +| --- | --- | +| Closed replay guard receipt | `71d6981518acad0ad0fd199f906c01ccfa867796aaf053cf6fa5f5a3df1193ae` | +| Final replay manifest | `c8a1b7715bae740c183aa592fca718849e35edf6722328c1de3d83f0cacafa77` | +| Cold baseline manifest | `0d745678802d8dae16750f57ebe779644237e1b2416f1cb78edf4c59d2163f0e` | +| Reviewed guard | `5a2bcd05665644a86ff2f3fc35a2e7e751d759412380dce07193f9b0c6f3c670` | +| Manifest content key, both runs | `134ab4ea39b434066e48258be345999425a6d23f3b5016680f457fd357be9edf` | + +Retained local evidence is in +`PolicyEngine/_recovered/pilot-runs/native19-required-20260912/run`: +`us-native-postclone-financial19-required-20260912.json`, +`native-financial-manifest.json`, and the reviewed harness and helper. +The coordinator's metadata postcheck is +`PolicyEngine/_recovered/scratch-backup/893/codex-takeover-20260912/native-required-replay-postcheck.json`. +It rechecked receipt consistency, exact owned-file hashes, reported bounds, +node hits and export identities without reading native data or generated Frame +payloads again. The actual payload comparison and final source/output checks ran +inside the reviewed native harness before its successful guard closure. + +The guard's expected optional dependency probes remain recorded separately from +unexpected refusals. The run enabled no engine, network, child processes or +automatic retry. Original cold and failed-attempt evidence remains retained. + +## Next acceptance + +The maintained integration includes later runtime repairs and the restored +PUF55 host. Its native candidate needs a fresh cold execution, required replay, +complete input coverage and full Frame export/readback on that identified source. +Calibration and complete-population acceptance follow separately. This replay +does not authorize reuse of the serialized receipt as a live population issuer. diff --git a/experiments/us-input-coverage25-20260910.json b/experiments/us-input-coverage25-20260910.json new file mode 100644 index 000000000..6ff56ed08 --- /dev/null +++ b/experiments/us-input-coverage25-20260910.json @@ -0,0 +1,125 @@ +{ + "api": { + "assigned_block_diagnostic_retained": "census_block_geoid", + "diagnostic_module": "packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py", + "employment_income_last_year_in_required_profiles": false, + "existing_release_gate_or_manifest_modified": false, + "historical_required_us_inputs_default_count": 163, + "national_cd_engine_omissions": [ + "block_geoid", + "tract_geoid" + ], + "national_congressional_district_diagnostic_default_count": 161, + "profile_module": "packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py", + "report_protocol": "microcosm.us.input_coverage_diagnostic.v1", + "signature": "diagnose_us_input_coverage(population: Population, *, compiled: CompiledGraph, manifest: RunManifest, profile: USInputProfile = USInputProfile.NATIONAL_CD) -> PopulationInputCoverage" + }, + "code": [ + { + "after_sha256": "88a8456ad02846d8f41af7ead743c86caa8d82b9b2c18d196feee663acb02a33", + "before_sha256": null, + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py" + }, + { + "after_sha256": "34455450437b5e4b7ce0377522d2bf2d815ac0a9d6e554b25c25d690e196e318", + "before_sha256": null, + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py" + }, + { + "after_sha256": "7271ca0197d1396dc42656f3220207b0ba96927c1cd11233543f72fcf57ce94d", + "before_sha256": null, + "path": "packages/microcosm-build/tests/test_us_population_input_coverage.py" + } + ], + "description": "Descriptive input coverage on a supplied actual graph Population and attached RunManifest; complete ownership, storage, origin/clone and writer-mask checks.", + "frozen_evidence": { + "cpu_seconds": 6.143905999999999, + "guard_sha256": "0fb8e3f9f25ec47d89d6a3f61c8483caa86ccdee83958f7e801a8be579c4a559", + "models": 0, + "peak_rss_bytes": 332939264, + "physical_postcheck_sha256": "017ccd3d1d97819c1d6332436449e0ba961a1873ce6962547be07743fe097fc7", + "receipt_sha256": "8fed087e9f44d87d47a2c3071653255c959cbfee53ca1166fd22994025ca8d43", + "resources": 0, + "selectors": { + "test_actual_masked_coverage_separates_origin_clone_grain_and_unknownness": 1, + "test_closed_profiles_match_manifest_without_changing_historical_default": 1, + "test_complete_current_ownership_refuses_wrong_carrier_or_writer": 5, + "test_declared_absence_is_not_an_applicability_exemption": 1, + "test_duplicate_grains_are_rejected_by_the_actual_frame_contract": 1, + "test_inconsistent_population_manifest_or_compiler_refuses": 5, + "test_last_manifest_borrow_mutation_refuses": 1, + "test_nonfinite_values_and_missing_grains_remain_unresolved": 1, + "test_population_version_must_be_an_actual_structural_version": 2, + "test_profile_omissions_do_not_omit_assigned_block_diagnostic": 1, + "test_required_replay_preserves_coverage_and_manifest_identity": 1, + "test_source_role_and_membership_are_checked_on_actual_rows": 3, + "test_unclaimed_physical_column_is_not_accepted_as_a_structural_carrier": 1, + "test_zero_weight_origin_and_zero_observations_remain_known": 1 + }, + "source_and_owned_physical_checks": 515, + "source_count": 507, + "source_map_sha256": "55af74ead417eeab1b0e977da24e91ed5de61e52cf5953cd060797f58bc40038", + "source_peer_proof_sha256": "e1b8ae0cadd118ca7d4cfd85af2f4fd707462d87ace779c94277a0ee87fbc4a3", + "source_peer_review_sha256": "5a3177a15b478da0a8dcda6030397483892b60ae4cfd7481aa10a4c889ad59e9", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 25 + }, + "unexpected_refusals": {}, + "wall_seconds": 6.524979708017781, + "xml_sha256": "e5ea4cc97ceddbd3ad9153b35ab14dd2e7eb3ff13e0d878972f85e9a3e72ade8" + }, + "limitations": { + "actual_source_ancestry_verified": false, + "applicability_complete": false, + "calibration_modules_rolled_back": false, + "complete_population_input_coverage_proven": false, + "host_integration_added": false, + "native_build_or_model_evaluation_verified": false, + "release_eligible": false, + "statistical_signal_verified": false, + "supplied_graph_consistency_only": true + }, + "maintained_integration_execution_verified": false, + "maintained_source_comparison": { + "baseline_eager_modules_compared": 59, + "comparison_is_not_recomputed_current_import_closure": true, + "different_calibration_imports_preserved": [ + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "397f02916de1535de3bd0c34fdd71a2e685fe37f1a61ff4d389a8343f9642d4b", + "maintained_sha256": "716018cfbb136c631ce56724e58048c94202d6d41b64a56c9ba901cc586fd253", + "module": "microcosm.calibrate", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py" + }, + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "c5084a0dd45617922f8efca08560f243195eb4456430c1d87258e6240f379ab4", + "maintained_sha256": "8a400c46e4c94773dc0d83122f173c8c3bd405694d411989b10f792597b94196", + "module": "microcosm.calibrate.exact_k", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/exact_k.py" + }, + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "379e4adbb3ed7cdd13f7c7c548da45163f2a50330c874ee51095d38243d392e6", + "maintained_sha256": "14608a5ee02132f06eaad4a59de3b051c8b8e1d07bd1768f4eb57f237173cef7", + "module": "microcosm.calibrate.gates", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/gates.py" + }, + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "c04df2f61544d4ba83ad467365333a69528ccf9b78913d449539603cd848b6cc", + "maintained_sha256": "925a1b6bae1c714de93df3ee44e26afeed7f3912377b93942f3a5173281d851a", + "module": "microcosm.calibrate.solve", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py" + } + ], + "identical": 53, + "maintained_CI_pending": true, + "new_owners": 2 + }, + "proposed_adoption_base": "151166377036a6f4703ed8effdcb328c7699e416", + "status": "25_frozen_scoped_controls_accepted" +} diff --git a/experiments/us-integrated-source-acceptance-20260912.md b/experiments/us-integrated-source-acceptance-20260912.md new file mode 100644 index 000000000..c01643a54 --- /dev/null +++ b/experiments/us-integrated-source-acceptance-20260912.md @@ -0,0 +1,72 @@ +# Integrated US source acceptance, 2026-09-12 + +The accepted current-income routing, interest and allocation-source changes +compose with the accepted fiscal target-snapshot host. This is an invented-source +integration check, not a native build, model evaluation, or release verdict. + +The integration preserves the full histories of current main +`a9cc63e737fd4619a304117dc2ec7dcd86d97901`, the income source branch +`4a8f1d9edeacc99a36a9b2e43175081a8757f0fa`, and the fiscal snapshot branch +`13791d75a7fce94d12697b8d5ce9bdf46f83ebab`. The merges needed no production +source edits: each runtime/calibration source difference from one accepted +parent is byte-identical to the other parent. In particular, the routing +NamedTuple declarations, private interest-reader seam, and unpublished farm +allocation-code distinction coexist. + +## Runtime classification + +The source-spine inventory now includes twelve already reviewed graph modules. +Seven qualify source observations, validate original-person joins, or declare +those joins' inputs: ACS income anchors, ASEC interest, ASEC unemployment, +current survey amounts, the health source and recode/attachment helper, and the +health graph fragment. These receive explicit source-provenance ownership +comments. Neither attachment uses source channel to choose a PUF-detail model; +it verifies the source-person identity and attaches to the same two clones. + +Fiscal measurement, PUF route attachment, the PUF composition host, and the +combined amount/health host retain source-spine scanning. Their reviewed dynamic +selectors index exact geography fields, numeric masks, typed artifact names, +producer keys, or retained node histories. Sixteen direct counterexamples verify +that provenance columns and factories still fail in each of those four modules. +The dense fiscal solver needs no exception. Scanner functions and classes, +registered population operators, and the physical source-accessor gate remain +unchanged. + +## Validation + +The frozen combined suite passed **138 tests**: routing 88, interest 40, and +allocation meaning 10. The first launch stopped during collection because the +handoff's routing count was 64; no tests executed in that attempt. The second +launch corrected only the expected count and owned scratch paths, preserving +all test and production bytes and the existing audit-control functions. + +The passing run took 216.86 seconds wall time and 211.50 seconds CPU, with +405,897,216 bytes peak RSS. All 1,011 source/owned hashes and 15 resource hashes +were unchanged; there were no unexpected refusals and Torch thread counts were +1/1. Limits remained 300 CPU seconds, 600 wall seconds, 8 GiB RSS, no children, +no network and no country engine. The existing denied optional timezone-resource +probe produced the sole warning. Native inputs remained inaccessible. + +The exact passing receipt SHA256 is +`21d4c5723558ad539c2bee4a70d3296b004dea2b2fe951be599a1c07f5b680f6`; +JUnit SHA256 is +`c9d6bd0420417ca6d246b25fbd48701374877a4f2924504e245533c91bf16d44`. +Only the source-spine test roster changed after this frozen source run; runtime, +resource and selected source-test bytes did not change. + +The seven existing runtime source scans first demonstrated the unclassified +modules and source-owner/dynamic-selector findings. Their other four checks +already passed. The full scanner is CPU-heavy in the existing +`atomic_block_support.py`; its first 120-second capped attempt was retained +without claiming a result, and the completed red run took 260.66 seconds. The +classification changes do not optimize or weaken this scanner. All seven green +checks passed in 238.44 seconds, with unchanged roster-test bytes during the +run. An observational helper wrapper recorded module names, timing and the exact +returned findings; it did not alter scanner inputs or findings. + +Local evidence is retained under +`codex-integrated-source-acceptance-20260912`: source-v1 preserves the count +refusal, source-v2 holds the frozen passing acceptance, and separate roster +receipts, source-equivalence proof and direct counterexamples delimit the test +inventory change. Fiscal numerical/callback acceptance remains the separately +reviewed 13791 parent evidence; no duplicate fiscal or PUF fixture was run here. diff --git a/experiments/us-launch-consolidation-20260913.md b/experiments/us-launch-consolidation-20260913.md new file mode 100644 index 000000000..0856abcd0 --- /dev/null +++ b/experiments/us-launch-consolidation-20260913.md @@ -0,0 +1,173 @@ +# US launch integration: 13 September 2026 + +This batch advances the checked survey graph, adds completion diagnostics and +repairs the identified CI failures. It does not certify a native release file. + +## Property income and tax inputs + +The opt-in 35-node host combines qualified ACS/ASEC property observations, +four conditional component models, signed reconciliation and clone attachment. +Its cold construction, required replay and retained-population checks pass. +The separately repeated PUF-recipient handoff passes after fixing an invalid +invented source record. The original failed attempt remains recorded in +[the 35-node host evidence](us-property-financial-host-20260912.md). + +The next opt-in adds three nodes: a receiving population version, four tax-leaf +rewrites, and a numerical gate. All 11 integration controls pass over actual +invented source owners, including cold construction and required replay of the +38-node graph. The run took 645.283 seconds wall time and 0.567 GB peak RSS. +Forty smaller numerical and declaration controls also pass. + +Ordinary interest supplies taxable and tax-exempt interest; dividends supply +qualified and non-qualified dividends. The maintained split fractions remain +explicit assumptions. Retirement-account earnings stay separate. Unknown +components stay unknown, including under-15 observations outside survey +reporting universes. Full preceding populations remain available and checked. +An incomplete numerical gate refuses PUF source qualification. A complete-source +PUF run through this extension remains unverified. See +[the host contract](../docs/us-atomic-property-financial.md) and +[the tax-leaf contract](../docs/us-property-tax-leaves.md). + +The host receipt SHA256 is +`6a1e5798d2182e25eb08ac6e073c71cec3a31c03f1f647df3b8d131222f1cb02`. +All frozen source and resource hashes remained unchanged; no country engine, +native source or full PUF run was executed in that test packet. + +## CI repairs + +- **Retained observations:** final checks now compare the complete detached + observation Frame with its verified manifest Frame after the last artifact + I/O. The previous test reproduced an escaping mutation. Five controls now + pass: actual cold/required cloning and late mutations of either copy with + both return modes. No source revalidation or ownership check was removed. +- **Country identities:** the AM, BE and UK expected fingerprints now include + the already reviewed solver snapshot changes. Replacing only that source + digest in a detached envelope reconstructs all three old fingerprints + exactly. Four controls pass; authored country resources are unchanged. +- **Installed wheels:** `microcosm-build[source-io]` declares h5py and PyTables + without a country engine. CI installs the built extras after checking base + imports and verifies an invented HDF read/write round trip. A fresh local + six-wheel installation passes these checks with no US or UK engine. Locked + dependency versions are unchanged. This smoke test does not replace the + full remote wheel suite. + +The following CI run exposed two remaining literal lockfile pins: the seed +diagnostic and Primary-QRF worker identity still expected the pre-extra lock. +Both now reference the reviewed lock. Two diagnostic controls and six worker +lock controls pass, including refusal of incorrect pins and preservation of +the explicitly named historical campaign boundary. The six worker controls +first reproduced three expected failures against the old constant. All 125 +locked package version/source records remain unchanged; only the source-I/O +optional metadata changed. Full remote CI is still pending. + +## Other integrated source work + +ASEC retirement qualification retains raw slot and aggregate observations, +including the annuity not-in-universe literal. Its comparison follows the +declared normalization without rewriting the source; 62 controls pass. +Retirement measurement and regularity assumptions still require an explicit +bridge before they can supply complete tax inputs. + +The conservative retirement candidate ledger now passes 80 focused controls +and independent review. It preserves source observations, distinguishes +assumption-dependent candidates from identified amounts, and refuses an +aggregate interval when applicable source evidence is unreadable or fails its +accounting checks. It supplies no fiscal inputs. An under-58 zero requirement +for the main DBTN aggregate remains unsupported by the qualified source +contract; its raw value and residual remain diagnostics. See +[the retirement ledger](../docs/us-current-asec-retirement-basis.md). + +An independent property-observation test confirms that excluding an ASEC record +from joint model fitting preserves its separately known interest and dividends +on both clones. A new pure completion diagnostic passes 29 controls and +independent review. It distinguishes known components, ACS anchor decomposition, +missing evidence, source contradictions and unsupported under-15 measurement. +Counts and DESIGN support refer to original records, with union-household +support reported separately. No amount is filled by this diagnostic. + +The optional connection to the existing projection node now passes 27 focused +controls and an actual invented 38-node cold/required acceptance. The projection +adds a typed private row artifact and an allowlisted public aggregate receipt, +without adding a numerical node or changing the ordinary three-artifact +attachment path. The focused controls cover default-option compatibility, +source-function lifetime checks and the real public graph serializers. + +Three full-host acceptance tests share one cold/required pair and invoke five +committed host controls plus the independently reviewed native-harness helpers. +They verify complete populations and ledgers, the typed completion artifact, +private/public separation, full Frame export and disk readback after final source +checks, and refusal of a changed retained artifact. The nine original people +become eighteen clone rows. Cold execution reuses nine prefix nodes; required +replay reuses all 38 nodes with the same manifest. The tax gate reports +`numeric_verified=true` and `complete=false`; it still refuses PUF admission. +The full exported Frame is 129,243 bytes. This is invented input evidence, +not a native coverage measurement. + +The run passed in 806.892 wall seconds and 793.124 CPU seconds, with peak RSS +585,924,608 bytes. All 1,043 frozen source, eight owned-file and fifteen resource +identities remained unchanged. There were no unexpected access refusals or +child processes. Root and independent review accepted source commit +`c06d29ea8f39a820f9055889c89ccf0f3a9e2511`; receipt SHA256 +`05b75c2668889a708c935a72e2a5381f68b86f676f03f7ac7a6722efd019f4d2` +and test XML SHA256 +`39b7e8701ff0674a6c1b156eeef9f81d6f571072345ca3aaa39e7427e5ffb46f` +bind this acceptance. The entire host test module was not rerun in that packet. +See [completion routing](../docs/us-property-completion-routing.md). + +The UK household-lineage helper preserves exact original identities through +declared selection and cloning. Forty-three controls pass, including large +integer IDs and graph operations. This is a supplied-lineage contract; it +does not certify a new native UK graph or replace the UK country work. + +## Hashing checks and measured scope + +The latest hashing changes preserve exact digests in 267 controls. Two additional +tests exercise actual invented source preparation and requalification, including +refusal after a retained Frame changes during the final source I/O. These pass +with no changed source/resource pins or unexpected access refusals. + +Six isolated processes compared the old and new helpers on 207,692 invented +rows, with three repetitions per process. The paired digests match exactly. +Median helper times were: + +| Helper | Before | After | +| --- | ---: | ---: | +| Population part streaming | 29.17 ms | 29.34 ms | +| Whole-series storage selection | 26.20 ms | 25.60 ms | +| Exact Python-float framing | 55.10 ms | 23.75 ms | + +The float helper is about 2.32 times faster in this workload. The other two +show little timing difference. Peak RSS differs by less than about 1.1 MB in +each pair; these runs establish no meaningful memory reduction. Helper timings +exclude input construction, while peak RSS includes imports and construction. +This is not a native-build speed or memory estimate. All mutation checks and +digest framing remain active. + +The parity receipt SHA256 is +`30fba8c4e2c91042e06d177ce51e9077219c743b28180c210fa630600ac1a197`; +the independent qualified-owner receipt is +`ceaf0946c07e7f0d333ca1d2673ad2ffb63d7ab81de2a70e8e62c5020640923c`. +The frozen benchmark comparison document has SHA256 +`db8d881b0b3bbd710acaca4a3b850d13b9855cc294f7cc1cda6837d20ee9fce5`. + +## Native run and remaining work + +The preceding native PUF attempt ended without a final receipt or PUF manifest. +Its stopping cause is unknown. It provides no verified native PUF output. +A new attempt needs a separately reviewed packet and resource budget. + +The immediate native milestone is the property/tax graph with completion +diagnostics. Its original-record counts will quantify the missing-input routes +before selecting a completion method or attempting PUF qualification. The +current PUF allocation code does not establish child-income coverage: it uses +existing positive per-person component shares across receiving unit members, +with a first-row fallback when all shares are zero. The HEAD/spouse restriction +belongs to Social Security conditioning. There is no admitted dependent-return +to child-ownership match, so neither a zero fallback nor a claim that the PUF +already covers the gap has been adopted. + +Remaining release work includes honest completion of applicable unknown inputs, +full PUF composition, a small native end-to-end build, engine and calibration +evaluation, national and congressional-district quality checks, progressive +local scale, and full/compact exports. The +[US/UK release path](../docs/us-uk-release-path.md) tracks those gates. diff --git a/experiments/us-native-atomic-financial-cold-20260910.json b/experiments/us-native-atomic-financial-cold-20260910.json new file mode 100644 index 000000000..6a6d1d586 --- /dev/null +++ b/experiments/us-native-atomic-financial-cold-20260910.json @@ -0,0 +1,104 @@ +{ + "age_calibration_acceptance": false, + "approval_sha256": "837bf6c06fbe186f615080cd844ce0e92eecbad387e800f6d7cc4bfa02b673ae", + "artifact_identity_verification": "pinned manifest metadata and internal producer/key/size relations only; no graph store, export or snapshot reopen", + "collected_tests": 1, + "compiled_nodes": 20, + "composed_prefix_store_hits": 10, + "cpu_seconds": 6591.938158, + "cross_process_source_authority_granted": false, + "current_checkout_certified": false, + "date": "2026-09-10", + "entity_rows": { + "clone": { + "household": 3168, + "person": 6928 + }, + "create": { + "household": 1584, + "person": 3464 + }, + "financial": { + "household": 3168, + "person": 6928 + } + }, + "features": [ + "survey_predictor_age", + "survey_predictor_employment_income", + "survey_predictor_self_employment_income", + "survey_predictor_is_female", + "survey_predictor_state_fips" + ], + "financial_cold_nodes": 10, + "financial_successor_acceptance": false, + "guard_before_after_identity_count": 509, + "guard_sha256": "5bffe33e4c60055070bb9ab9a51ad2e87bfbf86fcd843e1da0d6407d53debda1", + "head_coordinate": "e0718150d062f0fb34702d1443c701f0e1ac59e9", + "implementation_overlays": { + "acs_fingerprint_sharing": false, + "budget_detached_view_correction": false, + "compilation_cache": false + }, + "limits": { + "automatic_retry": false, + "children": 0, + "cpu_seconds": 7200, + "engine": false, + "network": false, + "numeric_threads": 1, + "peak_rss_bytes": 34359738368, + "wall_seconds": 10800 + }, + "manifest_metadata_sha256": "2168b79ff60b481636b4dacf504ff70f4f1934b92359d86cb95ce1ae78987b7a", + "model_quality_established": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_identity_verification": "config/staging/manifest/guard before-after metadata; no postcheck payload read, hash or stat", + "native_input_bytes": 3735332980, + "native_input_count": 10, + "owned_columns": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response" + ], + "peak_rss_bytes": 11452743680, + "postcheck_script_sha256": "ae9b4dacefbfb06c7571c81a55a87f50c080b9e3c27e4c3b39f8e4f878a95d1e", + "puf_enrichment_acceptance": false, + "receipt_sha256": "aea67b38a7a9af17a87b3bdce9e31c8033bdf2d14f7f01191cc7ec12d0d43eae", + "release_acceptance": false, + "remaining": [ + "Required cached replay on this exact native source and graph", + "Actual two-route PUF55 fitting and single finalization/attachment", + "Model-input coverage and fit-quality checks", + "National and congressional-district calibration/holdouts", + "Full and pruned export verification with exact-file dashboard" + ], + "required_replay_run": false, + "resource_map_sha256": "62a6329a8d84ad96410581f4e5043c4b62fb648a7aa7737a2ee8eed61a80a7bc", + "root_postcheck_sha256": "50eabf02b476f2fdc55e15812e374fd76cf6002feb7fd60f10810e4d2122ac8c", + "runtime_result": 0, + "schema_version": "microcosm.scoped-native-control.v1", + "scope": "one_selected_native_survey_atomic_financial_cold_development_pilot", + "source_count": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "cold_development_pilot_accepted", + "targets": [ + "survey_current_INT_VAL", + "survey_current_DIV_VAL", + "survey_current_CAP_VAL" + ], + "test_outcomes": [ + "passed" + ], + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_safe_owned": 505 + }, + "wall_seconds": 6607.312281332968 +} diff --git a/experiments/us-native-atomic-financial-required-replay-20260910.json b/experiments/us-native-atomic-financial-required-replay-20260910.json new file mode 100644 index 000000000..60ddc0bf6 --- /dev/null +++ b/experiments/us-native-atomic-financial-required-replay-20260910.json @@ -0,0 +1,137 @@ +{ + "age_calibration_acceptance": false, + "approval_sha256": "3dbd7c5f0e939e803d292ece30eaeee2ea88b9b9adabd8534d22d8841fa3d343", + "artifact_identity_verification": "pinned manifest metadata and internal producer/key/size relations only; no graph store, export or snapshot reopen", + "cold_exports_unchanged": true, + "cold_root_postcheck_sha256": "50eabf02b476f2fdc55e15812e374fd76cf6002feb7fd60f10810e4d2122ac8c", + "collected_tests": 1, + "compiled_nodes": 20, + "composed_prefix_store_hits": 10, + "cpu_seconds": 4442.82255, + "cross_process_source_authority_granted": false, + "current_checkout_certified": false, + "date": "2026-09-10", + "entity_rows": { + "clone": { + "household": 3168, + "person": 6928 + }, + "create": { + "household": 1584, + "person": 3464 + }, + "financial": { + "household": 3168, + "person": 6928 + } + }, + "features": [ + "survey_predictor_age", + "survey_predictor_employment_income", + "survey_predictor_self_employment_income", + "survey_predictor_is_female", + "survey_predictor_state_fips" + ], + "final_manifest_store_hits": 20, + "financial_cold_nodes": 0, + "financial_successor_acceptance": false, + "guard_before_after_identity_count": 520, + "guard_sha256": "49c8971db347181c888e17b69d33efed987bf32ba3d4e93b2c06c283b4460b4c", + "head_coordinate": "e0718150d062f0fb34702d1443c701f0e1ac59e9", + "implementation_overlays": { + "acs_fingerprint_sharing": false, + "budget_detached_view_correction": false, + "compilation_cache": false + }, + "limits": { + "automatic_retry": false, + "children": 0, + "cpu_seconds": 7200, + "engine": false, + "network": false, + "numeric_threads": 1, + "peak_rss_bytes": 34359738368, + "wall_seconds": 10800 + }, + "manifest_metadata_sha256": "2d7816bf1260b72008a5c774a27534412fd2e4f72cc23effe457a2429dd76ff3", + "model_quality_established": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_identity_verification": "config/staging/manifest/guard before-after metadata; no postcheck payload read, hash or stat", + "native_input_bytes": 3735332980, + "native_input_count": 10, + "owned_columns": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response" + ], + "peak_rss_bytes": 11312398336, + "postcheck_script_sha256": "a6514601c1a7e9c28e65ecad0e363822a67e373f96fa9421a3b204afefc87b63", + "puf_enrichment_acceptance": false, + "receipt_sha256": "7669f91957b1610e56d3284e013f2c2c4a113b610322975c2cb8c92274a80650", + "release_acceptance": false, + "remaining": [ + "Actual two-route PUF55 fitting and single finalization/attachment", + "Model-input coverage and fit-quality checks", + "National and congressional-district calibration/holdouts", + "Full and pruned export verification with exact-file dashboard" + ], + "request": { + "demographic_conditioning": true, + "fraction": [ + 1, + 1000 + ], + "geography_seed": 20260908, + "n_estimators": 2, + "resume": "require", + "seed": 20260908, + "selection_request": { + "declaration": "experiments/us-survey-allocation-declaration-v1-20260907.md", + "fraction": [ + 1, + 1000 + ], + "protocol": "microcosm.us.survey-population-request.v1", + "seed": 20260908 + }, + "selection_request_bytes": 168, + "selection_request_sha256": "33f8ec1279a7bac6a27302bf6ea230e704cbcabe02d6f85cdc9b22edaec1f745" + }, + "required_replay_comparison": { + "actual_stored_attach_reconstruction_required": true, + "all_ten_prefix_nodes_store_hits": true, + "all_twenty_final_nodes_store_hits": true, + "cold_manifest_key": "c9d9e2e4d2c04a294ece9b2325929b4344732a7e833a20c58d25eb9e95104567", + "four_non_manifest_export_bytes_equal": true, + "portable_manifest_content_equal": true, + "stored_create_clone_and_terminal_column_identities_equal": true + }, + "required_replay_run": true, + "resource_map_sha256": "62a6329a8d84ad96410581f4e5043c4b62fb648a7aa7737a2ee8eed61a80a7bc", + "root_postcheck_sha256": "596f29b8fd8aa3d4e59e3df80f863d33c443b526de191d7b2c6a77ac82b62d3c", + "runtime_result": 0, + "schema_version": "microcosm.scoped-native-control.v1", + "scope": "one_selected_native_survey_atomic_financial_required_replay", + "source_count": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "required_replay_development_pilot_accepted", + "targets": [ + "survey_current_INT_VAL", + "survey_current_DIV_VAL", + "survey_current_CAP_VAL" + ], + "test_outcomes": [ + "passed" + ], + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_safe_owned": 516 + }, + "wall_seconds": 4450.337948458036 +} diff --git a/experiments/us-native-postclone-financial-incomplete-20260910.json b/experiments/us-native-postclone-financial-incomplete-20260910.json new file mode 100644 index 000000000..aeecacf9c --- /dev/null +++ b/experiments/us-native-postclone-financial-incomplete-20260910.json @@ -0,0 +1,22 @@ +{ + "automatic_retry": false, + "cpu_limit_seconds": 7200, + "exact_exit_signal_available": false, + "final_guard_receipt_present": false, + "guard_sha256": "f46e5dc061cbb50f7941891e2db8f7159ebe034f736280279b330605a40ad07b", + "interpretation": "The process is absent and its last sampled CPU exceeded the reviewed limit. Final verification did not complete; stored intermediate artifacts are not an accepted output. Retained progress samples identify repeated source/catalogue checks, but do not measure an isolated performance cause.", + "last_observed_cpu_seconds": 7216.063893, + "last_observed_wall_seconds": 7315.9110773339635, + "native_acceptance": false, + "next_step": "Review and test a semantics-preserving verification optimization before proposing another native run.", + "peak_rss_bytes": 11515363328, + "postcheck_native_payloads_reopened": false, + "published_financial_manifest_present": false, + "release_acceptance": false, + "resource_progress_sha256": "eaa7f4dc175f055d7b3e6997f9c47b2a9a3573e041412338a487ccba06fc7d7c", + "root_incomplete_observation_sha256": "b4c704e61cec4f00cc4ce0340cb8504eb592e4773a10b473c5e9cbcf497eb2b7", + "schema_version": "microcosm.scoped-native-control.v1", + "scope": "One cold 1/1000 native postclone nineteen-node financial pilot", + "status": "incomplete_without_final_guard_receipt", + "wall_limit_seconds": 10800 +} diff --git a/experiments/us-postclone-budget-replay-fence4-20260910.json b/experiments/us-postclone-budget-replay-fence4-20260910.json new file mode 100644 index 000000000..517507d08 --- /dev/null +++ b/experiments/us-postclone-budget-replay-fence4-20260910.json @@ -0,0 +1,52 @@ +{ + "code_adopted_on_top_of": "1cebe3547543bf2917a7af7362bfbb22a05bf964", + "code_changes": [ + { + "after_sha256": "4010b6518b295ec7ad18d65e8a4af0e5d0eaa09fddce60e46f70a197e4d79706", + "before_sha256": "b3a5596f3862c998a079e5d5368418df8169d3ddd262921632f587dbaab9c3fd", + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py" + }, + { + "after_sha256": "281be16a58d8d72e7ee00a3eb4e62632b7a210f9c3b914b3187e48c775927d16", + "before_sha256": "91b3719210ca11e342b1bb0798ca615457b890fbe26f6be4b69c6c6aacff843f", + "path": "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py" + }, + { + "after_sha256": "0f79edc1ab92e16998b41baf9f1735501ef875fddc92a049237699d982ed70a1", + "before_sha256": null, + "path": "packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py" + } + ], + "description": "Include the actual geography replay dependency in the budget producer fence and strengthen the preclone block counterexample while retaining valid postclone owners.", + "earlier_70_control_evidence_unchanged": true, + "guard_sha256": "7424ca92c6aca7e3ad2e76cbb24232c65e2bda6af6af23c8542a5c28c5b8f0ea", + "native_dataset_accepted": false, + "physical_postcheck_counts": { + "models": 0, + "resources": 12, + "source_and_owned": 513 + }, + "physical_postcheck_sha256": "301c7b2812f030fbd329641ae4ec90c32e91f7117fec2eb2b9387e17a6b3ddc9", + "receipt_sha256": "24d783cac659a3a76a166259d61ecdf83a4bfb95ee85da0101f5415ba9412fa2", + "release_eligible": false, + "resource_use": { + "cpu_seconds": 156.80812, + "peak_rss_bytes": 431046656, + "wall_seconds": 161.48273254197557 + }, + "source_map_sha256": "315a018c3a3521a172b3de48bfffd6abaa38a72d37a02e246ef0009b17809fbe", + "source_review_sha256": "60e949960540f69e234ab132789993d95dfa2451ba5a755cb39af4bb0bcab245", + "status": "four_scoped_controls_accepted", + "tests": { + "errors": 0, + "failed": 0, + "passed": 4, + "selectors": [ + "test_budget_producer_refuses_replaced_replay_dependency (3 parametrizations)", + "test_atomic_budget_refuses_preclone_assignment_candidate" + ], + "skipped": 0 + }, + "unexpected_refusals": {}, + "xml_sha256": "41334743e72be57619c50839304d10e48d67ade99e7c74710baa4a5d8335679e" +} diff --git a/experiments/us-postclone-geography-70-controls-20260910.json b/experiments/us-postclone-geography-70-controls-20260910.json new file mode 100644 index 000000000..1abe29a64 --- /dev/null +++ b/experiments/us-postclone-geography-70-controls-20260910.json @@ -0,0 +1,197 @@ +{ + "adopted_production_sha256": { + "packages/microcosm-build/src/microcosm/build/atomic_geography.py": "ed20cdec84eebe46fda5ae213522b6b6699812f04cc6699411c83f27df0ac82a", + "packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py": "9972f125566c4b697f8f862d236f6ae8cd3eece43fddb5d82df46552b0723496", + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py": "c70cfb2cf8f7915666846e563fd3d729fcb814244fc4bc3b3292d064fd0fdd40", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "ecb883827ca47c1fb1f2bf3ff25328706d0cc5c187d7914749b3ede4bd3ced31", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py": "9aef169897cb0f91d1414d607c5025e8c9d80bcc84c6eef269a27570aa8c12b0", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "d407086f05311e721e8961707e9471f5abffd74cef100eee2620cc75294492fc", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py": "e8a92fc299d6a388a0ef9ebf26d842460aefda9370006a4b1d929c78983cba0e", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "291285c35b19ff9dfccffc37ac2e768a72021db4196f3f662d86d26626790253", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "f7169ef0631a25acb4ca2fea0b49e105a5ca97d8fe85da2ecb4a5bdd5669cbeb", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py": "d4e80f94ba79c4bad4b47d18fcb5daad9469d45aeb81dea64fba101554080f91", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "b3a5596f3862c998a079e5d5368418df8169d3ddd262921632f587dbaab9c3fd" + }, + "adopted_test_sha256": { + "packages/microcosm-build/tests/test_atomic_geography.py": "bc66ae7461a57ac21afd1bdeb59a11549a5b74c6b81274544de0c9423579f5c9", + "packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py": "9bfce65ca47f74a325e652684b7c47fcf4fae1561a465e8f87e7518786bd4efa", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py": "59adfd2af5225ad7d08aba2a55872546b9f252f61baea5f2647863b0b13a8c03", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py": "c2474f4943a2ccb0d89b77f74e1fa991b293bc977139da2c4f28b8626aed0c37", + "packages/microcosm-build/tests/test_us_postclone_geography_identity.py": "763d15b4d543b1e5b294d20a55169138d1a5d54bac15c11a9e9dc6fa93376c14", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "8dca207eb22b0709358d97c723c2367820e586f443462fd9039372145ed827ab", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "91b3719210ca11e342b1bb0798ca615457b890fbe26f6be4b69c6c6aacff843f" + }, + "adoption_exclusions": [ + "PUF nineteen-node predecessor bridge", + "optional ACS source cache/fingerprint overlays", + "split control collectors", + "native guard or payload changes" + ], + "assignment_identity": [ + "survey_geography_origin_key", + "household_support_clone_index" + ], + "compatibility": { + "default_no_geography_case_passed_separately": true, + "default_shared_gate_api_preserved": true, + "optional_typed_gate_artifact": "microcosm.geography.validation_result@1" + }, + "controls": { + "all_original_selected_function_and_fixture_asts_match_adopted_tests": true, + "candidate_cases": 70, + "candidate_split": [ + 45, + 8, + 5, + 5, + 1, + 4, + 2 + ], + "separate_default_compatibility_cases": 1, + "single_combined_suite_run": false + }, + "graph_nodes": { + "age_extension_total": 12, + "financial_extension_total": 19, + "survey_prefix": 9 + }, + "historical_native_note": "Accepted twenty-node cold/replay evidence assigned geography before cloning and remains separately versioned; it does not certify this new ordering.", + "model_quality_accepted": false, + "national_or_local_fiscal_calibration_accepted": false, + "native_full_puf_enrichment_accepted": false, + "native_postclone_survey_accepted": false, + "new_order_puf_host_accepted": false, + "original_candidate_source_map_sha256": "dd1a782dceb020fcf8d9ff5ba94fb32035aa01e8d670ea3529a42a01b26ce9da", + "release_eligible": false, + "runs": [ + { + "cases": 45, + "cpu_seconds": 3.744692, + "included_in_candidate70": true, + "label": "us-postclone-atomic-geography45-v2", + "model_sources": 0, + "physical_source_and_owned": 510, + "postcheck_sha256": "45efe59214d52901a9712304e8ca6f66175c13caaecba3ff773d7780bc2a5ffa", + "receipt_sha256": "f5d1390d9b5183d7f23a2e2492a2d24e06df44bb66cec6e33bc100e74659cfe3", + "resources": 12, + "source_count": 502, + "source_map_sha256": "a294308425861522235f68d025c3c5becf7c2922708ed70543ab54f6a5a15ba4", + "wall_seconds": 4.635587666998617, + "xml_sha256": "8fd2783fc1917170f7d6abcb7ee0c7154a794dcddf326b2c154d5d7b612a2f82" + }, + { + "cases": 8, + "cpu_seconds": 235.803439, + "included_in_candidate70": true, + "label": "us-postclone-prefix-budget8-v3", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "85695bedda8003c9278d2d2ac4fc8d4d6e659898c3f3f5e8d8efcae4b08d74ae", + "receipt_sha256": "3b3fe93a0fe65ba6fd63889afdf99a8f45ba4a37c11aa5e243557fa05352b07c", + "resources": 12, + "source_count": 503, + "source_map_sha256": "71812aeb4d1600223f90882d8652405876c05379acbe0828fbee41b24680d7a1", + "wall_seconds": 237.32238883303944, + "xml_sha256": "caffc1d0eefa51cb0a71338324a213224e67cac50692d1f62d6d61d3cdb328bf" + }, + { + "cases": 5, + "cpu_seconds": 345.624984, + "included_in_candidate70": true, + "label": "us-postclone-financial5", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "7e993eccb0085b7dfce203a50d43f868eb822068ab62b50f344f69ce8f478396", + "receipt_sha256": "e3d9e95210185976de73982f994b51386bf4aa1243cc10b4f8db893760e55a90", + "resources": 12, + "source_count": 503, + "source_map_sha256": "02ae2194338378ac652a4ee6c2049b68dd78a813962b665b010c947b5cab0a4b", + "wall_seconds": 352.15383283403935, + "xml_sha256": "224590dab0d79f68b30a8e50ef9e86e9144869d90521cd66c0689d2b21167ff5" + }, + { + "cases": 5, + "cpu_seconds": 203.630023, + "included_in_candidate70": true, + "label": "us-postclone-budget5", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "9056592f58e29a73ba8b9ac69314e78602015be112261212911ca19a517e1df0", + "receipt_sha256": "0d577da19dd0086c8d5c47a7fc91101ad708ac51a22c62d8a487f1684c3f1b47", + "resources": 12, + "source_count": 503, + "source_map_sha256": "d316393b1aadf99fefadc342dcd0a88575491665971c8a3908c07b14d7f81f05", + "wall_seconds": 206.07668850000482, + "xml_sha256": "6a189b65471483178fc3518bc68692cf1c88678ae214db5d4407e178394722a1" + }, + { + "cases": 1, + "cpu_seconds": 549.278067, + "included_in_candidate70": true, + "label": "us-postclone-age1", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "f2c6533c11be9b325e9a507e8356a1c516ca4b20a3f4aecc7721458a8480f9ff", + "receipt_sha256": "e3a513ba6483688fb30b6bd7199b690278f32a02bc49101c78385539447fd699", + "resources": 12, + "source_count": 503, + "source_map_sha256": "b8e4b18994687ea33bead77003ccaffc28d72de7b79d3a34316faa1c7f945b4f", + "wall_seconds": 552.7509316250216, + "xml_sha256": "621b6dff98084c6748eb4a57b3515ef98828ae634bda3362c1504a87c4aba937" + }, + { + "cases": 4, + "cpu_seconds": 438.682175, + "included_in_candidate70": true, + "label": "us-postclone-prefix-late4", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "f25adff98c4645dabb503cd155e2652c9988342fca1be108d8b5aea8ee77f90e", + "receipt_sha256": "973fad9735b877c2867bb37af9c60025e10ae7b91216d846b056b5164e39ee33", + "resources": 12, + "source_count": 503, + "source_map_sha256": "b6b9faabf3080fce9cb3c9d1ab9521ba69da615c12d0c2328d75a24f30c976ec", + "wall_seconds": 459.0505850840127, + "xml_sha256": "c0034d745588c5d3821fb73d7086ada5e44c334b2a3e0b52e4ae4116bbfff13d" + }, + { + "cases": 2, + "cpu_seconds": 458.346596, + "included_in_candidate70": true, + "label": "us-postclone-financial-late2", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "2ab808471fc7943f6a3e6a810480ef9a1dddc267a908f0df03fc11311915c37c", + "receipt_sha256": "306c8e250421248faf282a9ac89742a820684b2ba2f6072189426c8301bd5464", + "resources": 12, + "source_count": 503, + "source_map_sha256": "e62aac60cb960e323cbf127067b3c27aa22daf73cdc815e20b55683984ed3ce8", + "wall_seconds": 478.2829617500538, + "xml_sha256": "e303d11996dbe2289c612d5fc102f55da5724f3f259d2cd5de9610686c2d7bb4" + }, + { + "cases": 1, + "cpu_seconds": 28.545443, + "included_in_candidate70": false, + "label": "us-postclone-default1", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "157961870ad76565d8c3634a457e4ef8d5ac2245abd75e6b7dc789288cc13fae", + "receipt_sha256": "9c96d01404c8a48fac7b991c2c7d7742d586bdd2b6b754a694e31c451d1bd5ad", + "resources": 12, + "source_count": 503, + "source_map_sha256": "38d7f20d01526ff275c74a8fcd92404b1a572cadade7f896d602f974d275fe2d", + "wall_seconds": 29.92554637498688, + "xml_sha256": "d8da781a858400f3670031566d774d7148b1333cecd6feee543901e68383ec8f" + } + ], + "scope": "Complete initial survey clones before constrained block assignment; current financial and limited age-calibration interfaces on invented source/target fixtures", + "source_adoption_base_head": "94b7c2a5d73c1512efcc41425224bfbb7a04cbb7", + "source_peer_review_sha256": "08c53c53cbd63481279c7c003f536404ec0b2163151b152ff39a6a72b6a3f713", + "status": "invented_postclone_controls_accepted", + "test_only_corrections": { + "packages/microcosm-build/tests/test_atomic_geography.py": "bc66ae7461a57ac21afd1bdeb59a11549a5b74c6b81274544de0c9423579f5c9", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "91b3719210ca11e342b1bb0798ca615457b890fbe26f6be4b69c6c6aacff843f" + } +} diff --git a/experiments/us-property-financial-host-20260912.md b/experiments/us-property-financial-host-20260912.md new file mode 100644 index 000000000..87c12e0a0 --- /dev/null +++ b/experiments/us-property-financial-host-20260912.md @@ -0,0 +1,73 @@ +# Property-income integration in the US financial graph + +The existing checked financial host now accepts an explicit property-income +configuration. Its ordinary graph remains 19 nodes. The configured graph adds +16 operations: qualified donor and recipient projections, four conditional +fits and draws, signed reconciliation, and attachment to the complete initial +survey clones after atomic geography. + +This is development integration. It adds 24 component, draw, diagnostic and +knownness columns while retaining the preceding eight financial leaves. It does +not yet rebase their tax treatment, change the legacy capital-gain model's +conditioning, complete missing original-channel inputs, or certify native data. + +## Independent verification + +The invented-source host test executed an ordinary 19-node build, an extended +35-node build sharing the 19-node prefix, and required replay with all 35 cache +hits. The receiving population contains 18 people in 12 households. Eight +controls passed: default behavior, real extended execution/replay and issuance, +complete preceding-population preservation, source/clone identity and unknowns, +three retained-state mutation cases, and refusal of a copied run dataclass. + +The complete comparison includes every preceding entity table and column, +relationships, row order, schema, geography, metadata, design and current +weights, strata, owners and mass history. Fitting uses original design-weighted +donors; attachment shares the resulting property values across each original +person's two initial clones. Missing anchors and unresolved components stay +unknown. Negative property totals remain valid where the declared signed +component permits them. + +The run took 729.20 seconds and peaked at 584,957,952 bytes RSS. Source and +resource hashes matched before and after. Its receipt SHA256 is +`90137cd08aa151e9db92a8bc5c1c51c2448f8e3bceb6d3b54d6fdf3ff9530e23`. +This was **eight passes and one failure**, not a fully passing suite. The PUF +recipient test correctly refused an invalid invented Social Security observation: +the composed fixture had made a person age 14 without blanking that question. +The guard also denied a metadata lookup of its own error-output path during +failure formatting. Neither exception has been relabelled as a clean run. + +The fixture correction blanks that exact person's income literals before the +archive, catalogue, source pins and real preparation are constructed. It changes +no production source validator. Separate source/base/cold/required fixtures let +the focused PUF handoff test execute one real cold graph without repeating the +eight accepted controls. Independent review confirms their test-function bodies +and both production files are unchanged. The targeted handoff test then passed +through the actual source qualifier and both recipient graph declarations in +467.14 seconds, peaking at 545,980,416 bytes RSS. All 1,022 source and 15 resource +hashes remained identical, with no unexpected access refusals or child processes. +Its receipt SHA256 is +`9cd769de79b72a6e68069dc6c9f46db090d9cfe1a8e11d17e8d753a86bd5c388`. +Independent review approves the combined scope of eight earlier controls and +this corrected follow-up; it does not turn the earlier attempt into nine passes. + +## Verification and interpretation boundaries + +The runner checks actual training/model/raw-draw receipts against the qualified +donor and recipient before issuing either a cold or required-replay result. +Later checked views requalify the retained source owners and population contents +and require those verified artifact bytes to remain identical. They do not +perform another fit or replay the model applications. A descriptive copied +dataclass cannot acquire execution authority. + +The PUF host now derives its upstream node count and final writer from the +checked financial run. The new recipient test covers qualification and graph +dependencies; it does not execute the complete 261-node two-route PUF graph. +The separate ongoing native pilot uses its earlier frozen source and supplies +no acceptance for this new extension. + +Next steps are a separately verified tax-input rebase, explicit completion of +required unknown inputs, full PUF/enrichment execution, and candidate-specific +engine, calibration, geographic-support and quality checks. The public +[host contract](../docs/us-atomic-property-financial.md) and +[release plan](../docs/us-uk-release-path.md) describe those remaining boundaries. diff --git a/experiments/us-property-source-model-integration-20260912.md b/experiments/us-property-source-model-integration-20260912.md new file mode 100644 index 000000000..e192d72cb --- /dev/null +++ b/experiments/us-property-source-model-integration-20260912.md @@ -0,0 +1,82 @@ +# Property source and model integration + +12 September 2026. These are bounded code and invented-input checks. They do +not certify a native population or a releasable dataset. + +The integrated property model learns four jointly drawn components from +original ASEC household DESIGN weights: ordinary interest, retirement-account +earnings, dividends and signed broad property receipts. A visible signed +reconciliation operation aligns recipient components with qualified ACS INTP. +Negative and zero net totals retain offsetting positive components and losses. +Broad property receipts are not yet a pure rental-income tax input, and +retirement-account earnings are not withdrawals. + +The source composition retains original source Frames, native identities, +knownness, reporting routes, allocation flags and full exclusion diagnostics. +It selects original ASEC donor persons and original ACS persons with known +adult anchors. Model columns are separate from the selected Frames, ready for +explicit graph operations. It does not complete unknown anchors or attach +modeled values to clones by itself. + +## Accepted checks + +| Component | Exact implementation | Evidence | +| --- | --- | --- | +| Four-component graph | `e2e38a3c75175abfeea3b8f25f0412afb6c100fb` | 22 tests; actual weighted QRF, raw draws, signed reconciliation and required cache replay. A delegated training-implementation change invalidates the cache identity. Independent repair review approved. | +| DIV and visible survivor routes | `90bc1bfbc2592530c8fe914f5ed60b3bc7b2e1e3` | 82 source controls; exact printed domains, unknown zeros, source joins and final ownership checks. Independent review and root hash verification approved. | +| Conservative property donor basis | `bc278d9808e41820b175da4c1b036c283929d6d7` | 60 tests; signed observations, exact axes, source discrepancies, explicit unused-slot derivation and weighted exclusions. Independent review approved. | +| Original source composition | `9bc5932af214881bcc7931cda2847fab60163dfa` | 21 controls on the owner's freeze and 21 again after integrating the corrected basis and hardened source domains. The latter ran in 47.82 seconds with 516,046,848 bytes peak RSS. Root source, test and receipt review approved. | + +The integrated source fixture executed the actual four-stage survey graph: +CREATE, allocation, complete initial support cloning and ownership attachment. +It had 9 original persons in 6 households, 18 cloned persons, 1 eligible ASEC +donor and 3 eligible ACS recipients. It ran no QRF, PUF or country engine. +The separate graph tests above establish numerical graph execution, not a +completed country host combining those two pieces. + +The integrated source receipt is +`8fef8af66c88f701d2f62422f8513b92e28a2626af4a9fc8481eb96fc32cdf96`; +its XML is +`8164608b6d96ea492a133f11f560ee1c688f0524198c4614d1e4ac6ce4aa4ae9`. +All 1,017 frozen source files and 15 explicitly admitted resources remained +unchanged; no unexpected guarded access occurred. + +The complete module inventory and existing source/metadata scanners passed +on the nine affected modules. Only actual source joins receive original-source +access classification. The pure basis, fiscal policy and numerical graph do +not. Literal and accessor checks still reject injected source-channel reads +in the two modules with reviewed dynamic selectors. The scoped receipt is +`3543fa52f0440db58a6592c6dc313c54a83b6348bd2bfd42cda3bcc66f2eb828`. +An earlier attempt to run the entire legacy source suite reached its CPU cap; +it is preserved as incomplete, not counted as a passing full-suite run. + +## A source finding that changed donor selection + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=49) +states that SRVS_VAL includes unedited third and fourth sources as well as +the two edited visible slots. Therefore, two visible survivor sources outside +estate/trust code 8 do not establish complete absence of that overlap. + +The first property donor bridge now requires known survivor nonreceipt with +readable NIU slots. Positive receipts have an explicit additional-source-scope +exclusion and design-weight mass summary. The qualifier's two-slot clearance +remains diagnostic. No source observations are rewritten, no survivor dollars +are assigned to a property component, and conservation of the ACS anchor is +not presented as protection against selection bias. + +## Remaining integration + +The next stage adds these original-person components and their knownness to +the actual pre-PUF financial graph, with one ACS draw shared across both +initial clone rows. It will reuse the existing checked financial run owner. +The existing tax leaves and legacy capital-gain conditioning cannot be called +rebased until the replacement explicitly executes and passes verification. +Pension/disability/survivor source detail, withdrawal regularity, taxability, +missing-anchor completion and finer property tax categories remain separate +measurement or model work. + +The running native pilot uses an earlier frozen financial implementation. +It is not evidence for the newly integrated property code. Native PUF +acceptance, applicable fiscal inputs, national/state/CD calibration, held-out +quality checks, progressive scaling and exact full/compact exports remain +release requirements. No publication, default promotion or deployment occurred. diff --git a/experiments/us-puf-donor-boundaries-20260910.json b/experiments/us-puf-donor-boundaries-20260910.json new file mode 100644 index 000000000..802668e0a --- /dev/null +++ b/experiments/us-puf-donor-boundaries-20260910.json @@ -0,0 +1,120 @@ +{ + "calibration_acceptance": false, + "cold_and_required_replay_checked": true, + "controls": [ + { + "cpu_seconds": 3.567905, + "guard_sha256": "348bc2816d499e592fcf90515ff79175fdabf535bb51bd23a47a15f355265f3a", + "junit_sha256": "29cc6d7aac55b4f6a64cf4940ef2613c5880fa42a2e9c4eb1f5b113ea02d9219", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "name": "donor_validation", + "peak_rss_bytes": 439549952, + "receipt_sha256": "99a0efcb676ba96f5e485f204428120ce75e0f43b50f685c0eeace9d972c36ff", + "resource_map_sha256": "d58f2bc53f56cd36305f8c6f2855c9fa001153b2f51cd41c2cf430c0ea3d2683", + "root_postcheck_sha256": "e8c4a24705541e82b3fe834496a0bd32afd53627507bd75a055090f76d95b518", + "selected_test_cases": [ + "test_existing_profiles_share_exact_selected_donor_check[PufOutputProfile.FULL65]", + "test_existing_profiles_share_exact_selected_donor_check[PufOutputProfile.PUF59]", + "test_existing_profiles_share_exact_selected_donor_check[PufOutputProfile.PUF55_SURVEY_SS]", + "test_existing_profiles_share_exact_selected_donor_check[PufOutputProfile.PUF55_SURVEY_SS_NO_TOTAL]", + "test_complete_historical_donor_domains_are_retained[missing]", + "test_complete_historical_donor_domains_are_retained[numeric_string]", + "test_complete_historical_donor_domains_are_retained[unknown]", + "test_complete_historical_donor_domains_are_retained[nonfinite]", + "test_complete_historical_donor_domains_are_retained[integer_range]", + "test_complete_historical_donor_domains_are_retained[incidence_capacity]", + "test_complete_historical_donor_domains_are_retained[boolean_count]", + "test_complete_historical_donor_domains_are_retained[year]", + "test_donor_rejection_still_precedes_recipient_access", + "test_selected_values_are_detached_from_auxiliary_source_table", + "test_excluded_ss_outputs_are_not_accepted_as_extra_donor_columns", + "test_eight_route_selection_keeps_targets_recids_weights_and_six_money_values" + ], + "source_count": 498, + "source_map_sha256": "e2f500ab2055e34899d663f9792a2f97b1157ab087409072567f985d3874b853", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 16 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 506 + }, + "wall_seconds": 5.826573541969992 + }, + { + "cpu_seconds": 15.144223, + "guard_sha256": "b91941ff88c7dd446d650a68ee1ac015e9e39450f51e179f181b2cd75277a8a7", + "junit_sha256": "5cca9cf41e93c7b01bdc0beb5a42dc48b5ebd58e5cacde92e0f2b0435c4c5cce", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "name": "full65_chain", + "peak_rss_bytes": 625786880, + "receipt_sha256": "29d3799c065ccc5277bca30a643dd615bf9f425835bf552abd63300d13a65525", + "resource_map_sha256": "9fadfa1d4f77bf1510d04895bada88d7558a5652b09b6d9c22b9b2739ca9370f", + "root_postcheck_sha256": "8c5654c6bdceb7fd91624644a06808202fa1b2dc6ad94e925d11a1bf4519871b", + "selected_test_cases": [ + "test_real_full_65_target_graph_draw_chain_finalization_and_replay" + ], + "source_count": 498, + "source_map_sha256": "e2f500ab2055e34899d663f9792a2f97b1157ab087409072567f985d3874b853", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 506 + }, + "wall_seconds": 16.31964554201113 + }, + { + "cpu_seconds": 13.678566, + "guard_sha256": "09e9e78fba275090101d569a037bc15b2ac9f6ed13bd656fe3b42f949909e5fc", + "junit_sha256": "eea60075e8c7a64720d52b9afc63f1779184072c5b41623b7ce41f2cfee741bd", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "name": "survey_ss_puf55_chain", + "peak_rss_bytes": 628178944, + "receipt_sha256": "2546a4531c7c4099066cc346cd42115175972f35a8699614e1670ec4a8bcfb29", + "resource_map_sha256": "d8aef904d7ceb5067c0d4f39ccab0837ad241bd96eac34f7c0aecaa6fde54848", + "root_postcheck_sha256": "3a0180f061f10d873ae38de5e583bc7d8f1ef3c6262fea4665d2194b1d947f04", + "selected_test_cases": [ + "test_real_puf55_chain_preserves_survey_ss_and_replays" + ], + "source_count": 498, + "source_map_sha256": "e2f500ab2055e34899d663f9792a2f97b1157ab087409072567f985d3874b853", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 506 + }, + "wall_seconds": 14.389938707987312 + } + ], + "current_checkout_runtime_certification": false, + "full65_nodes": 133, + "native_execution": false, + "owner_sha256": "e0c9d34cd1815d323495e01ec5f94133c1b7da8da37d3fc714041ea38be677cc", + "release_acceptance": false, + "same_complete_fitted_model_donor_check_preserved": true, + "same_ordinary_donor_validation_preserved": true, + "scope": "invented_donor_validation_and_actual_full65_puf55_chain_regressions", + "scope_note": "Validation is on the pinned 498-file snapshot. Source adoption does not certify the later combined checkout or the prospective two-route host. The checks exercise actual graph fitting, raw draws, finalization, and required cache replay over invented donors; no native PUF or survey transfer is claimed.", + "selected_tests": 18, + "source_authority_issued": false, + "status": "frozen_donor_helper_extraction_accepted", + "survey_ss_puf55_nodes": 113, + "test_sha256": "8d84c4d379cfd75ad36ca232fcadcdf0e0bf899ad8c9b8f242319d0f3c8a8db0", + "two_route_host_execution": false +} diff --git a/experiments/us-puf55-canonical-create-47-controls-20260910.json b/experiments/us-puf55-canonical-create-47-controls-20260910.json new file mode 100644 index 000000000..0942a6a1b --- /dev/null +++ b/experiments/us-puf55-canonical-create-47-controls-20260910.json @@ -0,0 +1,52 @@ +{ + "cpu_seconds": 11.761269, + "current_checkout_runtime_certification": false, + "failed_predecessor_accepted": false, + "guard_sha256": "3dd78388b0426c92183db97aaa48516903b61ef9f4f646f35b9301c56662aa7a", + "junit_sha256": "0e6a0a3d4f77363c7ccd1705afd75e8082219ca4a24cfc6d36db383b77e2daab", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_access": false, + "observed_outcomes": { + "error": 0, + "failed": 0, + "passed": 47, + "skipped": 0 + }, + "peak_rss_bytes": 501907456, + "postcheck_script_sha256": "f1c8d68d9c4edd92391317683f308f599f6dbc771a4ef9c538acf7d6e4b679e0", + "preceding_stable47_failed_postcheck_sha256": "cb2e3a344b4fa2e7bef7b1e60cf085b8a583567e54c7c2c940345639bc9081ce", + "preserved_failed_predecessors": [ + "us-puf55-canonical-create-failed-20260910.json", + "us-puf55-canonical-cycle39-failed-20260910.json" + ], + "receipt_sha256": "8f7ac625296fbb22dcd976918b4718af14cbaf31288e528583d1fb49ad6e95c9", + "release_acceptance": false, + "resource_map_sha256": "57927f284c647abf03ea1e8370ddeb6035b4e68ff93af1e4f745a5027307e7ef", + "root_postcheck_sha256": "5617f7949e261d42975cb088671f83bda71c37d424e190a209766df5e4ce20ec", + "scope": "47_invented_canonical_donor_and_converter_controls", + "scope_note": "The exact frozen canonical donor CREATE and projection controls pass cold execution and required replay. Source adopted locally. Native donor, complete survey attachment, calibration and release acceptance remain separate.", + "source_adoption": true, + "source_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py": "a04c8dd2d89b4d3e5c563707587545299bd218aadf567ee7e772b9bbc5755311", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py": "6a8d07e38f5f0bff2619a7eb53fd0f34a10eed798f79de44c82e7740db923066", + "packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py": "675bd57728a6ef56e79a0a24c79713fbb98e454be385fae607528d57048ce76d", + "packages/microcosm-build/tests/test_us_puf59_canonical.py": "85a6d41e716e0d3f1794851dd6cc6007603834a22982f31058fa5bf6da31df6d" + }, + "source_graph_control_execution": "actual canonical CREATE cold and required replay with all 47 selected assertions and original fixture teardown passed", + "source_map_sha256": "dd16d99e9d0bbc3b1bf3aacddcd1159887758c2a8183bcee0333715270bcd9bb", + "status": "frozen_canonical_create_and_converter_controls_accepted", + "survey_attachment_host_execution": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 47 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 509 + }, + "wall_seconds": 14.356884333014023 +} diff --git a/experiments/us-puf55-canonical-create-failed-20260910.json b/experiments/us-puf55-canonical-create-failed-20260910.json new file mode 100644 index 000000000..e2d8251d6 --- /dev/null +++ b/experiments/us-puf55-canonical-create-failed-20260910.json @@ -0,0 +1,32 @@ +{ + "corrected_source_pending": true, + "cpu_seconds": 7.677668, + "date": "2026-09-10", + "failed_scope_accepted": false, + "failure_code": "PUF55_CANONICAL_SOURCE_LIVE_DEPTH", + "failure_stage": "kernel construction, while traversing cyclic function closure/mapping references", + "guard_sha256": "b50c65325ec901aada10368562ba481f3253f8d4fbdfba7c38fb6a5089de7612", + "independent_physical_postcheck": false, + "junit_sha256": "c9f6146ec4ada2ea277311ff8f9609207585acc7c12233ef6f9437f273a82e4d", + "native_input_access": false, + "passed_cases_reported": 1, + "peak_rss_bytes": 500629504, + "production_adopted": false, + "receipt_sha256": "fa34a78b323eda9873b24a0b11a8d7f84dc3696cd9e0eab0cfb8e409a1152da7", + "release_acceptance": false, + "result": 1, + "schema_version": "microcosm.failed-control-observation.v1", + "scope": "invented canonical donor CREATE and converter controls; no native input, current-checkout or release acceptance", + "source_before_after_equal_reported": true, + "source_count": 501, + "source_map_sha256": "6fdb0549fe8331db32450f3161a2d8cf1e00096c9245728e84360a08050277da", + "status": "failed_before_canonical_create", + "test_counts": { + "errors": 24, + "failures": 4, + "skipped": 0, + "tests": 29 + }, + "unexpected_refusals": {}, + "wall_seconds": 10.18145758396713 +} diff --git a/experiments/us-puf55-canonical-cycle39-failed-20260910.json b/experiments/us-puf55-canonical-cycle39-failed-20260910.json new file mode 100644 index 000000000..8f4142131 --- /dev/null +++ b/experiments/us-puf55-canonical-cycle39-failed-20260910.json @@ -0,0 +1,34 @@ +{ + "automatic_retry": false, + "cpu_seconds": 8.717074, + "date": "2026-09-10", + "exact_changed_module_or_member_identified": false, + "failed_scope_accepted": false, + "failure_code": "PUF55_CANONICAL_SOURCE_LIVE_STATE_CHANGED", + "failure_stage": "first executor implementation_hash after source fixture initializes, before CREATE execution", + "guard_sha256": "367e4de4c1587082d36dd4ae1f09847e7e8db3ef312ba33f5e867addf1119318", + "independent_physical_postcheck": false, + "junit_sha256": "fc00a99888c91d1d8fb6cf80ad381236a2c25d8b64cd0a95fa3d02a8951645d3", + "native_input_access": false, + "passed_cases_reported": 14, + "peak_rss_bytes": 507789312, + "predecessor_receipt_sha256": "fa34a78b323eda9873b24a0b11a8d7f84dc3696cd9e0eab0cfb8e409a1152da7", + "production_adopted": false, + "receipt_sha256": "ec5dfec245cc5cf1a29aebb9afe81f0768dcd79cefa9c28e2610af5c1c79c229", + "release_acceptance": false, + "result": 1, + "schema_version": "microcosm.failed-control-observation.v1", + "scope": "invented canonical donor CREATE and converter controls; no native input, current-checkout or release acceptance", + "source_before_after_equal_reported": true, + "source_count": 501, + "source_map_sha256": "027ab7d21adf3f74cac798e759eed27bd59d20a1b769839549e557d8c61ba0a8", + "status": "failed_before_canonical_create", + "test_counts": { + "errors": 24, + "failures": 1, + "skipped": 0, + "tests": 39 + }, + "unexpected_refusals": {}, + "wall_seconds": 11.895262666977942 +} diff --git a/experiments/us-puf55-checked-output-20260912.md b/experiments/us-puf55-checked-output-20260912.md new file mode 100644 index 000000000..7ce57a7f2 --- /dev/null +++ b/experiments/us-puf55-checked-output-20260912.md @@ -0,0 +1,83 @@ +# Retained PUF55 output verification + +On September 12, 2026, the PUF55 host gained an in-process checked run handle +for downstream graph stages. The actual `run_survey_puf55` execution retains +the handle only after its existing final source, artifact, numerical, and +complete-population checks. Constructing or copying `SurveyPuf55Run` does not +issue a handle. + +`check_survey_puf55_run(run)` and `run.checked_view()` return a descriptive +`CheckedSurveyPuf55Run(payload, digest, population)`. Downstream owners must +retain the original run, check it immediately before consumption, and recheck +it after their last relevant I/O before returning or exporting a successor. +A failed public check revokes that handle; restoring fields cannot reissue it. +The payload and view cannot authorize a reconstructed or copied run. + +## Retained checks + +The checker reuses the attachment boundary's real financial/source owners, +source and producer keys, donor resource declaration, source codecs, kernel +registry, recipient qualification, and receiving support. It reads declared +artifacts through the existing producer/type/store verifier, borrows the +boundary again, then checks the output without further external reads. + +The output checks preserve original object bindings, the compiled graph, +manifest JSON, exact attached population and ledger roster, heterogeneous +manifest contents, and the complete final and independently reconstructed +populations. Existing physical seals include values and missing backing +storage, axes, schema, memberships, weights and kinds, design anchors, strata, +owners, metadata, mass log, and ledger. The retained state holds final output +and expected populations, rather than all 245 observed node snapshots. + +Rechecking does not fit, execute a graph, decode model pickles, or reconstruct +the canonical donor. It verifies the current ancestry and identities of the +artifact bytes that actual execution already checked numerically. It still +performs source, implementation, and store checks; consumers should use it at +stage boundaries rather than inside calibration iterations. + +## Verification and independent review + +- Three public-constructor/API controls first failed against the absent API, + then passed against the implementation. The original red evidence remains. +- The first complete invented-source run passed 23 controls. Independent + review then reproduced a gap: adding an extra attached Frame or ledger key + did not change portable manifest JSON or the old expected-version seals. + The checker now requires the exact attachment roster, with separate + population and ledger regression cases. +- The corrected source passed all 25 controls through an actual invented + financial19 to PUF245 cold execution and required replay. It used 901.07 + CPU seconds, 910.28 wall seconds, and 740,917,248 bytes peak RSS. All 985 + source/owned-file and 15 resource hashes remained unchanged; the guard + recorded no unexpected refusals or child processes and one numeric thread. +- The 12 new test functions were then moved beside the existing host tests + without changing their decorated ASTs, any existing function/class AST, or + production source. Independent review approved the move. Collection found + 45 cases, including the 25 new cases, sharing one `composed` fixture + definition. The three cheap checks passed again. The combined 45-case suite + was collected, not rerun as one full suite. +- Ruff and the CI test inventory verifier pass. The source repair and test + move each received independent review with no remaining actionable finding. + +The first post-move guard recorded the expected 45-case collection and three +passing tests but returned nonzero because two pytest sessions doubled an +already-denied optional locale metadata probe from 12 to 24. Its evidence +remains. A new owned run adjusted only that denied-probe reporting ceiling; +it closed green without allowing the read or changing source or tests. + +Frozen production SHA256: +`5ba34ce40e19cb2bf22f9c9976576f9dc19e92b19a5a5cc1763569df7a40b796`. +Final colocated test module SHA256: +`bf8b53d229a61565337e2ee9766900158b2bde4edc3042c60c52fe09ecdc531f`. +Corrected 25-case receipt SHA256: +`e47f4408387fcfab6564f66ad9479c9c1cf32615862b2268ee854ba7781a4553`. +Post-move collection/cheap receipt SHA256: +`016a5defba66ce8d36372e890a000295d82f4825d1e516d477185eaccfc70633`. + +## Scope + +This is invented-source software acceptance. It grants no native source or +population admission, calibration acceptance, release eligibility, or +publication authorization. Graph node and artifact contracts are unchanged. +The attachment kernel hashes the host module, so this additive change moves +PUF producer identities on its own source revision. The separately frozen +d35 native pilot packet and its evidence remain unchanged. diff --git a/experiments/us-puf55-host-adoption-20260912.md b/experiments/us-puf55-host-adoption-20260912.md new file mode 100644 index 000000000..37a15fe78 --- /dev/null +++ b/experiments/us-puf55-host-adoption-20260912.md @@ -0,0 +1,101 @@ +# PUF55 host restoration on the reconciled graph + +This change restores the existing PUF55 composition over an issued nineteen-node +post-clone financial run. The shared ACS/ASEC survey population is cloned before +atomic geography is assigned. Recipient measurement follows financial completion; +a common canonical PUF donor supplies two disjoint conditioning routes, followed +by one placement into the full survey population. The original survey channel, +source identities, memberships, Social Security fields and knownness, geography, +design anchors and household weights are retained. + +`run_survey_puf55` accepts the actual financial run and original PUF source paths. +It requires the financial issuer, source keys, producer implementations and typed +model ancestry. Its explicit `resume="require"` call must hit every graph node. +There are 25 fixed-prefix/finalization nodes plus 110 per nonempty route: 245 +nodes for the two-route fixture. Each route fits and applies 55 targets. The +known survey report total adds a ninth predictor; unknown totals use eight +predictors without a zero fallback. The four Social Security components remain +owned by the survey completion stage. Prior wages are excluded. + +The restoration consists of two new production modules, the existing canonical +donor fixture's opt-in weighted-tail support, the full host controls and the tiny +heterogeneous-manifest controls. It incorporates the later corrected mutation +control, which changes the retained receiving DataFrame and proves its complete +population seal changed. All earlier fixture helpers and newer tests remain. +Ruff formatting/import order is the only difference from the preserved source +candidate's executable AST. No existing production module or implementation +inventory JSON was replaced. The new kernels' implementation hashes bind both +new host modules and their consumed canonical, recipient and numerical owners. + +## Current-source validation + +The source base is `9e61d0d04f5930aa72aef2faf83b2a9c7f2b4284`, including the +reconciled graph and keyed-draw corrections. Tests run against a frozen inventory +of 976 source/configuration files and 15 exact code resources, with no native +input, country engine, network or child process access. Numeric worker counts +are one. Runtime stores contain newly constructed invented inputs only. + +The 28 heterogeneous-manifest controls and three canonical fixture controls pass +together: 31 tests, zero failures/errors/skips, 5.97 wall seconds, 5.61 CPU seconds +and 437 MB peak RSS. The guard's source/owned/resource before-and-after checks +pass. The preceding run also passed its tests but failed its reporting gate +because optional pytest-owned current-symlink attempts were newly refused. That +failed record is preserved. A fresh run kept the symlinks refused and classified +only explicitly bounded attempts within its owned directory as expected. + +The two full-host positive controls pass: two tests, zero failures/errors/skips, +741.87 wall seconds, 734.55 CPU seconds and 739 MB peak RSS. They share one actual +financial fixture, use 64 invented PUF donors and two trees, and execute the +245-node cold extension plus required replay. The direct whole-cohort oracle +checks all outputs and non-owned storage. The finite limits were 1,800 CPU +seconds, 2,700 wall seconds and 8 GiB measured peak RSS. The guard exits zero, +records no unexpected refusals, and confirms all 984 source/owned hashes and +15 resource hashes stayed unchanged. This validates the restored composition on +the current frozen source, not the entire twenty-case host test file. + +An owned diagnostic hook successfully ran the existing NATIONAL_CD input-coverage +operation on the actual attached population within the fixture's checked +lifetime, bracketed by the normal purity and financial-owner checks. The fixture +contains 12 households and 18 people spanning ACS/ASEC and original/clone channels. +Of 161 required input names, 61 are present and 100 are absent; no ambiguous grain +or block-storage issue is reported, and all 12 household block values are known. +The absent roster includes survey demographics and disability, health coverage, +assets, take-up and several survey-owned income names. These include the four +canonical Social Security component names, which this PUF stage deliberately +does not write; the synthetic roster does not establish native input coverage. Prior wages remain excluded. +Known/unknown and writer/carried counts are recorded by source/clone. Applicability, +source ancestry, statistical signal and release eligibility remain unverified. + +The closed host receipt is SHA256 +`d2288ee0e321297875509ee962faef8ee886399a717cad47356921dada8be698`; +the descriptive synthetic coverage is +`d405c7d559a2f2141d83c076d8cef8bbe33f30ba403a7eaedf76cff1161cb7a6`. +Both live under the durable local adoption packet's `run-host2-v2` directory; +`ACCEPTANCE-SUMMARY.json` provides a bounded index without producer payloads. +An independent source review found no actionable defect in the five-file +restoration and confirmed the preserved source pins and executable AST parity. + +CI test-file partition verification passes with 487 tracked files. The new host +file enters fast `rest` and engine `us-am:build`; the heterogeneous-manifest file +enters fast `rest` and engine `us-p`. The canonical fixture file retains its +existing `us-am:build` placement. The classifier and country spec/seed pins are +unchanged. Ruff and staged whitespace checks pass. + +## Native and release work remains separate + +A new native harness must retain the actual financial run in process and provide +a writable store for new PUF artifacts. The retained historical financial replay +store remains read-only. Genuine PUF source validation, conditional-fit quality, +complete applicable input coverage, national/CD calibration, holdouts, pruning +and exact release export/readback are not established by these synthetic tests. +The host emits a development result with release eligibility false. + +`SurveyPuf55Run` is a public frozen dataclass, not a retained issuer. The runner +checks source/artifact ancestry and the complete output immediately before +return, but supplies no post-return `check_run` or issuance registry. The genuine +financial handle retained inside it authenticates the upstream financial run, +not subsequent mutations or copies of the final PUF population. Receipt booleans +cannot grant that authority. A downstream calibration bridge must consume the +actual checked output within an explicitly fenced host operation or implement +and test a retained output check; it cannot infer portable ownership from this +dataclass or receipt. This restoration adds no such admission mechanism. diff --git a/experiments/us-puf55-native-donor-20260909.json b/experiments/us-puf55-native-donor-20260909.json new file mode 100644 index 000000000..68dc87e6d --- /dev/null +++ b/experiments/us-puf55-native-donor-20260909.json @@ -0,0 +1,40 @@ +{ + "code_resources": 12, + "cpu_seconds": 8.784642, + "expected_test_file_counts": { + "test_native_puf55_projection.py": 1 + }, + "first_attempt": "Value checks passed, but removing the diagnostic plugin also removed collection accounting. Guard refused the run. The replacement retained exact collection accounting while suppressing private failure context; neither values nor access permissions changed.", + "input_artifact_sha256": "28cd44bd59218c1e77f14fa35c510ab550d2035527e3c139e7d19875c12ac11c", + "model_source_files": 5983, + "native_input_hash_verified_inside_guard_before_and_after": true, + "native_payload_reopened_by_postcheck": false, + "native_resources": 1, + "original_canonical_artifact_unchanged": true, + "outputs": 55, + "peak_rss_bytes": 1431109632, + "predictors": 9, + "qrf_fitted": false, + "receipt_sha256": "ce09455947f0efbca16014a23baa19037ac6450876c639568af6584c81bfc45c", + "release_eligible": false, + "report_sha256": "7ade9f442fe25f02e013b39afa5bf8077e89a75b43b4c6c7d0d2dc123f052766", + "result": 0, + "rows": 207692, + "scope": "Native canonical59 artifact projected to the survey-SS PUF55 donor; no recipient fit or whole-Population attachment.", + "source_and_code_before_after_current_equal": true, + "source_authority_granted": false, + "source_commit": "74814151a89e9ffc0ace982846152ef87623e73d", + "source_owned_files": 478, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "test_file_counts": { + "test_native_puf55_projection.py": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 11.048004040989326, + "whole_population_attachment": false +} diff --git a/experiments/us-puf55-numerical-output-seal-46-controls-20260910.json b/experiments/us-puf55-numerical-output-seal-46-controls-20260910.json new file mode 100644 index 000000000..17567c038 --- /dev/null +++ b/experiments/us-puf55-numerical-output-seal-46-controls-20260910.json @@ -0,0 +1,115 @@ +{ + "actual_model_chain_verified": true, + "approval_sha256": "359a56ef088584a149b04a6a4bfa5e047c97f6e90db368376cbcf67d1564756d", + "bridge_owner_sha256": "ca49788ebd67ea807b072cbfbf4817fd7051f967f77098d0e2f8586576add13f", + "cpu_seconds": 18.093259, + "current_checkout_runtime_certification": false, + "financial_fixture_execution": false, + "fixture_apply_target_calls": 220, + "fixture_donors_per_model": 4, + "fixture_fit_target_calls": 110, + "fixture_trees": 2, + "full65_and_puf55_chain_acceptance": false, + "graph_host_execution": false, + "guard_sha256": "5bc1ac74e18b6379d51cf34e55ada8afb17fbb11d06428ebaefd81b4a92baab6", + "host_receiving_value_seal_still_required": true, + "junit_sha256": "aa2825f0279ed6cdbec8b08cad1eebbc287d2bd6d813a538a68b26885c76f481", + "model_cache_metadata_scans": 1506, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "native_access": false, + "new_numerical_cases": 28, + "numerical_test_sha256": "7552d83574e000b5e5279e8155711f0d4435125a7e2fe95571ef8b4f9189db26", + "one_whole_cohort_finalizer_per_call": true, + "peak_rss_bytes": 680542208, + "peer_target_review_sha256": "a616d764c903fd90383f7e2b6147bd9f065ff4164e0b98f4ce315f8b646d7b42", + "postcheck_script_sha256": "c0a6e6f5ff02a9086488702b04a5d6c475c903108e6c933c654941141aa0bd59", + "preserved_failed_predecessor": { + "accepted": false, + "failed": 1, + "passed": 45, + "reason": "The new test addressed the source alias employment_income instead of finalized employment_income_before_lsr. It failed before applying its intended mutation.", + "receipt_sha256": "26bf35628a60abacbf765725ebc7cab3a0ed0afd844b1d71e9a153fa97ca6ef7", + "root_failed_postcheck_sha256": "709621c49cd300e83c96fe8100d5257c462b86e3b96b0dd29a4585cbd94c025b", + "tests": 46, + "unexpected_denied_traceback_labels": 22 + }, + "production_adopted": true, + "project_imports_or_runtime_reexecution": false, + "puf_fit_and_numerical_finalization": true, + "receipt_sha256": "b9e6349583a1959e918e491c1267c28535919fc2ef699d1608e9ea52b951f020", + "release_acceptance": false, + "resource_map_sha256": "8add0e99307b3cfe6ba7fb5cc4f68f711d5bf7f82ccde8213bd52ed04ab4ee80", + "root_postcheck_sha256": "e85d557093ac8b53ea4c16eac883b9dae2598889ca35cb56f8db0f9b0f270098", + "root_review_sha256": "73ae7e8ea10b317de40d2a11419aaaf15437b48cd689eb27a1ce6d4473cfc325", + "scope": "46_invented_two_route_numerical_and_raw_merge_controls", + "scope_note": "All 46 invented numerical/raw controls pass on the frozen source. The v2 numerical receipt seals the complete finalized candidate before return. The added test changes an owned finalized value during receipt encoding and requires refusal. Source is adopted locally; native donor/source admission, graph host ancestry, complete enrichment, calibration and release remain separate.", + "selected_test_cases": [ + "test_two_real_chains_finalize_once_on_the_complete_original_cohort", + "test_one_nonempty_route_uses_same_whole_cohort_finalizer[0]", + "test_one_nonempty_route_uses_same_whole_cohort_finalizer[1]", + "test_donor_parity_refuses_before_any_model_decode[recid_order]", + "test_donor_parity_refuses_before_any_model_decode[target]", + "test_donor_parity_refuses_before_any_model_decode[weight]", + "test_donor_parity_refuses_before_any_model_decode[capacity]", + "test_donor_parity_refuses_before_any_model_decode[shared_money]", + "test_donor_parity_refuses_before_any_model_decode[shared_status]", + "test_route_state_and_actual_model_consumption_are_bound[phase]", + "test_route_state_and_actual_model_consumption_are_bound[model_seed]", + "test_route_state_and_actual_model_consumption_are_bound[matrix]", + "test_route_state_and_actual_model_consumption_are_bound[producer]", + "test_route_state_and_actual_model_consumption_are_bound[raw_history]", + "test_route_state_and_actual_model_consumption_are_bound[last_model]", + "test_route_state_and_actual_model_consumption_are_bound[ninth_donor_value]", + "test_route_state_and_actual_model_consumption_are_bound[both_donor_targets]", + "test_route_state_and_actual_model_consumption_are_bound[both_recid_order]", + "test_route_state_and_actual_model_consumption_are_bound[both_weights]", + "test_empty_cohort_and_empty_route_roster_refuse_before_models[no_routes]", + "test_empty_cohort_and_empty_route_roster_refuse_before_models[no_clone_one]", + "test_last_model_return_cannot_change_a_retained_donor[target]", + "test_last_model_return_cannot_change_a_retained_donor[capacity]", + "test_model_donor_frame_matches_existing_single_profile_preparation", + "test_model_frame_constructor_return_cannot_coerce_a_donor_column", + "test_finalizer_return_cannot_change_retained_raw_or_donor[raw]", + "test_finalizer_return_cannot_change_retained_raw_or_donor[donor]", + "test_receipt_encoding_cannot_change_the_already_sealed_candidate", + "test_merge_restores_complete_receiving_order_and_preserves_source_values[False]", + "test_merge_restores_complete_receiving_order_and_preserves_source_values[True]", + "test_single_nonempty_route_omits_other_route[PufOutputProfile.PUF55_SURVEY_SS]", + "test_single_nonempty_route_omits_other_route[PufOutputProfile.PUF55_SURVEY_SS_NO_TOTAL]", + "test_refuses_inexact_recipient_partition[overlap]", + "test_refuses_inexact_recipient_partition[missing]", + "test_refuses_inexact_recipient_partition[extra]", + "test_refuses_inexact_recipient_partition[clone_zero]", + "test_refuses_changed_matrix_route_or_storage[matrix]", + "test_refuses_changed_matrix_route_or_storage[order]", + "test_refuses_changed_matrix_route_or_storage[mutable_bytes]", + "test_refuses_changed_matrix_route_or_storage[profile]", + "test_complete_chain_history_is_checked_before_merge[seed]", + "test_complete_chain_history_is_checked_before_merge[raw]", + "test_complete_chain_history_is_checked_before_merge[model_history]", + "test_complete_chain_history_is_checked_before_merge[target_history]", + "test_changed_detached_decoder_values_refuse", + "test_later_decoder_cannot_change_an_earlier_route_table" + ], + "source_adoption": true, + "source_and_typed_producer_authentication_still_required": true, + "source_count": 500, + "source_map_sha256": "67bdcfc92a36563de2b866b6d6b3bca88c127e90854977c681f56d61b6988e20", + "status": "frozen_numerical_output_seal_controls_accepted", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 46 + }, + "unchanged_raw_cases": 18, + "unexpected_refusals": {}, + "verification_passed": true, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 508 + }, + "wall_seconds": 21.00948112498736 +} diff --git a/experiments/us-puf55-population-controls-20260909.json b/experiments/us-puf55-population-controls-20260909.json new file mode 100644 index 000000000..dc8cb5d14 --- /dev/null +++ b/experiments/us-puf55-population-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 18.064019, + "independent_selected_value_oracle": "all 55 direct finalizer outputs compared by entity id", + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 630538240, + "predictor_count": 9, + "previous_receipt_sha256": "ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30", + "receipt_sha256": "e3a4d548f0209cfacd7d9e3beddb63c1b46204a3168eb9869d2b2eafe9cf2291", + "release_eligible": false, + "required_replay": "passed", + "result": 0, + "scope": "Invented PUF55 whole-Population attachment/replay and independent direct-finalizer oracle; no native recipient qualification or release acceptance.", + "source_and_owned_files": 486, + "ss_incumbent_bits_and_unknownness_preserved": true, + "tampered_ss_population_refusals": 2, + "target_count": 55, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 3 + }, + "unexpected_refusals": {}, + "wall_seconds": 20.73042149998946 +} diff --git a/experiments/us-puf55-public-recipient-controls-20260910.json b/experiments/us-puf55-public-recipient-controls-20260910.json new file mode 100644 index 000000000..a4866c0ce --- /dev/null +++ b/experiments/us-puf55-public-recipient-controls-20260910.json @@ -0,0 +1,85 @@ +{ + "actual_original_source_issuer_comparisons": true, + "actual_twenty_two_node_cold_and_required_replay": true, + "adopted_source_hashes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_survey_recipients.py": "888caefb31861bc5892d607ed3598c1ea86eb78ecf1e06035cf3f4f004c8cbd4", + "packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py": "b50b99d259c21ac71c4e07f6438465b6614f15ee937d496bac5a45720216246b", + "packages/microcosm-build/tests/test_us_puf55_survey_recipients.py": "fd7ff35d8e752ca7350c7022897f5afa806732ba7f1526f3ea2a1f26bf0a40b6" + }, + "calibration_acceptance": false, + "changed_source_files": [ + "packages/microcosm-build/tests/test_us_puf55_survey_recipients.py" + ], + "corrected_successor": { + "actual_model_chain_verified": false, + "actual_nan_storage_mutation_verified_by_test": true, + "approval_sha256": "fcc818febc58926eb12333217276daf3c41cfd665419b3583ef7d92422412a6a", + "cpu_seconds": 390.301149, + "financial_fixture_fit_and_graph_execution": true, + "full65_and_puf55_chain_acceptance": false, + "guard_sha256": "92b3cda8a663b073b55c5370a6a41d2a9143b30eb6a0c268ad7d9c561d1bd4c4", + "junit_sha256": "5fdb73added770d03cc56847234579f4dc4b7ac9cb11640337782d07d3966bd4", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "native_access": false, + "peak_rss_bytes": 539738112, + "postcheck_script_sha256": "f6d472f7e29dda2184df6b1819656c2ecf0679352b6517a44681d966b1fea90c", + "production_adopted": false, + "production_changes": false, + "project_imports_or_runtime_reexecution": false, + "puf_fit_or_finalization": false, + "receipt_sha256": "79d339ff7eb1fe9f9370d3f89976351612a8334bcce46ac91c4e3f549a05ba3d", + "recipient_test_sha256": "fd7ff35d8e752ca7350c7022897f5afa806732ba7f1526f3ea2a1f26bf0a40b6", + "release_acceptance": false, + "resource_map_sha256": "7200349a765cd20316304c73632608851b1320293f797c7c3da7e7daf2229742", + "root_review_sha256": "bb8d8adbea2e8579a39b69b32b774a0acaccf6b59d48d243f91164724fcc734a", + "scope": "five_invented_detached_recipient_output_mutation_controls", + "selected_test_cases": [ + "test_final_run_check_cannot_mutate_detached_recipient_outputs[person]", + "test_final_run_check_cannot_mutate_detached_recipient_outputs[matrix]", + "test_final_run_check_cannot_mutate_detached_recipient_outputs[columns_name]", + "test_final_run_check_cannot_mutate_detached_recipient_outputs[nan_bits]", + "test_final_run_check_cannot_mutate_detached_recipient_outputs[mutable_matrix]" + ], + "source_count": 499, + "source_map_sha256": "5d19ee15bcff9bd26458258bcff86fa18f03856afd93df32f214b599ab120695", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 5 + }, + "unexpected_refusals": {}, + "verification_passed": true, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 507 + }, + "wall_seconds": 392.8034432080458 + }, + "current_checkout_runtime_certification": false, + "distinct_passing_controls": 28, + "duplicated_passing_cases": 4, + "historical_run": { + "failed": 1, + "failed_case": "test_final_run_check_cannot_mutate_detached_recipient_outputs[nan_bits]", + "junit_sha256": "256569c21fa52f78f4e8e3e809b81fd738564aca32f02b2876e0d625168fe1ed", + "passed": 27, + "receipt_sha256": "c174e4051b711f3beee1a1aff20f8d7aced657b094cd4831ceec5b82fd224451", + "root_evidence_verification_sha256": "d70f8cb7edc2ab2d026fe5ca282e54d45576536d5cb48eb84d92a125ed2bae39", + "source_map_sha256": "9a4888a721f53a4203ed188b81642bed5b423616991ab4c191081afdd1f43af4", + "suite_accepted": false, + "tests": 28 + }, + "native_acceptance": false, + "new_nan_setup_asserts_actual_uint64_storage_change": true, + "passing_executions": 32, + "production_changed_between_runs": false, + "puf_fit_or_finalization": false, + "recipient_nodes_declare_original_survey_and_atomic_support_sources": true, + "release_acceptance": false, + "retained_source_files": 498, + "scope_note": "28 distinct public controls are covered by 27 passing cases in the preserved failed run plus five passing test-only successor cases, four overlapping. This is not a single all-green 28-case invocation. The only source change between these frozen runs makes the NaN test mutation observable; all production and original fixtures are identical. These runs use actual issuers and graphs over invented survey sources.", + "status": "public_recipient_controls_covered_across_preserved_run_and_test_only_successor" +} diff --git a/experiments/us-puf55-public-recipient-positives-20260910.json b/experiments/us-puf55-public-recipient-positives-20260910.json new file mode 100644 index 000000000..51cf6e145 --- /dev/null +++ b/experiments/us-puf55-public-recipient-positives-20260910.json @@ -0,0 +1,48 @@ +{ + "accepted_compilation_cache_overlay_sha256": "534042fdd4f2af5a238b556882d3b75c5495d9909e240bb07201ec48bd2bb4da", + "actual_issued_recipient_positive": true, + "actual_twenty_two_node_cold_required_replay": true, + "cpu_seconds": 331.268576, + "fable_strengthened_issuer_comparisons_included": false, + "frozen_budget_owner_sha256": "80418528db45266e7322a8984cc63f9414f6fa3c084119cd9d0f558f4a0e0260", + "guard_sha256": "45e7faf460d74210613e2898c4471d226a2b8d2e9f241d4c6945fcb7ec7cdb09", + "head": "640c1db49e6a70d47d58cea194e44df26c0762e5", + "junit_case_seconds": { + "test_actual_issued_financial_run_retains_source_reports_and_current_money": "104.759", + "test_actual_twenty_two_node_cold_required_replay_keeps_complete_population": "223.518" + }, + "junit_sha256": "de2befa8fe7f3878a681f4de9e4c81ad467d51fa7d75c33494258b0fc8c46d60", + "junit_timing_scope": "pytest per-case durations include assigned fixture setup/teardown; not isolated issuer or graph compute timings", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "original_fixture_teardowns_passed": true, + "original_twenty_node_cold_hit_assertion_present": false, + "peak_rss_bytes": 541048832, + "receipt_sha256": "ae02970da176ad92efacd831de8cf78e18ea1d96a811c124f2d268e6c5da71e4", + "release_acceptance": false, + "remaining_public_cases_executed": 0, + "remaining_public_controls": 24, + "resource_map_sha256": "c7e4193c952bd0d28d81db4371b51b51462870326fe59d5d26c13e4617f3e5b7", + "root_postcheck_sha256": "465b347efcc12e5f717a846207f2db1703b3c31648abefbe355d4274c0c61eab", + "scope": "2_invented_actual_public_puf55_recipient_issuer_and_graph_controls", + "scope_note": "Actual issuers and graph over invented survey originals; no native donor fitting, whole-cohort PUF finalization or enriched calibration.", + "selected_test_cases": [ + "test_us_puf55_recipients_public_positive.py::test_actual_issued_financial_run_retains_source_reports_and_current_money", + "test_us_puf55_recipients_public_positive.py::test_actual_twenty_two_node_cold_required_replay_keeps_complete_population" + ], + "source_count": 498, + "source_map_sha256": "504e045e6551895fabf8bc377fdfce1c69da9e70aa17989d2b771d5b66176c4b", + "status": "two_original_public_positives_accepted", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 506 + }, + "wall_seconds": 333.96333820803557 +} diff --git a/experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json b/experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json new file mode 100644 index 000000000..581263a6f --- /dev/null +++ b/experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json @@ -0,0 +1,28 @@ +{ + "code_resources": 12, + "control_accepted": true, + "cpu_seconds": 38.894051, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 517455872, + "publisher_provenance_established": false, + "receipt_sha256": "ac932822b7693129e3be97923bbc6798f8883f9fad0365b76b671117f7770f8e", + "release_eligible": false, + "result": 0, + "scope": "31 invented filer/joint-spouse Social Security report-sum measurement, missingness/refusal, authenticated retained-source callback controls, and isolated55-output/eight-predictor declarations. No native source reads, route-aware graph fitting, donor transport acceptance, beneficiary allocation, source admission or release acceptance.", + "source_and_owned_files": 498, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 31 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py": "f76bdc251811286e93f03d3aa9f5caac8548b60595739b5b17efce18d38e715e", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py": "06c4441f6397a69154f39ae905986a0a7928859cd38d7896fd1d8910ed5504cd", + "packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py": "6e7b63e2ac1f7bc2cd0cd6902a89facb91685f2c1f5c8b0d184a7e89143b3dce" + }, + "unexpected_refusals": {}, + "wall_seconds": 41.406911208992824 +} diff --git a/experiments/us-puf55-two-route-numerical-20260910.json b/experiments/us-puf55-two-route-numerical-20260910.json new file mode 100644 index 000000000..3c21f20a6 --- /dev/null +++ b/experiments/us-puf55-two-route-numerical-20260910.json @@ -0,0 +1,105 @@ +{ + "actual_model_chain_verified": true, + "approval_sha256": "f6e923a32416426702e4f1cb92ac91fbf7d07e6534285bd5116475f28a134918", + "bridge_owner_sha256": "ce1d0229965e3e8d2938ff74c205a0b43fcd9f997b795e6441ea9a80be7dfe7d", + "cpu_seconds": 15.591476, + "current_checkout_runtime_certification": false, + "fable_source_review_sha256": "5e9831d555950cda0151d19555aa5157fda944782eb071428166d999484866d7", + "financial_fixture_execution": false, + "fixture_apply_target_calls": 220, + "fixture_donors_per_model": 4, + "fixture_fit_target_calls": 110, + "fixture_trees": 2, + "full65_and_puf55_chain_acceptance": false, + "graph_host_execution": false, + "guard_sha256": "ab00e84bc478f48e2343f2e4da05b8e8777336fc07bc0141c5e7157f00a34f08", + "host_receiving_value_seal_still_required": true, + "junit_sha256": "7242c609dd524533c94848021c843cd9bd6638b9c252be173fa7d28d581b3762", + "model_cache_metadata_scans": 1506, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "native_access": false, + "new_numerical_cases": 27, + "numerical_test_sha256": "e3d98a911967e44009485b62dd7405a3628e087f2534d3a614e63e6e8aab1267", + "one_whole_cohort_finalizer_per_call": true, + "peak_rss_bytes": 680591360, + "postcheck_script_sha256": "eb3a753fd91d611fb9185fb73d5a3fb84f83c8e535fcf151a2d7c14f4958f1cd", + "production_adopted": true, + "project_imports_or_runtime_reexecution": false, + "puf_fit_and_numerical_finalization": true, + "receipt_sha256": "4dbc0e70ac4802e6d0a288d0f5ad2c0cab8f081659d1f6d8df3caae6597ae352", + "release_acceptance": false, + "resource_map_sha256": "0742c4e0c6ae9f202d0be2b801fef252bcdec716a45516f7033ab18c89eeda4d", + "root_postcheck_sha256": "7134b14292ba950bb133fd73a347bbccc61df6442f27fb83069e4889fad84519", + "root_review_sha256": "98e95235902ffca638061bf89ca30ebe1efe99d8cc49e56d4ce88ee0ffca2cca", + "scope": "45_invented_two_route_numerical_and_raw_merge_controls", + "scope_note": "Two real invented 55-target chains, exact merged whole-cohort finalization, single-route oracle equality, donor/model binding refusals, and post-finalizer mutation checks. The original18 raw-merger cases remain unchanged. This does not execute the canonical-source CREATE or the full receiving graph host; that host must authenticate producer ancestry before trusted model decoding and seal receiving table values across model/finalizer I/O.", + "selected_test_cases": [ + "test_two_real_chains_finalize_once_on_the_complete_original_cohort", + "test_one_nonempty_route_uses_same_whole_cohort_finalizer[0]", + "test_one_nonempty_route_uses_same_whole_cohort_finalizer[1]", + "test_donor_parity_refuses_before_any_model_decode[recid_order]", + "test_donor_parity_refuses_before_any_model_decode[target]", + "test_donor_parity_refuses_before_any_model_decode[weight]", + "test_donor_parity_refuses_before_any_model_decode[capacity]", + "test_donor_parity_refuses_before_any_model_decode[shared_money]", + "test_donor_parity_refuses_before_any_model_decode[shared_status]", + "test_route_state_and_actual_model_consumption_are_bound[phase]", + "test_route_state_and_actual_model_consumption_are_bound[model_seed]", + "test_route_state_and_actual_model_consumption_are_bound[matrix]", + "test_route_state_and_actual_model_consumption_are_bound[producer]", + "test_route_state_and_actual_model_consumption_are_bound[raw_history]", + "test_route_state_and_actual_model_consumption_are_bound[last_model]", + "test_route_state_and_actual_model_consumption_are_bound[ninth_donor_value]", + "test_route_state_and_actual_model_consumption_are_bound[both_donor_targets]", + "test_route_state_and_actual_model_consumption_are_bound[both_recid_order]", + "test_route_state_and_actual_model_consumption_are_bound[both_weights]", + "test_empty_cohort_and_empty_route_roster_refuse_before_models[no_routes]", + "test_empty_cohort_and_empty_route_roster_refuse_before_models[no_clone_one]", + "test_last_model_return_cannot_change_a_retained_donor[target]", + "test_last_model_return_cannot_change_a_retained_donor[capacity]", + "test_model_donor_frame_matches_existing_single_profile_preparation", + "test_model_frame_constructor_return_cannot_coerce_a_donor_column", + "test_finalizer_return_cannot_change_retained_raw_or_donor[raw]", + "test_finalizer_return_cannot_change_retained_raw_or_donor[donor]", + "test_merge_restores_complete_receiving_order_and_preserves_source_values[False]", + "test_merge_restores_complete_receiving_order_and_preserves_source_values[True]", + "test_single_nonempty_route_omits_other_route[PufOutputProfile.PUF55_SURVEY_SS]", + "test_single_nonempty_route_omits_other_route[PufOutputProfile.PUF55_SURVEY_SS_NO_TOTAL]", + "test_refuses_inexact_recipient_partition[overlap]", + "test_refuses_inexact_recipient_partition[missing]", + "test_refuses_inexact_recipient_partition[extra]", + "test_refuses_inexact_recipient_partition[clone_zero]", + "test_refuses_changed_matrix_route_or_storage[matrix]", + "test_refuses_changed_matrix_route_or_storage[order]", + "test_refuses_changed_matrix_route_or_storage[mutable_bytes]", + "test_refuses_changed_matrix_route_or_storage[profile]", + "test_complete_chain_history_is_checked_before_merge[seed]", + "test_complete_chain_history_is_checked_before_merge[raw]", + "test_complete_chain_history_is_checked_before_merge[model_history]", + "test_complete_chain_history_is_checked_before_merge[target_history]", + "test_changed_detached_decoder_values_refuse", + "test_later_decoder_cannot_change_an_earlier_route_table" + ], + "source_adoption": true, + "source_and_typed_producer_authentication_still_required": true, + "source_count": 500, + "source_map_sha256": "4bfe1790225e49820547aedd02f84203fdfd8658bfb881ac76258ffcbc3f5d2b", + "status": "frozen_two_route_numerical_bridge_accepted", + "test_correction_diff_sha256": "43f79383df5805133baaa38802d8ed61fd2317dc547fa679a81e086d63fe6782", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 45 + }, + "unchanged_raw_cases": 18, + "unexpected_refusals": {}, + "verification_passed": true, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 508 + }, + "wall_seconds": 16.730265708989464 +} diff --git a/experiments/us-puf55-two-route-values-controls-20260910.json b/experiments/us-puf55-two-route-values-controls-20260910.json new file mode 100644 index 000000000..5db3a77bc --- /dev/null +++ b/experiments/us-puf55-two-route-values-controls-20260910.json @@ -0,0 +1,38 @@ +{ + "attachment_performed": false, + "cpu_seconds": 3.8935099999999996, + "date": "2026-09-10", + "finalization_performed": false, + "fitted_model_or_donor_authority_verified": false, + "guard_sha256": "bde8f7ce2742b45a750e7705be6fc899b7f04f3d913d933d69594ff6724ca6e9", + "junit_sha256": "1619f4b57c5a39df909ee03f7c54d8dd05f634bdda68fe59d12185d012bf4431", + "native_acceptance": false, + "native_access": false, + "next_step": "Bind actual recipient and donor/model owners, execute both real routes, merge raw targets, finalize the whole clone cohort once, and attach once.", + "peak_rss_bytes": 438517760, + "peer_source_review_sha256": "df628b7d237decebbf252f0c51cc215d2801fa67a355068e05a0a0ae21ee70b1", + "postcheck_sha256": "15732db6c91ba14f8c9c012a0f4251c37e6cd1793287958ce0d7f92739e588c4", + "protocol": "microcosm.puf55-two-route-values-controls.v1", + "raw_merge_only": true, + "receipt_sha256": "d52648c33cf48d786a6ee7d4a29838a03e936700db8de05cb4ae0a5b476cac1e", + "release_acceptance": false, + "scope": "18_invented_puf55_raw_route_merge_controls", + "source_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py": "5ce739283cd80d37c17a3063dc0f8585f303411c21b2f5d87cc92ffb6345f9cd", + "packages/microcosm-build/tests/test_us_puf55_route_finalization.py": "f3638ceb41953af57f50c2a43b58eac7fb9dcc4f959832d4921da9ac236041d9" + }, + "source_or_population_admission": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 18 + }, + "typed_chain_metadata_synthetic": true, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 505 + }, + "wall_seconds": 4.931140874978155 +} diff --git a/experiments/us-source-string-seal-20260912.json b/experiments/us-source-string-seal-20260912.json new file mode 100644 index 000000000..c2ea1e348 --- /dev/null +++ b/experiments/us-source-string-seal-20260912.json @@ -0,0 +1,96 @@ +{ + "scope": "invented columns, exact old byte-stream parity; no native build timing claim", + "baseline_commit": "d5cbe60b2c6402648565f5139dbf7d94be209def", + "before_source_sha256": "b1c0ba9856714ce2131f73af6daa7731265a4fcc50c696d095a28f80a9d7cd03", + "after_source_sha256": "9e6ecca118942f7a76a2ef66de22abc082d3e19ca87bca67ca01477958c5b7e2", + "cases": [ + { + "case": "low_cardinality_python_strings", + "rows": 500000, + "dtype": ")>", + "exact_digest_equal": true, + "sha256": "44146e967c4e15ce95279827bcd34fed25b12731c223a3b2ffb16b50026df371", + "before_cpu_seconds": 0.15536000000000016, + "after_cpu_seconds": 0.0252460000000001, + "speedup": 6.1538461538461355, + "all_cpu_samples": { + "before": [ + 0.158949, + 0.1550100000000001, + 0.15536000000000016 + ], + "after": [ + 0.025262000000000118, + 0.024386999999999937, + 0.0252460000000001 + ] + } + }, + { + "case": "low_cardinality_arrow_strings", + "rows": 500000, + "dtype": ")>", + "exact_digest_equal": true, + "sha256": "03a54e7514b1419afbc6e092f3cd308e3c760be4f30c318e08ab85badc5f0241", + "before_cpu_seconds": 0.15287399999999995, + "after_cpu_seconds": 0.02783200000000008, + "speedup": 5.492742167289434, + "all_cpu_samples": { + "before": [ + 0.1512340000000001, + 0.1538870000000001, + 0.15287399999999995 + ], + "after": [ + 0.028616999999999893, + 0.02783200000000008, + 0.027589000000000086 + ] + } + }, + { + "case": "unique_source_keys", + "rows": 100000, + "dtype": ")>", + "exact_digest_equal": true, + "sha256": "3fa0e7051459014e6069c01acde5e7afc8de133e769a52e0a520fe75821a6d0b", + "before_cpu_seconds": 0.031774999999999665, + "after_cpu_seconds": 0.035517999999999716, + "speedup": 0.8946168140097955, + "all_cpu_samples": { + "before": [ + 0.03200500000000028, + 0.030060000000000198, + 0.031774999999999665 + ], + "after": [ + 0.035517999999999716, + 0.035597999999999796, + 0.03467199999999959 + ] + } + }, + { + "case": "numeric_control", + "rows": 500000, + "dtype": "dtype('int64')", + "exact_digest_equal": true, + "sha256": "d8577194aeb21cddc40a14a2e0a1f2f13dee18b0b9c26e83842214f3b98bb2ba", + "before_cpu_seconds": 0.0013450000000001516, + "after_cpu_seconds": 0.0013260000000001604, + "speedup": 1.0143288084464472, + "all_cpu_samples": { + "before": [ + 0.0014279999999997628, + 0.0013450000000001516, + 0.0013090000000000046 + ], + "after": [ + 0.0013320000000001109, + 0.0013260000000001604, + 0.0012980000000002434 + ] + } + } + ] +} diff --git a/experiments/us-source-string-seal-20260912.md b/experiments/us-source-string-seal-20260912.md new file mode 100644 index 000000000..2309362a3 --- /dev/null +++ b/experiments/us-source-string-seal-20260912.md @@ -0,0 +1,31 @@ +# Repeated source-string hashing + +The ASEC source validator now reuses the exact length-prefixed encoding of a +repeated string within one column's digest calculation. It still visits every +value and computes a new digest on every validation. The cache holds at most +1,024 entries and 256 KiB of encoded payload, then stops admitting new entries. +Only exact Python strings use it; other scalar families retain the original +codec. Dtype metadata, null sentinels, row order and all digest bytes retain +their prior meaning. + +Thirty invented-source tests pass, including eleven added byte-stream parity +and later-mutation controls. They cover Python and Arrow strings, distinct +Unicode encodings, nulls, more than 1,024 distinct values, strings exceeding the +payload budget, and the unchanged bytes, boolean, integer and floating object +paths. The suite took 1.860 seconds, with no failures, errors or skips. + +The [benchmark receipt](us-source-string-seal-20260912.json) compares the exact +pre-change function from `d5cbe60b2` with the new implementation. Each row reports +the median of three process-CPU measurements on invented columns: + +| Column | Rows | Before | After | Ratio | +| --- | ---: | ---: | ---: | ---: | +| Repeated Python strings | 500,000 | 0.15536 s | 0.02525 s | 6.15× faster | +| Repeated Arrow strings | 500,000 | 0.15287 s | 0.02783 s | 5.49× faster | +| Unique source keys | 100,000 | 0.03178 s | 0.03552 s | 12% slower | +| Integer control | 500,000 | 0.00135 s | 0.00133 s | Approximately unchanged | + +All four complete digests match exactly. These measurements do not establish +native build speed or quality. Source-file identity changes because executable +code changed; an old graph receipt is not acceptance of the new implementation. +The active native PUF pilot retains its original frozen source. diff --git a/experiments/us-survey-age-development-20260909.json b/experiments/us-survey-age-development-20260909.json new file mode 100644 index 000000000..54df4b623 --- /dev/null +++ b/experiments/us-survey-age-development-20260909.json @@ -0,0 +1,54 @@ +{ + "artifact_readback_and_final_owner_target_verification": "passed inside the unchanged reviewed harness; final manifest exists only after final verification", + "code_resource_pins_current_equal": true, + "code_resources": 12, + "compiled_order": [ + "survey_population.create", + "survey_population.allocate", + "combined_survey_puf_support_clone", + "combined_survey_puf_support_clone.owned", + "survey.age_count_matrix", + "survey.sampling_budget", + "survey.age_calibration" + ], + "cpu_seconds": 4115.737018, + "cross_process_source_authority_granted": false, + "entity_rows": { + "current": { + "household": 3168, + "person": 6928 + }, + "prepared": { + "household": 1584, + "person": 3464 + }, + "previous": { + "household": 3168, + "person": 6928 + } + }, + "export_count": 5, + "manifest_sha256": "2bdb68de84635c6cc2e0c25555887693304792e25d40052cd77996774cb7f669", + "native_counts": { + "acs": { + "households": 1529, + "persons": 3324 + }, + "asec": { + "households": 55, + "persons": 140 + } + }, + "native_payloads_read_by_postcheck": 0, + "owned_before_after_current_equal": true, + "peak_rss_bytes": 8260485120, + "receipt_sha256": "2aa76b5daca7bb0df1ac85d4db23d6b5482073d31d437418d46442ef01b002ca", + "release_eligible": false, + "result": 0, + "scope": "genuine seven-node ACS+ASEC age-development artifact acceptance; not full enrichment or release", + "source_before_after_current_equal": true, + "source_control_runtime_files": 473, + "source_head": "a9895d8a50886470f6dfe4b2b9d20c352c9871cf", + "unexpected_refusals": {}, + "wall_seconds": 4268.8438986669935 +} diff --git a/experiments/us-survey-catalogue-memo20-20260910.json b/experiments/us-survey-catalogue-memo20-20260910.json new file mode 100644 index 000000000..ff6b93661 --- /dev/null +++ b/experiments/us-survey-catalogue-memo20-20260910.json @@ -0,0 +1,32 @@ +{ + "code": [ + { + "maintained_sha256": "508d18bac53690f8841e3eb2770bc2aa6079b9180ad0a31c65c5566f3304a206", + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py" + }, + { + "maintained_sha256": "8a9742e108347916937317a0901a462c315d688b32795758972ff9554b07d32f", + "path": "packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py" + } + ], + "evidence": { + "cpu_seconds": 18.188797, + "model_sources": 0, + "peak_rss_bytes": 405815296, + "physical_postcheck_sha256": "fcd8dcdb6b706f65888b1e33add9c042302d51358597973db14901e0c70f17b8", + "receipt_sha256": "ee5279f9678d035b0d6ede176704b4d04196d1eea351adfba56510dba973f972", + "resources": 12, + "source_peer_review_sha256": "c5e0e1188fdac1e84929f46d8e13f5e42e33b286f86a121ddd16aa38222c7e1d", + "source_plus_owned": 514, + "tests_failed": 0, + "tests_passed": 20, + "unexpected_refusals": {}, + "wall_seconds": 18.8203088750015, + "xml_sha256": "04dec52f2e8ca14815b8368a7ac2023d2f08f752c94367f9a6d7b2189402077f" + }, + "maintained_full_CI_accepted": false, + "native_build_accepted": false, + "release_eligible": false, + "scope": "Reuse a sealed immutable ACS catalogue value within its issued preparation. All source and producer checks, complete Frame seals and final checks remain; replacing roots or mutable contents takes the original hashing path. Native performance remains to be measured.", + "status": "scoped_controls_accepted" +} diff --git a/experiments/us-survey-enrichment-integration-20260912.md b/experiments/us-survey-enrichment-integration-20260912.md new file mode 100644 index 000000000..3fef6cc29 --- /dev/null +++ b/experiments/us-survey-enrichment-integration-20260912.md @@ -0,0 +1,90 @@ +# Survey amount and health integration + +The release branch now includes the source-qualified health fragment from +`7cb1384be82ae4bd559577c5ebe142bf708e8570` and the fixed post-PUF enrichment host +from `99c35b0865e6f6cfe86ede15a10da91360a5e66b`. Their local integration commits +are `f92418a859e576ab7fd40919d3e9d75f41bd1ee4` and +`091261c0e979d0ee191ddbaf08a069b2554d7165`. Both cherry-picks were conflict-free. + +The preceding main merge incorporated shared graph PR #913, main revision +`a9cc63e737fd4619a304117dc2ec7dcd86d97901`, without changing the release branch's +existing source tree. The checked PUF API, eighth financial output, source-string +hashing optimization, ACS anchor qualification and signed reconciliation helper +are preserved. None of those building blocks is interpreted as native acceptance +of this integrated candidate. + +## Accepted combined stage + +The host retains the original checked PUF run and original survey source owners. +It adds four amounts and nine coverage fields to `survey_puf55.receiving`, with +health explicitly dependent on the amount attachment. Source reporting-universe, +knownness and allocation information remain distinct from numeric values. +The [source/model specification](../docs/us-current-survey-amount-successor.md) +records UC classification, PHIP_VAL rather than PHIP_VAL2, the conditional +health-cost chain, deliberate ACS clone-sharing and narrower coverage gaps. + +On the accepted prior implementation, the 18-person invented fixture executed +26 new nodes over a validated 245-node prefix. Required replay hit all 271 nodes; +complete Frame readback preserved parent columns, owners, geography, design +anchors and mass ledger. The diagnostic profile contained 74 of 161 input names, +leaving 87 missing. UC had ten known and eight unresolved values; all 18 had the +three medical-cost amounts and ESI, while narrower Medicaid coverage had eight +known and ten unresolved values. These are invented counts, not population +estimates, native coverage or evidence of statistical quality. + +Acceptance combines the first five passing tests from the main v3 attempt and +the corrected final parent-revocation test from a focused v4 attempt. The v3 +result remains failed: its last test named a nonexistent income column and +stopped before mutation. The corrected test uses the actual +`employment_income_before_lsr` column and proves permanent parent and child +revocation, including after restoring the original value. It passed without +repeating the already-passed required replay and Frame readback. There is no +claim of one all-green full v3 run. Independent source and evidence review +approved this six-control coverage at unchanged production bytes. + +| Preserved local record | SHA256 | +| --- | --- | +| Combined v3 receipt | `19495b1cf3e6aad2f60771c8dce557ccbdc6a78b98c1a1ad453229276ac7e4a4` | +| Combined v3 aggregate acceptance | `07ac957851b19546a5ea1d2123975b9fa616dc8fe94a75e171c93e2379a094da` | +| Focused v4 parent-revocation receipt | `42bbeeae05b01a737d3c50cf3b5569be37cab43d71378fd8e78872965697170c` | + +The v3 acceptance took 1,388.313 seconds with 917,979,136 bytes peak RSS; the +focused v4 control took 624.872 seconds with 885,669,888 bytes peak RSS. Both +retained 857 Python source pins and Torch thread settings of 1/1. These wrappers +are not the strict native pilot audit guard. The source-reviewed execution path +used invented fixtures; receipt flags alone do not prove native admission or +independent network/resource restrictions. Failed attempts remain preserved. + +## Checks after integration + +At integrated source `091261c0e979d0ee191ddbaf08a069b2554d7165`, 80 tests passed +with zero failures, errors or skips in 27.149 seconds wall time; peak RSS was +503,201,792 bytes. The selection includes all 32 amount controls, all 31 health +source/fragment controls, the actual invented UC source-qualification test and +the cheap financial/source controls, including the conserving interest +attachment. The health fragment executes cold and required replay. The expensive +financial/PUF/combined fixture was not selected again. + +All 477 Python files under the shard source directories retained their hashes; +Torch threads remained 1/1. CPU and wall limits were applied. This is bounded +invented integration evidence, not the native admission guard. + +| Integrated record | SHA256 | +| --- | --- | +| Receipt | `b2a764144afe952bba7a0f274cbf4f71ad6f0e46174743610127df7d42e30ba9` | +| JUnit result | `201fff9ac69b9259a9a7c11a291904dac97f082a92364521b622fc80d6805074` | + +Actual declaration functions, invoked with invented descriptive values, confirm +three financial model targets, eight financial output fields and ten financial +extension nodes. Each PUF route retains 55 fits and 55 applications. The amount +fragment has 14 nodes and health has 12; together they compile with the explicit +parent and attachment dependencies. The complete host therefore still declares +271 nodes. The added tax-exempt-interest output changes the existing attachment +and its source identity, not the target chain or node count. No issued parent, +native source or full graph execution is established by this declaration probe. + +Native PUF execution, original-channel tax-detail completion, remaining inputs, +full-candidate coverage, model evaluation, calibration and release verification +remain separate work. The ACS anchor qualifier and signed projection helper are +adopted source/numeric building blocks; the current enrichment host does not yet +invoke them to reconcile property or retirement components. diff --git a/experiments/us-survey-geography-graph-controls-20260909.json b/experiments/us-survey-geography-graph-controls-20260909.json new file mode 100644 index 000000000..6952299e9 --- /dev/null +++ b/experiments/us-survey-geography-graph-controls-20260909.json @@ -0,0 +1,21 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 313.767356, + "model_source_files": 5983, + "peak_rss_bytes": 439091200, + "receipt_sha256": "28a6b5c58a258c7b3129a48054a3360647c788f8b76d781cc3bc8fc35a73f997", + "release_eligible": false, + "result": 0, + "scope": "Observed-geography graph node over actual invented original-source issuers: cold/replay/unknownness/binding/mutation controls. No native atomic assignment or release acceptance.", + "source_and_owned_files": 486, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 14 + }, + "unexpected_refusals": {}, + "wall_seconds": 319.43621275003534 +} diff --git a/experiments/us-survey-geography-source-controls-20260909.json b/experiments/us-survey-geography-source-controls-20260909.json new file mode 100644 index 000000000..f34e40267 --- /dev/null +++ b/experiments/us-survey-geography-source-controls-20260909.json @@ -0,0 +1,21 @@ +{ + "before_after_and_frozen_source_control_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 178.432832, + "model_source_files": 5983, + "peak_rss_bytes": 430325760, + "receipt_sha256": "1900801b1bbcea89a3747f656410e500b1328519694c3351100562bba61cab08", + "release_eligible": false, + "result": 0, + "scope": "Invented source qualification only; no native atomic assignment, engine, data or release acceptance.", + "source_and_control_files": 484, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 11 + }, + "unexpected_refusals": {}, + "wall_seconds": 182.49943441699725 +} diff --git a/experiments/us-survey-interest-conservation-20260912.md b/experiments/us-survey-interest-conservation-20260912.md new file mode 100644 index 000000000..97bd273a0 --- /dev/null +++ b/experiments/us-survey-interest-conservation-20260912.md @@ -0,0 +1,32 @@ +# Conserving survey interest before PUF enrichment + +The financial graph now retains the tax-exempt remainder of the same interest +total used for its taxable component. Previously it attached only the taxable +part, leaving the original survey support's exempt component unresolved after +the clone1-only PUF attachment. + +The new eighth financial output is `INT_VAL - taxable_interest_income`, computed +before source-key alignment to both clones. ASEC uses its qualified current +amount; ACS uses the existing modeled interest total. The original taxable +calculation and legacy seven-leaf helper are unchanged. The graph's judgment +metadata identifies the split as an assumption, not separately observed +taxable and exempt amounts. It does not establish that the modeled ACS interest +total reconciles to the broader observed ACS property-income aggregate. + +Two new numerical controls first reproduced the missing column. After repair, +all 25 predictor and atomic-financial tests passed in 294.225 seconds, including +nonconstant and zero-regime invented source runs, source-key permutation, +conservation and cold/required graph replay. + +Three full PUF controls then passed in 617.283 seconds: the actual two-route +cold/required extension, independent whole-cohort finalizer comparison and +non-owned storage preservation, and retained checked-parent validation without +re-execution. The original survey complement remains identical through the +PUF attachment; clone1 retains its existing PUF ownership. All tests use +invented sources. No failure, error or skip occurred in either green run. + +Independent source review found no actionable defect. Ruff and CI test inventory +verification pass. The executable source SHA256 is +`0df64e5f7768b326f43d7b0ac927ccb24c89de8e3d21556a9139628a369b5358`. +These checks do not certify a native candidate, fiscal calibration or release. +The running native PUF pilot remains on its earlier frozen source. diff --git a/experiments/us-survey-numeric-diagnostics27-20260910.json b/experiments/us-survey-numeric-diagnostics27-20260910.json new file mode 100644 index 000000000..c74a4409e --- /dev/null +++ b/experiments/us-survey-numeric-diagnostics27-20260910.json @@ -0,0 +1,38 @@ +{ + "code": [ + { + "maintained_sha256": "22ff28977582c3b903ecf5d52b791a9f864a6a3c3df3b64541571e9199876b81", + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py" + }, + { + "maintained_sha256": "4d72fa2260af1178c889819295f39f3c2a0b79c37152eaf14e83cb7483b907aa", + "path": "packages/microcosm-build/tests/test_us_survey_calibration.py" + } + ], + "evidence": { + "cpu_seconds": 3.194314, + "model_sources": 0, + "peak_rss_bytes": 414924800, + "physical_postcheck_sha256": "e8a6f475e8bb8564c919ccfcbfce28182abd5b57e61029d0dab40404bae45d31", + "receipt_sha256": "50c1a09d4e71cffb00a375a703cb04623521a612478c34b2ac558c907bd4188b", + "resources": 0, + "source_peer_review_sha256": "4bfde0c677cecde4f2912a0a524c937292d33ee40ba35c10d5d29d34fa4c7b71", + "source_plus_owned": 512, + "tests_failed": 0, + "tests_passed": 27, + "unexpected_refusals": {}, + "wall_seconds": 3.6059712909627706, + "xml_sha256": "e8dc6d33a24fcf4c0a226216cb6be574b599c73f4b16cb666bb9db1724d35ba1" + }, + "maintained_full_CI_accepted": false, + "native_build_accepted": false, + "release_eligible": false, + "scope": "Independently reconstruct four closed solver options when verifying calibration diagnostics: no supplied gate initialization, nonzero-count budget basis, no feasible-draw upper bound and no budget search. Real grouped Adam over five invented households and four altered-option refusals pass against maintained calibration source. Complete age graph and native calibration remain separate.", + "status": "scoped_controls_accepted", + "test_formatting": { + "accepted_sha256": "e8a5fd5eea56ac357c5e919de0206d7ea44ff77b5dbf3809bea214b5d0e01109", + "change": "Ruff parenthesized one import; no AST change.", + "complete_AST_equal": true, + "maintained_sha256": "4d72fa2260af1178c889819295f39f3c2a0b79c37152eaf14e83cb7483b907aa" + } +} diff --git a/packages/microcosm-build/README.md b/packages/microcosm-build/README.md index dcc65be05..d2ea129cd 100644 --- a/packages/microcosm-build/README.md +++ b/packages/microcosm-build/README.md @@ -49,6 +49,11 @@ The `us` extra adds the rules engine for formula/export checks. Country source loaders are not Python dependencies: source stages are declared in packaged JSON manifests and executed by shared Microcosm runtimes. +The `source-io` extra installs h5py and PyTables for HDF survey input without a +country rules engine. US source preparation also uses `microcosm-frame[us]` +for tax-unit construction. These extras provide readers and constructors; +they do not download or authorize any source data. + Country namespaces under `microcosm.build.us` and `microcosm.build.uk` are resource packages only. They may contain specs and data artifacts, but no Python modules; guard tests enforce this so country content stays declarative. diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index 8f64f0a5d..e9ee60538 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -26,6 +26,8 @@ dependencies = [ ] [project.optional-dependencies] +# Survey HDF inputs can be prepared without installing a country rules engine. +source-io = ["h5py>=3", "tables>=3"] # The US extra adds the rules engine for formula-owned export checks. Source # content is declared in packaged manifests and interpreted by shared Microcosm # runtimes; country source loaders must not depend on incumbent data packages. diff --git a/packages/microcosm-build/src/microcosm/build/atomic_geography.py b/packages/microcosm-build/src/microcosm/build/atomic_geography.py new file mode 100644 index 000000000..dc419490f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/atomic_geography.py @@ -0,0 +1,591 @@ +"""Country-neutral atomic geography support, assignment and functional lookups. + +Publisher adapters supply one row per atomic area. Country declarations choose +constraints, sampling stages and output names. Observed geography is never +overwritten. This module does not acquire sources or certify their authority. +""" + +from __future__ import annotations + +import bisect +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from io import BytesIO +from types import MappingProxyType +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo + +import numpy as np +import pandas as pd + +from microcosm.graph.canonical import canonical_json +from microcosm.graph.randomness import keyed_uniform + +RELATIONS = frozenset({"exact", "best_fit", "official_tabulation", "inferred_modal"}) +_U53 = 2**53 + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("Atomic geography: " + message) + + +def _text(value: object) -> bool: + return isinstance(value, str) and bool(value) and value.strip() == value + + +def _unique_json(pairs): + result = {} + for key, value in pairs: + _require(key not in result, "duplicate metadata key") + result[key] = value + return result + + +@dataclass(frozen=True) +class AtomicSupport: + """Validated, immutable arrays decoded from a pinned support artifact.""" + + metadata: Mapping + arrays: Mapping[str, np.ndarray] + sha256: str + + +def encode_atomic_support(metadata: Mapping, arrays: Mapping[str, np.ndarray]) -> bytes: + """Create deterministic NPZ bytes; validate through the same decoder as readers.""" + _require("metadata_json" not in arrays, "reserved metadata array") + output = BytesIO() + members = {**arrays, "metadata_json": np.asarray(canonical_json(metadata).decode())} + with ZipFile(output, "w", compression=ZIP_DEFLATED) as archive: + for name, array in sorted(members.items()): + _require(_text(name) and name.isidentifier(), "invalid array name") + member = BytesIO() + np.lib.format.write_array(member, np.asarray(array), allow_pickle=False) + info = ZipInfo(name + ".npy", date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = ZIP_DEFLATED + archive.writestr(info, member.getvalue()) + payload = output.getvalue() + decode_atomic_support(payload) + return payload + + +def decode_atomic_support(payload: bytes) -> AtomicSupport: + """Refuse ambiguous mappings, object arrays, untyped columns and invalid weights.""" + _require(type(payload) is bytes, "support requires immutable bytes") + with np.load(BytesIO(payload), allow_pickle=False) as archive: + _require(len(set(archive.files)) == len(archive.files), "duplicate array") + meta_array = archive["metadata_json"] + _require( + meta_array.shape == () and meta_array.dtype.kind == "U", "metadata encoding" + ) + metadata = json.loads(str(meta_array.item()), object_pairs_hook=_unique_json) + _require( + isinstance(metadata, dict) + and set(metadata) + == {"version", "system", "level", "code_system", "vintage", "columns"} + and type(metadata["version"]) is int + and metadata["version"] == 1, + "metadata schema", + ) + _require( + all( + _text(metadata[k]) + for k in ("system", "level", "code_system", "vintage") + ), + "area identity", + ) + columns = metadata["columns"] + _require( + isinstance(columns, dict) and "area" in columns and bool(columns), + "column metadata", + ) + _require(set(archive.files) == {"metadata_json", *columns}, "undeclared array") + arrays = {} + for name, description in columns.items(): + _require( + _text(name) and name.isidentifier() and name != "metadata_json", + "column name", + ) + _require(isinstance(description, dict), "column description") + array = archive[name] + _require( + array.ndim == 1 and len(array) > 0, "nonempty one-dimensional support" + ) + kind = description.get("kind") + if kind == "code": + _require( + set(description) == {"kind", "source", "vintage", "relation"}, + "code metadata", + ) + _require( + array.dtype.kind == "U" and np.all(np.char.str_len(array) > 0), + "nonempty string codes", + ) + _require(np.all(np.char.strip(array) == array), "whitespace in codes") + _require( + _text(description["vintage"]) + and description["relation"] in RELATIONS, + "mapping relation or vintage", + ) + else: + _require( + kind == "weight" + and set(description) == {"kind", "source", "basis"}, + "weight metadata", + ) + _require( + array.dtype.kind in "iu" and np.all(array >= 0), + "nonnegative integer weights", + ) + _require(_text(description["basis"]), "sampling basis") + # Python accumulation cannot wrap before the bound is checked. + total = sum(int(value) for value in array) + _require(0 < total <= _U53, "weight total outside exact sampling range") + array = array.astype(np.int64) + _require(_text(description["source"]), "source identity") + # Bytes-backed arrays cannot be made writable by changing flags. + arrays[name] = np.frombuffer(array.tobytes(), dtype=array.dtype) + area = arrays["area"] + _require(columns["area"]["kind"] == "code", "atomic area must be a code") + _require( + columns["area"]["vintage"] == metadata["vintage"], "atomic vintage mismatch" + ) + _require( + columns["area"]["relation"] == "exact", "atomic identity must be exact" + ) + _require( + all(len(a) == len(area) for a in arrays.values()), "array lengths differ" + ) + _require(len(np.unique(area)) == len(area), "atomic codes must be unique") + _require( + any(c["kind"] == "weight" for c in columns.values()), + "missing sampling weights", + ) + frozen = { + **metadata, + "columns": MappingProxyType( + {k: MappingProxyType(v) for k, v in columns.items()} + ), + } + return AtomicSupport( + MappingProxyType(frozen), + MappingProxyType(arrays), + hashlib.sha256(payload).hexdigest(), + ) + + +def validate_assignment_spec(spec: Mapping) -> dict: + """Normalize a graph parameter mapping without accepting undeclared options. + + All geography code columns use strings, preserving leading zeroes. A later + engine adapter may declare a separate storage conversion when needed. + """ + value = json.loads(canonical_json(spec)) + _require( + set(value) == {"version", "identity", "stream", "outputs", "systems"} + and value["version"] == 1 + and type(value["version"]) is int, + "assignment schema", + ) + identity, outputs, systems = value["identity"], value["outputs"], value["systems"] + _require( + isinstance(identity, list) + and bool(identity) + and all(_text(c) for c in identity) + and len(set(identity)) == len(identity), + "stable identity columns", + ) + _require( + isinstance(outputs, dict) + and set(outputs) == {"area", "system", "basis"} + and all(_text(c) for c in outputs.values()) + and len(set(outputs.values())) == 3, + "assignment outputs", + ) + keyed_uniform(stream=tuple(value["stream"]), keys=[]) + _require(isinstance(systems, list) and bool(systems), "area systems") + ids, layers = set(), {} + for system in systems: + _require( + isinstance(system, dict) + and set(system) + == { + "id", + "level", + "code_system", + "vintage", + "source", + "selector", + "constraints", + "observed_area", + "stages", + "layers", + }, + "system schema", + ) + _require( + all( + _text(system[k]) + for k in ("id", "level", "code_system", "vintage", "source") + ), + "system identity", + ) + _require(system["id"] not in ids, "duplicate system") + ids.add(system["id"]) + selector = system["selector"] + _require( + isinstance(selector, dict) + and all( + _text(c) + and isinstance(v, list) + and bool(v) + and all(_text(x) for x in v) + and len(set(v)) == len(v) + for c, v in selector.items() + ), + "system selector", + ) + _require( + system["observed_area"] is None or _text(system["observed_area"]), + "observed atomic column", + ) + constraints = system["constraints"] + _require(isinstance(constraints, list), "constraints") + seen = set() + for c in constraints: + _require( + isinstance(c, dict) + and set(c) == {"input", "support", "required"} + and _text(c["input"]) + and _text(c["support"]) + and type(c["required"]) is bool, + "constraint schema", + ) + _require( + c["support"] not in seen and c["support"] != "area", + "duplicate or atomic constraint", + ) + seen.add(c["support"]) + stages = system["stages"] + _require(isinstance(stages, list) and bool(stages), "sampling stages") + _require( + all( + isinstance(s, dict) + and set(s) == {"level", "weight"} + and _text(s["level"]) + and _text(s["weight"]) + for s in stages + ), + "stage schema", + ) + _require( + stages[-1]["level"] == "area" + and len({s["level"] for s in stages}) == len(stages), + "stages must end at the unique atomic level", + ) + _require(isinstance(system["layers"], list), "layers") + local_layers = set() + for layer in system["layers"]: + _require( + isinstance(layer, dict) + and set(layer) == {"input", "output", "vintage", "relation", "source"} + and all(_text(x) for x in layer.values()) + and layer["relation"] in RELATIONS, + "layer schema", + ) + name = layer["output"] + _require( + name not in outputs.values() and name not in local_layers, + "duplicate layer output", + ) + local_layers.add(name) + layers.setdefault(name, []).append(layer) + required = set(identity) + for system in systems: + required.update(system["selector"]) + required.update(c["input"] for c in system["constraints"]) + if system["observed_area"]: + required.add(system["observed_area"]) + _require( + not required.intersection({*outputs.values(), *layers}), + "observed or identity columns would be overwritten", + ) + return value + + +def _bind(spec: Mapping, supports: Mapping[str, AtomicSupport]): + spec = validate_assignment_spec(spec) + _require( + set(supports) == {s["id"] for s in spec["systems"]}, "support systems differ" + ) + for system in spec["systems"]: + support = supports[system["id"]] + _require( + all( + support.metadata[k] == system[v] + for k, v in ( + ("system", "id"), + ("level", "level"), + ("code_system", "code_system"), + ("vintage", "vintage"), + ) + ), + "support identity differs from declaration", + ) + columns = support.metadata["columns"] + for c in system["constraints"]: + _require( + c["support"] in columns and columns[c["support"]]["kind"] == "code", + "unknown constraint", + ) + for s in system["stages"]: + _require( + s["level"] in columns + and columns[s["level"]]["kind"] == "code" + and s["weight"] in columns + and columns[s["weight"]]["kind"] == "weight", + "unknown stage column", + ) + for layer in system["layers"]: + description = columns.get(layer["input"], {}) + _require( + description.get("kind") == "code" + and all( + description[k] == layer[k] + for k in ("source", "vintage", "relation") + ), + "layer source, relation or vintage differs", + ) + return spec + + +def _route(households: pd.DataFrame, spec: Mapping) -> dict[str, np.ndarray]: + masks, claimed = {}, np.zeros(len(households), dtype=np.int64) + for system in spec["systems"]: + mask = np.ones(len(households), dtype=bool) + for name, accepted in system["selector"].items(): + mask &= households[name].isin(accepted).to_numpy() + masks[system["id"]] = np.flatnonzero(mask) + claimed += mask + _require(np.all(claimed == 1), "every household must match exactly one area system") + return masks + + +def _constraints(row, system) -> tuple: + values = [] + for c in system["constraints"]: + value = row[c["input"]] + if pd.isna(value): + _require(not c["required"], "missing required observed geography") + else: + _require(_text(value), "observed codes require nonempty strings") + values.append((c["support"], value)) + return tuple(sorted(values)) + + +class _SupportIndex: + """Index each observed-column pattern once, rather than scanning per household.""" + + def __init__(self, support): + self.support = support + self.groupings = {} + self.area_index = pd.Index(support.arrays["area"]) + self.draw_cells = {} + + def rows(self, constraints): + columns = tuple(k for k, _ in constraints) + if columns not in self.groupings: + if columns: + table = pd.DataFrame({c: self.support.arrays[c] for c in columns}) + # Always group by a list: normalize pandas' single-column key below. + groups = table.groupby(list(columns), sort=False, observed=True).indices + self.groupings[columns] = { + (k,) if len(columns) == 1 and not isinstance(k, tuple) else k: v + for k, v in groups.items() + } + else: + self.groupings[columns] = {(): np.arange(len(self.area_index))} + key = tuple(v for _, v in constraints) + _require( + key in self.groupings[columns], + "observed constraints have no common support", + ) + return self.groupings[columns][key] + + def pick(self, constraints, stage, u): + key = (constraints, stage["level"], stage["weight"]) + if key not in self.draw_cells: + rows = self.rows(constraints) + levels, inverse = np.unique( + self.support.arrays[stage["level"]][rows], return_inverse=True + ) + weights = np.zeros(len(levels), dtype=np.int64) + np.add.at(weights, inverse, self.support.arrays[stage["weight"]][rows]) + cumulative = np.cumsum(weights).tolist() + _require(cumulative[-1] > 0, "constraint cell has zero sampling mass") + self.draw_cells[key] = levels, cumulative + levels, cumulative = self.draw_cells[key] + # Integer inverse CDF: no floating-point rounding across a cell boundary. + threshold = (int(u * _U53) * cumulative[-1]) // _U53 + return str(levels[bisect.bisect_right(cumulative, threshold)]) + + +def assign_atomic( + households: pd.DataFrame, spec: Mapping, supports: Mapping[str, AtomicSupport] +) -> pd.DataFrame: + """Select atomic areas without changing row order, ids or observed geography.""" + spec = _bind(spec, supports) + _require( + not set(spec["outputs"].values()).intersection(households.columns), + "assignment output already exists", + ) + _require( + not households[spec["identity"]].isna().any().any() + and not households.duplicated(spec["identity"]).any(), + "null or duplicate draw identity", + ) + routing = _route(households, spec) + result = pd.DataFrame(index=households.index) + for name in spec["outputs"].values(): + result[name] = pd.array([pd.NA] * len(households), dtype="string") + definition = hashlib.sha256(canonical_json(spec)).hexdigest() + for system in spec["systems"]: + positions = routing[system["id"]] + support = supports[system["id"]] + index = _SupportIndex(support) + identities = list( + households.iloc[positions][spec["identity"]].itertuples( + index=False, name=None + ) + ) + draws = [ + keyed_uniform( + stream=tuple(spec["stream"]), + keys=[ + ( + *identity, + system["id"], + stage["level"], + definition, + support.sha256, + ) + for identity in identities + ], + ) + for stage in system["stages"] + ] + for local, position in enumerate(positions): + row = households.iloc[position] + constraints = _constraints(row, system) + area = row[system["observed_area"]] if system["observed_area"] else None + observed = area is not None and not pd.isna(area) + if observed: + _require(_text(area), "invalid observed atomic code") + match = index.area_index.get_indexer([area])[0] + _require( + match >= 0 + and all( + support.arrays[c][match] == value for c, value in constraints + ), + "observed atomic code is unsupported or conflicts", + ) + else: + for stage, uniforms in zip(system["stages"], draws, strict=True): + area = index.pick(constraints, stage, uniforms[local]) + constraints = tuple( + sorted({**dict(constraints), stage["level"]: area}.items()) + ) + for key, value in ( + ("area", area), + ("system", system["id"]), + ("basis", "observed" if observed else "assigned"), + ): + result.iloc[position, result.columns.get_loc(spec["outputs"][key])] = ( + value + ) + return result + + +def derive_geography( + households: pd.DataFrame, spec: Mapping, supports: Mapping[str, AtomicSupport] +) -> pd.DataFrame: + """Functional area lookups, including explicitly identified non-nesting conventions.""" + spec = _bind(spec, supports) + routing = _route(households, spec) + result = pd.DataFrame(index=households.index) + for name in sorted( + {layer["output"] for s in spec["systems"] for layer in s["layers"]} + ): + result[name] = pd.array([pd.NA] * len(households), dtype="string") + for system in spec["systems"]: + positions = routing[system["id"]] + rows = households.iloc[positions] + support = supports[system["id"]] + assigned_system = rows[spec["outputs"]["system"]] + _require( + assigned_system.notna().all() and assigned_system.eq(system["id"]).all(), + "assigned system disagrees with routing", + ) + locations = pd.Index(support.arrays["area"]).get_indexer( + rows[spec["outputs"]["area"]] + ) + _require(np.all(locations >= 0), "missing atomic mapping") + for layer in system["layers"]: + result.iloc[positions, result.columns.get_loc(layer["output"])] = ( + support.arrays[layer["input"]][locations] + ) + return result + + +def validate_geography( + households: pd.DataFrame, spec: Mapping, supports: Mapping[str, AtomicSupport] +) -> dict: + """Recheck constraints and functional mappings on a full or pruned population. + + Draws are not repeated. Initial support clones receive their own assignment + after expansion; subsequent views retain it. Structural lineage verification + remains the executor's responsibility. This gate is not a population-quality + certificate. + """ + spec = _bind(spec, supports) + expected = derive_geography(households, spec, supports) + for name in expected: + _require( + households[name].astype("string").equals(expected[name]), + "derived geography differs from atomic mapping", + ) + routing = _route(households, spec) + for system in spec["systems"]: + support = supports[system["id"]] + rows = households.iloc[routing[system["id"]]] + positions = pd.Index(support.arrays["area"]).get_indexer( + rows[spec["outputs"]["area"]] + ) + for (_, row), position in zip(rows.iterrows(), positions, strict=True): + _require( + all( + support.arrays[c][position] == v + for c, v in _constraints(row, system) + ), + "assigned area violates observed geography", + ) + original = row[system["observed_area"]] if system["observed_area"] else None + observed = original is not None and not pd.isna(original) + _require( + row[spec["outputs"]["basis"]] + == ("observed" if observed else "assigned"), + "assignment basis differs", + ) + if observed: + _require( + row[spec["outputs"]["area"]] == original, + "observed atomic area changed", + ) + return { + "outcome": "pass", + "scope": "atomic_geography_mapping_integrity", + "households": len(households), + "support_sha256": {k: v.sha256 for k, v in sorted(supports.items())}, + "definition_sha256": hashlib.sha256(canonical_json(spec)).hexdigest(), + } diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/__init__.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/__init__.py new file mode 100644 index 000000000..2812fd3a5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/__init__.py @@ -0,0 +1,74 @@ +"""The US congressional-district published-benchmark contract, as executable rules. + +This package implements protocol ``us-cd-published-benchmark-v4`` (SHA-256 +``f7574a2a00734ce64ceb9c101ebc52469a176d37ee32b02c140d3ea2e6c568ea``) under an +approval whose scope is **invented-fixture implementation only**. Candidate +scoring and production ledger registration are explicitly not approved, and +nothing here performs either: no candidate, population, ACS or Census artifact +is read, no ledger file is created or appended, and no release, promotion or +publication decision is produced. + +The approved document itself ships as a packaged resource beside +:mod:`~microcosm.build.cd_benchmark.protocol` and is hash-verified on load. Every +threshold, vocabulary and payload roster is compiled out of those bytes rather +than retyped, so the code cannot drift from the contract without a test failing. +Paths named inside the document are locators recording what was inspected; this +package never opens them. + +Import cost: the modules below are pure and depend only on the standard library +and this package. The two source adapters the contract pins are reached through +:mod:`~microcosm.build.cd_benchmark.universe`, which defers those imports to call +time, so importing this package does not load ``microcosm.build.us_runtime`` and +does not load PolicyEngine. + +Layout: + +* :mod:`~microcosm.build.cd_benchmark.canonical` — canonical JSON and digests. +* :mod:`~microcosm.build.cd_benchmark.reasons` — the closed vocabularies. +* :mod:`~microcosm.build.cd_benchmark.protocol` — compile the approved document. +* :mod:`~microcosm.build.cd_benchmark.metrics` — point and categorical checks. +* :mod:`~microcosm.build.cd_benchmark.reference` — reference validity and precision. +* :mod:`~microcosm.build.cd_benchmark.support` — origin support and concentration. +* :mod:`~microcosm.build.cd_benchmark.origin` — typed original-source origin keys. +* :mod:`~microcosm.build.cd_benchmark.approval` — injected approval and registration. +* :mod:`~microcosm.build.cd_benchmark.ledger` — in-memory ledger records and states. +* :mod:`~microcosm.build.cd_benchmark.binding` — candidate binding slots. +* :mod:`~microcosm.build.cd_benchmark.universe` — the pinned source adapters. +* :mod:`~microcosm.build.cd_benchmark.evaluation` — the six axes and one reduction. +""" + +from __future__ import annotations + +from microcosm.build.cd_benchmark.protocol import ( + APPROVED_PROTOCOL_ID, + APPROVED_PROTOCOL_SHA256, + CompiledProtocol, + ProtocolError, + canonical_protocol, + compile_protocol, +) +from microcosm.build.cd_benchmark.reasons import ( + AXES, + BenchmarkVerdict, + ExposureStatus, + MappingStatus, + PrecisionStatus, + Reason, + SupportVerdict, +) + +__all__ = [ + "APPROVED_PROTOCOL_ID", + "APPROVED_PROTOCOL_SHA256", + "AXES", + "BenchmarkVerdict", + "CompiledProtocol", + "ExposureStatus", + "MappingStatus", + "PrecisionStatus", + "ProtocolError", + "Reason", + "SupportVerdict", + "canonical_protocol", + "compile_protocol", +] diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/canonical.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/canonical.py new file mode 100644 index 000000000..15108b494 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/canonical.py @@ -0,0 +1,115 @@ +"""Canonical JSON bytes and digests, exactly as the protocol defines them. + +``/identity/encoding`` and ``/exposure_ledger/format`` require UTF-8 canonical +JSON with sorted keys, compact separators, no BOM, no nonfinite values and no +trailing LF inside the hashed bytes, and they additionally reject duplicate +object keys. The shared aggregate-JSON reader in +:mod:`microcosm.build.candidate_quality` enforces the first set but accepts +duplicate keys (``json.loads`` keeps the last one), so this module supplies the +stricter loader the contract names rather than relaxing the contract to match an +existing reader. The protocol also fixes ``ensure_ascii=false``, which that +reader leaves at the escaping default; for ASCII payloads the two encoders agree +byte for byte apart from its trailing newline, and a contract test pins that. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from typing import Any + +__all__ = [ + "CanonicalJsonError", + "SHA256_HEX", + "canonical_bytes", + "digest", + "is_sha256_hex", + "strict_load", +] + +SHA256_HEX = re.compile(r"[0-9a-f]{64}\Z") + + +class CanonicalJsonError(ValueError): + """Bytes are not the canonical JSON the protocol requires.""" + + +def is_sha256_hex(value: object) -> bool: + """Return whether ``value`` is a lowercase 64-character hex digest.""" + return isinstance(value, str) and SHA256_HEX.fullmatch(value) is not None + + +def _pairs(items: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in items: + if key in result: + raise CanonicalJsonError(f"Duplicate JSON key {key!r}.") + result[key] = value + return result + + +def _nonfinite(token: str) -> None: + raise CanonicalJsonError(f"Nonfinite JSON value {token}.") + + +def strict_load(data: bytes) -> Any: + """Parse canonical JSON bytes, refusing BOM, duplicate keys and nonfinite. + + Raises: + CanonicalJsonError: On any of those, or on invalid UTF-8/JSON. + """ + if not isinstance(data, bytes): + raise CanonicalJsonError("Canonical JSON input must be bytes.") + if data.startswith(b"\xef\xbb\xbf"): + raise CanonicalJsonError("Canonical JSON must not begin with a BOM.") + try: + text = data.decode("utf-8") + value = json.loads(text, object_pairs_hook=_pairs, parse_constant=_nonfinite) + except CanonicalJsonError: + raise + except (UnicodeError, ValueError, RecursionError) as error: + raise CanonicalJsonError(f"Invalid canonical JSON: {error}") from None + _assert_finite(value) + return value + + +def _assert_finite(value: object) -> None: + if isinstance(value, bool): + return + if isinstance(value, float | int): + try: + finite = math.isfinite(value) + except OverflowError: + finite = False + if not finite: + raise CanonicalJsonError("JSON numbers must be finite.") + elif isinstance(value, dict): + for child in value.values(): + _assert_finite(child) + elif isinstance(value, list): + for child in value: + _assert_finite(child) + + +def canonical_bytes(value: object) -> bytes: + """Serialize to the protocol's canonical UTF-8 JSON, with no trailing LF.""" + try: + text = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + except (TypeError, ValueError, RecursionError) as error: + raise CanonicalJsonError(f"Value is not canonical JSON: {error}") from None + return text.encode("utf-8") + + +def digest(data: bytes) -> str: + """Return the lowercase hex SHA-256 of exact bytes.""" + if not isinstance(data, bytes): + raise CanonicalJsonError("Digest input must be bytes.") + return hashlib.sha256(data).hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py new file mode 100644 index 000000000..2912a5e32 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py @@ -0,0 +1,186 @@ +"""Typed original-source household origin keys, exactly as ``v4`` encodes them. + +``/source_household_origin/key_encoding`` fixes the encoding: +``SHA256(UTF8(domain) + one NUL byte + canonical payload bytes)``, lowercase +hex, where the payload is a canonical UTF-8 JSON array of four ordered typed +pairs. Each pair is a two-element array whose first element is the literal type +name. Integers are JSON integers, never booleans, and raw strings receive no +normalization, trimming or coercion. + +The two arms differ in their third and fourth components, and this module names +them per arm rather than generically, so a survey vintage cannot be filed as an +income cohort year: + +* ACS — ``arm=ACS_PUMS``, authenticated original household *archive* member + identity, **survey vintage**, and the **raw ``SERIALNO`` string** with its + leading and lexical characters preserved. +* ASEC — ``arm=ASEC``, authenticated original household member identity, + **income cohort year**, and the **native integer ``H_SEQ``**. + +Two limits carried from the document. A generated ``source_household_id`` or a +normalized ``household_id`` is positional and cannot replace this identity +(``/source_household_origin/ACS/not_equivalent``). Cohort or vintage +distinguishes statistical source records; it does not prove different real +households across years, and an origin key does not establish independent +households across surveys or independent imputation donors. + +Physical staging paths, recompressed archive hashes and clone or cache +filenames are not identities: the authenticated source manifest supplies +``canonical_member_id`` and the original member bytes hash, and aliases must +resolve to that same member before hashing. Typed components stay private in +authenticated lineage; public diagnostics use counts and digests only. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from enum import StrEnum + +from microcosm.build.cd_benchmark.canonical import canonical_bytes, is_sha256_hex + +__all__ = [ + "AcsHouseholdOrigin", + "AsecHouseholdOrigin", + "HouseholdOrigin", + "ORIGIN_DOMAIN", + "OriginKeyError", + "SourceArm", + "SourceMember", + "origin_key", + "origin_payload", + "origin_payload_bytes", +] + +#: ``/source_household_origin/key_encoding/domain``. +ORIGIN_DOMAIN = "microcosm.statistical_source_household.v1" + + +class OriginKeyError(ValueError): + """The typed origin components are not what the encoding accepts.""" + + +class SourceArm(StrEnum): + """The two arms with a registered origin definition. + + ``/source_household_origin/other_arms``: any other arm needs its own + registered, authenticated source-specific definition; there is deliberately + no default to a synthetic row id here. + """ + + ACS_PUMS = "ACS_PUMS" + ASEC = "ASEC" + + +def _identifier(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise OriginKeyError(f"{label} must be a nonempty string.") + return value + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise OriginKeyError(f"{label} must be a JSON integer, not a boolean.") + return value + + +@dataclass(frozen=True) +class SourceMember: + """Canonical original member identity from the authenticated manifest.""" + + canonical_member_id: str + member_sha256: str + + def __post_init__(self) -> None: + _identifier(self.canonical_member_id, "canonical_member_id") + if not is_sha256_hex(self.member_sha256): + raise OriginKeyError( + "member_sha256 must be a lowercase 64-character hex digest." + ) + + def as_payload(self) -> dict[str, str]: + """The member pair's object value, with lexicographically sorted keys.""" + return { + "canonical_member_id": self.canonical_member_id, + "member_sha256": self.member_sha256, + } + + +@dataclass(frozen=True) +class AcsHouseholdOrigin: + """ACS PUMS origin: archive member identity, survey vintage, raw SERIALNO.""" + + member: SourceMember + survey_vintage: int + raw_serialno: str + + arm = SourceArm.ACS_PUMS + + def __post_init__(self) -> None: + if not isinstance(self.member, SourceMember): + raise OriginKeyError("member must be an authenticated SourceMember.") + _integer(self.survey_vintage, "survey_vintage") + # No normalization, trimming or coercion: an integer SERIALNO would be a + # coercion of the published string, and leading characters are meaning. + if not isinstance(self.raw_serialno, str) or not self.raw_serialno: + raise OriginKeyError("raw_serialno must be the nonempty published string.") + + +@dataclass(frozen=True) +class AsecHouseholdOrigin: + """ASEC origin: member identity, income cohort year, native integer H_SEQ.""" + + member: SourceMember + income_cohort_year: int + native_h_seq: int + + arm = SourceArm.ASEC + + def __post_init__(self) -> None: + if not isinstance(self.member, SourceMember): + raise OriginKeyError("member must be an authenticated SourceMember.") + _integer(self.income_cohort_year, "income_cohort_year") + # The shipped ASEC adapter's NATIVE_KEY invariant is H_SEQ > 0; a zero or + # negative sequence is not a native household key. + if _integer(self.native_h_seq, "native_h_seq") <= 0: + raise OriginKeyError("native_h_seq must be a positive native key.") + + +HouseholdOrigin = AcsHouseholdOrigin | AsecHouseholdOrigin + + +def origin_payload(origin: HouseholdOrigin) -> list[list[object]]: + """Return the four ordered typed pairs for ``origin``. + + Raises: + OriginKeyError: If ``origin`` is not a registered arm's origin. + """ + if isinstance(origin, AcsHouseholdOrigin): + return [ + ["string", str(SourceArm.ACS_PUMS)], + ["member", origin.member.as_payload()], + ["integer", origin.survey_vintage], + ["string", origin.raw_serialno], + ] + if isinstance(origin, AsecHouseholdOrigin): + return [ + ["string", str(SourceArm.ASEC)], + ["member", origin.member.as_payload()], + ["integer", origin.income_cohort_year], + ["integer", origin.native_h_seq], + ] + raise OriginKeyError( + "Only registered ACS and ASEC origins have a defined encoding; another " + "arm requires its own registered source-specific origin definition." + ) + + +def origin_payload_bytes(origin: HouseholdOrigin) -> bytes: + """Canonical payload bytes: sorted object keys, compact, no BOM or LF.""" + return canonical_bytes(origin_payload(origin)) + + +def origin_key(origin: HouseholdOrigin) -> str: + """Return the lowercase hex origin key for one original source household.""" + payload = origin_payload_bytes(origin) + return hashlib.sha256(ORIGIN_DOMAIN.encode("utf-8") + b"\x00" + payload).hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/protocol.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/protocol.py new file mode 100644 index 000000000..18c24024c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/protocol.py @@ -0,0 +1,871 @@ +"""Compile the approved canonical CD published-benchmark protocol document. + +The exact approved bytes ship as a packaged resource beside this module. Nothing +here reads a path named inside the document: ``/identity/legacy_design_parents``, +``/reference_authority`` and the code locators are *locators* recording what was +inspected when the contract was written, never inputs this module opens. + +Compilation is total and closed. Every threshold is carried as an exact +:class:`~decimal.Decimal` parsed from the document's decimal string, never as a +float; every vocabulary is compared against the document rather than assumed; +unknown block shapes refuse compilation. :func:`compile_protocol` will compile +any candidate document so that a successor can be inspected, but only bytes +whose SHA-256 equals :data:`APPROVED_PROTOCOL_SHA256` compile to a protocol with +``canonical`` set, and approval is resolved from that digest through an injected +authority (:mod:`microcosm.build.cd_benchmark.approval`) — never from a field +inside the document and never from a caller's label. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from enum import StrEnum +from functools import lru_cache +from importlib import resources +from types import MappingProxyType +from typing import Any + +from microcosm.build.cd_benchmark.canonical import ( + CanonicalJsonError, + canonical_bytes, + digest, + is_sha256_hex, + strict_load, +) +from microcosm.build.cd_benchmark.reasons import ExposureStatus, Reason + +__all__ = [ + "APPROVED_PROTOCOL_BYTES", + "APPROVED_PROTOCOL_ID", + "APPROVED_PROTOCOL_SHA256", + "CompiledProtocol", + "DistributionBlock", + "PartitionRules", + "PointBlock", + "PrecisionKind", + "ProtocolError", + "ReferenceBin", + "RESERVED_FAMILIES", + "ScopeRules", + "SupportBounds", + "assert_vocabulary_closed", + "assign_bin", + "canonical_protocol", + "compile_protocol", + "load_canonical_bytes", +] + +#: Digest and byte count of the approved canonical document, verified against +#: the review directory's plan, its independent response and the peer review. +APPROVED_PROTOCOL_SHA256 = ( + "f7574a2a00734ce64ceb9c101ebc52469a176d37ee32b02c140d3ea2e6c568ea" +) +APPROVED_PROTOCOL_BYTES = 86626 +APPROVED_PROTOCOL_ID = "us-cd-published-benchmark-v4" +_RESOURCE = "us-cd-published-benchmark-v4.json" + +#: ``/exposure_ledger/reservation``: these table families and every derivative +#: stay out of fitting, tuning and selection. +RESERVED_FAMILIES = frozenset({"B19001", "B25003"}) + +_FAMILY_ROOT = re.compile(r"([A-Z][0-9]{4,5})(?:_|\Z)") + + +class ProtocolError(ValueError): + """The document is not a compilable CD published-benchmark contract.""" + + +class PrecisionKind(StrEnum): + """Which ``/reference_status/share_precision`` rule a distribution uses.""" + + PUBLISHED_PERCENTAGE_MOE = "published_percentage_moe" + SUBSET_SHARE_APPROXIMATION = "subset_share_approximation" + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ProtocolError(f"{label} must be a JSON object.") + return value + + +def _text(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise ProtocolError(f"{label} must be a nonempty string.") + return value + + +def _flag(value: object, label: str) -> bool: + if not isinstance(value, bool): + raise ProtocolError(f"{label} must be a JSON boolean.") + return value + + +def _decimal(value: object, label: str) -> Decimal: + """Parse an exact decimal-string threshold; floats are refused.""" + if not isinstance(value, str): + raise ProtocolError(f"{label} must be an exact decimal string.") + try: + parsed = Decimal(value) + except InvalidOperation: + raise ProtocolError(f"{label} is not a decimal: {value!r}.") from None + if not parsed.is_finite(): + raise ProtocolError(f"{label} must be finite.") + return parsed + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ProtocolError(f"{label} must be a JSON integer.") + return value + + +def _list(container: Mapping[str, Any], key: str, label: str) -> list[Any]: + """Read a required roster, refusing an absent key and a non-list value. + + Compilation is total: an absent section leaks no ``KeyError`` and a string + in a roster's place is never iterated per character into field names. + """ + if key not in container: + raise ProtocolError(f"{label} is required and absent.") + value = container[key] + if not isinstance(value, list): + raise ProtocolError(f"{label} must be a JSON array.") + return value + + +def _bound(value: object, label: str) -> Decimal | None: + """A bin edge: an exact integer, or ``None`` for an open side.""" + if value is None: + return None + return Decimal(_integer(value, label)) + + +@dataclass(frozen=True) +class ReferenceBin: + """One published bin, or the protocol-constructed ``unclassified`` residual. + + ``lower_inclusive``/``upper_exclusive`` follow the document's boundary rule: + lower inclusive, upper exclusive, ``None`` meaning minus/plus infinity. The + residual bin carries no publisher variable and a reference share of exactly + zero; it is a protocol construct, not a publisher cell. + """ + + id: str + publisher_variable: str | None + publisher_label: str | None + lower_inclusive: Decimal | None + upper_exclusive: Decimal | None + reference_share: Decimal | None + unclassified: bool + authority: str | None + + +@dataclass(frozen=True) +class PointBlock: + """A mandatory scalar block compared by relative discrepancy.""" + + id: str + family: str + dataset: str + unit: str + universe: str + mandatory: bool + rule: str + variables: tuple[str, ...] + point_relative_bound: Decimal + reference_relative_moe_ceiling: Decimal + controlled_reference: str | None + exposure_families: frozenset[str] + count_once: bool + duplicate_reference_aliases: tuple[str, ...] + + @property + def reserved(self) -> bool: + """Whether any exposure family is a reserved holdout family.""" + return bool(self.exposure_families & RESERVED_FAMILIES) + + +@dataclass(frozen=True) +class DistributionBlock: + """A mandatory categorical block compared by per-bin share and total variation. + + ``independent_votes`` is ``False`` where the document says so (tenure's owner + and renter bins) and ``None`` where it says nothing. Undeclared is not a + licence to treat bins as independent votes: ``/claim`` withholds that + licence for overlapping cells, and ``/status_contract/no_rescue`` forbids + rescuing a block with them. + """ + + id: str + family: str + dataset: str + unit: str + mandatory: bool + denominator_variable: str + bins: tuple[ReferenceBin, ...] + max_each_bin_discrepancy_pp: Decimal + max_total_variation: Decimal + reference_moe_ceiling_pp: Decimal + precision_kind: PrecisionKind + precision_variables: tuple[str, ...] + exposure_families: frozenset[str] + independent_votes: bool | None + open_upper: bool + boundary_rule: str | None + additional_bounds: MappingProxyType + complement_only_when: str | None + + @property + def reserved(self) -> bool: + """Whether any exposure family is a reserved holdout family.""" + return bool(self.exposure_families & RESERVED_FAMILIES) + + @property + def unclassified_bin(self) -> ReferenceBin: + """The protocol-constructed residual bin.""" + return next(item for item in self.bins if item.unclassified) + + @property + def known_bins(self) -> tuple[ReferenceBin, ...]: + """Published bins, excluding the residual.""" + return tuple(item for item in self.bins if not item.unclassified) + + +@dataclass(frozen=True) +class SupportBounds: + """``/support`` floors and ceilings, as exact decimals and integers.""" + + minimum_positive_source_household_origins: int + minimum_positive_origins_per_positive_reference_bin: int + minimum_origin_cluster_concentration_ess: Decimal + maximum_origin_denominator_share: Decimal + maximum_origin_positive_bin_share: Decimal + nonnegative_finite_weights_required: bool + required_for_every_district_and_relevant_denominator: bool + + +@dataclass(frozen=True) +class PartitionRules: + """``/categorical_partition`` closure and residual-mass rules.""" + + maximum_unclassified_fraction: Decimal + comparison: str + renormalize_known_bins: bool + unknown_bin_id: str + applies_to: frozenset[str] + + +@dataclass(frozen=True) +class ScopeRules: + """``/scope`` — the predeclared geographic and product scope.""" + + country: str + area: str + congress: int + district_count: int + acs_year: int + acs_product: str + puerto_rico: str + allowed_missing_mandatory_block_fraction: Decimal + allowed_missing_mandatory_district_fraction: Decimal + dc_canonical_id: str + dc_published_id: str + published_and_canonical_geography_ids_retained: bool + + +@dataclass(frozen=True) +class CompiledProtocol: + """A compiled contract document plus the digest of the bytes it came from.""" + + sha256: str + size_bytes: int + protocol_id: str + schema: str + status: str + parent_sha256: str + hash_scope: str + point_blocks: tuple[PointBlock, ...] + distribution_blocks: tuple[DistributionBlock, ...] + partition: PartitionRules + support: SupportBounds + scope: ScopeRules + reason_codes: frozenset[str] + exposure_status_values: frozenset[str] + independent_axes: tuple[str, ...] + scope_reduction_order: tuple[str, ...] + scoring_permitted_in_document: bool + approval_installed_in_document: bool + reference_inventory_sha256: str + block_definition_hashes: Mapping[str, str] + pinned_code_sha256: Mapping[str, str] + ledger_event_types: frozenset[str] + ledger_record_fields: frozenset[str] + ledger_snapshot_fields: tuple[str, ...] + attempt_payload_fields: tuple[tuple[str, tuple[str, ...] | None], ...] + completion_payload_fields: tuple[tuple[str, tuple[str, ...] | None], ...] + registration_payload_fields: tuple[str, ...] + quality_required_fields: tuple[str, ...] + amendment_change_fields: tuple[str, ...] + required_binding_slots: tuple[str, ...] + + @property + def canonical(self) -> bool: + """Whether these bytes are the exact approved canonical document.""" + return ( + self.sha256 == APPROVED_PROTOCOL_SHA256 + and self.size_bytes == APPROVED_PROTOCOL_BYTES + ) + + @property + def blocks(self) -> tuple[PointBlock | DistributionBlock, ...]: + """Every mandatory block, point blocks first, in document order.""" + return (*self.point_blocks, *self.distribution_blocks) + + def block(self, block_id: str) -> PointBlock | DistributionBlock: + """Return the block with ``block_id``. + + Raises: + ProtocolError: If no block carries that id. + """ + for item in self.blocks: + if item.id == block_id: + return item + raise ProtocolError(f"Unknown block id {block_id!r}.") + + +def _families(block: dict[str, Any]) -> frozenset[str]: + """Reference table-family roots whose exposure this block consumes.""" + declared = block.get("exposure_families") + if declared is not None: + if not isinstance(declared, list) or not declared: + raise ProtocolError("exposure_families must be a nonempty list.") + return frozenset(_text(item, "exposure family") for item in declared) + match = _FAMILY_ROOT.match(_text(block.get("family"), "block family")) + if match is None: + raise ProtocolError( + f"Block family {block.get('family')!r} names no published table root " + "and declares no exposure_families." + ) + return frozenset({match.group(1)}) + + +def _bin(item: object) -> ReferenceBin: + data = _object(item, "distribution bin") + identifier = _text(data.get("id"), "bin id") + unclassified = "authority" in data + if unclassified: + if data.get("publisher_variable") is not None: + raise ProtocolError("The residual bin cannot name a publisher variable.") + share = _decimal(data.get("reference_share"), "residual reference_share") + if share != 0: + raise ProtocolError("The residual bin's reference share must be zero.") + return ReferenceBin( + id=identifier, + publisher_variable=None, + publisher_label=None, + lower_inclusive=None, + upper_exclusive=None, + reference_share=share, + unclassified=True, + authority=_text(data.get("authority"), "residual bin authority"), + ) + variable = ( + data["publisher_variable"] if "publisher_variable" in data else identifier + ) + return ReferenceBin( + id=identifier, + publisher_variable=_text(variable, "bin publisher variable"), + publisher_label=data.get("publisher_label"), + lower_inclusive=_bound(data.get("lower_inclusive"), "bin lower_inclusive"), + upper_exclusive=_bound(data.get("upper_exclusive"), "bin upper_exclusive"), + reference_share=None, + unclassified=False, + authority=None, + ) + + +def _point_block(block: dict[str, Any]) -> PointBlock: + ceilings = [ + key + for key in ( + "ordinary_reference_relative_moe_ceiling", + "reference_relative_moe_ceiling", + ) + if key in block + ] + if len(ceilings) != 1: + raise ProtocolError( + "A point block declares exactly one relative reference MOE ceiling." + ) + variables = block.get("variables") + if not isinstance(variables, list) or not variables: + raise ProtocolError("A point block declares its reference variables.") + aliases = block.get("duplicate_reference_aliases", []) + if not isinstance(aliases, list): + raise ProtocolError("duplicate_reference_aliases must be a list.") + return PointBlock( + id=_text(block.get("id"), "block id"), + family=_text(block.get("family"), "block family"), + dataset=_text(block.get("dataset"), "block dataset"), + unit=_text(block.get("unit"), "block unit"), + universe=_text(block.get("universe"), "block universe"), + mandatory=_flag(block.get("mandatory"), "block mandatory"), + rule=_text(block.get("rule"), "block rule"), + variables=tuple(_text(item, "block variable") for item in variables), + point_relative_bound=_decimal( + block.get("point_relative_bound"), "point_relative_bound" + ), + reference_relative_moe_ceiling=_decimal(block[ceilings[0]], ceilings[0]), + controlled_reference=block.get("controlled_reference"), + exposure_families=_families(block), + count_once=bool(block.get("count_once", False)), + duplicate_reference_aliases=tuple( + _text(item, "duplicate reference alias") for item in aliases + ), + ) + + +_PRECISION_CEILINGS = { + "reference_published_moe_ceiling_pp": PrecisionKind.PUBLISHED_PERCENTAGE_MOE, + "reference_approximate_moe_ceiling_pp": PrecisionKind.SUBSET_SHARE_APPROXIMATION, +} + + +def _distribution_block(block: dict[str, Any]) -> DistributionBlock: + present = [key for key in _PRECISION_CEILINGS if key in block] + if len(present) != 1: + raise ProtocolError( + "A distribution block declares exactly one reference precision ceiling." + ) + bins = tuple(_bin(item) for item in block.get("bins", [])) + if len(bins) < 2 or sum(item.unclassified for item in bins) != 1: + raise ProtocolError("A distribution declares its bins plus one residual bin.") + if not bins[-1].unclassified: + raise ProtocolError("The residual bin is declared last.") + identifiers = [item.id for item in bins] + if len(set(identifiers)) != len(identifiers): + raise ProtocolError("Duplicate bin ids.") + extra = { + key: _decimal(value, key) + for key, value in block.items() + if key.startswith("max_") + and key.endswith("_pp") + and key not in {"max_each_bin_discrepancy_pp"} + } + if set(extra) - {"max_renter_discrepancy_pp"}: + raise ProtocolError("Unsupported additional distribution bound.") + if "max_renter_discrepancy_pp" in extra and "renter" not in identifiers: + raise ProtocolError("The renter bound requires a renter bin.") + precision_variables = block.get("precision_variables", []) + if not isinstance(precision_variables, list): + raise ProtocolError("precision_variables must be a JSON array.") + return DistributionBlock( + id=_text(block.get("id"), "block id"), + family=_text(block.get("family"), "block family"), + dataset=_text(block.get("dataset"), "block dataset"), + unit=_text(block.get("unit"), "block unit"), + mandatory=_flag(block.get("mandatory"), "block mandatory"), + denominator_variable=_text(block.get("denominator"), "block denominator"), + bins=bins, + max_each_bin_discrepancy_pp=_decimal( + block.get("max_each_bin_discrepancy_pp"), "max_each_bin_discrepancy_pp" + ), + max_total_variation=_decimal( + block.get("max_total_variation"), "max_total_variation" + ), + reference_moe_ceiling_pp=_decimal(block[present[0]], present[0]), + precision_kind=_PRECISION_CEILINGS[present[0]], + precision_variables=tuple( + _text(item, "precision variable") for item in precision_variables + ), + exposure_families=_families(block), + independent_votes=block.get("owner_and_renter_independent_votes"), + open_upper=bool(block.get("open_upper_age", False)), + boundary_rule=block.get("boundary_rule"), + additional_bounds=MappingProxyType(extra), + complement_only_when=block.get("owner_is_renter_complement_only_when"), + ) + + +def compile_protocol(raw: bytes) -> CompiledProtocol: + """Compile canonical protocol bytes into typed, exact rules. + + Args: + raw: The exact document bytes. Their SHA-256 is the protocol identity; + the document carries no self-hash member (``/identity/hash_scope``). + + Raises: + ProtocolError: On any missing, mistyped or unknown-shaped section. + """ + try: + document = _object(strict_load(raw), "protocol document") + except CanonicalJsonError as error: + raise ProtocolError(str(error)) from None + identity = _object(document.get("identity"), "/identity") + if identity.get("algorithm") != "sha256": + raise ProtocolError("Only sha256 protocol identity is supported.") + parent = _text(identity.get("protocol_parent_sha256"), "protocol_parent_sha256") + if not is_sha256_hex(parent): + raise ProtocolError("protocol_parent_sha256 must be a lowercase digest.") + blocks = document.get("blocks") + if not isinstance(blocks, list) or not blocks: + raise ProtocolError("/blocks must be a nonempty list.") + point, distribution = [], [] + for item in blocks: + block = _object(item, "block") + if "bins" in block: + distribution.append(_distribution_block(block)) + else: + point.append(_point_block(block)) + identifiers = [item.id for item in (*point, *distribution)] + if len(set(identifiers)) != len(identifiers): + raise ProtocolError("Duplicate block ids.") + + partition_doc = _object( + document.get("categorical_partition"), "/categorical_partition" + ) + unknown_bin = _object(partition_doc.get("unknown_bin"), "unknown_bin") + partition = PartitionRules( + maximum_unclassified_fraction=_decimal( + partition_doc.get("maximum_unclassified_fraction_of_known_denominator"), + "maximum_unclassified_fraction_of_known_denominator", + ), + comparison=_text( + partition_doc.get("maximum_unclassified_fraction_comparison"), + "maximum_unclassified_fraction_comparison", + ), + renormalize_known_bins=_flag( + partition_doc.get("renormalize_known_bins"), "renormalize_known_bins" + ), + unknown_bin_id=_text(unknown_bin.get("id"), "unknown bin id"), + applies_to=frozenset( + _text(item, "partition applies_to") + for item in _list( + partition_doc, "applies_to", "/categorical_partition/applies_to" + ) + ), + ) + support_doc = _object(document.get("support"), "/support") + support = SupportBounds( + minimum_positive_source_household_origins=_integer( + support_doc.get("minimum_positive_source_household_origins"), + "minimum_positive_source_household_origins", + ), + minimum_positive_origins_per_positive_reference_bin=_integer( + support_doc.get("minimum_positive_origins_per_positive_reference_bin"), + "minimum_positive_origins_per_positive_reference_bin", + ), + minimum_origin_cluster_concentration_ess=_decimal( + support_doc.get("minimum_origin_cluster_concentration_ESS"), + "minimum_origin_cluster_concentration_ESS", + ), + maximum_origin_denominator_share=_decimal( + support_doc.get("maximum_origin_denominator_share"), + "maximum_origin_denominator_share", + ), + maximum_origin_positive_bin_share=_decimal( + support_doc.get("maximum_origin_positive_bin_share"), + "maximum_origin_positive_bin_share", + ), + nonnegative_finite_weights_required=_flag( + support_doc.get("nonnegative_finite_weights_required"), + "nonnegative_finite_weights_required", + ), + required_for_every_district_and_relevant_denominator=_flag( + support_doc.get("required_for_every_district_and_relevant_denominator"), + "required_for_every_district_and_relevant_denominator", + ), + ) + scope_doc = _object(document.get("scope"), "/scope") + alias = _object(scope_doc.get("DC_identity_alias"), "DC_identity_alias") + scope = ScopeRules( + country=_text(scope_doc.get("country"), "scope country"), + area=_text(scope_doc.get("area"), "scope area"), + congress=_integer(scope_doc.get("congress"), "scope congress"), + district_count=_integer(scope_doc.get("district_count"), "district_count"), + acs_year=_integer(scope_doc.get("acs_year"), "acs_year"), + acs_product=_text(scope_doc.get("acs_product"), "acs_product"), + puerto_rico=_text(scope_doc.get("puerto_rico"), "puerto_rico"), + allowed_missing_mandatory_block_fraction=_decimal( + scope_doc.get("allowed_missing_mandatory_block_fraction"), + "allowed_missing_mandatory_block_fraction", + ), + allowed_missing_mandatory_district_fraction=_decimal( + scope_doc.get("allowed_missing_mandatory_district_fraction"), + "allowed_missing_mandatory_district_fraction", + ), + dc_canonical_id=_text(alias.get("canonical"), "DC canonical id"), + dc_published_id=_text(alias.get("published"), "DC published id"), + published_and_canonical_geography_ids_retained=_flag( + scope_doc.get("published_and_canonical_geography_ids_retained"), + "published_and_canonical_geography_ids_retained", + ), + ) + status_doc = _object(document.get("status_contract"), "/status_contract") + ledger_doc = _object(document.get("exposure_ledger"), "/exposure_ledger") + approval_doc = _object(document.get("approval"), "/approval") + return CompiledProtocol( + sha256=digest(raw), + size_bytes=len(raw), + protocol_id=_text(document.get("protocol_id"), "/protocol_id"), + schema=_text(document.get("schema"), "/schema"), + status=_text(document.get("status"), "/status"), + parent_sha256=parent, + hash_scope=_text(identity.get("hash_scope"), "/identity/hash_scope"), + point_blocks=tuple(point), + distribution_blocks=tuple(distribution), + partition=partition, + support=support, + scope=scope, + reason_codes=frozenset(_object(status_doc.get("reason_codes"), "reason_codes")), + exposure_status_values=frozenset( + _object(ledger_doc.get("exposure_status_values"), "exposure_status_values") + ), + independent_axes=tuple( + _text(item, "independent axis") + for item in _list( + status_doc, + "independent_axes_required", + "/status_contract/independent_axes_required", + ) + ), + scope_reduction_order=tuple( + _text(item, "scope reduction step") + for item in _list( + status_doc, + "scope_reduction_order", + "/status_contract/scope_reduction_order", + ) + ), + scoring_permitted_in_document=_flag( + approval_doc.get("scoring_permitted"), "/approval/scoring_permitted" + ), + approval_installed_in_document=_flag( + approval_doc.get("installed"), "/approval/installed" + ), + reference_inventory_sha256=_inventory_digest(document, scope_doc), + block_definition_hashes=MappingProxyType( + { + _text(_object(item, "block").get("id"), "block id"): digest( + canonical_bytes(item) + ) + for item in blocks + } + ), + pinned_code_sha256=MappingProxyType(_pinned_code(document)), + ledger_event_types=frozenset( + _object(ledger_doc.get("other_event_types"), "other_event_types") + ), + ledger_record_fields=frozenset( + _object(ledger_doc.get("record_fields"), "record_fields") + ), + ledger_snapshot_fields=tuple( + _text(item, "snapshot field") + for item in _list( + ledger_doc, "snapshot_fields", "/exposure_ledger/snapshot_fields" + ) + ), + attempt_payload_fields=_payload_fields( + _list( + ledger_doc, + "evaluation_attempt_payload", + "/exposure_ledger/evaluation_attempt_payload", + ) + ), + completion_payload_fields=_payload_fields( + _list( + ledger_doc, + "evaluation_completion_payload", + "/exposure_ledger/evaluation_completion_payload", + ) + ), + registration_payload_fields=tuple( + _text(item, "registration payload field") + for item in _list( + ledger_doc, + "protocol_registration_payload", + "/exposure_ledger/protocol_registration_payload", + ) + ), + quality_required_fields=tuple( + _text(item, "quality required field") + for item in _list( + ledger_doc, + "quality_required_fields", + "/exposure_ledger/quality_required_fields", + ) + ), + amendment_change_fields=tuple( + _text(item, "amendment change field") + for item in _list( + _object(document.get("protocol_amendments"), "/protocol_amendments"), + "required_change_fields", + "/protocol_amendments/required_change_fields", + ) + ), + required_binding_slots=tuple( + _text(item, "binding slot") + for item in _list( + _object( + document.get("candidate_binding_requirements"), + "/candidate_binding_requirements", + ), + "must_authenticate_before_scoring", + "/candidate_binding_requirements/must_authenticate_before_scoring", + ) + ), + ) + + +def _payload_fields( + declared: object, +) -> tuple[tuple[str, tuple[str, ...] | None], ...]: + """Split a declared payload roster into names and any pinned values. + + The document writes a constrained field as ``name=value`` and an + alternative as ``a_or_b``: ``attempt_state=started`` and + ``terminal_state=completed_or_failed``. Everything else is an unconstrained + required field name. + """ + if not isinstance(declared, list) or not declared: + raise ProtocolError("A ledger payload roster must be a nonempty list.") + fields: list[tuple[str, tuple[str, ...] | None]] = [] + for item in declared: + entry = _text(item, "ledger payload field") + name, separator, value = entry.partition("=") + if not separator: + fields.append((name, None)) + continue + fields.append((name, tuple(value.split("_or_")))) + names = [name for name, _ in fields] + if len(set(names)) != len(names): + raise ProtocolError("Duplicate ledger payload field names.") + return tuple(fields) + + +def _inventory_digest(document: dict[str, Any], scope_doc: dict[str, Any]) -> str: + """The reference inventory digest, required to agree in both places.""" + authority = _object(document.get("reference_authority"), "/reference_authority") + inventory = _object(authority.get("inventory"), "reference inventory") + required = _object(scope_doc.get("required_district_set"), "required_district_set") + primary = _text(inventory.get("sha256"), "reference inventory sha256") + secondary = _text(required.get("inventory_sha256"), "district inventory sha256") + if not is_sha256_hex(primary) or primary != secondary: + raise ProtocolError( + "The reference inventory digest must be a lowercase digest and must " + "agree between /reference_authority and /scope/required_district_set." + ) + return primary + + +def _pinned_code(document: dict[str, Any]) -> dict[str, str]: + """Source-file digests the contract inspected, keyed by file name. + + These are locators recording what was read when the contract was written. + Nothing in this package opens the paths beside them; the digests exist so a + binding can be checked against the adapter it was reviewed against, and a + changed adapter is a scope question rather than an automatic pin refresh. + """ + sections = ( + _object( + _object(document.get("reference_authority"), "/reference_authority").get( + "HU_adapter_clarification" + ), + "HU_adapter_clarification", + ), + _object( + _object(document.get("candidate_semantics"), "/candidate_semantics").get( + "definition_authority" + ), + "definition_authority", + ), + ) + pins: dict[str, str] = {} + for section in sections: + for item in section.get("code", []): + entry = _object(item, "pinned code entry") + name = _text(entry.get("path"), "pinned code path").rsplit("/", 1)[-1] + value = _text(entry.get("sha256"), "pinned code sha256") + if not is_sha256_hex(value): + raise ProtocolError(f"Pinned code digest for {name} is not a digest.") + if pins.setdefault(name, value) != value: + raise ProtocolError(f"Conflicting pinned code digests for {name}.") + if not pins: + raise ProtocolError("The document pins no inspected source code.") + return pins + + +def load_canonical_bytes() -> bytes: + """Read the packaged approved document and verify its digest and size. + + Raises: + ProtocolError: If the packaged bytes are not the approved document. + """ + raw = resources.files(__package__).joinpath(_RESOURCE).read_bytes() + if len(raw) != APPROVED_PROTOCOL_BYTES or digest(raw) != APPROVED_PROTOCOL_SHA256: + raise ProtocolError("Packaged protocol resource is not the approved document.") + return raw + + +@lru_cache(maxsize=1) +def canonical_protocol() -> CompiledProtocol: + """Return the compiled approved canonical protocol. + + Raises: + ProtocolError: If the packaged document is not the approved bytes, does + not identify itself as the approved protocol, or has drifted from + the closed reason/exposure vocabularies this package implements. + """ + compiled = compile_protocol(load_canonical_bytes()) + if not compiled.canonical or compiled.protocol_id != APPROVED_PROTOCOL_ID: + raise ProtocolError("Packaged protocol is not the approved canonical contract.") + assert_vocabulary_closed(compiled) + return compiled + + +def assert_vocabulary_closed(compiled: CompiledProtocol) -> None: + """Check this package's closed vocabularies equal the document's. + + Raises: + ProtocolError: On any code this package would emit that the document + does not define, or any document code this package cannot emit. + """ + for name, implemented, declared in ( + ("reason", {item.value for item in Reason}, set(compiled.reason_codes)), + ( + "exposure status", + {item.value for item in ExposureStatus}, + set(compiled.exposure_status_values), + ), + ): + if implemented != declared: + raise ProtocolError( + f"Closed {name} vocabulary differs from the document: " + f"only in code {sorted(implemented - declared)}, " + f"only in document {sorted(declared - implemented)}." + ) + + +def assign_bin(block: DistributionBlock, value: Decimal) -> str | None: + """Return the bin id for ``value`` under the declared boundary rule. + + Lower bounds are inclusive and upper bounds exclusive; a ``None`` lower or + upper bound is minus or plus infinity. No rounding is applied — a monetary + amount of ``9999.5`` stays below a ``10000`` cutpoint. Returns ``None`` when + no published bin covers the value, which the caller may treat as + unclassified mass only under known membership and an approved mapping. + """ + if not isinstance(value, Decimal) or not value.is_finite(): + raise ProtocolError("Bin assignment needs a finite Decimal value.") + for item in block.known_bins: + if item.lower_inclusive is None and item.upper_exclusive is None: + raise ProtocolError(f"Bin {item.id!r} declares no numeric boundary.") + if item.lower_inclusive is not None and value < item.lower_inclusive: + continue + if item.upper_exclusive is not None and value >= item.upper_exclusive: + continue + return item.id + return None diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/reasons.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/reasons.py new file mode 100644 index 000000000..855d938b4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/reasons.py @@ -0,0 +1,135 @@ +"""Closed reason, exposure and status vocabularies for the CD benchmark protocol. + +Every member here is checked against the canonical protocol document by +:func:`microcosm.build.cd_benchmark.protocol.assert_vocabulary_closed`, so the +code's vocabulary is verified equal to the approved document's rather than +merely resembling it. Per ``/status_contract/reason_encoding`` the primary +reason namespace is closed: unknown codes refuse validation, several +independently established reasons may coexist, and block/bin/metric identifiers +and raw source detail are separate bounded evidence, never a replacement code. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from enum import StrEnum + +__all__ = [ + "AXES", + "BenchmarkVerdict", + "ExposureStatus", + "MappingStatus", + "PrecisionStatus", + "Reason", + "SupportVerdict", + "dedupe", +] + + +class Reason(StrEnum): + """The 21 primary reason codes at ``/status_contract/reason_codes``.""" + + CANDIDATE_IDENTITY_UNAVAILABLE = "candidate_identity_unavailable" + DENOMINATOR_UNKNOWN = "denominator_unknown" + EXPOSURE_HISTORY_UNKNOWN = "exposure_history_unknown" + GQ_WEIGHT_BRIDGE_MISSING = "gq_weight_bridge_missing" + IDENTITY_MISMATCH = "identity_mismatch" + INVALID_REDUCER_INPUT = "invalid_reducer_input" + LEDGER_BUSY = "ledger_busy" + LEDGER_INTEGRITY_UNKNOWN = "ledger_integrity_unknown" + MAPPING_UNAPPROVED = "mapping_unapproved" + PROTOCOL_UNAPPROVED = "protocol_unapproved" + PUBLISHED_BENCHMARK_NONCONFORMITY = "published_benchmark_nonconformity" + REFERENCE_INVALID = "reference_invalid" + REFERENCE_PRECISION_INADEQUATE = "reference_precision_inadequate" + REQUIRED_CELL_OR_DISTRICT_MISSING = "required_cell_or_district_missing" + SOURCE_DOMAIN_REFUSED = "source_domain_refused" + SUPPORT_BOUND_EXCEEDED = "support_bound_exceeded" + UNCLASSIFIED_FRACTION_EXCEEDED = "unclassified_fraction_exceeded" + UNRESOLVED_SOURCE_LINEAGE = "unresolved_source_lineage" + VACANCY_BRIDGE_MISSING = "vacancy_bridge_missing" + ZERO_CANDIDATE_DENOMINATOR = "zero_candidate_denominator" + ZERO_REFERENCE_DENOMINATOR = "zero_reference_denominator" + + +class ExposureStatus(StrEnum): + """The four values at ``/exposure_ledger/exposure_status_values``.""" + + EXPOSURE_HISTORY_UNKNOWN = "exposure_history_unknown" + FRESH_SAME_SOURCE_HOLDOUT_ELIGIBLE = "fresh_same_source_holdout_eligible" + NOT_RESERVED_CONFORMITY = "not_reserved_conformity" + POST_EXPOSURE_CONFORMITY = "post_exposure_conformity" + + +class BenchmarkVerdict(StrEnum): + """Scope-level outcomes named by ``/status_contract``. + + There is deliberately no "release", "certified" or "passing" member: a + conforming verdict is operational published-benchmark conformity for the + stated scope and nothing else (``/claim``). + """ + + BENCHMARK_CONFORMING = "benchmark_conforming" + BENCHMARK_NONCONFORMING = "benchmark_nonconforming" + SCOPE_INCOMPLETE = "scope_incomplete" + + +class PrecisionStatus(StrEnum): + """The reference-precision axis, independent of the metric axis. + + ``CONTROLLED_UNKNOWN`` is the ``/reference_status/controlled_estimate`` + class: the point comparison stays valid, no variance is claimed, and no + adequacy is asserted. It is neither an adequacy verdict nor a failure. + """ + + ADEQUATE = "adequate" + INADEQUATE = "inadequate" + CONTROLLED_UNKNOWN = "controlled_unknown" + UNAVAILABLE = "unavailable" + + +class MappingStatus(StrEnum): + """Whether an authenticated producer bound the required mapping slots.""" + + APPROVED = "approved" + UNAPPROVED = "unapproved" + + +class SupportVerdict(StrEnum): + """Origin-support outcome, independent of the benchmark metric. + + ``NOT_APPLICABLE`` is the ``/support/reference_zero_unknown_bin_carrier_floor`` + case: the carrier floor does not apply to a bin whose published reference + share is zero, so mass and origin diagnostics are retained without a bound. + """ + + WITHIN_BOUNDS = "within_bounds" + BOUND_EXCEEDED = "bound_exceeded" + UNDETERMINED = "undetermined" + NOT_APPLICABLE = "not_applicable" + + +#: ``/status_contract/independent_axes_required`` — reported separately and +#: never collapsed into one scalar. +AXES = ( + "benchmark_metric_status", + "reference_precision_status", + "mapping_status", + "support_status", + "exposure_status", + "scope_completeness", +) + + +def dedupe(items: Iterable[Reason]) -> tuple[Reason, ...]: + """Deduplicate reasons, preserving first-seen order. + + Raises: + TypeError: If any item is outside the closed :class:`Reason` namespace. + """ + seen: dict[Reason, None] = {} + for item in items: + if not isinstance(item, Reason): + raise TypeError(f"Unknown primary reason code {item!r}.") + seen.setdefault(item) + return tuple(seen) diff --git a/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py b/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py index ac26729c6..0e4cf984a 100644 --- a/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py @@ -41,10 +41,15 @@ "write_frame_checkpoint", ] -FRAME_CHECKPOINT_SCHEMA_VERSION = 3 +FRAME_CHECKPOINT_SCHEMA_VERSION = 4 +_NULLABLE_BOOLEAN_SCHEMA_VERSION = 3 _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION = 2 _SUPPORTED_FRAME_CHECKPOINT_SCHEMA_VERSIONS = frozenset( - {_LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION, FRAME_CHECKPOINT_SCHEMA_VERSION} + { + _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION, + _NULLABLE_BOOLEAN_SCHEMA_VERSION, + FRAME_CHECKPOINT_SCHEMA_VERSION, + } ) _ARTIFACT_KIND = "populace_frame_checkpoint" @@ -56,6 +61,20 @@ _ENCODING_TIMEDELTA = "timedelta64" _ENCODING_OBJECT = "object_scalars_v1" _ENCODING_NULLABLE_BOOLEAN = "nullable_boolean_v1" +_ENCODING_NULLABLE_INTEGER = "nullable_integer_v1" +_NULLABLE_INTEGER_DTYPES = { + str(dtype): dtype + for dtype in ( + pd.Int8Dtype(), + pd.Int16Dtype(), + pd.Int32Dtype(), + pd.Int64Dtype(), + pd.UInt8Dtype(), + pd.UInt16Dtype(), + pd.UInt32Dtype(), + pd.UInt64Dtype(), + ) +} _TAG_NONE = 0 _TAG_PD_NA = 1 @@ -346,25 +365,21 @@ def _checkpoint_metadata( ) strata_spec = _series_spec(frame.strata, label="strata") - uses_nullable_boolean = any( - column.get("encoding") == _ENCODING_NULLABLE_BOOLEAN - for table in table_specs - for column in table["columns"] - ) or any( - table["index"].get("encoding") == _ENCODING_NULLABLE_BOOLEAN + encodings = { + spec.get("encoding") for table in table_specs - ) - uses_nullable_boolean = uses_nullable_boolean or ( - strata_spec.get("encoding") == _ENCODING_NULLABLE_BOOLEAN - ) + for spec in [*table["columns"], table["index"]] + } | {strata_spec.get("encoding")} + if _ENCODING_NULLABLE_INTEGER in encodings: + version = FRAME_CHECKPOINT_SCHEMA_VERSION + elif _ENCODING_NULLABLE_BOOLEAN in encodings: + version = _NULLABLE_BOOLEAN_SCHEMA_VERSION + else: + version = _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION schema = frame.schema return { "artifact_kind": _ARTIFACT_KIND, - "schema_version": ( - FRAME_CHECKPOINT_SCHEMA_VERSION - if uses_nullable_boolean - else _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION - ), + "schema_version": version, "schema": { "person_entity": schema.person_entity, "group_entities": list(schema.group_entities), @@ -420,6 +435,15 @@ def _series_spec(series: pd.Series, *, label: str) -> dict[str, object]: "encoding": _ENCODING_NULLABLE_BOOLEAN, "has_null_mask": bool(series.isna().any()), } + if ( + str(dtype) in _NULLABLE_INTEGER_DTYPES + and dtype == (_NULLABLE_INTEGER_DTYPES[str(dtype)]) + ): + return { + "dtype": str(dtype), + "encoding": _ENCODING_NULLABLE_INTEGER, + "has_null_mask": bool(series.isna().any()), + } if isinstance(dtype, pd.api.extensions.ExtensionDtype) and not isinstance( dtype, pd.StringDtype ): @@ -561,6 +585,19 @@ def _write_series(group: Any, series: pd.Series, spec: Mapping[str, object]) -> null_mask.astype(np.uint8, copy=False), ) return + if encoding == _ENCODING_NULLABLE_INTEGER: + dtype = _NULLABLE_INTEGER_DTYPES[str(spec["dtype"])] + null_mask = series.isna().to_numpy(dtype=np.bool_, copy=False) + # Keep exact integer width and observed values, including UInt64 IDs. + # Hidden extension-array storage is not part of the logical payload. + values = series.to_numpy(dtype=dtype.numpy_dtype, na_value=0, copy=True) + values[null_mask] = 0 + _write_numpy_dataset(group, "values", values) + if spec.get("has_null_mask") is True: + _write_numpy_dataset( + group, "null_mask", null_mask.astype(np.uint8, copy=False) + ) + return raise RuntimeError(f"Unknown checkpoint series encoding {encoding!r}.") @@ -641,6 +678,55 @@ def _read_series( pd.arrays.BooleanArray(values, mask, copy=False), copy=False, ) + elif encoding == _ENCODING_NULLABLE_INTEGER: + integer_dtype = _NULLABLE_INTEGER_DTYPES.get(dtype) + if integer_dtype is None: + raise ValueError( + f"Frame checkpoint {path} {label!r} nullable integer encoding " + f"has unsupported declared dtype {dtype!r}." + ) + has_null_mask = spec.get("has_null_mask") + if type(has_null_mask) is not bool: + raise ValueError( + f"Frame checkpoint {path} {label!r} nullable integer " + "has_null_mask must be a boolean." + ) + values = _read_numpy_dataset(group, "values", path) + if values.ndim != 1 or values.dtype != integer_dtype.numpy_dtype: + raise ValueError( + f"Frame checkpoint {path} {label!r} nullable integer values " + f"must be a one-dimensional {integer_dtype.numpy_dtype} array." + ) + if has_null_mask: + if "null_mask" not in group: + raise ValueError( + f"Frame checkpoint {path} {label!r} is missing its null mask." + ) + null_mask = _read_numpy_dataset(group, "null_mask", path) + if ( + null_mask.ndim != 1 + or null_mask.dtype != np.dtype(np.uint8) + or len(null_mask) != len(values) + or ((null_mask != 0) & (null_mask != 1)).any() + or not null_mask.any() + ): + raise ValueError( + f"Frame checkpoint {path} {label!r} null mask must be a " + "one-dimensional uint8 0/1 array aligned to its values." + ) + mask = null_mask.astype(np.bool_, copy=False) + if (values[mask] != 0).any(): + raise ValueError( + f"Frame checkpoint {path} {label!r} null mask covers " + "noncanonical nonzero integer storage." + ) + else: + if "null_mask" in group: + raise ValueError( + f"Frame checkpoint {path} {label!r} has an unexpected null mask." + ) + mask = np.zeros(len(values), dtype=np.bool_) + series = pd.Series(pd.arrays.IntegerArray(values, mask, copy=False), copy=False) else: raise ValueError( f"Frame checkpoint {path} has unknown encoding {encoding!r} for {label!r}." @@ -838,39 +924,62 @@ def _read_metadata(root: Any, path: Path) -> dict[str, Any]: f"Frame checkpoint {path} schema version is {version!r}; expected " f"one of {sorted(_SUPPORTED_FRAME_CHECKPOINT_SCHEMA_VERSIONS)}." ) - nullable_spec_count = 0 + boolean_spec_count = 0 + integer_spec_count = 0 for label, spec in _checkpoint_series_specs(metadata): dtype = spec.get("dtype") encoding = spec.get("encoding") - declares_nullable = dtype == "boolean" - uses_nullable_encoding = encoding == _ENCODING_NULLABLE_BOOLEAN + declares_boolean = dtype == "boolean" + uses_boolean_encoding = encoding == _ENCODING_NULLABLE_BOOLEAN + declares_integer = isinstance(dtype, str) and dtype in _NULLABLE_INTEGER_DTYPES + uses_integer_encoding = encoding == _ENCODING_NULLABLE_INTEGER + if version < FRAME_CHECKPOINT_SCHEMA_VERSION and ( + declares_integer or uses_integer_encoding + ): + raise ValueError( + f"Frame checkpoint {path} schema version {version} cannot carry " + f"nullable integer spec {label!r}." + ) if version == _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION and ( - declares_nullable or uses_nullable_encoding + declares_boolean or uses_boolean_encoding ): raise ValueError( f"Frame checkpoint {path} schema version 2 cannot carry nullable " f"boolean spec {label!r}." ) - if version == FRAME_CHECKPOINT_SCHEMA_VERSION and ( - declares_nullable != uses_nullable_encoding + if version >= _NULLABLE_BOOLEAN_SCHEMA_VERSION and ( + declares_boolean != uses_boolean_encoding ): raise ValueError( - f"Frame checkpoint {path} schema version 3 nullable boolean spec " - f"{label!r} must pair declared dtype 'boolean' with encoding " + f"Frame checkpoint {path} schema version {version} nullable boolean " + f"spec {label!r} must pair declared dtype 'boolean' with encoding " f"{_ENCODING_NULLABLE_BOOLEAN!r}." ) - if uses_nullable_encoding: - nullable_spec_count += 1 + if declares_integer != uses_integer_encoding: + raise ValueError( + f"Frame checkpoint {path} schema version {version} nullable integer " + f"spec {label!r} must pair a supported pandas integer dtype " + f"with encoding {_ENCODING_NULLABLE_INTEGER!r}." + ) + if uses_boolean_encoding or uses_integer_encoding: + family = "boolean" if uses_boolean_encoding else "integer" if type(spec.get("has_null_mask")) is not bool: raise ValueError( - f"Frame checkpoint {path} {label!r} nullable boolean " + f"Frame checkpoint {path} {label!r} nullable {family} " "has_null_mask must be a boolean." ) - if version == FRAME_CHECKPOINT_SCHEMA_VERSION and nullable_spec_count == 0: + boolean_spec_count += uses_boolean_encoding + integer_spec_count += uses_integer_encoding + if version == _NULLABLE_BOOLEAN_SCHEMA_VERSION and boolean_spec_count == 0: raise ValueError( f"Frame checkpoint {path} schema version 3 requires at least one " "nullable boolean spec." ) + if version == FRAME_CHECKPOINT_SCHEMA_VERSION and integer_spec_count == 0: + raise ValueError( + f"Frame checkpoint {path} schema version 4 requires at least one " + "nullable integer spec." + ) return metadata diff --git a/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py new file mode 100644 index 000000000..a245992a8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py @@ -0,0 +1,281 @@ +"""Typed graph nodes for the shared atomic-area geography contract. + +Countries supply declarations and normalized source bytes. Import, assignment, +derivation and the integrity gate remain independently inspectable operations. +These nodes do not change weights, rows, memberships or observed columns. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence + +import pandas as pd + +from microcosm.build import atomic_geography as geography +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import load_source_bytes +from microcosm.graph.kernel import KernelRole +from microcosm.graph.randomness import keyed_uniform + +ATOMIC_SUPPORT_TYPE = ArtifactType("microcosm.geography.atomic_support_npz", 1) +ATOMIC_GEOGRAPHY_VALIDATION_TYPE = ArtifactType( + "microcosm.geography.validation_result", 1 +) +_DEPENDENCIES = ("numpy", "pandas") + + +class _GeographyKernel(KernelBase): + def implementation_hash(self) -> str: + return source_hash( + type(self), + geography, + canonical_json, + keyed_uniform, + load_source_bytes, + dependencies=_DEPENDENCIES, + ) + + +class AtomicSupportImportKernel(_GeographyKernel): + ref = "geography.support_import@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + if set(context.params) != {"source", "system"} or set(context.sources) != { + context.params["source"] + }: + raise ValueError( + "Atomic geography import requires its sole declared source." + ) + payload = load_source_bytes( + "raw-bytes-v1", context.sources[context.params["source"]] + ) + support = geography.decode_atomic_support(payload) + if support.metadata["system"] != context.params["system"]: + raise ValueError("Atomic geography source belongs to a different system.") + return KernelResult( + artifacts={"support": payload}, + receipt={ + "support_sha256": support.sha256, + "areas": len(support.arrays["area"]), + "metadata": support.metadata, + }, + ) + + +def _inputs(context): + if set(context.params) != {"definition", "stream"}: + raise ValueError( + "Atomic geography requires its canonical definition and stream." + ) + spec = geography.validate_assignment_spec(json.loads(context.params["definition"])) + if ( + canonical_json(spec).decode() != context.params["definition"] + or tuple(spec["stream"]) != context.params["stream"] + ): + raise ValueError("Atomic geography definition or stream differs.") + if set(context.artifacts) != {s["id"] for s in spec["systems"]}: + raise ValueError( + "Atomic geography requires exactly its declared support artifacts." + ) + supports = {} + for name, value in context.artifacts.items(): + if value.type != ATOMIC_SUPPORT_TYPE: + raise ValueError("Atomic geography artifact type differs.") + supports[name] = geography.decode_atomic_support(value.payload) + return context.tables["household"], spec, supports + + +def _result(context, frame, receipt): + if {(o.entity, o.column, o.dtype) for o in context.node.outputs} != { + ("household", c, "string") for c in frame + }: + raise ValueError("Atomic geography owned outputs differ from the declaration.") + index = pd.Index(context.tables["household"]["household_id"], name="household_id") + return KernelResult( + columns={ + ("household", c): pd.Series(frame[c].array, index=index) for c in frame + }, + receipt=receipt, + ) + + +class AtomicAssignKernel(_GeographyKernel): + ref = "geography.assign_atomic@1" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + seed_source=SeedSource.KEYED, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + households, spec, supports = _inputs(context) + output = geography.assign_atomic(households, spec, supports) + return _result( + context, + output, + { + "scope": "atomic_area_assignment", + "households": len(output), + "support_sha256": {k: v.sha256 for k, v in supports.items()}, + }, + ) + + +class AtomicDeriveKernel(_GeographyKernel): + ref = "geography.derive@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + households, spec, supports = _inputs(context) + output = geography.derive_geography(households, spec, supports) + return _result( + context, + output, + { + "scope": "atomic_area_functional_lookup", + "layers": {s["id"]: s["layers"] for s in spec["systems"]}, + }, + ) + + +class AtomicGeographyGateKernel(_GeographyKernel): + ref = "geography.gate@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + role=KernelRole.GATE, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + households, spec, supports = _inputs(context) + outputs = context.node.artifact_outputs + expected = (ArtifactOutput("validation", ATOMIC_GEOGRAPHY_VALIDATION_TYPE),) + if outputs not in ((), expected): + raise ValueError( + "Atomic geography validation artifact declaration differs." + ) + receipt = geography.validate_geography(households, spec, supports) + return KernelResult( + receipt=receipt, + artifacts={"validation": canonical_json(receipt)} if outputs else {}, + ) + + +def atomic_geography_nodes( + spec: Mapping, + columns: Sequence[Owned], + *, + base: str, + prefix: str = "geography", + emit_validation_artifact: bool = False, +) -> tuple[Node, ...]: + """Append shared nodes to an existing population, refusing column rewrites. + + SourceRef declarations use each system's ``source`` and ``raw-bytes-v1``. + ``identity`` must be stable across the country's sampling rungs; this builder + cannot establish that property merely from a column name. + + The optional typed gate result orders downstream consumers. Its bytes alone + do not authenticate their receiving population or source ancestry. + """ + if type(emit_validation_artifact) is not bool: + raise ValueError("Atomic geography validation artifact flag must be boolean.") + spec = geography.validate_assignment_spec(spec) + inventory = {(o.entity, o.column): o for o in columns} + if len(inventory) != len(columns): + raise ValueError("Atomic geography input inventory repeats columns.") + inputs = set(spec["identity"]) + for system in spec["systems"]: + inputs.update(system["selector"]) + inputs.update(c["input"] for c in system["constraints"]) + if system["observed_area"]: + inputs.add(system["observed_area"]) + assignment = tuple(spec["outputs"].values()) + layers = tuple( + sorted({layer["output"] for s in spec["systems"] for layer in s["layers"]}) + ) + if any(("household", c) not in inventory for c in inputs): + raise ValueError("Atomic geography is missing a declared household input.") + if any(("household", c) in inventory for c in (*assignment, *layers)): + raise ValueError("Atomic geography refuses to overwrite existing columns.") + imports, artifacts = [], [] + for i, system in enumerate(spec["systems"]): + name = f"{prefix}.support.{i}" + imports.append( + Node( + id=name, + kernel=AtomicSupportImportKernel.ref, + population=base, + sources=(system["source"],), + params={"source": system["source"], "system": system["id"]}, + artifact_outputs=(ArtifactOutput("support", ATOMIC_SUPPORT_TYPE),), + ) + ) + artifacts.append( + ArtifactInput(system["id"], name, "support", ATOMIC_SUPPORT_TYPE) + ) + + def node(suffix, kernel, read, write=()): + return Node( + id=f"{prefix}.{suffix}", + kernel=kernel.ref, + population=base, + inputs=(Slice("household", tuple(sorted(read))),), + outputs=tuple(Owned("household", c, "string") for c in write), + params={ + "definition": canonical_json(spec).decode(), + "stream": tuple(spec["stream"]), + }, + artifact_inputs=tuple(artifacts), + artifact_outputs=( + (ArtifactOutput("validation", ATOMIC_GEOGRAPHY_VALIDATION_TYPE),) + if suffix == "gate" and emit_validation_artifact + else () + ), + ) + + return ( + *imports, + node("assign", AtomicAssignKernel, inputs, assignment), + node("derive", AtomicDeriveKernel, inputs | set(assignment), layers), + node("gate", AtomicGeographyGateKernel, inputs | set(assignment) | set(layers)), + ) + + +def register_atomic_geography_kernels(registry: KernelRegistry) -> None: + for kernel in ( + AtomicSupportImportKernel(), + AtomicAssignKernel(), + AtomicDeriveKernel(), + AtomicGeographyGateKernel(), + ): + registry.register(kernel) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index ca3c0357b..758ec425f 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -359,8 +359,8 @@ "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "20058e544f6034cee2e76d3b864cf86b6230c8dce042931dfe9247d8ac1329c4", - "seed_protocol": "553d5e0bd5afa93146dfbf8dca684926ee773dbfc89a559434528af5b4eb1d97", + "seed_map": "da07a54ab2bc4e297ac7d0693a8559b359de230787757e4b6e26dcb619c8e5a0", + "seed_protocol": "5f93b3ec98ada30338b06ad978a6e1b47d0f9f916af89418500cea435235ad06", "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/survey_allocation.py b/packages/microcosm-build/src/microcosm/build/survey_allocation.py new file mode 100644 index 000000000..417689e5a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/survey_allocation.py @@ -0,0 +1,202 @@ +"""Domain allocation of supplied design weights, before support cloning. + +This numerical primitive performs no source classification, subsampling, frame +mutation or graph activation. Callers must supply an aligned household axis and +reviewed domain declarations. A row can be a physical housing unit or a cluster +of people; explicit unit counts keep its statistical grain out of its weight. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from fractions import Fraction +from numbers import Integral +from types import MappingProxyType + +import numpy as np + +from microcosm.frame import WeightKind, Weights + + +class SurveyAllocationError(ValueError): + """A supplied allocation contract cannot represent its household axis.""" + + +def _label(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise SurveyAllocationError(f"{field} must be a nonempty string") + return value + + +@dataclass(frozen=True) +class SurveyDomain: + """One declared population/period and its prespecified source shares. + + Shares must be exact Fractions summing to one. Unit and basis labels are + declarations, not evidence of survey equivalence or source authentication. + """ + + key: str + statistical_unit: str + period: str + basis: str + source_shares: Mapping[str, Fraction] + + def __post_init__(self) -> None: + for field in ("key", "statistical_unit", "period", "basis"): + _label(getattr(self, field), field) + if not isinstance(self.source_shares, Mapping) or not self.source_shares: + raise SurveyAllocationError("source_shares must be a nonempty mapping") + shares = {} + for source, share in self.source_shares.items(): + _label(source, "source channel") + if not isinstance(share, Fraction) or not 0 <= share <= 1: + raise SurveyAllocationError( + "shares must be Fractions between zero and one" + ) + shares[source] = share + if sum(shares.values(), Fraction()) != 1: + raise SurveyAllocationError("domain shares must sum exactly to one") + object.__setattr__( + self, "source_shares", MappingProxyType(dict(sorted(shares.items()))) + ) + + +@dataclass(frozen=True) +class DomainSourceEstimate: + """Separate estimates in one declared unit, period and source domain.""" + + domain: str + source: str + statistical_unit: str + period: str + basis: str + source_share: Fraction + rows: int + design_estimate: float + allocated_estimate: float + + +@dataclass(frozen=True) +class SurveyAllocation: + """Output weights aligned to input row IDs; estimates never pool units.""" + + row_ids: tuple[int, ...] + weights: Weights + estimates: tuple[DomainSourceEstimate, ...] + + +def _axis(values: Sequence, size: int, field: str) -> tuple: + if isinstance(values, (str, bytes)): + raise SurveyAllocationError(f"{field} must be a row-aligned sequence") + result = tuple(values) + if len(result) != size: + raise SurveyAllocationError(f"{field} must match the household weight axis") + return result + + +def allocate_domain_weights( + design_weights: Weights, + *, + row_ids: Sequence[int], + source_channels: Sequence[str], + domain_keys: Sequence[str | None], + unit_counts: Sequence[int], + domains: Sequence[SurveyDomain], +) -> SurveyAllocation: + """Multiply each household weight by its declared domain/source share. + + ``unit_counts`` is mandatory: one for a housing unit or individual GQ + placeholder, the number of covered people for a person-counting cluster. + It affects accounting, never the household weight multiplier. No source + total is normalized to another source. Unknown domains and positive-share + cells lacking positive design support refuse. Known exclusions must be + selected explicitly before this operation; rows are never dropped here. + + The declarations do not authenticate their inputs. A production graph + adapter must bind IDs, source membership, coverage and period assumptions + to the actual population and use a new allocation context/receipt version. + """ + if ( + not isinstance(design_weights, Weights) + or design_weights.kind is not WeightKind.DESIGN + ): + raise SurveyAllocationError("allocation requires DESIGN weights") + size = len(design_weights.values) + ids = _axis(row_ids, size, "row_ids") + if any( + isinstance(x, bool) or not isinstance(x, Integral) or not 0 < x <= 2**63 - 1 + for x in ids + ): + raise SurveyAllocationError( + "row_ids must be positive int64 household identities" + ) + if len(set(ids)) != size: + raise SurveyAllocationError("household identities must be unique") + sources = _axis(source_channels, size, "source_channels") + keys = _axis(domain_keys, size, "domain_keys") + counts = _axis(unit_counts, size, "unit_counts") + if any( + isinstance(x, bool) or not isinstance(x, Integral) or not 0 < x <= 2**53 + for x in counts + ): + raise SurveyAllocationError( + "unit_counts must be positive exactly representable integers" + ) + declared = {} + for domain in domains: + if not isinstance(domain, SurveyDomain) or domain.key in declared: + raise SurveyAllocationError( + "domains must be unique SurveyDomain declarations" + ) + declared[domain.key] = domain + cells: dict[tuple[str, str], list[int]] = {} + for position, (key, source) in enumerate(zip(keys, sources, strict=True)): + _label(source, "source channel") + if not isinstance(key, str) or key not in declared: + raise SurveyAllocationError("unknown or undeclared coverage domain") + if source not in declared[key].source_shares: + raise SurveyAllocationError( + "source is not declared for its coverage domain" + ) + cells.setdefault((key, source), []).append(position) + incoming = design_weights.values + allocated = np.zeros(size, dtype=np.float64) + unit_array = np.asarray(counts, dtype=np.float64) + estimates = [] + for key, domain in sorted(declared.items()): + for source, share in domain.source_shares.items(): + positions = np.asarray(cells.get((key, source), ()), dtype=np.int64) + values = incoming[positions] + if share > 0 and not np.any(values > 0): + raise SurveyAllocationError( + "positive-share domain/source lacks sampled support" + ) + with np.errstate(over="ignore", under="ignore", invalid="ignore"): + output = values * float(share) + design_total = float(np.sum(values * unit_array[positions])) + allocated_total = float(np.sum(output * unit_array[positions])) + if not np.isfinite(design_total) or not np.isfinite(allocated_total): + raise SurveyAllocationError("domain estimate overflow") + if share > 0 and np.any((values > 0) & (output == 0)): + raise SurveyAllocationError("positive allocated weight underflow") + allocated[positions] = output + estimates.append( + DomainSourceEstimate( + key, + source, + domain.statistical_unit, + domain.period, + domain.basis, + share, + len(positions), + design_total, + allocated_total, + ) + ) + return SurveyAllocation( + tuple(int(x) for x in ids), + Weights(allocated, WeightKind.IMPORTANCE), + tuple(estimates), + ) diff --git a/packages/microcosm-build/src/microcosm/build/survey_domain_sample.py b/packages/microcosm-build/src/microcosm/build/survey_domain_sample.py new file mode 100644 index 000000000..5d28fd57e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/survey_domain_sample.py @@ -0,0 +1,369 @@ +"""Whole-household domain sampling composed with survey-share allocation. + +Coverage columns are supplied declarations bound to this Frame's household +axis. This module does not authenticate their survey meaning, align periods, +classify sources, or authorize a production graph. It never normalizes a +sampled domain estimate to another estimate or to a mixed-unit weight sum. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Sequence +from dataclasses import dataclass +from fractions import Fraction +from numbers import Integral + +import numpy as np + +from microcosm.build.survey_allocation import ( + DomainSourceEstimate, + SurveyAllocationError, + SurveyDomain, + allocate_domain_weights, +) +from microcosm.frame import Frame, MassChange, WeightKind, Weights + +PROTOCOL = "microcosm.survey-domain-sample-allocation.v1" + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _id_digest(ids: Sequence[int]) -> str: + return hashlib.sha256(np.asarray(ids, dtype=" tuple[DomainHouseholdSelection, ...]: + """Use the same per-cell draw for native catalogues and full Frames. + + ``cells`` contains (domain, source) pairs, including declared empty cells. + IDs must be uniformly positive int64 integers or nonempty literal strings. + They are unique within each source, and sort numerically or lexically, + respectively. No ID conversion, source-share exclusion, classification, + weighting, or claim of full-source completeness occurs here. A source owner + must establish the complete eligible axis before calling this function. + + Each populated cell retains max(1, floor(fraction * N)) rows. Empty cells + have no inclusion probability; full selection consumes no random draw. + The existing protocol and PCG64 seed construction remain unchanged. + """ + if not isinstance(fraction, Fraction) or not 0 < fraction <= 1: + raise SurveyAllocationError("fraction must be a Fraction in (0, 1]") + if type(seed) is not int or not 0 <= seed < 2**64: + raise SurveyAllocationError("seed must be an unsigned 64-bit integer") + axes = (row_ids, source_channels, domain_keys, cells) + if any(isinstance(axis, (str, bytes)) for axis in axes): + raise SurveyAllocationError("selection inputs must be row sequences") + ids, sources, domains, declared = (tuple(axis) for axis in axes) + if len(ids) != len(sources) or len(ids) != len(domains): + raise SurveyAllocationError("selection axes must have equal lengths") + integer_ids = all( + not isinstance(value, bool) + and isinstance(value, Integral) + and 0 < value <= 2**63 - 1 + for value in ids + ) + string_ids = all(type(value) is str and bool(value.strip()) for value in ids) + if not integer_ids and not string_ids: + raise SurveyAllocationError( + "selection IDs must be uniformly positive int64 or literal strings" + ) + if any( + not isinstance(value, str) or not value.strip() + for value in (*sources, *domains) + ): + raise SurveyAllocationError("selection source/domain labels must be nonempty") + if any( + type(cell) is not tuple + or len(cell) != 2 + or any(not isinstance(label, str) or not label.strip() for label in cell) + for cell in declared + ): + raise SurveyAllocationError("selection cells require (domain, source) pairs") + if len(set(declared)) != len(declared): + raise SurveyAllocationError("selection cells must be unique") + if len(set(zip(sources, ids, strict=True))) != len(ids): + raise SurveyAllocationError("selection identities must be unique within source") + groups: dict[tuple[str, str], list[int]] = {cell: [] for cell in declared} + for position, cell in enumerate(zip(domains, sources, strict=True)): + if cell not in groups: + raise SurveyAllocationError("selection row has an undeclared cell") + groups[cell].append(position) + result = [] + for (domain, source), group in sorted(groups.items()): + positions = np.asarray(sorted(group, key=ids.__getitem__), dtype=np.int64) + eligible = len(positions) + if not eligible: + chosen, probability = (), None + else: + count = max(1, fraction.numerator * eligible // fraction.denominator) + probability = Fraction(count, eligible) + if count == eligible: + chosen = positions + else: + material = _json([PROTOCOL, seed, source, domain]) + cell_seed = int.from_bytes(hashlib.sha256(material).digest(), "big") + rng = np.random.Generator(np.random.PCG64(cell_seed)) + chosen = np.sort(rng.choice(positions, size=count, replace=False)) + result.append( + DomainHouseholdSelection( + domain, source, eligible, tuple(int(p) for p in chosen), probability + ) + ) + return tuple(result) + + +@dataclass(frozen=True) +class SampledDomainCell: + """One source/domain estimate in its declared unit, period and basis.""" + + full: DomainSourceEstimate + selected_households: int + inclusion_probability: Fraction | None + selected_household_ids_sha256: str + sampled_allocated_estimate: float + + +@dataclass(frozen=True) +class DomainSampleAllocation: + """A sampled IMPORTANCE Frame and separate, immutable domain accounting.""" + + frame: Frame + cells: tuple[SampledDomainCell, ...] + fraction: Fraction + seed: int + input_axis_sha256: str + domain_declarations_sha256: str + numpy_version: str + coverage_columns: tuple[str, str, str] + + def receipt(self) -> dict[str, object]: + """Describe the executed arithmetic without claiming source admission.""" + return { + "protocol": PROTOCOL, + "coverage_status": "declared_frame_columns_only", + "release_eligible": False, + "coverage_columns": dict( + zip( + ("source", "domain", "unit_count"), + self.coverage_columns, + strict=True, + ) + ), + "fraction": [self.fraction.numerator, self.fraction.denominator], + "seed": self.seed, + "rng": {"family": "PCG64", "numpy_version": self.numpy_version}, + "input_axis_sha256": self.input_axis_sha256, + "domain_declarations_sha256": self.domain_declarations_sha256, + "cells": [ + { + "domain": cell.full.domain, + "source": cell.full.source, + "statistical_unit": cell.full.statistical_unit, + "period": cell.full.period, + "basis": cell.full.basis, + "source_share": [ + cell.full.source_share.numerator, + cell.full.source_share.denominator, + ], + "eligible_households": cell.full.rows, + "selected_households": cell.selected_households, + "inclusion_probability": ( + None + if cell.inclusion_probability is None + else [ + cell.inclusion_probability.numerator, + cell.inclusion_probability.denominator, + ] + ), + "selected_household_ids_sha256": cell.selected_household_ids_sha256, + "full_design_estimate": cell.full.design_estimate, + "full_allocated_estimate": cell.full.allocated_estimate, + "sampled_allocated_estimate": cell.sampled_allocated_estimate, + } + for cell in self.cells + ], + } + + +def sample_and_allocate_domains( + frame: Frame, + *, + source_column: str, + domain_column: str, + unit_count_column: str, + domains: Sequence[SurveyDomain], + fraction: Fraction, + seed: int, +) -> DomainSampleAllocation: + """Sample each declared source/domain cell, then compose its multipliers. + + Each populated cell retains ``max(1, floor(fraction * N))`` households by + simple random sampling without replacement. Its output household weight + is ``source_design_weight * domain_source_share * N / n``. Unit counts + affect estimates only. Small domains therefore remain represented, with + their own actual inclusion probability recorded; no totals are forced to + match. Full selection does not draw. + + PCG64 streams are separated by protocol, source and domain. Sorted household + IDs make selection independent of row order and unrelated cells. The + returned Frame preserves complete selected lineages through Frame.select. + It never labels the sampling/allocation result DESIGN. + + A positive-share cell whose random sample has no positive weight refuses. + Callers must not search seeds to evade that refusal. Coverage exclusions and + period harmonization must be resolved explicitly before this operation. + """ + if not isinstance(frame, Frame): + raise TypeError("domain sampling requires a Frame") + frame.revalidate() + if frame.weighted_entities != ("household",): + raise SurveyAllocationError("domain sampling requires household-only weights") + if not isinstance(fraction, Fraction) or not 0 < fraction <= 1: + raise SurveyAllocationError("fraction must be a Fraction in (0, 1]") + if type(seed) is not int or not 0 <= seed < 2**64: + raise SurveyAllocationError("seed must be an unsigned 64-bit integer") + names = (source_column, domain_column, unit_count_column) + if any(not isinstance(name, str) or not name for name in names): + raise SurveyAllocationError("coverage column names must be nonempty strings") + if len(set(names)) != 3 or "household_id" in names: + raise SurveyAllocationError("coverage columns must be distinct from identities") + household = frame.table("household") + if not set(names).issubset(household): + raise SurveyAllocationError("household coverage columns are missing") + ids = household["household_id"].to_numpy() + sources = tuple(household[source_column]) + keys = tuple(household[domain_column]) + counts = tuple(household[unit_count_column]) + domains = tuple(domains) + weights = frame.weights_for("household") + full = allocate_domain_weights( + weights, + row_ids=ids, + source_channels=sources, + domain_keys=keys, + unit_counts=counts, + domains=domains, + ) + axis = hashlib.sha256() + for row, source, key, count, weight in zip( + ids, sources, keys, counts, weights.values, strict=True + ): + axis.update(_json([int(row), source, key, int(count), float(weight)])) + axis.update(b"\n") + declarations = [ + { + "key": domain.key, + "unit": domain.statistical_unit, + "period": domain.period, + "basis": domain.basis, + "shares": { + source: [share.numerator, share.denominator] + for source, share in domain.source_shares.items() + }, + } + for domain in sorted(domains, key=lambda domain: domain.key) + ] + selection = select_domain_households( + row_ids=ids, + source_channels=sources, + domain_keys=keys, + cells=tuple((estimate.domain, estimate.source) for estimate in full.estimates), + fraction=fraction, + seed=seed, + ) + selected_positions = [] + output_by_id = {} + cells = [] + for estimate, cell in zip(full.estimates, selection, strict=True): + chosen = np.asarray(cell.positions, dtype=np.int64) + selected = len(chosen) + probability = cell.inclusion_probability + if selected: + values = full.weights.values[chosen] + if estimate.source_share > 0 and not np.any(values > 0): + raise SurveyAllocationError( + "positive-share domain/source lacks sampled positive weight" + ) + with np.errstate(over="ignore", invalid="ignore"): + output = values * float(1 / probability) + sample_estimate = float( + np.sum(output * np.asarray([counts[p] for p in chosen])) + ) + if not np.isfinite(output).all() or not np.isfinite(sample_estimate): + raise SurveyAllocationError( + "sampled domain weight or estimate overflow" + ) + selected_positions.extend(chosen.tolist()) + output_by_id.update(zip(ids[chosen].tolist(), output.tolist(), strict=True)) + selected_ids = np.sort(ids[chosen]) + else: + selected, probability, sample_estimate = 0, None, 0.0 + selected_ids = () + cells.append( + SampledDomainCell( + estimate, + selected, + probability, + _id_digest(selected_ids), + sample_estimate, + ) + ) + selected_ids = ids[np.asarray(selected_positions, dtype=np.int64)] + if len(selected_ids) == len(ids): + sampled = frame + else: + membership = frame.schema.membership_column("household") + sampled = frame.select(frame.person[membership].isin(selected_ids).to_numpy()) + realized = sampled.table("household")["household_id"].to_numpy() + if set(realized) != set(selected_ids): + raise SurveyAllocationError("whole-household sample differs from selection") + result = sampled.with_weights( + "household", + Weights( + np.asarray([output_by_id[int(row)] for row in realized]), + WeightKind.IMPORTANCE, + ), + mass=MassChange( + factor=None, + reason=f"{PROTOCOL}: domain source shares and inverse household inclusion", + ), + ) + return DomainSampleAllocation( + result, + tuple(cells), + fraction, + seed, + axis.hexdigest(), + hashlib.sha256(_json(declarations)).hexdigest(), + np.__version__, + names, + ) diff --git a/packages/microcosm-build/src/microcosm/build/table_identity.py b/packages/microcosm-build/src/microcosm/build/table_identity.py new file mode 100644 index 000000000..7dcc9725f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/table_identity.py @@ -0,0 +1,365 @@ +"""Native table identity without constructing a population Frame. + +The canonical_scalar_v1 values codec is factored unchanged from the US late +producer runtime. Its domain labels and optional string normalization remain +stable for existing receipts. native_table_identity additionally binds a closed +native dtype/schema descriptor, including pandas string storage and NA kind. +""" + +from __future__ import annotations + +import hashlib +import json +import struct +from collections.abc import Sequence +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes + +TABLE_VALUES_DIGEST_CODEC = "canonical_scalar_v1" +_TABLE_DIGEST_CHUNK_ROWS = 65_536 + + +def _digest_part( + digest, + *, + domain: str, + payload: bytes | bytearray | memoryview | np.ndarray, +) -> None: + """Append one length-framed, domain-separated byte field to a digest.""" + + domain_bytes = domain.encode("utf-8") + payload_view = memoryview(payload) + if payload_view.format != "B" or payload_view.ndim != 1: + payload_view = payload_view.cast("B") + digest.update(struct.pack(" np.ndarray: + """Return a contiguous, explicitly little-endian numeric byte source.""" + + array = np.asarray(values) + array = array.astype(array.dtype.newbyteorder("<"), copy=False) + return np.ascontiguousarray(array) + + +def _scalar_bytes(value: object) -> bytes: + """Encode one supported object scalar without lossy intermediary hashes.""" + + missing = pd.isna(value) + if isinstance(missing, (bool, np.bool_)) and bool(missing): + return b"null" + if isinstance(value, (bool, np.bool_)): + return b"bool\x01" if bool(value) else b"bool\x00" + if isinstance(value, (int, np.integer)): + return b"integer\x00" + str(int(value)).encode("ascii") + if isinstance(value, (float, np.floating)): + if isinstance(value, np.floating) and value.dtype.itemsize > 8: + raise TypeError( + "US late-producer content digest does not support object " + f"floating scalar {value.dtype!s}." + ) + return b"float64\x00" + struct.pack(" 16: + raise TypeError( + "US late-producer content digest does not support object " + f"complex scalar {value.dtype!s}." + ) + numeric = complex(value) + return b"complex128\x00" + struct.pack(" None: + """Stream framed string or object scalars in bounded-memory chunks.""" + + for chunk_index, start in enumerate( + range(0, len(values), _TABLE_DIGEST_CHUNK_ROWS) + ): + stop = min(start + _TABLE_DIGEST_CHUNK_ROWS, len(values)) + lengths = np.zeros(stop - start, dtype=" None: + """Hash one ordered logical Series with explicit dtype and null domains.""" + + dtype = series.dtype + missing = series.isna().to_numpy(dtype=bool) + _digest_part( + digest, + domain=f"{domain}/dtype", + payload=str(dtype).encode("utf-8"), + ) + _digest_part( + digest, + domain=f"{domain}/row_count", + payload=struct.pack(" str: + """Hash ordered table scalars directly with typed, null-aware framing.""" + + values = ( + canonicalize_table_string_dtypes( + table, + boundary="late-producer content digest", + table_name="declared_surface", + ) + if normalize_strings + else table + ) + if isinstance(values.index, pd.MultiIndex): + index_levels = [ + pd.Series(values.index.get_level_values(level), copy=False) + for level in range(values.index.nlevels) + ] + else: + index_levels = [pd.Series(values.index, copy=False)] + header = { + "codec": TABLE_VALUES_DIGEST_CODEC, + "columns": [str(column) for column in values.columns], + "dtypes": [ + str(values.iloc[:, index].dtype) for index in range(values.shape[1]) + ], + "index_type": type(values.index).__name__, + "index_dtype": str(values.index.dtype), + "index_level_dtypes": [str(level.dtype) for level in index_levels], + "index_names": [ + None if name is None else str(name) for name in values.index.names + ], + } + digest = hashlib.sha256() + _digest_part( + digest, + domain="late_table_digest_codec", + payload=TABLE_VALUES_DIGEST_CODEC.encode("ascii"), + ) + _digest_part( + digest, + domain="late_table_header_json", + payload=json.dumps( + header, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8"), + ) + for level_index, level in enumerate(index_levels): + _digest_series_values( + digest, + level, + domain=f"index_level/{level_index}", + ) + for column_index in range(values.shape[1]): + _digest_series_values( + digest, + values.iloc[:, column_index], + domain=f"column/{column_index}", + ) + return digest.hexdigest() + + +def _native_dtype(dtype: object) -> dict[str, str]: + if isinstance(dtype, pd.StringDtype): + return { + "kind": "pandas_string", + "storage": dtype.storage, + "na_value": "pd.NA" if dtype.na_value is pd.NA else "nan", + } + if isinstance(dtype, np.dtype) and dtype.kind in "biufO": + return {"kind": "numpy", "dtype": dtype.str} + if isinstance( + dtype, + ( + pd.BooleanDtype, + pd.Int8Dtype, + pd.Int16Dtype, + pd.Int32Dtype, + pd.Int64Dtype, + pd.UInt8Dtype, + pd.UInt16Dtype, + pd.UInt32Dtype, + pd.UInt64Dtype, + pd.Float32Dtype, + pd.Float64Dtype, + ), + ): + return {"kind": "pandas_nullable", "dtype": str(dtype)} + raise TypeError(f"Unsupported native table identity dtype: {dtype!r}.") + + +def native_table_identity(table: pd.DataFrame) -> dict[str, Any]: + """Bind native values, order, index and a closed exact dtype schema. + + This narrow identity supports numeric, boolean and string donor tables, + including nullable numeric columns. Categorical/temporal/custom extension + dtypes and MultiIndex axes require a separately declared contract. Nulls + retain their positions; individual floating NaN payload bits are canonical. + No normalization or coercion is performed on the caller's table. + """ + if not isinstance(table, pd.DataFrame): + raise TypeError("Native table identity requires a pandas DataFrame.") + if isinstance(table.index, pd.MultiIndex) or isinstance( + table.columns, pd.MultiIndex + ): + raise TypeError("Native table identity does not support MultiIndex axes.") + if table.columns.has_duplicates or not all( + isinstance(x, str) and x for x in table.columns + ): + raise TypeError( + "Native table identity requires unique nonempty string columns." + ) + for axis in (table.index, table.columns): + if axis.name is not None and not isinstance(axis.name, str): + raise TypeError( + "Native table identity requires string or absent axis names." + ) + series = [ + pd.Series(table.index), + *(table.iloc[:, i] for i in range(table.shape[1])), + ] + for values in series: + if values.dtype == object and not all( + isinstance(x, str) for x in values.dropna() + ): + raise TypeError("Native object dtype may contain only strings or nulls.") + identity = { + "schema": "microcosm.native_table_identity.v1", + "values_sha256": table_values_sha256(table, normalize_strings=False), + "index": { + "type": type(table.index).__name__, + "name": table.index.name, + "dtype": _native_dtype(table.index.dtype), + }, + "column_axis": { + "type": type(table.columns).__name__, + "name": table.columns.name, + "dtype": _native_dtype(table.columns.dtype), + }, + "columns": [ + {"name": name, "dtype": _native_dtype(table[name].dtype)} + for name in table.columns + ], + } + identity["sha256"] = hashlib.sha256( + json.dumps( + identity, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + return identity diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index 2629e4507..bc5f899e6 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -716,6 +716,10 @@ def _expression(adapter: Any, entity: str, expression: str) -> np.ndarray: total: np.ndarray | None = None for part in parts: values = _column(adapter, entity, part) + # NumPy adds bool arrays with logical-OR semantics. A sum of indicator + # terms counts their True values, so widen only boolean operands. + if values.dtype.kind == "b": + values = values.astype(np.int64) total = values.copy() if total is None else total + values if total is None: raise ValueError(f"unsupported value_expression {expression!r}") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 187310e40..05a835787 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -1,552 +1,564 @@ -"""UK build helpers for Microcosm-owned local-geography artifacts.""" +"""UK build helpers, resolved from their defining modules on explicit access. -from microcosm.build.uk_runtime.age_tail import ( - UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE, - UK_AGE_TAIL_BANDS, - UK_AGE_TAIL_DECLARED_SEEDS, - UK_AGE_TOP_CODE, - UKAgeTailStageTransform, - disaggregate_uk_age_top_code, - load_uk_age_tail_band_populations, -) -from microcosm.build.uk_runtime.battery_bindings import ( - UK_GATE_REGISTRY, - UKGateBinding, -) -from microcosm.build.uk_runtime.calibration_run import ( - load_bound_spine_sidecar, - runtime_provenance, - spine_provenance_from_sidecar, -) -from microcosm.build.uk_runtime.cgt_calibration import ( - UK_CGT_ANNUAL_EXEMPT_AMOUNTS, - UK_CGT_GAINS_AMOUNT_COLUMN, - UK_CGT_SOURCE_COLUMN, - UK_CGT_TAXPAYER_COUNT_COLUMN, - UKCGTTargetMaterialization, - materialize_uk_cgt_calibration_frame, - uk_cgt_annual_exempt_amount, -) -from microcosm.build.uk_runtime.content_identity import ( - uk_frame_content_identity, -) -from microcosm.build.uk_runtime.diagnostics import ( - UK_DIAGNOSTICS_SCHEMA_VERSION, - UK_TARGET_GEOGRAPHY_LEVELS, - uk_calibration_diagnostics_payload, - uk_fit_by_family, - uk_support_limited_misses, - uk_weakest_areas_by_fit, - uk_weakest_families, - uk_weight_summary, - uk_zero_weight_strata, - write_uk_calibration_diagnostics, -) -from microcosm.build.uk_runtime.firm_generation import ( - EMPLOYMENT_BANDS, - HMRC_BAND_COLUMNS, - INPUT_FILES, - VAT_LIABILITY_BANDS, - VINTAGES, - AxiomVATRuleEvaluator, - UKFirmCalibrationResult, - UKFirmGenerationConfig, - UKFirmGenerationResult, - UKFirmSourceData, - UKFirmTargetLayout, - UKFirmValidationReport, - UKFirmVATRuleEvaluator, - assign_employment, - assign_vat_flags, - build_firm_target_matrix, - employment_band_name, - generate_base_firms, - generate_input_values, - generate_uk_firm_population, - hmrc_band_name, - map_to_hmrc_band_indices, - optimize_firm_weights, - read_uk_firm_source_data, - solve_firm_weights, - target_diagnostics, - uk_firm_source_data_from_frames, - uk_firm_source_data_from_ledger_facts, - validate_uk_firm_population, - write_uk_firm_population, -) -from microcosm.build.uk_runtime.fiscal_targets import ( - UK_CGT_REQUIRED_COLUMNS, - UK_CGT_TARGET_COVERAGE_REQUIREMENTS, - UK_CGT_TARGET_SPECS, - UK_FISCAL_TARGET_REGISTRY, -) -from microcosm.build.uk_runtime.frs_council_tax import ( - FRS_COUNCIL_TAX_OUTPUT_COLUMNS, - UKFRSCouncilTaxStageTransform, - add_frs_council_tax, - derive_council_tax, -) -from microcosm.build.uk_runtime.frs_disability import ( - FRS_DISABILITY_OUTPUT_COLUMNS, - UK_INTERNAL_DISABILITY_REPORTED_COLUMNS, - UKDWPDisabilityCategoryRates, - UKDWPDisabilityFlagRates, - UKFRSDisabilityStageTransform, - add_frs_disability, - derive_frs_disability, - uk_dwp_disability_category_rates, - uk_dwp_disability_flag_rates, -) -from microcosm.build.uk_runtime.frs_education import ( - EDUCQUAL_MAP, - FRS_EDUCATION_OUTPUT_COLUMNS, - UKFRSEducationStageTransform, - add_frs_education, - derive_current_education, - derive_frs_education, -) -from microcosm.build.uk_runtime.frs_education_grants import ( - FRS_EDUCATION_GRANT_OUTPUT_COLUMNS, - FRS_EDUCATION_GRANT_REWRITES, - UK_EDUCATION_GRANT_CAPACITY_PREDICTORS, - UKDSAPolicy, - UKFRSEducationGrantSplitStageTransform, - add_frs_education_grant_split, - allocate_reported_education_grants, - disabled_students_allowance_capacity, - uk_dsa_policy, -) -from microcosm.build.uk_runtime.frs_employment import ( - FRS_EMPLOYMENT_OUTPUT_COLUMNS, - UKFRSEmploymentStageTransform, - add_frs_employment, - derive_frs_employment, -) -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( - FRS_HMRC_INCPBEN_COLUMN, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, - FRS_HMRC_PAY_COLUMN, - FRS_HMRC_RETAINED_LEAF_COLUMNS, - FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, - FRS_HMRC_UBISJA_COLUMN, - UKFRSHMRCRetainedLeavesResult, - UKFRSHMRCRetainedLeavesStageTransform, - retain_uk_frs_hmrc_leaves, -) -from microcosm.build.uk_runtime.frs_legacy_proxies import ( - FRS_LEGACY_PROXY_OUTPUT_COLUMNS, - UK_LEGACY_PROXY_PREDICTORS, - UKFRSLegacyProxiesStageTransform, - UKLegacyJSAPolicy, - derive_frs_legacy_proxies, - uk_legacy_jsa_policy, -) -from microcosm.build.uk_runtime.frs_release import ( - UK_YEAR_RULES, - UKFRSRelease, - load_uk_frs_release, - resolve_uk_year_rule, -) -from microcosm.build.uk_runtime.geography_ladder import ( - GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR, - GEOGRAPHY_LADDER_VINTAGES_ATTR, - UK_ENGLAND_WALES_REGION_CODES, - UK_GEOGRAPHY_LADDER_COLUMNS, - UK_LONDON_REGION_CODE, - UK_OA_LADDER_DERIVED_LAYERS, - UK_OA_LADDER_KIND, - UK_OA_LADDER_SCHEMA_VERSION, - UkOaLadder, - assign_uk_geography_ladder, - expected_uk_ladder_area_support, - load_uk_oa_ladder, - uk_geography_ladder_assignment_summary, - uk_geography_ladder_gate, - uk_region_mix, -) -from microcosm.build.uk_runtime.geography_sources import ( - ENGLAND_LAD_REGION_URL, - ENGLAND_WALES_OA2021_COUNT, - EW_OA_CONSTITUENCY_URL, - EW_OA_HIERARCHY_URL, - EW_OA_HOUSEHOLDS_URL, - EW_OA_LAD23_URL, - EW_OA_POPULATION_URL, - EW_OA_WARD_URL, - LAD23_ITL_URL, - NI_DZ2021_COUNT, - NI_DZ_GEOJSON_ZIP_URL, - NI_DZ_HOUSEHOLDS_CSV_URL, - NI_DZ_LOOKUP_SHEET, - NI_DZ_PARLCON24_LOOKUP_XLSX_URL, - NI_DZ_POPULATION_CSV_URL, - NI_PARLCON24_COUNT, - SCOTLAND_CENSUS_INDEX_ZIP_URL, - SCOTLAND_OA2022_COUNT, - SCOTLAND_OA_CONSTITUENCY_URL, - SCOTLAND_OA_DZ_IZ_URL, - SCOTLAND_OA_LAU_ITL_URL, - SCOTLAND_OA_POPULATION_URL, - build_complete_uk_geography_crosswalk, - build_england_wales_crosswalk, - build_great_britain_crosswalk, - build_northern_ireland_crosswalk, - build_official_uk_geography_crosswalk, - build_scotland_crosswalk, - load_england_lad_region_lookup, - load_england_wales_oa_constituencies, - load_england_wales_oa_hierarchy, - load_england_wales_oa_households, - load_england_wales_oa_population, - load_england_wales_oa_ward_lookup, - load_ew_oa_lad23_lookup, - load_lad_itl_lookup, - load_ni_dz_hierarchy, - load_ni_dz_households, - load_ni_dz_parlcon24_lookup, - load_ni_dz_population, - load_ni_dz_ward_lookup, - load_scotland_oa_constituencies, - load_scotland_oa_dz_iz_lookup, - load_scotland_oa_households, - load_scotland_oa_lau_lookup, - load_scotland_oa_population, - load_scotland_oa_ward_lookup, - update_england_wales_lad_codes, - write_geography_crosswalk, -) -from microcosm.build.uk_runtime.hmrc_calibration import ( - DEFAULT_HMRC_CALIBRATION_EPOCHS, - DEFAULT_HMRC_CALIBRATION_LEARNING_RATE, - DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR, - DEFAULT_HMRC_MAX_WEIGHT_RATIO, - HMRC_ASSESSABLE_INCOME_COLUMN, - HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN, - HMRC_TAXPAYER_COLUMN, - UKHMRCIncomeCalibration, - UKHMRCTargetMaterialization, - calibrate_uk_hmrc_income, - materialize_uk_hmrc_calibration_frame, -) -from microcosm.build.uk_runtime.hmrc_income import ( - HMRC_SPI_BUILD_PERIOD, - HMRC_SPI_COLLATED_ODS_URL, - HMRC_SPI_INCOME_COMPONENTS, - HMRC_SPI_PUBLICATION_URL, - HMRC_SPI_SOURCE_VINTAGE, - HMRC_SPI_TARGET_RECORD_COUNT, - HMRCIncomeBandTargetRecord, - HMRCIncomeSourceProvenance, - HMRCIncomeTargetSet, - materialize_hmrc_spi_income_band_targets, - verify_hmrc_spi_collated_ods, -) -from microcosm.build.uk_runtime.hmrc_replay import ( - CANONICAL_HMRC_FACT_FENCES, - FULL_FRS_TI_BAND_FENCE_ID, - HMRCFactFence, - HMRCReplayDiagnosticAggregate, - HMRCReplayFact, - HMRCReplayReport, - build_conservative_hmrc_replay_report, - classify_hmrc_replay_targets, - write_hmrc_replay_report, -) -from microcosm.build.uk_runtime.hmrc_source_contract import ( - HMRC_DISTRIBUTIONAL_INPUTS, - UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE, - assert_uk_hmrc_income_source_contract_current, -) -from microcosm.build.uk_runtime.ladder_targets import ( - constituency_household_targets, - ladder_target_provenance, - ladder_vs_chronicle_household_dispersion, - local_authority_household_targets, -) -from microcosm.build.uk_runtime.ledger_targets import ( - UK_CENSUS_HOUSEHOLDS_TARGET_ID, - UK_CROSS_GRAIN_BRIDGES, - UK_CROSS_GRAIN_GRAIN_PRECEDENCE, - UK_CROSS_GRAIN_RULE, - UKFrameTargetAdapter, - UKLedgerTargetCompilation, - apply_uk_cross_grain_reconciliation, - compile_uk_local_target_registry, - compile_uk_target_registry, - load_uk_local_area_crosswalk, - materialize_uk_ledger_targets, - uk_census_household_uprating, - uk_ledger_households_total, - uk_local_target_surface, -) -from microcosm.build.uk_runtime.local_doctrine import ( - UK_LOCAL_CLONE_COUNT, - UK_LOCAL_MAX_WEIGHT_RATIO, - UK_LOCAL_SOLVE_DOCTRINE, - UK_LOCAL_SOLVE_EPOCHS, - UK_LOCAL_TARGET_LOSS_CAP, - UK_LOCAL_TARGET_WEIGHT_RULE, - UKLocalSolveDoctrine, - uk_local_doctrine_with_overrides, - uk_local_target_loss_weights, -) -from microcosm.build.uk_runtime.local_geography import ( - align_area_targets, -) -from microcosm.build.uk_runtime.local_rowwise import ( - UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, - UK_LOCAL_HOLDOUT_FOLDS, - UK_LOCAL_HOLDOUT_SEED, - UKRowwiseDoctrineSolve, - UKRowwiseLocalMatrix, - UKRowwiseNationalRows, - build_uk_rowwise_local_matrix, - build_uk_rowwise_local_surface_matrix, - past_cap_census, - require_adjudicated_uk_local_binding, - rotated_uk_local_holdout, - rowwise_area_support_summary, - rowwise_calibration_mass_reason, - solve_uk_rowwise_weights_under_doctrine, - uk_area_support_summary, - uk_ladder_area_support_summary, -) -from microcosm.build.uk_runtime.local_target_census import ( - CENSUS_KIND, - CENSUS_RESOURCE, - CENSUS_SCHEMA_VERSION, - METRIC_STATUS_BOUND_IN_CODE, - SOURCE_STATUS_DOCUMENTED_UNPINNED, - assert_uk_local_target_census_current, - build_uk_local_target_census, - committed_uk_local_target_census_path, - load_uk_local_target_census, - write_uk_local_target_census, -) -from microcosm.build.uk_runtime.local_targets import ( - AGE_BANDS, - AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL, - AREA_TYPES, - COUNTRY_TO_REGION, - INCOME_VARIABLES, - LA_EXTRA_METRICS, - area_groups_from_codes, - compute_household_metrics, - metric_names, - metric_names_from_target_profile, - metric_tables_by_area_group, -) -from microcosm.build.uk_runtime.national_calibration import ( - CalibrationFrameAdapter, - drop_injected_measure_inputs, - inject_measure_inputs, - prepare_uk_target_frame, -) -from microcosm.build.uk_runtime.national_doctrine import ( - UK_NATIONAL_L0_LAMBDA, - UK_NATIONAL_LEARNING_RATE, - UK_NATIONAL_MASS_RULE, - UK_NATIONAL_MAX_WEIGHT_RATIO, - UK_NATIONAL_SEED, - UK_NATIONAL_SOLVE_DOCTRINE, - UK_NATIONAL_SOLVE_EPOCHS, - UK_NATIONAL_TARGET_LOSS_CAP, - UK_NATIONAL_TARGET_WEIGHT_RULE, - UKNationalSolveDoctrine, - uk_doctrine_with_overrides, - uk_national_target_loss_weights, -) -from microcosm.build.uk_runtime.national_frame import ( - UK_NATIONAL_SCHEMA, - UKNationalStage, - UKStagingProvenance, - load_uk_national_frame, - uk_household_weight_kind, - uk_national_frame, - uk_time_period, - validate_uk_national_frame, - write_uk_national_frame, -) -from microcosm.build.uk_runtime.national_sampling import ( - sample_uk_spine_frame, - uk_spine_source_family_units, -) -from microcosm.build.uk_runtime.oa_ladder_sources import ( - LADDER_OA_COLUMNS, - assemble_uk_oa_ladder, - concat_uk_ladder_frames, - join_uk_oa_ladder_layers, -) -from microcosm.build.uk_runtime.parity_reference import ( - EFRS_PARITY_KNOWN_GAPS_RESOURCE, - EFRS_PARITY_REFERENCE_RESOURCE, - EfrsParityKnownGap, - EfrsParityReference, - EfrsParitySource, - load_efrs_parity_known_gaps, - load_efrs_parity_reference, -) -from microcosm.build.uk_runtime.release_identity import ( - UK_DENSE_RELEASE_ID, - UK_RELEASE_TIER_CPS_TRANSFER, - UK_RELEASE_TIER_FRS, - UK_RELEASE_TIERS, - UKReleaseIdentity, - apply_uk_release_identity, - format_uk_release_id, - validate_uk_release_tier, -) -from microcosm.build.uk_runtime.release_input_coverage import ( - RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS, - UK_LOADER_INPUT_ALIASES, - UK_RELEASE_INPUT_COVERAGE_RESOURCE, - PolicyEngineUKCoverageEngine, - UKEffectiveMassCoveragePolicy, - UKReleaseInputColumn, - UKReleaseInputCoverageManifest, - assert_uk_release_input_coverage_build_stages, - assert_uk_release_input_coverage_manifest_current, - load_uk_release_input_coverage_manifest, - uk_release_input_coverage_gate, - uk_release_input_coverage_required_columns, - uk_release_input_coverage_reviewed_exclusions, -) -from microcosm.build.uk_runtime.rowwise_dataset import ( - ARTIFACT_CLONE_INDEX_COLUMN, - BENUNIT_ID_COLUMNS, - HOUSEHOLD_ID_COLUMNS, - MASS_CONSERVATION_RELATIVE_TOLERANCE, - PERSON_ID_COLUMNS, - POOL_SOURCE_LINEAGE_COLUMN, - UK_SINGLE_YEAR_TABLES, - UKLadderRowwiseDatasetResult, - UKRowwiseDatasetResult, - apply_uk_source_lineage_modulus, - clone_uk_dataset_tables_with_ladder_geography, - clone_uk_dataset_tables_with_rowwise_geography, - clone_uk_dataset_with_ladder_geography, - clone_uk_dataset_with_rowwise_geography, - ladder_clone_index_column, - load_uk_rowwise_dataset, - read_uk_single_year_weight_metadata, - validate_uk_ladder_rowwise_dataset_tables, - validate_uk_rowwise_dataset_tables, - write_uk_rowwise_dataset, -) -from microcosm.build.uk_runtime.rowwise_geography import ( - AREA_TYPE_TO_CROSSWALK_COLUMN, - CROSSWALK_COLUMNS, - FRS_REGION_TO_COUNTRY, - FRS_REGION_TO_REGION_CODE, - ROWWISE_GEOGRAPHY_COLUMNS, - RowwiseGeographyAssignment, - assign_household_geography, - clone_entity_frame, - expected_uk_rowwise_area_support, - geography_coverage_summary, - id_multiplier_for_values, - prepare_geography_crosswalk, - validate_geography_coverage, -) -from microcosm.build.uk_runtime.size_evaluation import ( - PRE_REGISTERED_OUTCOMES_V1, - RowwiseRun, - area_support_tables, - dense_reference_deltas, - fit_tables, - footprint, - frozen_vs_recomputed, - gate_table, - load_run, - paired_targets, - run_acceptance, - summarize, - weight_tables, -) -from microcosm.build.uk_runtime.spi_income import ( - SPI_DONOR_DOI, - SPI_DONOR_FILENAME, - SPI_DONOR_SHA256, - SPI_DONOR_SIZE_BYTES, - SPI_DONOR_UKDS_STUDY, - SPI_DONOR_VINTAGE, - SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS, - UKSPIIncomeImputationResult, - assert_frs_hmrc_auxiliary_crosswalk_available, - derive_hmrc_income_auxiliaries, - impute_uk_spi_income_support, - verify_spi_donor_identity, -) -from microcosm.build.uk_runtime.spi_spine import ( - EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN, - UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS, - UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME, - UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME, - UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND, - UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS, - UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS, - UK_SPI_INCOME_SPINE_REWRITE_COLUMNS, - UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS, - UKFRSHMRCSpineLeavesResult, - UKFRSHMRCSpineLeavesStageTransform, - UKSPIIncomeSpineResult, - UKSPIIncomeSpineStageTransform, - UKSPISupportChannelStageTransform, -) -from microcosm.build.uk_runtime.spi_support import ( - BASE_FRS_SUPPORT_CHANNEL, - DEFAULT_SPI_PRIOR_MASS_SHARE, - DEFAULT_SPI_SUPPORT_HOUSEHOLDS, - FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS, - FRS_ONLY_SPI_FILL_PERSON_COLUMNS, - FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS, - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - SPI_INCOME_COMPONENT_COLUMNS, - SPI_INCOME_IMPUTATION_COLUMNS, - SPI_PRIOR_MASS_CHANGE_REASON, - SPI_REPLACEMENT_STRATA_COLUMNS, - SPI_SYNTHETIC_SUPPORT_CHANNEL, - UK_SPI_SUPPORT_STAGE_NAME, - UKSPISupportResult, - build_uk_spi_support_channel, - create_uk_spi_support_tables, - fill_support_channel_from_source, - replace_uk_spi_support_tables, - support_channel_column, - support_clone_index_column, - support_source_id_column, -) -from microcosm.build.uk_runtime.stage_checkpoints import ( - UK_FRAME_METADATA_KEY, - load_uk_stage_checkpoint, - load_uk_stage_predecessor, - uk_stage_metadata, -) -from microcosm.build.uk_runtime.terminal_gates import ( - UK_DEFAULT_ZERO_WEIGHT_STRATA, - UK_MAX_TARGET_ABS_RELATIVE_ERROR, - UKZeroWeightStratumDeclaration, - uk_degenerate_release_surface_gate, - uk_export_surface_gate, - uk_target_fit_gate, - uk_target_surface_gate, - uk_weight_ess_gate, - uk_weight_ratio_gate, - uk_zero_weight_strata_gate, -) -from microcosm.build.uk_runtime.weighted_integrity import ( - UKInputMassParityPolicy, - UKInputMassReference, - UKQRFTailConcentrationPolicy, - load_uk_input_mass_reference, - load_uk_local_area_support_exclusion_register, - load_uk_reviewed_exclusion_register, - uk_input_mass_parity_gate, - uk_input_mass_totals, - uk_qrf_tail_concentration_columns, - uk_qrf_tail_concentration_gate, -) +Direct helper imports do not initialize unrelated calibration or source loaders. +The public export roster is unchanged; requesting every export with ``import *`` +still imports every defining module. Use ``dir`` or ``__all__`` for discovery: +``vars`` contains only exports already resolved in this process. +""" + +from importlib import import_module as _import_module +from types import MappingProxyType as _MappingProxyType +from typing import TYPE_CHECKING as _TYPE_CHECKING +from typing import Any as _Any + +if _TYPE_CHECKING: + from microcosm.build.uk_runtime.age_tail import ( + UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE, + UK_AGE_TAIL_BANDS, + UK_AGE_TAIL_DECLARED_SEEDS, + UK_AGE_TOP_CODE, + UKAgeTailStageTransform, + disaggregate_uk_age_top_code, + load_uk_age_tail_band_populations, + ) + from microcosm.build.uk_runtime.battery_bindings import ( + UK_GATE_REGISTRY, + UKGateBinding, + ) + from microcosm.build.uk_runtime.calibration_run import ( + load_bound_spine_sidecar, + runtime_provenance, + spine_provenance_from_sidecar, + ) + from microcosm.build.uk_runtime.cgt_calibration import ( + UK_CGT_ANNUAL_EXEMPT_AMOUNTS, + UK_CGT_GAINS_AMOUNT_COLUMN, + UK_CGT_SOURCE_COLUMN, + UK_CGT_TAXPAYER_COUNT_COLUMN, + UKCGTTargetMaterialization, + materialize_uk_cgt_calibration_frame, + uk_cgt_annual_exempt_amount, + ) + from microcosm.build.uk_runtime.content_identity import ( + uk_frame_content_identity, + ) + from microcosm.build.uk_runtime.diagnostics import ( + UK_DIAGNOSTICS_SCHEMA_VERSION, + UK_TARGET_GEOGRAPHY_LEVELS, + uk_calibration_diagnostics_payload, + uk_fit_by_family, + uk_support_limited_misses, + uk_weakest_areas_by_fit, + uk_weakest_families, + uk_weight_summary, + uk_zero_weight_strata, + write_uk_calibration_diagnostics, + ) + from microcosm.build.uk_runtime.firm_generation import ( + EMPLOYMENT_BANDS, + HMRC_BAND_COLUMNS, + INPUT_FILES, + VAT_LIABILITY_BANDS, + VINTAGES, + AxiomVATRuleEvaluator, + UKFirmCalibrationResult, + UKFirmGenerationConfig, + UKFirmGenerationResult, + UKFirmSourceData, + UKFirmTargetLayout, + UKFirmValidationReport, + UKFirmVATRuleEvaluator, + assign_employment, + assign_vat_flags, + build_firm_target_matrix, + employment_band_name, + generate_base_firms, + generate_input_values, + generate_uk_firm_population, + hmrc_band_name, + map_to_hmrc_band_indices, + optimize_firm_weights, + read_uk_firm_source_data, + solve_firm_weights, + target_diagnostics, + uk_firm_source_data_from_frames, + uk_firm_source_data_from_ledger_facts, + validate_uk_firm_population, + write_uk_firm_population, + ) + from microcosm.build.uk_runtime.fiscal_targets import ( + UK_CGT_REQUIRED_COLUMNS, + UK_CGT_TARGET_COVERAGE_REQUIREMENTS, + UK_CGT_TARGET_SPECS, + UK_FISCAL_TARGET_REGISTRY, + ) + from microcosm.build.uk_runtime.frs_council_tax import ( + FRS_COUNCIL_TAX_OUTPUT_COLUMNS, + UKFRSCouncilTaxStageTransform, + add_frs_council_tax, + derive_council_tax, + ) + from microcosm.build.uk_runtime.frs_disability import ( + FRS_DISABILITY_OUTPUT_COLUMNS, + UK_INTERNAL_DISABILITY_REPORTED_COLUMNS, + UKDWPDisabilityCategoryRates, + UKDWPDisabilityFlagRates, + UKFRSDisabilityStageTransform, + add_frs_disability, + derive_frs_disability, + uk_dwp_disability_category_rates, + uk_dwp_disability_flag_rates, + ) + from microcosm.build.uk_runtime.frs_education import ( + EDUCQUAL_MAP, + FRS_EDUCATION_OUTPUT_COLUMNS, + UKFRSEducationStageTransform, + add_frs_education, + derive_current_education, + derive_frs_education, + ) + from microcosm.build.uk_runtime.frs_education_grants import ( + FRS_EDUCATION_GRANT_OUTPUT_COLUMNS, + FRS_EDUCATION_GRANT_REWRITES, + UK_EDUCATION_GRANT_CAPACITY_PREDICTORS, + UKDSAPolicy, + UKFRSEducationGrantSplitStageTransform, + add_frs_education_grant_split, + allocate_reported_education_grants, + disabled_students_allowance_capacity, + uk_dsa_policy, + ) + from microcosm.build.uk_runtime.frs_employment import ( + FRS_EMPLOYMENT_OUTPUT_COLUMNS, + UKFRSEmploymentStageTransform, + add_frs_employment, + derive_frs_employment, + ) + from microcosm.build.uk_runtime.frs_hmrc_leaves import ( + FRS_HMRC_INCPBEN_COLUMN, + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, + FRS_HMRC_PAY_COLUMN, + FRS_HMRC_RETAINED_LEAF_COLUMNS, + FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, + FRS_HMRC_UBISJA_COLUMN, + UKFRSHMRCRetainedLeavesResult, + UKFRSHMRCRetainedLeavesStageTransform, + retain_uk_frs_hmrc_leaves, + ) + from microcosm.build.uk_runtime.frs_legacy_proxies import ( + FRS_LEGACY_PROXY_OUTPUT_COLUMNS, + UK_LEGACY_PROXY_PREDICTORS, + UKFRSLegacyProxiesStageTransform, + UKLegacyJSAPolicy, + derive_frs_legacy_proxies, + uk_legacy_jsa_policy, + ) + from microcosm.build.uk_runtime.frs_release import ( + UK_YEAR_RULES, + UKFRSRelease, + load_uk_frs_release, + resolve_uk_year_rule, + ) + from microcosm.build.uk_runtime.geography_ladder import ( + GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR, + GEOGRAPHY_LADDER_VINTAGES_ATTR, + UK_ENGLAND_WALES_REGION_CODES, + UK_GEOGRAPHY_LADDER_COLUMNS, + UK_LONDON_REGION_CODE, + UK_OA_LADDER_DERIVED_LAYERS, + UK_OA_LADDER_KIND, + UK_OA_LADDER_SCHEMA_VERSION, + UkOaLadder, + assign_uk_geography_ladder, + expected_uk_ladder_area_support, + load_uk_oa_ladder, + uk_geography_ladder_assignment_summary, + uk_geography_ladder_gate, + uk_region_mix, + ) + from microcosm.build.uk_runtime.geography_sources import ( + ENGLAND_LAD_REGION_URL, + ENGLAND_WALES_OA2021_COUNT, + EW_OA_CONSTITUENCY_URL, + EW_OA_HIERARCHY_URL, + EW_OA_HOUSEHOLDS_URL, + EW_OA_LAD23_URL, + EW_OA_POPULATION_URL, + EW_OA_WARD_URL, + LAD23_ITL_URL, + NI_DZ2021_COUNT, + NI_DZ_GEOJSON_ZIP_URL, + NI_DZ_HOUSEHOLDS_CSV_URL, + NI_DZ_LOOKUP_SHEET, + NI_DZ_PARLCON24_LOOKUP_XLSX_URL, + NI_DZ_POPULATION_CSV_URL, + NI_PARLCON24_COUNT, + SCOTLAND_CENSUS_INDEX_ZIP_URL, + SCOTLAND_OA2022_COUNT, + SCOTLAND_OA_CONSTITUENCY_URL, + SCOTLAND_OA_DZ_IZ_URL, + SCOTLAND_OA_LAU_ITL_URL, + SCOTLAND_OA_POPULATION_URL, + build_complete_uk_geography_crosswalk, + build_england_wales_crosswalk, + build_great_britain_crosswalk, + build_northern_ireland_crosswalk, + build_official_uk_geography_crosswalk, + build_scotland_crosswalk, + load_england_lad_region_lookup, + load_england_wales_oa_constituencies, + load_england_wales_oa_hierarchy, + load_england_wales_oa_households, + load_england_wales_oa_population, + load_england_wales_oa_ward_lookup, + load_ew_oa_lad23_lookup, + load_lad_itl_lookup, + load_ni_dz_hierarchy, + load_ni_dz_households, + load_ni_dz_parlcon24_lookup, + load_ni_dz_population, + load_ni_dz_ward_lookup, + load_scotland_oa_constituencies, + load_scotland_oa_dz_iz_lookup, + load_scotland_oa_households, + load_scotland_oa_lau_lookup, + load_scotland_oa_population, + load_scotland_oa_ward_lookup, + update_england_wales_lad_codes, + write_geography_crosswalk, + ) + from microcosm.build.uk_runtime.hmrc_calibration import ( + DEFAULT_HMRC_CALIBRATION_EPOCHS, + DEFAULT_HMRC_CALIBRATION_LEARNING_RATE, + DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR, + DEFAULT_HMRC_MAX_WEIGHT_RATIO, + HMRC_ASSESSABLE_INCOME_COLUMN, + HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN, + HMRC_TAXPAYER_COLUMN, + UKHMRCIncomeCalibration, + UKHMRCTargetMaterialization, + calibrate_uk_hmrc_income, + materialize_uk_hmrc_calibration_frame, + ) + from microcosm.build.uk_runtime.hmrc_income import ( + HMRC_SPI_BUILD_PERIOD, + HMRC_SPI_COLLATED_ODS_URL, + HMRC_SPI_INCOME_COMPONENTS, + HMRC_SPI_PUBLICATION_URL, + HMRC_SPI_SOURCE_VINTAGE, + HMRC_SPI_TARGET_RECORD_COUNT, + HMRCIncomeBandTargetRecord, + HMRCIncomeSourceProvenance, + HMRCIncomeTargetSet, + materialize_hmrc_spi_income_band_targets, + verify_hmrc_spi_collated_ods, + ) + from microcosm.build.uk_runtime.hmrc_replay import ( + CANONICAL_HMRC_FACT_FENCES, + FULL_FRS_TI_BAND_FENCE_ID, + HMRCFactFence, + HMRCReplayDiagnosticAggregate, + HMRCReplayFact, + HMRCReplayReport, + build_conservative_hmrc_replay_report, + classify_hmrc_replay_targets, + write_hmrc_replay_report, + ) + from microcosm.build.uk_runtime.hmrc_source_contract import ( + HMRC_DISTRIBUTIONAL_INPUTS, + UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE, + assert_uk_hmrc_income_source_contract_current, + ) + from microcosm.build.uk_runtime.ladder_targets import ( + constituency_household_targets, + ladder_target_provenance, + ladder_vs_chronicle_household_dispersion, + local_authority_household_targets, + ) + from microcosm.build.uk_runtime.ledger_targets import ( + UK_CENSUS_HOUSEHOLDS_TARGET_ID, + UK_CROSS_GRAIN_BRIDGES, + UK_CROSS_GRAIN_GRAIN_PRECEDENCE, + UK_CROSS_GRAIN_RULE, + UKFrameTargetAdapter, + UKLedgerTargetCompilation, + apply_uk_cross_grain_reconciliation, + compile_uk_local_target_registry, + compile_uk_target_registry, + load_uk_local_area_crosswalk, + materialize_uk_ledger_targets, + uk_census_household_uprating, + uk_ledger_households_total, + uk_local_target_surface, + ) + from microcosm.build.uk_runtime.local_doctrine import ( + UK_LOCAL_CLONE_COUNT, + UK_LOCAL_MAX_WEIGHT_RATIO, + UK_LOCAL_SOLVE_DOCTRINE, + UK_LOCAL_SOLVE_EPOCHS, + UK_LOCAL_TARGET_LOSS_CAP, + UK_LOCAL_TARGET_WEIGHT_RULE, + UKLocalSolveDoctrine, + uk_local_doctrine_with_overrides, + uk_local_target_loss_weights, + ) + from microcosm.build.uk_runtime.local_geography import ( + align_area_targets, + ) + from microcosm.build.uk_runtime.local_rowwise import ( + UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, + UK_LOCAL_HOLDOUT_FOLDS, + UK_LOCAL_HOLDOUT_SEED, + UKRowwiseDoctrineSolve, + UKRowwiseLocalMatrix, + UKRowwiseNationalRows, + build_uk_rowwise_local_matrix, + build_uk_rowwise_local_surface_matrix, + past_cap_census, + require_adjudicated_uk_local_binding, + rotated_uk_local_holdout, + rowwise_area_support_summary, + rowwise_calibration_mass_reason, + solve_uk_rowwise_weights_under_doctrine, + uk_area_support_summary, + uk_ladder_area_support_summary, + ) + from microcosm.build.uk_runtime.local_target_census import ( + CENSUS_KIND, + CENSUS_RESOURCE, + CENSUS_SCHEMA_VERSION, + METRIC_STATUS_BOUND_IN_CODE, + SOURCE_STATUS_DOCUMENTED_UNPINNED, + assert_uk_local_target_census_current, + build_uk_local_target_census, + committed_uk_local_target_census_path, + load_uk_local_target_census, + write_uk_local_target_census, + ) + from microcosm.build.uk_runtime.local_targets import ( + AGE_BANDS, + AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL, + AREA_TYPES, + COUNTRY_TO_REGION, + INCOME_VARIABLES, + LA_EXTRA_METRICS, + area_groups_from_codes, + compute_household_metrics, + metric_names, + metric_names_from_target_profile, + metric_tables_by_area_group, + ) + from microcosm.build.uk_runtime.national_calibration import ( + CalibrationFrameAdapter, + drop_injected_measure_inputs, + inject_measure_inputs, + prepare_uk_target_frame, + ) + from microcosm.build.uk_runtime.national_doctrine import ( + UK_NATIONAL_L0_LAMBDA, + UK_NATIONAL_LEARNING_RATE, + UK_NATIONAL_MASS_RULE, + UK_NATIONAL_MAX_WEIGHT_RATIO, + UK_NATIONAL_SEED, + UK_NATIONAL_SOLVE_DOCTRINE, + UK_NATIONAL_SOLVE_EPOCHS, + UK_NATIONAL_TARGET_LOSS_CAP, + UK_NATIONAL_TARGET_WEIGHT_RULE, + UKNationalSolveDoctrine, + uk_doctrine_with_overrides, + uk_national_target_loss_weights, + ) + from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, + UKNationalStage, + UKStagingProvenance, + load_uk_national_frame, + uk_household_weight_kind, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, + write_uk_national_frame, + ) + from microcosm.build.uk_runtime.national_sampling import ( + sample_uk_spine_frame, + uk_spine_source_family_units, + ) + from microcosm.build.uk_runtime.oa_ladder_sources import ( + LADDER_OA_COLUMNS, + assemble_uk_oa_ladder, + concat_uk_ladder_frames, + join_uk_oa_ladder_layers, + ) + from microcosm.build.uk_runtime.parity_reference import ( + EFRS_PARITY_KNOWN_GAPS_RESOURCE, + EFRS_PARITY_REFERENCE_RESOURCE, + EfrsParityKnownGap, + EfrsParityReference, + EfrsParitySource, + load_efrs_parity_known_gaps, + load_efrs_parity_reference, + ) + from microcosm.build.uk_runtime.release_identity import ( + UK_DENSE_RELEASE_ID, + UK_RELEASE_TIER_CPS_TRANSFER, + UK_RELEASE_TIER_FRS, + UK_RELEASE_TIERS, + UKReleaseIdentity, + apply_uk_release_identity, + format_uk_release_id, + validate_uk_release_tier, + ) + from microcosm.build.uk_runtime.release_input_coverage import ( + RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS, + UK_LOADER_INPUT_ALIASES, + UK_RELEASE_INPUT_COVERAGE_RESOURCE, + PolicyEngineUKCoverageEngine, + UKEffectiveMassCoveragePolicy, + UKReleaseInputColumn, + UKReleaseInputCoverageManifest, + assert_uk_release_input_coverage_build_stages, + assert_uk_release_input_coverage_manifest_current, + load_uk_release_input_coverage_manifest, + uk_release_input_coverage_gate, + uk_release_input_coverage_required_columns, + uk_release_input_coverage_reviewed_exclusions, + ) + from microcosm.build.uk_runtime.rowwise_dataset import ( + ARTIFACT_CLONE_INDEX_COLUMN, + BENUNIT_ID_COLUMNS, + HOUSEHOLD_ID_COLUMNS, + MASS_CONSERVATION_RELATIVE_TOLERANCE, + PERSON_ID_COLUMNS, + POOL_SOURCE_LINEAGE_COLUMN, + UK_SINGLE_YEAR_TABLES, + UKLadderRowwiseDatasetResult, + UKRowwiseDatasetResult, + apply_uk_source_lineage_modulus, + clone_uk_dataset_tables_with_ladder_geography, + clone_uk_dataset_tables_with_rowwise_geography, + clone_uk_dataset_with_ladder_geography, + clone_uk_dataset_with_rowwise_geography, + ladder_clone_index_column, + load_uk_rowwise_dataset, + read_uk_single_year_weight_metadata, + validate_uk_ladder_rowwise_dataset_tables, + validate_uk_rowwise_dataset_tables, + write_uk_rowwise_dataset, + ) + from microcosm.build.uk_runtime.rowwise_geography import ( + AREA_TYPE_TO_CROSSWALK_COLUMN, + CROSSWALK_COLUMNS, + FRS_REGION_TO_COUNTRY, + FRS_REGION_TO_REGION_CODE, + ROWWISE_GEOGRAPHY_COLUMNS, + RowwiseGeographyAssignment, + assign_household_geography, + clone_entity_frame, + expected_uk_rowwise_area_support, + geography_coverage_summary, + id_multiplier_for_values, + prepare_geography_crosswalk, + validate_geography_coverage, + ) + from microcosm.build.uk_runtime.size_evaluation import ( + PRE_REGISTERED_OUTCOMES_V1, + RowwiseRun, + area_support_tables, + dense_reference_deltas, + fit_tables, + footprint, + frozen_vs_recomputed, + gate_table, + load_run, + paired_targets, + run_acceptance, + summarize, + weight_tables, + ) + from microcosm.build.uk_runtime.spi_income import ( + SPI_DONOR_DOI, + SPI_DONOR_FILENAME, + SPI_DONOR_SHA256, + SPI_DONOR_SIZE_BYTES, + SPI_DONOR_UKDS_STUDY, + SPI_DONOR_VINTAGE, + SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS, + UKSPIIncomeImputationResult, + assert_frs_hmrc_auxiliary_crosswalk_available, + derive_hmrc_income_auxiliaries, + impute_uk_spi_income_support, + verify_spi_donor_identity, + ) + from microcosm.build.uk_runtime.spi_spine import ( + EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN, + UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS, + UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME, + UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME, + UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND, + UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS, + UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS, + UK_SPI_INCOME_SPINE_REWRITE_COLUMNS, + UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS, + UKFRSHMRCSpineLeavesResult, + UKFRSHMRCSpineLeavesStageTransform, + UKSPIIncomeSpineResult, + UKSPIIncomeSpineStageTransform, + UKSPISupportChannelStageTransform, + ) + from microcosm.build.uk_runtime.spi_support import ( + BASE_FRS_SUPPORT_CHANNEL, + DEFAULT_SPI_PRIOR_MASS_SHARE, + DEFAULT_SPI_SUPPORT_HOUSEHOLDS, + FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS, + FRS_ONLY_SPI_FILL_PERSON_COLUMNS, + FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS, + HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, + SPI_INCOME_COMPONENT_COLUMNS, + SPI_INCOME_IMPUTATION_COLUMNS, + SPI_PRIOR_MASS_CHANGE_REASON, + SPI_REPLACEMENT_STRATA_COLUMNS, + SPI_SYNTHETIC_SUPPORT_CHANNEL, + UK_SPI_SUPPORT_STAGE_NAME, + UKSPISupportResult, + build_uk_spi_support_channel, + create_uk_spi_support_tables, + fill_support_channel_from_source, + replace_uk_spi_support_tables, + support_channel_column, + support_clone_index_column, + support_source_id_column, + ) + from microcosm.build.uk_runtime.stage_checkpoints import ( + UK_FRAME_METADATA_KEY, + load_uk_stage_checkpoint, + load_uk_stage_predecessor, + uk_stage_metadata, + ) + from microcosm.build.uk_runtime.terminal_gates import ( + UK_DEFAULT_ZERO_WEIGHT_STRATA, + UK_MAX_TARGET_ABS_RELATIVE_ERROR, + UKZeroWeightStratumDeclaration, + uk_degenerate_release_surface_gate, + uk_export_surface_gate, + uk_target_fit_gate, + uk_target_surface_gate, + uk_weight_ess_gate, + uk_weight_ratio_gate, + uk_zero_weight_strata_gate, + ) + from microcosm.build.uk_runtime.weighted_integrity import ( + UKInputMassParityPolicy, + UKInputMassReference, + UKQRFTailConcentrationPolicy, + load_uk_input_mass_reference, + load_uk_local_area_support_exclusion_register, + load_uk_reviewed_exclusion_register, + uk_input_mass_parity_gate, + uk_input_mass_totals, + uk_qrf_tail_concentration_columns, + uk_qrf_tail_concentration_gate, + ) __all__ = [ "PRE_REGISTERED_OUTCOMES_V1", @@ -1006,3 +1018,1793 @@ "uk_weight_ratio_gate", "uk_zero_weight_strata_gate", ] + + +_EXPORTS = _MappingProxyType( + { + "UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE": ( + "microcosm.build.uk_runtime.age_tail", + "UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE", + ), + "UK_AGE_TAIL_BANDS": ( + "microcosm.build.uk_runtime.age_tail", + "UK_AGE_TAIL_BANDS", + ), + "UK_AGE_TAIL_DECLARED_SEEDS": ( + "microcosm.build.uk_runtime.age_tail", + "UK_AGE_TAIL_DECLARED_SEEDS", + ), + "UK_AGE_TOP_CODE": ("microcosm.build.uk_runtime.age_tail", "UK_AGE_TOP_CODE"), + "UKAgeTailStageTransform": ( + "microcosm.build.uk_runtime.age_tail", + "UKAgeTailStageTransform", + ), + "disaggregate_uk_age_top_code": ( + "microcosm.build.uk_runtime.age_tail", + "disaggregate_uk_age_top_code", + ), + "load_uk_age_tail_band_populations": ( + "microcosm.build.uk_runtime.age_tail", + "load_uk_age_tail_band_populations", + ), + "UK_GATE_REGISTRY": ( + "microcosm.build.uk_runtime.battery_bindings", + "UK_GATE_REGISTRY", + ), + "UKGateBinding": ( + "microcosm.build.uk_runtime.battery_bindings", + "UKGateBinding", + ), + "load_bound_spine_sidecar": ( + "microcosm.build.uk_runtime.calibration_run", + "load_bound_spine_sidecar", + ), + "runtime_provenance": ( + "microcosm.build.uk_runtime.calibration_run", + "runtime_provenance", + ), + "spine_provenance_from_sidecar": ( + "microcosm.build.uk_runtime.calibration_run", + "spine_provenance_from_sidecar", + ), + "UK_CGT_ANNUAL_EXEMPT_AMOUNTS": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_ANNUAL_EXEMPT_AMOUNTS", + ), + "UK_CGT_GAINS_AMOUNT_COLUMN": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_GAINS_AMOUNT_COLUMN", + ), + "UK_CGT_SOURCE_COLUMN": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_SOURCE_COLUMN", + ), + "UK_CGT_TAXPAYER_COUNT_COLUMN": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_TAXPAYER_COUNT_COLUMN", + ), + "UKCGTTargetMaterialization": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UKCGTTargetMaterialization", + ), + "materialize_uk_cgt_calibration_frame": ( + "microcosm.build.uk_runtime.cgt_calibration", + "materialize_uk_cgt_calibration_frame", + ), + "uk_cgt_annual_exempt_amount": ( + "microcosm.build.uk_runtime.cgt_calibration", + "uk_cgt_annual_exempt_amount", + ), + "uk_frame_content_identity": ( + "microcosm.build.uk_runtime.content_identity", + "uk_frame_content_identity", + ), + "UK_DIAGNOSTICS_SCHEMA_VERSION": ( + "microcosm.build.uk_runtime.diagnostics", + "UK_DIAGNOSTICS_SCHEMA_VERSION", + ), + "UK_TARGET_GEOGRAPHY_LEVELS": ( + "microcosm.build.uk_runtime.diagnostics", + "UK_TARGET_GEOGRAPHY_LEVELS", + ), + "uk_calibration_diagnostics_payload": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_calibration_diagnostics_payload", + ), + "uk_fit_by_family": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_fit_by_family", + ), + "uk_support_limited_misses": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_support_limited_misses", + ), + "uk_weakest_areas_by_fit": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_weakest_areas_by_fit", + ), + "uk_weakest_families": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_weakest_families", + ), + "uk_weight_summary": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_weight_summary", + ), + "uk_zero_weight_strata": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_zero_weight_strata", + ), + "write_uk_calibration_diagnostics": ( + "microcosm.build.uk_runtime.diagnostics", + "write_uk_calibration_diagnostics", + ), + "EMPLOYMENT_BANDS": ( + "microcosm.build.uk_runtime.firm_generation", + "EMPLOYMENT_BANDS", + ), + "HMRC_BAND_COLUMNS": ( + "microcosm.build.uk_runtime.firm_generation", + "HMRC_BAND_COLUMNS", + ), + "INPUT_FILES": ("microcosm.build.uk_runtime.firm_generation", "INPUT_FILES"), + "VAT_LIABILITY_BANDS": ( + "microcosm.build.uk_runtime.firm_generation", + "VAT_LIABILITY_BANDS", + ), + "VINTAGES": ("microcosm.build.uk_runtime.firm_generation", "VINTAGES"), + "AxiomVATRuleEvaluator": ( + "microcosm.build.uk_runtime.firm_generation", + "AxiomVATRuleEvaluator", + ), + "UKFirmCalibrationResult": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmCalibrationResult", + ), + "UKFirmGenerationConfig": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmGenerationConfig", + ), + "UKFirmGenerationResult": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmGenerationResult", + ), + "UKFirmSourceData": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmSourceData", + ), + "UKFirmTargetLayout": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmTargetLayout", + ), + "UKFirmValidationReport": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmValidationReport", + ), + "UKFirmVATRuleEvaluator": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmVATRuleEvaluator", + ), + "assign_employment": ( + "microcosm.build.uk_runtime.firm_generation", + "assign_employment", + ), + "assign_vat_flags": ( + "microcosm.build.uk_runtime.firm_generation", + "assign_vat_flags", + ), + "build_firm_target_matrix": ( + "microcosm.build.uk_runtime.firm_generation", + "build_firm_target_matrix", + ), + "employment_band_name": ( + "microcosm.build.uk_runtime.firm_generation", + "employment_band_name", + ), + "generate_base_firms": ( + "microcosm.build.uk_runtime.firm_generation", + "generate_base_firms", + ), + "generate_input_values": ( + "microcosm.build.uk_runtime.firm_generation", + "generate_input_values", + ), + "generate_uk_firm_population": ( + "microcosm.build.uk_runtime.firm_generation", + "generate_uk_firm_population", + ), + "hmrc_band_name": ( + "microcosm.build.uk_runtime.firm_generation", + "hmrc_band_name", + ), + "map_to_hmrc_band_indices": ( + "microcosm.build.uk_runtime.firm_generation", + "map_to_hmrc_band_indices", + ), + "optimize_firm_weights": ( + "microcosm.build.uk_runtime.firm_generation", + "optimize_firm_weights", + ), + "read_uk_firm_source_data": ( + "microcosm.build.uk_runtime.firm_generation", + "read_uk_firm_source_data", + ), + "solve_firm_weights": ( + "microcosm.build.uk_runtime.firm_generation", + "solve_firm_weights", + ), + "target_diagnostics": ( + "microcosm.build.uk_runtime.firm_generation", + "target_diagnostics", + ), + "uk_firm_source_data_from_frames": ( + "microcosm.build.uk_runtime.firm_generation", + "uk_firm_source_data_from_frames", + ), + "uk_firm_source_data_from_ledger_facts": ( + "microcosm.build.uk_runtime.firm_generation", + "uk_firm_source_data_from_ledger_facts", + ), + "validate_uk_firm_population": ( + "microcosm.build.uk_runtime.firm_generation", + "validate_uk_firm_population", + ), + "write_uk_firm_population": ( + "microcosm.build.uk_runtime.firm_generation", + "write_uk_firm_population", + ), + "UK_CGT_REQUIRED_COLUMNS": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_CGT_REQUIRED_COLUMNS", + ), + "UK_CGT_TARGET_COVERAGE_REQUIREMENTS": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_CGT_TARGET_COVERAGE_REQUIREMENTS", + ), + "UK_CGT_TARGET_SPECS": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_CGT_TARGET_SPECS", + ), + "UK_FISCAL_TARGET_REGISTRY": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_FISCAL_TARGET_REGISTRY", + ), + "FRS_COUNCIL_TAX_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_council_tax", + "FRS_COUNCIL_TAX_OUTPUT_COLUMNS", + ), + "UKFRSCouncilTaxStageTransform": ( + "microcosm.build.uk_runtime.frs_council_tax", + "UKFRSCouncilTaxStageTransform", + ), + "add_frs_council_tax": ( + "microcosm.build.uk_runtime.frs_council_tax", + "add_frs_council_tax", + ), + "derive_council_tax": ( + "microcosm.build.uk_runtime.frs_council_tax", + "derive_council_tax", + ), + "FRS_DISABILITY_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_disability", + "FRS_DISABILITY_OUTPUT_COLUMNS", + ), + "UK_INTERNAL_DISABILITY_REPORTED_COLUMNS": ( + "microcosm.build.uk_runtime.frs_disability", + "UK_INTERNAL_DISABILITY_REPORTED_COLUMNS", + ), + "UKDWPDisabilityCategoryRates": ( + "microcosm.build.uk_runtime.frs_disability", + "UKDWPDisabilityCategoryRates", + ), + "UKDWPDisabilityFlagRates": ( + "microcosm.build.uk_runtime.frs_disability", + "UKDWPDisabilityFlagRates", + ), + "UKFRSDisabilityStageTransform": ( + "microcosm.build.uk_runtime.frs_disability", + "UKFRSDisabilityStageTransform", + ), + "add_frs_disability": ( + "microcosm.build.uk_runtime.frs_disability", + "add_frs_disability", + ), + "derive_frs_disability": ( + "microcosm.build.uk_runtime.frs_disability", + "derive_frs_disability", + ), + "uk_dwp_disability_category_rates": ( + "microcosm.build.uk_runtime.frs_disability", + "uk_dwp_disability_category_rates", + ), + "uk_dwp_disability_flag_rates": ( + "microcosm.build.uk_runtime.frs_disability", + "uk_dwp_disability_flag_rates", + ), + "EDUCQUAL_MAP": ("microcosm.build.uk_runtime.frs_education", "EDUCQUAL_MAP"), + "FRS_EDUCATION_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_education", + "FRS_EDUCATION_OUTPUT_COLUMNS", + ), + "UKFRSEducationStageTransform": ( + "microcosm.build.uk_runtime.frs_education", + "UKFRSEducationStageTransform", + ), + "add_frs_education": ( + "microcosm.build.uk_runtime.frs_education", + "add_frs_education", + ), + "derive_current_education": ( + "microcosm.build.uk_runtime.frs_education", + "derive_current_education", + ), + "derive_frs_education": ( + "microcosm.build.uk_runtime.frs_education", + "derive_frs_education", + ), + "FRS_EDUCATION_GRANT_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_education_grants", + "FRS_EDUCATION_GRANT_OUTPUT_COLUMNS", + ), + "FRS_EDUCATION_GRANT_REWRITES": ( + "microcosm.build.uk_runtime.frs_education_grants", + "FRS_EDUCATION_GRANT_REWRITES", + ), + "UK_EDUCATION_GRANT_CAPACITY_PREDICTORS": ( + "microcosm.build.uk_runtime.frs_education_grants", + "UK_EDUCATION_GRANT_CAPACITY_PREDICTORS", + ), + "UKDSAPolicy": ( + "microcosm.build.uk_runtime.frs_education_grants", + "UKDSAPolicy", + ), + "UKFRSEducationGrantSplitStageTransform": ( + "microcosm.build.uk_runtime.frs_education_grants", + "UKFRSEducationGrantSplitStageTransform", + ), + "add_frs_education_grant_split": ( + "microcosm.build.uk_runtime.frs_education_grants", + "add_frs_education_grant_split", + ), + "allocate_reported_education_grants": ( + "microcosm.build.uk_runtime.frs_education_grants", + "allocate_reported_education_grants", + ), + "disabled_students_allowance_capacity": ( + "microcosm.build.uk_runtime.frs_education_grants", + "disabled_students_allowance_capacity", + ), + "uk_dsa_policy": ( + "microcosm.build.uk_runtime.frs_education_grants", + "uk_dsa_policy", + ), + "FRS_EMPLOYMENT_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_employment", + "FRS_EMPLOYMENT_OUTPUT_COLUMNS", + ), + "UKFRSEmploymentStageTransform": ( + "microcosm.build.uk_runtime.frs_employment", + "UKFRSEmploymentStageTransform", + ), + "add_frs_employment": ( + "microcosm.build.uk_runtime.frs_employment", + "add_frs_employment", + ), + "derive_frs_employment": ( + "microcosm.build.uk_runtime.frs_employment", + "derive_frs_employment", + ), + "FRS_HMRC_INCPBEN_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_INCPBEN_COLUMN", + ), + "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN", + ), + "FRS_HMRC_PAY_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_PAY_COLUMN", + ), + "FRS_HMRC_RETAINED_LEAF_COLUMNS": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_RETAINED_LEAF_COLUMNS", + ), + "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME", + ), + "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN", + ), + "FRS_HMRC_UBISJA_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_UBISJA_COLUMN", + ), + "UKFRSHMRCRetainedLeavesResult": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "UKFRSHMRCRetainedLeavesResult", + ), + "UKFRSHMRCRetainedLeavesStageTransform": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "UKFRSHMRCRetainedLeavesStageTransform", + ), + "retain_uk_frs_hmrc_leaves": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "retain_uk_frs_hmrc_leaves", + ), + "FRS_LEGACY_PROXY_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "FRS_LEGACY_PROXY_OUTPUT_COLUMNS", + ), + "UK_LEGACY_PROXY_PREDICTORS": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "UK_LEGACY_PROXY_PREDICTORS", + ), + "UKFRSLegacyProxiesStageTransform": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "UKFRSLegacyProxiesStageTransform", + ), + "UKLegacyJSAPolicy": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "UKLegacyJSAPolicy", + ), + "derive_frs_legacy_proxies": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "derive_frs_legacy_proxies", + ), + "uk_legacy_jsa_policy": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "uk_legacy_jsa_policy", + ), + "UK_YEAR_RULES": ("microcosm.build.uk_runtime.frs_release", "UK_YEAR_RULES"), + "UKFRSRelease": ("microcosm.build.uk_runtime.frs_release", "UKFRSRelease"), + "load_uk_frs_release": ( + "microcosm.build.uk_runtime.frs_release", + "load_uk_frs_release", + ), + "resolve_uk_year_rule": ( + "microcosm.build.uk_runtime.frs_release", + "resolve_uk_year_rule", + ), + "GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR": ( + "microcosm.build.uk_runtime.geography_ladder", + "GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR", + ), + "GEOGRAPHY_LADDER_VINTAGES_ATTR": ( + "microcosm.build.uk_runtime.geography_ladder", + "GEOGRAPHY_LADDER_VINTAGES_ATTR", + ), + "UK_ENGLAND_WALES_REGION_CODES": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_ENGLAND_WALES_REGION_CODES", + ), + "UK_GEOGRAPHY_LADDER_COLUMNS": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_GEOGRAPHY_LADDER_COLUMNS", + ), + "UK_LONDON_REGION_CODE": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_LONDON_REGION_CODE", + ), + "UK_OA_LADDER_DERIVED_LAYERS": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_OA_LADDER_DERIVED_LAYERS", + ), + "UK_OA_LADDER_KIND": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_OA_LADDER_KIND", + ), + "UK_OA_LADDER_SCHEMA_VERSION": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_OA_LADDER_SCHEMA_VERSION", + ), + "UkOaLadder": ("microcosm.build.uk_runtime.geography_ladder", "UkOaLadder"), + "assign_uk_geography_ladder": ( + "microcosm.build.uk_runtime.geography_ladder", + "assign_uk_geography_ladder", + ), + "expected_uk_ladder_area_support": ( + "microcosm.build.uk_runtime.geography_ladder", + "expected_uk_ladder_area_support", + ), + "load_uk_oa_ladder": ( + "microcosm.build.uk_runtime.geography_ladder", + "load_uk_oa_ladder", + ), + "uk_geography_ladder_assignment_summary": ( + "microcosm.build.uk_runtime.geography_ladder", + "uk_geography_ladder_assignment_summary", + ), + "uk_geography_ladder_gate": ( + "microcosm.build.uk_runtime.geography_ladder", + "uk_geography_ladder_gate", + ), + "uk_region_mix": ( + "microcosm.build.uk_runtime.geography_ladder", + "uk_region_mix", + ), + "ENGLAND_LAD_REGION_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "ENGLAND_LAD_REGION_URL", + ), + "ENGLAND_WALES_OA2021_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "ENGLAND_WALES_OA2021_COUNT", + ), + "EW_OA_CONSTITUENCY_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_CONSTITUENCY_URL", + ), + "EW_OA_HIERARCHY_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_HIERARCHY_URL", + ), + "EW_OA_HOUSEHOLDS_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_HOUSEHOLDS_URL", + ), + "EW_OA_LAD23_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_LAD23_URL", + ), + "EW_OA_POPULATION_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_POPULATION_URL", + ), + "EW_OA_WARD_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_WARD_URL", + ), + "LAD23_ITL_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "LAD23_ITL_URL", + ), + "NI_DZ2021_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ2021_COUNT", + ), + "NI_DZ_GEOJSON_ZIP_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_GEOJSON_ZIP_URL", + ), + "NI_DZ_HOUSEHOLDS_CSV_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_HOUSEHOLDS_CSV_URL", + ), + "NI_DZ_LOOKUP_SHEET": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_LOOKUP_SHEET", + ), + "NI_DZ_PARLCON24_LOOKUP_XLSX_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_PARLCON24_LOOKUP_XLSX_URL", + ), + "NI_DZ_POPULATION_CSV_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_POPULATION_CSV_URL", + ), + "NI_PARLCON24_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_PARLCON24_COUNT", + ), + "SCOTLAND_CENSUS_INDEX_ZIP_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_CENSUS_INDEX_ZIP_URL", + ), + "SCOTLAND_OA2022_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA2022_COUNT", + ), + "SCOTLAND_OA_CONSTITUENCY_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_CONSTITUENCY_URL", + ), + "SCOTLAND_OA_DZ_IZ_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_DZ_IZ_URL", + ), + "SCOTLAND_OA_LAU_ITL_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_LAU_ITL_URL", + ), + "SCOTLAND_OA_POPULATION_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_POPULATION_URL", + ), + "build_complete_uk_geography_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_complete_uk_geography_crosswalk", + ), + "build_england_wales_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_england_wales_crosswalk", + ), + "build_great_britain_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_great_britain_crosswalk", + ), + "build_northern_ireland_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_northern_ireland_crosswalk", + ), + "build_official_uk_geography_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_official_uk_geography_crosswalk", + ), + "build_scotland_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_scotland_crosswalk", + ), + "load_england_lad_region_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_lad_region_lookup", + ), + "load_england_wales_oa_constituencies": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_constituencies", + ), + "load_england_wales_oa_hierarchy": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_hierarchy", + ), + "load_england_wales_oa_households": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_households", + ), + "load_england_wales_oa_population": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_population", + ), + "load_england_wales_oa_ward_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_ward_lookup", + ), + "load_ew_oa_lad23_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ew_oa_lad23_lookup", + ), + "load_lad_itl_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_lad_itl_lookup", + ), + "load_ni_dz_hierarchy": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_hierarchy", + ), + "load_ni_dz_households": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_households", + ), + "load_ni_dz_parlcon24_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_parlcon24_lookup", + ), + "load_ni_dz_population": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_population", + ), + "load_ni_dz_ward_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_ward_lookup", + ), + "load_scotland_oa_constituencies": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_constituencies", + ), + "load_scotland_oa_dz_iz_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_dz_iz_lookup", + ), + "load_scotland_oa_households": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_households", + ), + "load_scotland_oa_lau_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_lau_lookup", + ), + "load_scotland_oa_population": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_population", + ), + "load_scotland_oa_ward_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_ward_lookup", + ), + "update_england_wales_lad_codes": ( + "microcosm.build.uk_runtime.geography_sources", + "update_england_wales_lad_codes", + ), + "write_geography_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "write_geography_crosswalk", + ), + "DEFAULT_HMRC_CALIBRATION_EPOCHS": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_CALIBRATION_EPOCHS", + ), + "DEFAULT_HMRC_CALIBRATION_LEARNING_RATE": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_CALIBRATION_LEARNING_RATE", + ), + "DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR", + ), + "DEFAULT_HMRC_MAX_WEIGHT_RATIO": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_MAX_WEIGHT_RATIO", + ), + "HMRC_ASSESSABLE_INCOME_COLUMN": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "HMRC_ASSESSABLE_INCOME_COLUMN", + ), + "HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN", + ), + "HMRC_TAXPAYER_COLUMN": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "HMRC_TAXPAYER_COLUMN", + ), + "UKHMRCIncomeCalibration": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "UKHMRCIncomeCalibration", + ), + "UKHMRCTargetMaterialization": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "UKHMRCTargetMaterialization", + ), + "calibrate_uk_hmrc_income": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "calibrate_uk_hmrc_income", + ), + "materialize_uk_hmrc_calibration_frame": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "materialize_uk_hmrc_calibration_frame", + ), + "HMRC_SPI_BUILD_PERIOD": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_BUILD_PERIOD", + ), + "HMRC_SPI_COLLATED_ODS_URL": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_COLLATED_ODS_URL", + ), + "HMRC_SPI_INCOME_COMPONENTS": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_INCOME_COMPONENTS", + ), + "HMRC_SPI_PUBLICATION_URL": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_PUBLICATION_URL", + ), + "HMRC_SPI_SOURCE_VINTAGE": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_SOURCE_VINTAGE", + ), + "HMRC_SPI_TARGET_RECORD_COUNT": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_TARGET_RECORD_COUNT", + ), + "HMRCIncomeBandTargetRecord": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRCIncomeBandTargetRecord", + ), + "HMRCIncomeSourceProvenance": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRCIncomeSourceProvenance", + ), + "HMRCIncomeTargetSet": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRCIncomeTargetSet", + ), + "materialize_hmrc_spi_income_band_targets": ( + "microcosm.build.uk_runtime.hmrc_income", + "materialize_hmrc_spi_income_band_targets", + ), + "verify_hmrc_spi_collated_ods": ( + "microcosm.build.uk_runtime.hmrc_income", + "verify_hmrc_spi_collated_ods", + ), + "CANONICAL_HMRC_FACT_FENCES": ( + "microcosm.build.uk_runtime.hmrc_replay", + "CANONICAL_HMRC_FACT_FENCES", + ), + "FULL_FRS_TI_BAND_FENCE_ID": ( + "microcosm.build.uk_runtime.hmrc_replay", + "FULL_FRS_TI_BAND_FENCE_ID", + ), + "HMRCFactFence": ("microcosm.build.uk_runtime.hmrc_replay", "HMRCFactFence"), + "HMRCReplayDiagnosticAggregate": ( + "microcosm.build.uk_runtime.hmrc_replay", + "HMRCReplayDiagnosticAggregate", + ), + "HMRCReplayFact": ("microcosm.build.uk_runtime.hmrc_replay", "HMRCReplayFact"), + "HMRCReplayReport": ( + "microcosm.build.uk_runtime.hmrc_replay", + "HMRCReplayReport", + ), + "build_conservative_hmrc_replay_report": ( + "microcosm.build.uk_runtime.hmrc_replay", + "build_conservative_hmrc_replay_report", + ), + "classify_hmrc_replay_targets": ( + "microcosm.build.uk_runtime.hmrc_replay", + "classify_hmrc_replay_targets", + ), + "write_hmrc_replay_report": ( + "microcosm.build.uk_runtime.hmrc_replay", + "write_hmrc_replay_report", + ), + "HMRC_DISTRIBUTIONAL_INPUTS": ( + "microcosm.build.uk_runtime.hmrc_source_contract", + "HMRC_DISTRIBUTIONAL_INPUTS", + ), + "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE": ( + "microcosm.build.uk_runtime.hmrc_source_contract", + "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE", + ), + "assert_uk_hmrc_income_source_contract_current": ( + "microcosm.build.uk_runtime.hmrc_source_contract", + "assert_uk_hmrc_income_source_contract_current", + ), + "constituency_household_targets": ( + "microcosm.build.uk_runtime.ladder_targets", + "constituency_household_targets", + ), + "ladder_target_provenance": ( + "microcosm.build.uk_runtime.ladder_targets", + "ladder_target_provenance", + ), + "ladder_vs_chronicle_household_dispersion": ( + "microcosm.build.uk_runtime.ladder_targets", + "ladder_vs_chronicle_household_dispersion", + ), + "local_authority_household_targets": ( + "microcosm.build.uk_runtime.ladder_targets", + "local_authority_household_targets", + ), + "UK_CENSUS_HOUSEHOLDS_TARGET_ID": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CENSUS_HOUSEHOLDS_TARGET_ID", + ), + "UK_CROSS_GRAIN_BRIDGES": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CROSS_GRAIN_BRIDGES", + ), + "UK_CROSS_GRAIN_GRAIN_PRECEDENCE": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CROSS_GRAIN_GRAIN_PRECEDENCE", + ), + "UK_CROSS_GRAIN_RULE": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CROSS_GRAIN_RULE", + ), + "UKFrameTargetAdapter": ( + "microcosm.build.uk_runtime.ledger_targets", + "UKFrameTargetAdapter", + ), + "UKLedgerTargetCompilation": ( + "microcosm.build.uk_runtime.ledger_targets", + "UKLedgerTargetCompilation", + ), + "apply_uk_cross_grain_reconciliation": ( + "microcosm.build.uk_runtime.ledger_targets", + "apply_uk_cross_grain_reconciliation", + ), + "compile_uk_local_target_registry": ( + "microcosm.build.uk_runtime.ledger_targets", + "compile_uk_local_target_registry", + ), + "compile_uk_target_registry": ( + "microcosm.build.uk_runtime.ledger_targets", + "compile_uk_target_registry", + ), + "load_uk_local_area_crosswalk": ( + "microcosm.build.uk_runtime.ledger_targets", + "load_uk_local_area_crosswalk", + ), + "materialize_uk_ledger_targets": ( + "microcosm.build.uk_runtime.ledger_targets", + "materialize_uk_ledger_targets", + ), + "uk_census_household_uprating": ( + "microcosm.build.uk_runtime.ledger_targets", + "uk_census_household_uprating", + ), + "uk_ledger_households_total": ( + "microcosm.build.uk_runtime.ledger_targets", + "uk_ledger_households_total", + ), + "uk_local_target_surface": ( + "microcosm.build.uk_runtime.ledger_targets", + "uk_local_target_surface", + ), + "UK_LOCAL_CLONE_COUNT": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_CLONE_COUNT", + ), + "UK_LOCAL_MAX_WEIGHT_RATIO": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_MAX_WEIGHT_RATIO", + ), + "UK_LOCAL_SOLVE_DOCTRINE": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_SOLVE_DOCTRINE", + ), + "UK_LOCAL_SOLVE_EPOCHS": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_SOLVE_EPOCHS", + ), + "UK_LOCAL_TARGET_LOSS_CAP": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_TARGET_LOSS_CAP", + ), + "UK_LOCAL_TARGET_WEIGHT_RULE": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_TARGET_WEIGHT_RULE", + ), + "UKLocalSolveDoctrine": ( + "microcosm.build.uk_runtime.local_doctrine", + "UKLocalSolveDoctrine", + ), + "uk_local_doctrine_with_overrides": ( + "microcosm.build.uk_runtime.local_doctrine", + "uk_local_doctrine_with_overrides", + ), + "uk_local_target_loss_weights": ( + "microcosm.build.uk_runtime.local_doctrine", + "uk_local_target_loss_weights", + ), + "align_area_targets": ( + "microcosm.build.uk_runtime.local_geography", + "align_area_targets", + ), + "UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE": ( + "microcosm.build.uk_runtime.local_rowwise", + "UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE", + ), + "UK_LOCAL_HOLDOUT_FOLDS": ( + "microcosm.build.uk_runtime.local_rowwise", + "UK_LOCAL_HOLDOUT_FOLDS", + ), + "UK_LOCAL_HOLDOUT_SEED": ( + "microcosm.build.uk_runtime.local_rowwise", + "UK_LOCAL_HOLDOUT_SEED", + ), + "UKRowwiseDoctrineSolve": ( + "microcosm.build.uk_runtime.local_rowwise", + "UKRowwiseDoctrineSolve", + ), + "UKRowwiseLocalMatrix": ( + "microcosm.build.uk_runtime.local_rowwise", + "UKRowwiseLocalMatrix", + ), + "UKRowwiseNationalRows": ( + "microcosm.build.uk_runtime.local_rowwise", + "UKRowwiseNationalRows", + ), + "build_uk_rowwise_local_matrix": ( + "microcosm.build.uk_runtime.local_rowwise", + "build_uk_rowwise_local_matrix", + ), + "build_uk_rowwise_local_surface_matrix": ( + "microcosm.build.uk_runtime.local_rowwise", + "build_uk_rowwise_local_surface_matrix", + ), + "past_cap_census": ( + "microcosm.build.uk_runtime.local_rowwise", + "past_cap_census", + ), + "require_adjudicated_uk_local_binding": ( + "microcosm.build.uk_runtime.local_rowwise", + "require_adjudicated_uk_local_binding", + ), + "rotated_uk_local_holdout": ( + "microcosm.build.uk_runtime.local_rowwise", + "rotated_uk_local_holdout", + ), + "rowwise_area_support_summary": ( + "microcosm.build.uk_runtime.local_rowwise", + "rowwise_area_support_summary", + ), + "rowwise_calibration_mass_reason": ( + "microcosm.build.uk_runtime.local_rowwise", + "rowwise_calibration_mass_reason", + ), + "solve_uk_rowwise_weights_under_doctrine": ( + "microcosm.build.uk_runtime.local_rowwise", + "solve_uk_rowwise_weights_under_doctrine", + ), + "uk_area_support_summary": ( + "microcosm.build.uk_runtime.local_rowwise", + "uk_area_support_summary", + ), + "uk_ladder_area_support_summary": ( + "microcosm.build.uk_runtime.local_rowwise", + "uk_ladder_area_support_summary", + ), + "CENSUS_KIND": ( + "microcosm.build.uk_runtime.local_target_census", + "CENSUS_KIND", + ), + "CENSUS_RESOURCE": ( + "microcosm.build.uk_runtime.local_target_census", + "CENSUS_RESOURCE", + ), + "CENSUS_SCHEMA_VERSION": ( + "microcosm.build.uk_runtime.local_target_census", + "CENSUS_SCHEMA_VERSION", + ), + "METRIC_STATUS_BOUND_IN_CODE": ( + "microcosm.build.uk_runtime.local_target_census", + "METRIC_STATUS_BOUND_IN_CODE", + ), + "SOURCE_STATUS_DOCUMENTED_UNPINNED": ( + "microcosm.build.uk_runtime.local_target_census", + "SOURCE_STATUS_DOCUMENTED_UNPINNED", + ), + "assert_uk_local_target_census_current": ( + "microcosm.build.uk_runtime.local_target_census", + "assert_uk_local_target_census_current", + ), + "build_uk_local_target_census": ( + "microcosm.build.uk_runtime.local_target_census", + "build_uk_local_target_census", + ), + "committed_uk_local_target_census_path": ( + "microcosm.build.uk_runtime.local_target_census", + "committed_uk_local_target_census_path", + ), + "load_uk_local_target_census": ( + "microcosm.build.uk_runtime.local_target_census", + "load_uk_local_target_census", + ), + "write_uk_local_target_census": ( + "microcosm.build.uk_runtime.local_target_census", + "write_uk_local_target_census", + ), + "AGE_BANDS": ("microcosm.build.uk_runtime.local_targets", "AGE_BANDS"), + "AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL": ( + "microcosm.build.uk_runtime.local_targets", + "AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL", + ), + "AREA_TYPES": ("microcosm.build.uk_runtime.local_targets", "AREA_TYPES"), + "COUNTRY_TO_REGION": ( + "microcosm.build.uk_runtime.local_targets", + "COUNTRY_TO_REGION", + ), + "INCOME_VARIABLES": ( + "microcosm.build.uk_runtime.local_targets", + "INCOME_VARIABLES", + ), + "LA_EXTRA_METRICS": ( + "microcosm.build.uk_runtime.local_targets", + "LA_EXTRA_METRICS", + ), + "area_groups_from_codes": ( + "microcosm.build.uk_runtime.local_targets", + "area_groups_from_codes", + ), + "compute_household_metrics": ( + "microcosm.build.uk_runtime.local_targets", + "compute_household_metrics", + ), + "metric_names": ("microcosm.build.uk_runtime.local_targets", "metric_names"), + "metric_names_from_target_profile": ( + "microcosm.build.uk_runtime.local_targets", + "metric_names_from_target_profile", + ), + "metric_tables_by_area_group": ( + "microcosm.build.uk_runtime.local_targets", + "metric_tables_by_area_group", + ), + "CalibrationFrameAdapter": ( + "microcosm.build.uk_runtime.national_calibration", + "CalibrationFrameAdapter", + ), + "drop_injected_measure_inputs": ( + "microcosm.build.uk_runtime.national_calibration", + "drop_injected_measure_inputs", + ), + "inject_measure_inputs": ( + "microcosm.build.uk_runtime.national_calibration", + "inject_measure_inputs", + ), + "prepare_uk_target_frame": ( + "microcosm.build.uk_runtime.national_calibration", + "prepare_uk_target_frame", + ), + "UK_NATIONAL_L0_LAMBDA": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_L0_LAMBDA", + ), + "UK_NATIONAL_LEARNING_RATE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_LEARNING_RATE", + ), + "UK_NATIONAL_MASS_RULE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_MASS_RULE", + ), + "UK_NATIONAL_MAX_WEIGHT_RATIO": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_MAX_WEIGHT_RATIO", + ), + "UK_NATIONAL_SEED": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_SEED", + ), + "UK_NATIONAL_SOLVE_DOCTRINE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_SOLVE_DOCTRINE", + ), + "UK_NATIONAL_SOLVE_EPOCHS": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_SOLVE_EPOCHS", + ), + "UK_NATIONAL_TARGET_LOSS_CAP": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_TARGET_LOSS_CAP", + ), + "UK_NATIONAL_TARGET_WEIGHT_RULE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_TARGET_WEIGHT_RULE", + ), + "UKNationalSolveDoctrine": ( + "microcosm.build.uk_runtime.national_doctrine", + "UKNationalSolveDoctrine", + ), + "uk_doctrine_with_overrides": ( + "microcosm.build.uk_runtime.national_doctrine", + "uk_doctrine_with_overrides", + ), + "uk_national_target_loss_weights": ( + "microcosm.build.uk_runtime.national_doctrine", + "uk_national_target_loss_weights", + ), + "UK_NATIONAL_SCHEMA": ( + "microcosm.build.uk_runtime.national_frame", + "UK_NATIONAL_SCHEMA", + ), + "UKNationalStage": ( + "microcosm.build.uk_runtime.national_frame", + "UKNationalStage", + ), + "UKStagingProvenance": ( + "microcosm.build.uk_runtime.national_frame", + "UKStagingProvenance", + ), + "load_uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "load_uk_national_frame", + ), + "uk_household_weight_kind": ( + "microcosm.build.uk_runtime.national_frame", + "uk_household_weight_kind", + ), + "uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "uk_national_frame", + ), + "uk_time_period": ( + "microcosm.build.uk_runtime.national_frame", + "uk_time_period", + ), + "validate_uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "validate_uk_national_frame", + ), + "write_uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "write_uk_national_frame", + ), + "sample_uk_spine_frame": ( + "microcosm.build.uk_runtime.national_sampling", + "sample_uk_spine_frame", + ), + "uk_spine_source_family_units": ( + "microcosm.build.uk_runtime.national_sampling", + "uk_spine_source_family_units", + ), + "LADDER_OA_COLUMNS": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "LADDER_OA_COLUMNS", + ), + "assemble_uk_oa_ladder": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "assemble_uk_oa_ladder", + ), + "concat_uk_ladder_frames": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "concat_uk_ladder_frames", + ), + "join_uk_oa_ladder_layers": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "join_uk_oa_ladder_layers", + ), + "EFRS_PARITY_KNOWN_GAPS_RESOURCE": ( + "microcosm.build.uk_runtime.parity_reference", + "EFRS_PARITY_KNOWN_GAPS_RESOURCE", + ), + "EFRS_PARITY_REFERENCE_RESOURCE": ( + "microcosm.build.uk_runtime.parity_reference", + "EFRS_PARITY_REFERENCE_RESOURCE", + ), + "EfrsParityKnownGap": ( + "microcosm.build.uk_runtime.parity_reference", + "EfrsParityKnownGap", + ), + "EfrsParityReference": ( + "microcosm.build.uk_runtime.parity_reference", + "EfrsParityReference", + ), + "EfrsParitySource": ( + "microcosm.build.uk_runtime.parity_reference", + "EfrsParitySource", + ), + "load_efrs_parity_known_gaps": ( + "microcosm.build.uk_runtime.parity_reference", + "load_efrs_parity_known_gaps", + ), + "load_efrs_parity_reference": ( + "microcosm.build.uk_runtime.parity_reference", + "load_efrs_parity_reference", + ), + "UK_DENSE_RELEASE_ID": ( + "microcosm.build.uk_runtime.release_identity", + "UK_DENSE_RELEASE_ID", + ), + "UK_RELEASE_TIER_CPS_TRANSFER": ( + "microcosm.build.uk_runtime.release_identity", + "UK_RELEASE_TIER_CPS_TRANSFER", + ), + "UK_RELEASE_TIER_FRS": ( + "microcosm.build.uk_runtime.release_identity", + "UK_RELEASE_TIER_FRS", + ), + "UK_RELEASE_TIERS": ( + "microcosm.build.uk_runtime.release_identity", + "UK_RELEASE_TIERS", + ), + "UKReleaseIdentity": ( + "microcosm.build.uk_runtime.release_identity", + "UKReleaseIdentity", + ), + "apply_uk_release_identity": ( + "microcosm.build.uk_runtime.release_identity", + "apply_uk_release_identity", + ), + "format_uk_release_id": ( + "microcosm.build.uk_runtime.release_identity", + "format_uk_release_id", + ), + "validate_uk_release_tier": ( + "microcosm.build.uk_runtime.release_identity", + "validate_uk_release_tier", + ), + "RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS": ( + "microcosm.build.uk_runtime.release_input_coverage", + "RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS", + ), + "UK_LOADER_INPUT_ALIASES": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UK_LOADER_INPUT_ALIASES", + ), + "UK_RELEASE_INPUT_COVERAGE_RESOURCE": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UK_RELEASE_INPUT_COVERAGE_RESOURCE", + ), + "PolicyEngineUKCoverageEngine": ( + "microcosm.build.uk_runtime.release_input_coverage", + "PolicyEngineUKCoverageEngine", + ), + "UKEffectiveMassCoveragePolicy": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UKEffectiveMassCoveragePolicy", + ), + "UKReleaseInputColumn": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UKReleaseInputColumn", + ), + "UKReleaseInputCoverageManifest": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UKReleaseInputCoverageManifest", + ), + "assert_uk_release_input_coverage_build_stages": ( + "microcosm.build.uk_runtime.release_input_coverage", + "assert_uk_release_input_coverage_build_stages", + ), + "assert_uk_release_input_coverage_manifest_current": ( + "microcosm.build.uk_runtime.release_input_coverage", + "assert_uk_release_input_coverage_manifest_current", + ), + "load_uk_release_input_coverage_manifest": ( + "microcosm.build.uk_runtime.release_input_coverage", + "load_uk_release_input_coverage_manifest", + ), + "uk_release_input_coverage_gate": ( + "microcosm.build.uk_runtime.release_input_coverage", + "uk_release_input_coverage_gate", + ), + "uk_release_input_coverage_required_columns": ( + "microcosm.build.uk_runtime.release_input_coverage", + "uk_release_input_coverage_required_columns", + ), + "uk_release_input_coverage_reviewed_exclusions": ( + "microcosm.build.uk_runtime.release_input_coverage", + "uk_release_input_coverage_reviewed_exclusions", + ), + "ARTIFACT_CLONE_INDEX_COLUMN": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "ARTIFACT_CLONE_INDEX_COLUMN", + ), + "BENUNIT_ID_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "BENUNIT_ID_COLUMNS", + ), + "HOUSEHOLD_ID_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "HOUSEHOLD_ID_COLUMNS", + ), + "MASS_CONSERVATION_RELATIVE_TOLERANCE": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "MASS_CONSERVATION_RELATIVE_TOLERANCE", + ), + "PERSON_ID_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "PERSON_ID_COLUMNS", + ), + "POOL_SOURCE_LINEAGE_COLUMN": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "POOL_SOURCE_LINEAGE_COLUMN", + ), + "UK_SINGLE_YEAR_TABLES": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "UK_SINGLE_YEAR_TABLES", + ), + "UKLadderRowwiseDatasetResult": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "UKLadderRowwiseDatasetResult", + ), + "UKRowwiseDatasetResult": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "UKRowwiseDatasetResult", + ), + "apply_uk_source_lineage_modulus": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "apply_uk_source_lineage_modulus", + ), + "clone_uk_dataset_tables_with_ladder_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_tables_with_ladder_geography", + ), + "clone_uk_dataset_tables_with_rowwise_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_tables_with_rowwise_geography", + ), + "clone_uk_dataset_with_ladder_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_with_ladder_geography", + ), + "clone_uk_dataset_with_rowwise_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_with_rowwise_geography", + ), + "ladder_clone_index_column": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "ladder_clone_index_column", + ), + "load_uk_rowwise_dataset": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "load_uk_rowwise_dataset", + ), + "read_uk_single_year_weight_metadata": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "read_uk_single_year_weight_metadata", + ), + "validate_uk_ladder_rowwise_dataset_tables": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "validate_uk_ladder_rowwise_dataset_tables", + ), + "validate_uk_rowwise_dataset_tables": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "validate_uk_rowwise_dataset_tables", + ), + "write_uk_rowwise_dataset": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "write_uk_rowwise_dataset", + ), + "AREA_TYPE_TO_CROSSWALK_COLUMN": ( + "microcosm.build.uk_runtime.rowwise_geography", + "AREA_TYPE_TO_CROSSWALK_COLUMN", + ), + "CROSSWALK_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_geography", + "CROSSWALK_COLUMNS", + ), + "FRS_REGION_TO_COUNTRY": ( + "microcosm.build.uk_runtime.rowwise_geography", + "FRS_REGION_TO_COUNTRY", + ), + "FRS_REGION_TO_REGION_CODE": ( + "microcosm.build.uk_runtime.rowwise_geography", + "FRS_REGION_TO_REGION_CODE", + ), + "ROWWISE_GEOGRAPHY_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_geography", + "ROWWISE_GEOGRAPHY_COLUMNS", + ), + "RowwiseGeographyAssignment": ( + "microcosm.build.uk_runtime.rowwise_geography", + "RowwiseGeographyAssignment", + ), + "assign_household_geography": ( + "microcosm.build.uk_runtime.rowwise_geography", + "assign_household_geography", + ), + "clone_entity_frame": ( + "microcosm.build.uk_runtime.rowwise_geography", + "clone_entity_frame", + ), + "expected_uk_rowwise_area_support": ( + "microcosm.build.uk_runtime.rowwise_geography", + "expected_uk_rowwise_area_support", + ), + "geography_coverage_summary": ( + "microcosm.build.uk_runtime.rowwise_geography", + "geography_coverage_summary", + ), + "id_multiplier_for_values": ( + "microcosm.build.uk_runtime.rowwise_geography", + "id_multiplier_for_values", + ), + "prepare_geography_crosswalk": ( + "microcosm.build.uk_runtime.rowwise_geography", + "prepare_geography_crosswalk", + ), + "validate_geography_coverage": ( + "microcosm.build.uk_runtime.rowwise_geography", + "validate_geography_coverage", + ), + "PRE_REGISTERED_OUTCOMES_V1": ( + "microcosm.build.uk_runtime.size_evaluation", + "PRE_REGISTERED_OUTCOMES_V1", + ), + "RowwiseRun": ("microcosm.build.uk_runtime.size_evaluation", "RowwiseRun"), + "area_support_tables": ( + "microcosm.build.uk_runtime.size_evaluation", + "area_support_tables", + ), + "dense_reference_deltas": ( + "microcosm.build.uk_runtime.size_evaluation", + "dense_reference_deltas", + ), + "fit_tables": ("microcosm.build.uk_runtime.size_evaluation", "fit_tables"), + "footprint": ("microcosm.build.uk_runtime.size_evaluation", "footprint"), + "frozen_vs_recomputed": ( + "microcosm.build.uk_runtime.size_evaluation", + "frozen_vs_recomputed", + ), + "gate_table": ("microcosm.build.uk_runtime.size_evaluation", "gate_table"), + "load_run": ("microcosm.build.uk_runtime.size_evaluation", "load_run"), + "paired_targets": ( + "microcosm.build.uk_runtime.size_evaluation", + "paired_targets", + ), + "run_acceptance": ( + "microcosm.build.uk_runtime.size_evaluation", + "run_acceptance", + ), + "summarize": ("microcosm.build.uk_runtime.size_evaluation", "summarize"), + "weight_tables": ( + "microcosm.build.uk_runtime.size_evaluation", + "weight_tables", + ), + "SPI_DONOR_DOI": ("microcosm.build.uk_runtime.spi_income", "SPI_DONOR_DOI"), + "SPI_DONOR_FILENAME": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_FILENAME", + ), + "SPI_DONOR_SHA256": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_SHA256", + ), + "SPI_DONOR_SIZE_BYTES": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_SIZE_BYTES", + ), + "SPI_DONOR_UKDS_STUDY": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_UKDS_STUDY", + ), + "SPI_DONOR_VINTAGE": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_VINTAGE", + ), + "SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS", + ), + "UKSPIIncomeImputationResult": ( + "microcosm.build.uk_runtime.spi_income", + "UKSPIIncomeImputationResult", + ), + "assert_frs_hmrc_auxiliary_crosswalk_available": ( + "microcosm.build.uk_runtime.spi_income", + "assert_frs_hmrc_auxiliary_crosswalk_available", + ), + "derive_hmrc_income_auxiliaries": ( + "microcosm.build.uk_runtime.spi_income", + "derive_hmrc_income_auxiliaries", + ), + "impute_uk_spi_income_support": ( + "microcosm.build.uk_runtime.spi_income", + "impute_uk_spi_income_support", + ), + "verify_spi_donor_identity": ( + "microcosm.build.uk_runtime.spi_income", + "verify_spi_donor_identity", + ), + "EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN": ( + "microcosm.build.uk_runtime.spi_spine", + "EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN", + ), + "UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS", + ), + "UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME", + ), + "UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME", + ), + "UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND", + ), + "UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS", + ), + "UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS", + ), + "UK_SPI_INCOME_SPINE_REWRITE_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_INCOME_SPINE_REWRITE_COLUMNS", + ), + "UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS", + ), + "UKFRSHMRCSpineLeavesResult": ( + "microcosm.build.uk_runtime.spi_spine", + "UKFRSHMRCSpineLeavesResult", + ), + "UKFRSHMRCSpineLeavesStageTransform": ( + "microcosm.build.uk_runtime.spi_spine", + "UKFRSHMRCSpineLeavesStageTransform", + ), + "UKSPIIncomeSpineResult": ( + "microcosm.build.uk_runtime.spi_spine", + "UKSPIIncomeSpineResult", + ), + "UKSPIIncomeSpineStageTransform": ( + "microcosm.build.uk_runtime.spi_spine", + "UKSPIIncomeSpineStageTransform", + ), + "UKSPISupportChannelStageTransform": ( + "microcosm.build.uk_runtime.spi_spine", + "UKSPISupportChannelStageTransform", + ), + "BASE_FRS_SUPPORT_CHANNEL": ( + "microcosm.build.uk_runtime.spi_support", + "BASE_FRS_SUPPORT_CHANNEL", + ), + "DEFAULT_SPI_PRIOR_MASS_SHARE": ( + "microcosm.build.uk_runtime.spi_support", + "DEFAULT_SPI_PRIOR_MASS_SHARE", + ), + "DEFAULT_SPI_SUPPORT_HOUSEHOLDS": ( + "microcosm.build.uk_runtime.spi_support", + "DEFAULT_SPI_SUPPORT_HOUSEHOLDS", + ), + "FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS", + ), + "FRS_ONLY_SPI_FILL_PERSON_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "FRS_ONLY_SPI_FILL_PERSON_COLUMNS", + ), + "FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS", + ), + "HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN": ( + "microcosm.build.uk_runtime.spi_support", + "HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN", + ), + "SPI_INCOME_COMPONENT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_INCOME_COMPONENT_COLUMNS", + ), + "SPI_INCOME_IMPUTATION_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_INCOME_IMPUTATION_COLUMNS", + ), + "SPI_PRIOR_MASS_CHANGE_REASON": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_PRIOR_MASS_CHANGE_REASON", + ), + "SPI_REPLACEMENT_STRATA_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_REPLACEMENT_STRATA_COLUMNS", + ), + "SPI_SYNTHETIC_SUPPORT_CHANNEL": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_SYNTHETIC_SUPPORT_CHANNEL", + ), + "UK_SPI_SUPPORT_STAGE_NAME": ( + "microcosm.build.uk_runtime.spi_support", + "UK_SPI_SUPPORT_STAGE_NAME", + ), + "UKSPISupportResult": ( + "microcosm.build.uk_runtime.spi_support", + "UKSPISupportResult", + ), + "build_uk_spi_support_channel": ( + "microcosm.build.uk_runtime.spi_support", + "build_uk_spi_support_channel", + ), + "create_uk_spi_support_tables": ( + "microcosm.build.uk_runtime.spi_support", + "create_uk_spi_support_tables", + ), + "fill_support_channel_from_source": ( + "microcosm.build.uk_runtime.spi_support", + "fill_support_channel_from_source", + ), + "replace_uk_spi_support_tables": ( + "microcosm.build.uk_runtime.spi_support", + "replace_uk_spi_support_tables", + ), + "support_channel_column": ( + "microcosm.build.uk_runtime.spi_support", + "support_channel_column", + ), + "support_clone_index_column": ( + "microcosm.build.uk_runtime.spi_support", + "support_clone_index_column", + ), + "support_source_id_column": ( + "microcosm.build.uk_runtime.spi_support", + "support_source_id_column", + ), + "UK_FRAME_METADATA_KEY": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "UK_FRAME_METADATA_KEY", + ), + "load_uk_stage_checkpoint": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "load_uk_stage_checkpoint", + ), + "load_uk_stage_predecessor": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "load_uk_stage_predecessor", + ), + "uk_stage_metadata": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "uk_stage_metadata", + ), + "UK_DEFAULT_ZERO_WEIGHT_STRATA": ( + "microcosm.build.uk_runtime.terminal_gates", + "UK_DEFAULT_ZERO_WEIGHT_STRATA", + ), + "UK_MAX_TARGET_ABS_RELATIVE_ERROR": ( + "microcosm.build.uk_runtime.terminal_gates", + "UK_MAX_TARGET_ABS_RELATIVE_ERROR", + ), + "UKZeroWeightStratumDeclaration": ( + "microcosm.build.uk_runtime.terminal_gates", + "UKZeroWeightStratumDeclaration", + ), + "uk_degenerate_release_surface_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_degenerate_release_surface_gate", + ), + "uk_export_surface_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_export_surface_gate", + ), + "uk_target_fit_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_target_fit_gate", + ), + "uk_target_surface_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_target_surface_gate", + ), + "uk_weight_ess_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_weight_ess_gate", + ), + "uk_weight_ratio_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_weight_ratio_gate", + ), + "uk_zero_weight_strata_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_zero_weight_strata_gate", + ), + "UKInputMassParityPolicy": ( + "microcosm.build.uk_runtime.weighted_integrity", + "UKInputMassParityPolicy", + ), + "UKInputMassReference": ( + "microcosm.build.uk_runtime.weighted_integrity", + "UKInputMassReference", + ), + "UKQRFTailConcentrationPolicy": ( + "microcosm.build.uk_runtime.weighted_integrity", + "UKQRFTailConcentrationPolicy", + ), + "load_uk_input_mass_reference": ( + "microcosm.build.uk_runtime.weighted_integrity", + "load_uk_input_mass_reference", + ), + "load_uk_local_area_support_exclusion_register": ( + "microcosm.build.uk_runtime.weighted_integrity", + "load_uk_local_area_support_exclusion_register", + ), + "load_uk_reviewed_exclusion_register": ( + "microcosm.build.uk_runtime.weighted_integrity", + "load_uk_reviewed_exclusion_register", + ), + "uk_input_mass_parity_gate": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_input_mass_parity_gate", + ), + "uk_input_mass_totals": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_input_mass_totals", + ), + "uk_qrf_tail_concentration_columns": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_qrf_tail_concentration_columns", + ), + "uk_qrf_tail_concentration_gate": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_qrf_tail_concentration_gate", + ), + } +) + +# Reload clears public aliases, retaining the real defining modules and submodules. +for _export_name in __all__: + globals().pop(_export_name, None) +del _export_name + + +def __getattr__(name: str) -> _Any: + """Resolve and cache the exact public object, preserving import failures.""" + try: + module_name, attribute = _EXPORTS[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + value = getattr(_import_module(module_name), attribute) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """Discover declared exports without importing their defining modules.""" + return sorted(set(globals()) | set(__all__)) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py new file mode 100644 index 000000000..a6fedc9d6 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py @@ -0,0 +1,388 @@ +"""Convert supplied UK ladder arrays for shared atomic geography operators. + +This module does not acquire sources, admit native data, calibrate, or assign a +location. Source identities and mapping classifications are explicit caller +claims; publisher authentication belongs to the host. Initial spine/support +clones must exist before the resulting declaration is executed. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Mapping +from numbers import Integral + +import numpy as np + +from microcosm.build.atomic_geography import ( + RELATIONS, + decode_atomic_support, + encode_atomic_support, + validate_assignment_spec, +) +from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE +from microcosm.graph.canonical import canonical_json + +SYSTEMS = ( + "uk_ew_output_area_2021", + "uk_scotland_output_area_2022", + "uk_ni_data_zone_2021", +) +SOURCES = {system: system + "_support" for system in SYSTEMS} +IDENTITY_COLUMN = "geography_household_key" +_INPUT_COLUMNS = ( + "oa_code", + "population", + "households", + "constituency_code", + "region_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "itl3_code", +) +_COUNT_COLUMNS = frozenset({"population", "households"}) +_CODE = re.compile(r"[EWSN][0-9]{8}") +_WARD = re.compile(r"[EWSN][0-9]{2}[0-9RS][0-9]{5}") +_ITL = re.compile(r"TL[C-N][0-9A-Z]{2}") +_LIMIT = 2**53 + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("UK atomic support: " + message) + + +def _text(value: object) -> bool: + return type(value) is str and bool(value) and value.strip() == value + + +def _profile(system: str) -> tuple[str, str, tuple[str, ...], str]: + _require(system in SYSTEMS, "unknown area system") + if system == SYSTEMS[0]: + regions = tuple( + name + for name in FRS_REGION_TO_REGION_CODE + if name not in {"SCOTLAND", "NORTHERN_IRELAND"} + ) + return "output_area", "2021_census", regions, "EW" + if system == SYSTEMS[1]: + return "output_area", "2022_census", ("SCOTLAND",), "S" + return "data_zone", "2021_census", ("NORTHERN_IRELAND",), "N" + + +def _metadata(descriptions: Mapping, *, vintage: str, system: str) -> dict: + _require(type(descriptions) is dict, "column metadata must be a plain mapping") + _require(set(descriptions) == set(_INPUT_COLUMNS), "exact column metadata required") + result = {} + for column in _INPUT_COLUMNS: + item = descriptions[column] + _require(type(item) is dict, "plain column description required") + expected = ( + {"kind", "source", "basis"} + if column in _COUNT_COLUMNS + else {"kind", "source", "vintage", "relation"} + ) + _require(set(item) == expected, "column description keys differ") + _require( + all(_text(value) for value in item.values()), "nonempty metadata required" + ) + if column in _COUNT_COLUMNS: + _require(item["kind"] == "weight", "count metadata must describe weight") + else: + _require( + item["kind"] == "code" and item["relation"] in RELATIONS, + "code mapping classification", + ) + result[column] = dict(item) + _require( + result["oa_code"]["vintage"] == vintage + and result["oa_code"]["relation"] == "exact", + "atomic identity/vintage differs", + ) + if system == SYSTEMS[2]: + _require( + result["constituency_code"]["relation"] != "inferred_modal", + "NI postcode-modal mapping is not admitted by this adapter", + ) + return result + + +def _counts(array: np.ndarray, *, positive: bool) -> np.ndarray: + _require( + array.dtype.kind in "iu" or array.dtype == np.dtype("float64"), "count dtype" + ) + if array.dtype.kind == "f": + _require(np.isfinite(array).all(), "non-finite count") + _require(np.equal(array, np.floor(array)).all(), "non-integral count") + _require( + np.all(array > 0 if positive else array >= 0) and np.all(array <= _LIMIT), + "count range", + ) + integers = [int(value) for value in array] + _require(0 < sum(integers) <= _LIMIT, "count total outside exact sampling range") + converted = np.asarray(integers, dtype=np.int64) + _require( + np.array_equal(converted.astype(array.dtype), array), + "count conversion is not exact", + ) + return converted + + +def assemble_uk_atomic_area_support( + *, + system: str, + arrays: Mapping[str, np.ndarray], + column_metadata: Mapping, +) -> bytes: + """Normalize one complete nation system from supplied legacy ladder columns. + + Rows are sorted by atomic code, never dropped. Counts remain exact; old + float64 count arrays are accepted only when integral and exactly representable. + Each declared FRS region must have positive household sampling mass. The + caller supplies source/vintage/relation evidence; syntax is not authority. + NI legacy ward_code means district electoral area, and lsoa_code aliases DZ. + """ + level, vintage, regions, prefixes = _profile(system) + descriptions = _metadata(column_metadata, vintage=vintage, system=system) + _require( + type(arrays) is dict and set(arrays) == set(_INPUT_COLUMNS), + "exact arrays required", + ) + values = {} + for column in _INPUT_COLUMNS: + array = arrays[column] + _require( + type(array) is np.ndarray and array.ndim == 1 and len(array) > 0, + "nonempty one-dimensional arrays required", + ) + if column in _COUNT_COLUMNS: + values[column] = _counts(array, positive=column == "population") + continue + _require(array.dtype.kind == "U", "codes must be Unicode arrays") + _require( + np.all(np.char.str_len(array) > 0) + and np.all(np.char.strip(array) == array), + "blank or padded code", + ) + pattern = ( + _ITL if column == "itl3_code" else _WARD if column == "ward_code" else _CODE + ) + _require( + all(pattern.fullmatch(str(value)) for value in array), "invalid code syntax" + ) + if column != "itl3_code": + _require( + all(str(value)[0] in prefixes for value in array), "foreign nation code" + ) + values[column] = array.copy() + count = len(values["oa_code"]) + _require( + all(len(array) == count for array in values.values()), "array lengths differ" + ) + _require(len(np.unique(values["oa_code"])) == count, "duplicate atomic code") + nations = [str(value)[0] for value in values["oa_code"]] + for column in _INPUT_COLUMNS: + if column not in {*_COUNT_COLUMNS, "itl3_code"}: + _require( + [str(value)[0] for value in values[column]] == nations, + "mapping crosses a nation boundary", + ) + if system == SYSTEMS[2]: + _require( + np.array_equal(values["lsoa_code"], values["oa_code"]), + "NI legacy lsoa_code must alias its atomic Data Zone", + ) + region_names = {FRS_REGION_TO_REGION_CODE[name]: name for name in regions} + _require( + set(values["region_code"].tolist()) == set(region_names), + "incomplete or foreign FRS region coverage", + ) + for code in region_names: + _require( + sum(int(v) for v in values["households"][values["region_code"] == code]) + > 0, + "region has no household sampling mass", + ) + order = np.argsort(values["oa_code"], kind="stable") + values = {column: array[order] for column, array in values.items()} + normalized = {"area": values["oa_code"]} + normalized.update({k: v for k, v in values.items() if k != "oa_code"}) + columns = {"area": descriptions["oa_code"]} + columns.update({k: v for k, v in descriptions.items() if k != "oa_code"}) + normalized["frs_region"] = np.asarray( + [region_names[code] for code in values["region_code"]], dtype="U" + ) + region_rule = canonical_json( + { + "mapping": dict(FRS_REGION_TO_REGION_CODE), + "input": descriptions["region_code"], + } + ) + columns["frs_region"] = { + "kind": "code", + "source": "sha256:" + hashlib.sha256(region_rule).hexdigest(), + "vintage": "frs_region_enum_v1", + "relation": "exact", + } + for width in (3, 4): + column = "itl1_code" if width == 3 else "itl2_code" + normalized[column] = np.asarray([code[:width] for code in values["itl3_code"]]) + columns[column] = { + **descriptions["itl3_code"], + "source": descriptions["itl3_code"]["source"] + f"#prefix-{width}", + "relation": "exact", + } + return encode_atomic_support( + { + "version": 1, + "system": system, + "level": level, + "code_system": "uk_gss", + "vintage": vintage, + "columns": columns, + }, + normalized, + ) + + +def _layers(system: str) -> tuple[tuple[str, str], ...]: + shared = tuple( + (column, column) + for column in _INPUT_COLUMNS + if column not in {"oa_code", *_COUNT_COLUMNS} + ) + aliases = (("area", "oa_code"),) + if system == SYSTEMS[0]: + native = (("area", "output_area_code"),) + elif system == SYSTEMS[1]: + native = ( + ("area", "output_area_code"), + ("lsoa_code", "data_zone_code"), + ("msoa_code", "intermediate_zone_code"), + ) + else: + native = ( + ("area", "data_zone_code"), + ("msoa_code", "super_data_zone_code"), + ("ward_code", "district_electoral_area_code"), + ) + return ( + *shared, + *aliases, + *native, + ("itl1_code", "itl1_code"), + ("itl2_code", "itl2_code"), + ) + + +def uk_atomic_assignment_definition( + supports: Mapping[str, bytes], + *, + seed: int, +) -> dict: + """Declare shared assignment after the complete initial spine/support roster. + + Source names are fixed by system, not file paths. The host must bind admitted + bytes to those SourceRefs and authenticate the supplied full-spine identity. + This function neither creates a graph barrier nor admits a native artifact. + """ + _require( + type(supports) is dict and set(supports) == set(SYSTEMS), + "three systems required", + ) + _require( + isinstance(seed, Integral) + and not isinstance(seed, (bool, np.bool_)) + and 0 <= seed < 2**32, + "seed must be a bounded integer", + ) + systems = [] + for system in SYSTEMS: + _require(type(supports[system]) is bytes, "immutable support bytes required") + support = decode_atomic_support(supports[system]) + level, vintage, regions, _ = _profile(system) + _require( + all( + support.metadata[k] == value + for k, value in ( + ("system", system), + ("level", level), + ("code_system", "uk_gss"), + ("vintage", vintage), + ) + ), + "support system identity differs", + ) + expected_columns = { + "area", + *(_INPUT_COLUMNS[1:]), + "frs_region", + "itl1_code", + "itl2_code", + } + _require(set(support.arrays) == expected_columns, "support columns differ") + # The declaration accepts this adapter's normalized form. This closes + # forged derived aliases/region normalization without claiming source + # authority from a self-consistent payload. + restored_arrays = { + column: support.arrays["area" if column == "oa_code" else column] + for column in _INPUT_COLUMNS + } + restored_metadata = { + column: dict( + support.metadata["columns"]["area" if column == "oa_code" else column] + ) + for column in _INPUT_COLUMNS + } + _require( + assemble_uk_atomic_area_support( + system=system, arrays=restored_arrays, column_metadata=restored_metadata + ) + == supports[system], + "support is not the canonical UK adapter output", + ) + systems.append( + { + "id": system, + "level": level, + "code_system": "uk_gss", + "vintage": vintage, + "source": SOURCES[system], + "selector": {"region": list(regions)}, + "constraints": [ + {"input": "region", "support": "frs_region", "required": True} + ], + "observed_area": None, + "stages": [ + {"level": "constituency_code", "weight": "households"}, + {"level": "area", "weight": "population"}, + ], + "layers": [ + { + "input": column, + "output": output, + **{ + k: support.metadata["columns"][column][k] + for k in ("source", "vintage", "relation") + }, + } + for column, output in _layers(system) + ], + } + ) + return validate_assignment_spec( + { + "version": 1, + "identity": [IDENTITY_COLUMN], + "stream": ["sha256-u53-v1", "uk-post-clone-atomic-area-v1", 0, int(seed)], + "outputs": { + "area": "atomic_area_code", + "system": "atomic_area_system", + "basis": "atomic_area_basis", + }, + "systems": systems, + } + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py new file mode 100644 index 000000000..1d8435b8b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py @@ -0,0 +1,85 @@ +"""Explicit post-clone household keys for UK atomic assignment. + +The host supplies original-source identity and ordered structural branches. +This helper neither infers ancestry from financial values/ID offsets nor +authenticates the supplied lineage. Source/EXPAND receipts remain authoritative. +""" + +from __future__ import annotations + +from numbers import Integral + +import numpy as np + +from microcosm.graph.canonical import canonical_json + +_BRANCHES = ( + "spi_support_channel", + "cgt_incidence_clone", + "cgt_band_donors", + "geographic_support", +) + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("UK geography identity: " + message) + + +def household_draw_key( + *, + source: str, + source_vintage: str, + source_household_id: int, + clone_path: tuple[tuple[str, int], ...], +) -> str: + """Encode an original-source ID and explicit ordered clone branch path. + + Every full-spine household needs a distinct key. Branch ordinals are + supplied lineage identities, never sample row counters. Original branches + should use their declared ordinal consistently; growing a pool must not + change an existing path. Calibration K, weights, income, and final offset + IDs are deliberately not inputs. Exact IDs above 2**53 remain distinct. + """ + _require( + all( + type(value) is str + and value + and value.strip() == value + and len(value) <= 256 + for value in (source, source_vintage) + ), + "qualified source and vintage required", + ) + _require( + isinstance(source_household_id, Integral) + and not isinstance(source_household_id, (bool, np.bool_)) + and 0 < source_household_id < 2**63, + "original source household ID must be an exact positive int64", + ) + _require(type(clone_path) is tuple, "explicit immutable clone path required") + path = [] + previous = -1 + for branch in clone_path: + _require( + type(branch) is tuple and len(branch) == 2, + "named branch/ordinal pair required", + ) + name, ordinal = branch + _require(type(name) is str and name in _BRANCHES, "unknown structural branch") + position = _BRANCHES.index(name) + _require(position > previous, "clone branches must be unique and ordered") + _require( + isinstance(ordinal, Integral) + and not isinstance(ordinal, (bool, np.bool_)) + and 0 <= ordinal < 2**32, + "clone ordinal must be a bounded exact integer", + ) + path.append([name, str(int(ordinal))]) + previous = position + return ( + "uk-household-v1:" + + canonical_json( + [source, source_vintage, str(int(source_household_id)), path] + ).decode() + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_lineage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_lineage.py new file mode 100644 index 000000000..dcca8ac98 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_lineage.py @@ -0,0 +1,212 @@ +"""Project supplied source/structural lineage to stable UK geography keys. + +This is a pure descriptive adapter, not a source owner or graph admission gate. +The host must bind roots, every before/after axis, EXPAND parent receipt and +explicit ordinal to its actual retained observations before and after use. +No identifier is reconstructed from offsets, weights, financial flags or order. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from numbers import Integral + +import numpy as np +import pandas as pd + +from . import atomic_household_identity as identity +from .atomic_area_support import IDENTITY_COLUMN + +_ROOT_COLUMNS = ("household_id", "source", "source_vintage", "source_household_id") + + +@dataclass(frozen=True) +class HouseholdExpansion: + """One checked EXPAND's household axis and explicitly supplied ordinals. + + parent_pairs is the immutable household portion of the shared executor's + ``receipt['expand']`` mapping. child_ordinals names each new household's + declared branch ordinal; its position in the receipt is not an ordinal. + Existing households retain their previous path without adding a zero step. + Constructing this object does not authenticate any of these claims. + """ + + branch: str + before_ids: tuple[int, ...] + after_ids: tuple[int, ...] + parent_pairs: tuple[tuple[int, int], ...] + child_ordinals: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class HouseholdSelection: + """A checked FILTER's explicit household axes; no new identity is created.""" + + before_ids: tuple[int, ...] + after_ids: tuple[int, ...] + + +def _require(condition, reason): + if not condition: + raise ValueError("UK atomic household lineage: " + reason) + + +def _integer(value, *, ordinal=False): + _require( + isinstance(value, Integral) and not isinstance(value, (bool, np.bool_)), + "exact integer required", + ) + value = int(value) + _require( + 0 <= value < 2**32 if ordinal else 0 < value < 2**63, + "ordinal or ID outside its exact declared range", + ) + return value + + +def _axis(values): + _require(type(values) is tuple, "axes must be immutable tuples") + ids = tuple(_integer(value) for value in values) + _require(len(ids) == len(set(ids)), "duplicate household ID") + return ids + + +def _pairs(values, *, ordinals=False): + _require(type(values) is tuple, "pairs must be an immutable tuple") + pairs = {} + for row in values: + _require(type(row) is tuple and len(row) == 2, "exact immutable pair required") + target = _integer(row[0]) + source = _integer(row[1], ordinal=ordinals) + _require(target not in pairs, "duplicate pair target") + pairs[target] = source + return pairs + + +def _roots(roots): + _require(type(roots) is pd.DataFrame, "a plain root description table is required") + _require( + roots.columns.is_unique + and set(roots.columns) == set(_ROOT_COLUMNS) + and len(roots) > 0, + "exact nonempty root description columns required", + ) + for column in ("household_id", "source_household_id"): + _require(roots[column].dtype == np.dtype("int64"), "root IDs must be int64") + # Only immutable validated scalar values survive this copy. pandas row + # labels, column ordering and physical row order do not enter the key. + copied = roots.loc[:, list(_ROOT_COLUMNS)].copy(deep=True) + result = {} + initial_keys = set() + for row in copied.itertuples(index=False, name=None): + household_id, source, vintage, source_id = row + household_id = _integer(household_id) + source_id = _integer(source_id) + _require(household_id not in result, "duplicate root household ID") + key = identity.household_draw_key( + source=source, + source_vintage=vintage, + source_household_id=source_id, + clone_path=(), + ) + _require(key not in initial_keys, "duplicate original source identity") + initial_keys.add(key) + result[household_id] = (source, vintage, source_id, ()) + return result + + +def project_atomic_household_keys( + roots: pd.DataFrame, + *, + steps: tuple[HouseholdExpansion | HouseholdSelection, ...], + final_ids: np.ndarray, +) -> pd.DataFrame: + """Return exact final household IDs and canonical geography keys. + + Membership of each before axis must equal the preceding result; physical + ordering may differ because it is not identity. Every added row must have + one known preceding parent and one explicit ordinal unique for that parent + in this branch. Branch stages must follow the existing UK branch ordering. + A selection may only remove IDs; unexplained arrivals or losses refuse. + Empty selection results are valid descriptions, not viable build evidence. + + The output follows final_ids and is detached. All supplied descriptions + remain unissued; this checks internal consistency, not source authenticity, + complete-population ancestry, a sampling rule or release eligibility. + """ + current = _roots(roots) + _require(type(steps) is tuple, "steps must be an immutable tuple") + _require( + type(final_ids) is np.ndarray + and final_ids.dtype == np.dtype("int64") + and final_ids.ndim == 1, + "final household axis must be a one-dimensional int64 ndarray", + ) + final = _axis(tuple(final_ids)) + previous_branch = -1 + for step in steps: + _require( + type(step) in (HouseholdExpansion, HouseholdSelection), + "unsupported structural description", + ) + before = _axis(step.before_ids) + after = _axis(step.after_ids) + _require( + set(before) == set(current), "before axis differs from retained history" + ) + if type(step) is HouseholdSelection: + _require(set(after) <= set(before), "selection contains new households") + current = {target: current[target] for target in after} + continue + _require( + type(step.branch) is str and step.branch in identity._BRANCHES, + "unknown UK structural branch", + ) + position = identity._BRANCHES.index(step.branch) + _require(position > previous_branch, "repeated or reordered structural branch") + previous_branch = position + _require(set(before) <= set(after), "expansion removed incumbent households") + arrivals = set(after) - set(before) + parents = _pairs(step.parent_pairs) + ordinals = _pairs(step.child_ordinals, ordinals=True) + _require( + set(parents) == arrivals == set(ordinals), + "parents and ordinals must cover exactly the new households", + ) + _require(set(parents.values()) <= set(before), "parent absent before expansion") + branch_ids = [(parents[target], ordinals[target]) for target in arrivals] + _require( + len(branch_ids) == len(set(branch_ids)), + "repeated ordinal for one parent in this branch", + ) + extended = dict(current) + for target in arrivals: + source, vintage, source_id, path = current[parents[target]] + new_path = (*path, (step.branch, ordinals[target])) + # Reuse the existing complete path contract; no alternate key + # encoding or static branch roster is introduced here. + identity.household_draw_key( + source=source, + source_vintage=vintage, + source_household_id=source_id, + clone_path=new_path, + ) + extended[target] = (source, vintage, source_id, new_path) + current = {target: extended[target] for target in after} + _require(set(final) == set(current), "final axis differs from completed history") + keys = [ + identity.household_draw_key( + source=current[target][0], + source_vintage=current[target][1], + source_household_id=current[target][2], + clone_path=current[target][3], + ) + for target in final + ] + _require(len(keys) == len(set(keys)), "final geography keys are not unique") + return pd.DataFrame( + { + "household_id": np.asarray(final, dtype=np.int64), + IDENTITY_COLUMN: pd.array(keys, dtype=pd.StringDtype(storage="python")), + } + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 9437b99e8..60b2d5d28 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -24,7 +24,9 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass, field +from importlib import import_module from importlib.resources import files +from threading import RLock from microcosm.build.plan import DonorSpec, Stage, StagePlan from microcosm.build.source_manifest import ( @@ -35,1061 +37,12 @@ load_source_manifest, load_support_spine_manifest, ) -from microcosm.build.us_runtime.adult_care import ( - US_ADULT_CARE_CHILD_QUALIFYING_AGE_LIMIT, - US_ADULT_CARE_EARNED_INCOME_SOURCES, - US_ADULT_CARE_OUTPUT_COLUMNS, - US_ADULT_CARE_REQUIRED_SOURCE_COLUMNS, - US_ADULT_CARE_STAGE_NAME, - derive_us_adult_care_from_manifest, - us_adult_care_signal_gate, - us_adult_care_stage_spec, - us_adult_care_summary, - with_us_adult_care_inputs, -) -from microcosm.build.us_runtime.alimony import ( - ALIMONY_ASEC_ARCHIVED_DERIVATION_URL, - ALIMONY_PUF_ARCHIVED_DERIVATION_URL, - STRIKE_BENEFITS_ASEC_ARCHIVED_DERIVATION_URL, - US_ALIMONY_NONCONSTANT_PERSON_COLUMNS, - US_ALIMONY_OUTPUT_COLUMNS, - US_ALIMONY_STAGE_NAME, - US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS, - derive_us_alimony_from_asec, - derive_us_alimony_from_puf, - us_alimony_signal_gate, - us_alimony_stage_spec, - us_alimony_summary, -) -from microcosm.build.us_runtime.asec_checkpoint import ( - ASEC_RAW_STAGE_ARTIFACT_KIND, - ASEC_RAW_STAGE_CHECKPOINT_FILENAME, - ASEC_RAW_STAGE_OPERATOR_STATUS, - ASEC_RAW_STAGE_SCHEMA_VERSION, - ASEC_RAW_STAGE_STAGE, - load_asec_pre_clone_checkpoint, - load_asec_raw_stage_checkpoint, -) -from microcosm.build.us_runtime.asec_pool import ( - AsecSource, - build_pooled_asec_unit_frame, - load_asec_h5_tables, - pool_asec_sources, -) -from microcosm.build.us_runtime.capital_gain_details import ( - CAPITAL_GAIN_DETAILS_ARCHIVED_DERIVATION_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_EXPORT_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_IMPUTATION_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_PERSON_ALLOCATION_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_PUF_ARTIFACT_URL, - US_CAPITAL_GAIN_DETAILS_NONCONSTANT_PERSON_COLUMNS, - US_CAPITAL_GAIN_DETAILS_NONCONSTANT_TAX_UNIT_COLUMNS, - US_CAPITAL_GAIN_DETAILS_OUTPUT_COLUMNS, - US_CAPITAL_GAIN_DETAILS_STAGE_NAME, - derive_us_capital_gain_details_from_puf, - us_capital_gain_details_signal_gate, - us_capital_gain_details_stage_spec, - us_capital_gain_details_summary, -) -from microcosm.build.us_runtime.casualty_losses import ( - US_CASUALTY_LOSS_NONCONSTANT_PERSON_COLUMNS, - US_CASUALTY_LOSS_OUTPUT_COLUMNS, - US_CASUALTY_LOSS_STAGE_NAME, - derive_us_casualty_loss_from_puf, - us_casualty_loss_signal_gate, - us_casualty_loss_stage_spec, - us_casualty_loss_summary, -) -from microcosm.build.us_runtime.child_support import ( - CHILD_SUPPORT_ARCHIVED_PUF_IMPUTATION_URL, - CHILD_SUPPORT_ARCHIVED_PUF_OUTPUTS_URL, - CHILD_SUPPORT_EXPENSE_ARCHIVED_DERIVATION_URL, - CHILD_SUPPORT_RECEIVED_ARCHIVED_DERIVATION_URL, - US_CHILD_SUPPORT_NONCONSTANT_PERSON_COLUMNS, - US_CHILD_SUPPORT_OUTPUT_COLUMNS, - US_CHILD_SUPPORT_REQUIRED_SOURCE_COLUMNS, - US_CHILD_SUPPORT_STAGE_NAME, - derive_us_child_support_from_asec, - derive_us_child_support_from_manifest, - impute_us_child_support_to_puf_support_from_manifest, - us_child_support_signal_gate, - us_child_support_stage_spec, - us_child_support_summary, - with_us_child_support_inputs, -) -from microcosm.build.us_runtime.childcare import ( - US_CHILDCARE_OUTPUT_COLUMNS, - US_CHILDCARE_REQUIRED_SOURCE_COLUMNS, - US_CHILDCARE_STAGE_NAME, - derive_us_childcare_from_manifest, - impute_us_childcare_to_puf_support_from_manifest, - us_childcare_signal_gate, - us_childcare_stage_spec, - us_childcare_summary, - with_us_childcare_inputs, -) -from microcosm.build.us_runtime.congressional_district_geography import ( - CONGRESSIONAL_DISTRICT_GEOID_COLUMN, - SOI_CONGRESSIONAL_DISTRICT_RECORD_SET_ID, - assign_congressional_districts_to_households, - congressional_district_assignment_summary, - congressional_district_distribution_from_ledger_facts, - with_household_congressional_districts, -) -from microcosm.build.us_runtime.congressional_district_vintage import ( - CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, - CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, - CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, - CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, - DEFAULT_CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_RESOURCE, - SOURCE_CONGRESSIONAL_DISTRICT_PREFIX, - default_congressional_district_vintage_crosswalk_path, - load_congressional_district_vintage_crosswalk, - load_default_congressional_district_vintage_crosswalk, - translate_congressional_district_facts_to_current_vintage, -) -from microcosm.build.us_runtime.congressional_district_vintage_crosswalk import ( - CROSSWALK_BASIS_BLOCK_POPULATION, - build_cd_vintage_crosswalk_rows, - normalize_district_code, - parse_baf_cd_layer, - parse_national_cd_bef_districts, -) -from microcosm.build.us_runtime.cps_carried import ( - CPS_CARRIED_FORMULA_OWNED_COLUMNS, - CPS_CARRIED_PERSON_INPUTS, - CPS_CARRIED_SPM_UNIT_INPUTS, - CPS_REPORTED_TANF_AMOUNT_RAW_COLUMN, - CPS_REPORTED_TANF_TYPE_RAW_COLUMN, - CPS_REPORTED_WIC_RAW_COLUMN, - US_REPORTED_COVERAGE_PERSON_INPUTS, - US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS, - WIC_CARRIER_ADJUDICATION_URL, - derive_us_cps_carried_inputs, - reported_tanf_enrollment_by_spm_unit, - reported_wic_receipt_carrier, -) -from microcosm.build.us_runtime.demographics import ( - AGE_BANDS, - DEMOGRAPHICS_SCHEMA_VERSION, - AgeBand, - compute_age_distribution, - demographics_payload, - write_demographics, -) -from microcosm.build.us_runtime.disability_benefits import ( - DISABILITY_BENEFITS_ARCHIVED_DERIVATION_URL, - DISABILITY_BENEFITS_ARCHIVED_PUF_IMPUTATION_URL, - DISABILITY_BENEFITS_ARCHIVED_PUF_OUTPUTS_URL, - DISABILITY_BENEFITS_ARCHIVED_SOURCE_COLUMNS_URL, - US_DISABILITY_BENEFITS_NONCONSTANT_PERSON_COLUMNS, - US_DISABILITY_BENEFITS_OUTPUT_COLUMNS, - US_DISABILITY_BENEFITS_REQUIRED_SOURCE_COLUMNS, - US_DISABILITY_BENEFITS_STAGE_NAME, - derive_us_disability_benefits_from_asec, - derive_us_disability_benefits_from_manifest, - impute_us_disability_benefits_to_puf_support_from_manifest, - us_disability_benefits_signal_gate, - us_disability_benefits_stage_spec, - us_disability_benefits_summary, - with_us_disability_benefits, -) -from microcosm.build.us_runtime.domestic_production import ( - DOMESTIC_PRODUCTION_ALD_ARCHIVED_DERIVATION_URL, - DOMESTIC_PRODUCTION_ALD_ARCHIVED_EXPORT_URL, - DOMESTIC_PRODUCTION_ALD_ARCHIVED_IMPUTATION_URL, - DOMESTIC_PRODUCTION_ALD_ARCHIVED_PUF_ARTIFACT_URL, - US_DOMESTIC_PRODUCTION_ALD_NONCONSTANT_TAX_UNIT_COLUMNS, - US_DOMESTIC_PRODUCTION_ALD_OUTPUT_COLUMNS, - US_DOMESTIC_PRODUCTION_ALD_STAGE_NAME, - derive_us_domestic_production_ald_from_puf, - us_domestic_production_ald_signal_gate, - us_domestic_production_ald_stage_spec, - us_domestic_production_ald_summary, -) -from microcosm.build.us_runtime.education_assistance_source import ( - ASEC_EDUCATION_ASSISTANCE_ARCHIVES, - ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS, - fetch_asec_education_assistance_source, - fill_asec_education_assistance_source, - load_asec_education_assistance_sources, -) -from microcosm.build.us_runtime.education_inputs import ( - US_AOTC_ELIGIBILITY_OUTPUT_COLUMNS, - US_EDUCATION_INPUTS_NONCONSTANT_PERSON_COLUMNS, - US_EDUCATION_INPUTS_OUTPUT_COLUMNS, - US_EDUCATION_INPUTS_OWNED_OUTPUT_COLUMNS, - US_EDUCATION_INPUTS_REQUIRED_SOURCE_COLUMNS, - US_EDUCATION_INPUTS_STAGE_NAME, - derive_us_education_inputs_from_manifest, - us_education_inputs_signal_gate, - us_education_inputs_stage_spec, - us_education_inputs_summary, - with_us_education_inputs, -) -from microcosm.build.us_runtime.educator_expenses import ( - EDUCATOR_EXPENSE_ARCHIVED_ALLOCATION_URL, - EDUCATOR_EXPENSE_ARCHIVED_DERIVATION_URL, - EDUCATOR_EXPENSE_ARCHIVED_EXPORT_URL, - EDUCATOR_EXPENSE_ARCHIVED_PUF_IMPUTATION_URL, - US_EDUCATOR_EXPENSE_NONCONSTANT_PERSON_COLUMNS, - US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS, - US_EDUCATOR_EXPENSE_STAGE_NAME, - derive_us_educator_expense_from_puf, - us_educator_expense_signal_gate, - us_educator_expense_stage_spec, - us_educator_expense_summary, -) -from microcosm.build.us_runtime.eligibility_inputs import ( - US_ELIGIBILITY_INPUTS_NONCONSTANT_PERSON_COLUMNS, - US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS, - US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS, - US_ELIGIBILITY_INPUTS_STAGE_NAME, - derive_us_eligibility_inputs_from_manifest, - us_eligibility_inputs_signal_gate, - us_eligibility_inputs_stage_spec, - us_eligibility_inputs_summary, - with_us_eligibility_inputs, -) -from microcosm.build.us_runtime.energy_subsidy import ( - ENERGY_SUBSIDY_ARCHIVED_CPS_DERIVATION_URL, - ENERGY_SUBSIDY_ARCHIVED_PUF_IMPUTATION_URL, - US_ENERGY_SUBSIDY_OUTPUT_COLUMNS, - US_ENERGY_SUBSIDY_REQUIRED_SOURCE_COLUMNS, - US_ENERGY_SUBSIDY_STAGE_NAME, - derive_us_energy_subsidy_from_manifest, - impute_us_energy_subsidy_to_puf_support_from_manifest, - us_energy_subsidy_signal_gate, - us_energy_subsidy_stage_spec, - us_energy_subsidy_summary, - with_us_energy_subsidy_input, -) -from microcosm.build.us_runtime.farm_business_income import ( - FARM_BUSINESS_INCOME_ARCHIVED_CPS_FARM_INCOME_URL, - FARM_BUSINESS_INCOME_ARCHIVED_DERIVATION_URL, - FARM_BUSINESS_INCOME_ARCHIVED_EXPORT_URL, - FARM_BUSINESS_INCOME_ARCHIVED_IMPUTATION_URL, - FARM_BUSINESS_INCOME_ARCHIVED_OVERRIDE_URL, - FARM_BUSINESS_INCOME_ARCHIVED_PUF_ARTIFACT_URL, - US_FARM_BUSINESS_INCOME_NONCONSTANT_PERSON_COLUMNS, - US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS, - US_FARM_BUSINESS_INCOME_STAGE_NAME, - derive_us_farm_business_income_from_puf, - us_farm_business_income_signal_gate, - us_farm_business_income_stage_spec, - us_farm_business_income_summary, -) -from microcosm.build.us_runtime.fiscal_targets import ( - SOI_VARIABLE_MAP, - US_FISCAL_LEDGER_PARITY_REGISTRY, - US_FISCAL_LEDGER_PARITY_REPORT, - US_FISCAL_MACRO_REALISM_BANDS, - US_FISCAL_TARGET_COVERAGE_REQUIREMENTS, - US_FISCAL_TARGET_LEDGER_REFERENCES, - US_FISCAL_TARGET_REFERENCES, - US_FISCAL_TARGET_REGISTRY, - US_FISCAL_TARGET_SPECS, - US_FISCAL_TARGET_SUPPORT_EXCLUSIONS, - US_JCT_TAX_EXPENDITURE_REFORMS, - US_JCT_TAX_EXPENDITURE_TARGET_REFERENCES, - US_JCT_TAX_EXPENDITURE_TARGET_SPECS, - US_SOI_FISCAL_TARGET_REFERENCES, - US_SOI_FISCAL_TARGET_SPECS, - US_STATE_INCOME_TAX_TARGET_REFERENCES, - US_STATE_INCOME_TAX_TARGET_SPECS, - SimpleTaxExpenditureReform, - compile_us_fiscal_target_registry, -) -from microcosm.build.us_runtime.form_4952 import ( - FORM_4952_ARCHIVED_DERIVATION_URL, - FORM_4952_ARCHIVED_EXPORT_URL, - FORM_4952_ARCHIVED_IMPUTATION_URL, - FORM_4952_ARCHIVED_PERSON_ALLOCATION_URL, - FORM_4952_ARCHIVED_PUF_ARTIFACT_URL, - US_FORM_4952_NONCONSTANT_PERSON_COLUMNS, - US_FORM_4952_OUTPUT_COLUMNS, - US_FORM_4952_STAGE_NAME, - derive_us_form_4952_election_from_puf, - us_form_4952_election_signal_gate, - us_form_4952_election_stage_spec, - us_form_4952_election_summary, -) -from microcosm.build.us_runtime.geography_ladder import ( - GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR, - GEOGRAPHY_LADDER_VINTAGES_ATTR, - US_BLOCK_LADDER_DERIVED_LAYERS, - US_BLOCK_LADDER_KIND, - US_BLOCK_LADDER_SCHEMA_VERSION, - US_GEOGRAPHY_LADDER_COLUMNS, - US_NYC_COUNTY_FIPS, - UsBlockLadder, - assign_us_geography_ladder, - load_us_block_ladder, - us_geography_ladder_assignment_summary, - us_geography_ladder_gate, - with_household_us_geography_ladder, -) -from microcosm.build.us_runtime.hours_worked import ( - US_HOURS_WORKED_NONCONSTANT_PERSON_COLUMNS, - US_HOURS_WORKED_OUTPUT_COLUMNS, - US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS, - US_HOURS_WORKED_POOL_OUTPUT_COLUMNS, - US_HOURS_WORKED_REQUIRED_SOURCE_COLUMNS, - US_HOURS_WORKED_STAGE_NAME, - derive_us_hours_worked_from_manifest, - us_hours_worked_signal_gate, - us_hours_worked_stage_spec, - us_hours_worked_summary, - with_us_hours_worked_inputs, -) -from microcosm.build.us_runtime.housing_inputs import ( - ACS_2022_RENT_ARTIFACT_SHA256, - HOUSING_INPUTS_ARCHIVED_ACS_DERIVATION_URL, - HOUSING_INPUTS_ARCHIVED_CPS_RENT_URL, - HOUSING_INPUTS_ARCHIVED_CPS_SPM_URL, - HOUSING_INPUTS_ARCHIVED_PUF_IMPUTATION_URL, - HOUSING_TAKE_UP_ARCHIVED_DERIVATION_URL, - HOUSING_TAKE_UP_ARCHIVED_HUD_ETL_URL, - HOUSING_TAKE_UP_ARCHIVED_PARAMETER_URL, - US_HOUSING_HOUSEHOLD_OUTPUT_COLUMNS, - US_HOUSING_INPUTS_OUTPUT_COLUMNS, - US_HOUSING_INPUTS_STAGE_NAME, - US_HOUSING_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_HOUSING_NONCONSTANT_PERSON_COLUMNS, - US_HOUSING_NONCONSTANT_SPM_UNIT_COLUMNS, - US_HOUSING_PERSON_OUTPUT_COLUMNS, - US_HOUSING_REQUIRED_HOUSEHOLD_SOURCE_COLUMNS, - US_HOUSING_REQUIRED_PERSON_SOURCE_COLUMNS, - US_HOUSING_SPM_UNIT_OUTPUT_COLUMNS, - derive_us_housing_inputs, - impute_us_housing_assistance_to_puf_support, - impute_us_pre_subsidy_rent, - load_acs_2022_rent_donor, - us_housing_inputs_signal_gate, - us_housing_inputs_stage_spec, - us_housing_inputs_summary, - with_us_housing_inputs, -) -from microcosm.build.us_runtime.immigration import ( - IMMIGRATION_STATUS_VALUES, - SSN_CARD_TYPE_VALUES, - US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS, - US_IMMIGRATION_OUTPUT_COLUMNS, - US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS, - US_IMMIGRATION_STAGE_NAME, - UndocumentedControls, - derive_us_immigration_status_from_manifest, - us_immigration_composition_gate, - us_immigration_composition_summary, - us_immigration_stage_spec, - with_us_immigration_inputs, -) -from microcosm.build.us_runtime.input_mass import ( - us_input_mass_totals, -) -from microcosm.build.us_runtime.medicaid_take_up import ( - US_MEDICAID_ENROLLMENT_SUBSTITUTIONS, - US_MEDICAID_ENROLLMENT_TARGET_ROLE, - US_MEDICAID_ENROLLMENT_TARGET_TABLE, - US_MEDICAID_ENROLLMENT_TOLERANCE, - US_MEDICAID_TAKE_UP_ANCHOR, - US_MEDICAID_TAKE_UP_STAGE, - US_MEDICAID_TAKE_UP_VARIABLE, - MedicaidEnrollmentSubstitution, - apply_us_medicaid_enrollment_substitutions, - us_medicaid_source_person_table, - us_medicaid_take_up_diagnostics, - us_medicaid_take_up_gate, - with_us_medicaid_take_up, - write_us_medicaid_take_up_diagnostics, -) -from microcosm.build.us_runtime.medicare_take_up import ( - MEDICARE_TAKE_UP_ARCHIVED_CLONE_URL, - MEDICARE_TAKE_UP_ARCHIVED_DERIVATION_URL, - MEDICARE_TAKE_UP_ARCHIVED_EXPORT_URL, - MEDICARE_TAKE_UP_ARCHIVED_SOURCE_COLUMNS_URL, - US_MEDICARE_TAKE_UP_NONCONSTANT_PERSON_COLUMNS, - US_MEDICARE_TAKE_UP_OUTPUT_COLUMNS, - US_MEDICARE_TAKE_UP_REQUIRED_SOURCE_COLUMNS, - US_MEDICARE_TAKE_UP_STAGE_NAME, - derive_us_medicare_take_up_from_manifest, - us_medicare_take_up_signal_gate, - us_medicare_take_up_stage_spec, - us_medicare_take_up_summary, - with_us_medicare_take_up_input, -) -from microcosm.build.us_runtime.misc_itemized import ( - US_MISC_ITEMIZED_NONCONSTANT_PERSON_COLUMNS, - US_MISC_ITEMIZED_OUTPUT_COLUMNS, - US_MISC_ITEMIZED_STAGE_NAME, - derive_us_misc_itemized_from_puf, - us_misc_itemized_signal_gate, - us_misc_itemized_stage_spec, - us_misc_itemized_summary, -) -from microcosm.build.us_runtime.nonzero_shares import ( - nonzero_share, - us_nonzero_shares, -) -from microcosm.build.us_runtime.operator_boundary import ( - FORMULA_OWNED_SOURCE_COLUMNS, - PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, - assert_operator_free_source_frame, -) -from microcosm.build.us_runtime.org_wages import ( - BLS_STATE_UNION_REPRESENTATION_RATE_2024, - FLSA_EXECUTIVE_ADMINISTRATIVE_PROFESSIONAL_OCCUPATION_CODES, - FLSA_OVERTIME_OCCUPATION_CODES, - ORG_2024_DONOR_CONTENT_SHA256, - ORG_2024_DONOR_FILENAME, - ORG_PREDICTORS, - US_ORG_WAGES_NONCONSTANT_PERSON_COLUMNS, - US_ORG_WAGES_OUTPUT_COLUMNS, - US_ORG_WAGES_REQUIRED_SOURCE_COLUMNS, - US_ORG_WAGES_STAGE_NAME, - derive_flsa_overtime_premium, - derive_us_org_occupation_inputs, - fetch_org_2024_donor, - impute_us_org_wages, - load_org_2024_donor, - us_org_wages_signal_gate, - us_org_wages_stage_spec, - us_org_wages_summary, - with_us_org_wages_inputs, -) -from microcosm.build.us_runtime.other_health_insurance import ( - OTHER_HEALTH_INSURANCE_ARCHIVED_DERIVATION_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_IMPUTATION_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_OUTPUTS_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_PREDICTORS_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_SPLICE_URL, - US_OTHER_HEALTH_INSURANCE_MODELED_PREMIUM_VARIABLES, - US_OTHER_HEALTH_INSURANCE_NONCONSTANT_PERSON_COLUMNS, - US_OTHER_HEALTH_INSURANCE_OUTPUT_COLUMNS, - US_OTHER_HEALTH_INSURANCE_REQUIRED_SOURCE_COLUMNS, - US_OTHER_HEALTH_INSURANCE_STAGE_NAME, - US_OTHER_HEALTH_INSURANCE_STAGE_OUTPUT_COLUMNS, - US_SE_HEALTH_ATTRIBUTION_OUTPUT_COLUMNS, - US_SE_HEALTH_MEDICARE_AGE_THRESHOLD, - US_SE_HEALTH_SELF_EMPLOYMENT_INCOME_SOURCES, - attribute_us_se_health_premiums, - attribute_us_se_health_premiums_from_manifest, - derive_us_other_health_insurance_from_asec, - derive_us_other_health_insurance_from_manifest, - impute_us_other_health_insurance_to_puf_support_from_manifest, - us_other_health_insurance_signal_gate, - us_other_health_insurance_stage_spec, - us_other_health_insurance_summary, - with_us_other_health_insurance_inputs, -) -from microcosm.build.us_runtime.parity_reference import ( - ECPS_PARITY_KNOWN_GAPS_RESOURCE, - ECPS_PARITY_REFERENCE_RESOURCE, - EcpsParityReference, - EcpsParitySource, - ParityKnownGap, - load_ecps_parity_known_gaps, - load_ecps_parity_reference, -) -from microcosm.build.us_runtime.pregnancy import ( - US_PREGNANCY_NONCONSTANT_PERSON_COLUMNS, - US_PREGNANCY_OUTPUT_COLUMN, - US_PREGNANCY_REQUIRED_SOURCE_COLUMNS, - US_PREGNANCY_STAGE_NAME, - derive_us_pregnancy_from_manifest, - us_pregnancy_signal_gate, - us_pregnancy_stage_spec, - us_pregnancy_summary, - with_us_pregnancy_inputs, -) -from microcosm.build.us_runtime.prior_year_income import ( - PRIOR_YEAR_INCOME_ARCHIVED_DERIVATION_URL, - PRIOR_YEAR_INCOME_ARCHIVED_FINALIZER_URL, - PRIOR_YEAR_INCOME_ARCHIVED_FORMULA_OUTPUT_URL, - PRIOR_YEAR_INCOME_ARCHIVED_PUF_IMPUTATION_URL, - PRIOR_YEAR_INCOME_ARCHIVED_PUF_OUTPUTS_URL, - PRIOR_YEAR_INCOME_ARCHIVED_PUF_SPLICE_URL, - US_PRIOR_YEAR_INCOME_FORMULA_OWNED_OUTPUT_COLUMNS, - US_PRIOR_YEAR_INCOME_NONCONSTANT_PERSON_COLUMNS, - US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS, - US_PRIOR_YEAR_INCOME_PERSISTED_OUTPUT_COLUMNS, - US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS, - US_PRIOR_YEAR_INCOME_STAGE_NAME, - derive_us_prior_year_income_from_manifest, - impute_us_prior_year_income_to_puf_support_from_manifest, - us_prior_year_income_signal_gate, - us_prior_year_income_source_reconciliation_gate, - us_prior_year_income_stage_spec, - us_prior_year_income_summary, - with_us_prior_year_income_inputs, -) -from microcosm.build.us_runtime.public_assistance_type_source import ( - ASEC_PUBLIC_ASSISTANCE_TYPE_AUDIT_PINS, - ASEC_PUBLIC_ASSISTANCE_TYPE_INCOME_YEARS, - PAW_TYPE_TANF_CODES, - PAW_TYPE_VALID_CODES, - fill_asec_public_assistance_type_source, - load_asec_public_assistance_type_sources, -) -from microcosm.build.us_runtime.puf_capital_gains_tail import ( - PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN, - PUF_CAPITAL_GAINS_TAIL_MANIFEST_SCHEMA_VERSION, - PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_POSITIVE_MASS_FIVE_X_TARGET, - PUF_CAPITAL_GAINS_TAIL_QUANTILE, - PUF_CAPITAL_GAINS_TAIL_STAGE_NAME, - PUF_CAPITAL_GAINS_TAIL_SUPPORT_CHANNEL, - PUF_CAPITAL_GAINS_TAIL_SUPPORT_CONTRACT_VERSION, - PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN, - assert_puf_capital_gains_tail_survives_selection, - puf_capital_gains_tail_concentration_gate, - puf_capital_gains_tail_support_contract_identity, - puf_capital_gains_tail_terminal_support_receipt, - select_puf_capital_gains_tail_donors, - transfer_puf_capital_gains_tail, - validate_puf_capital_gains_tail_manifest, - validate_puf_capital_gains_tail_terminal_support_receipt, - write_puf_capital_gains_tail_manifest, -) -from microcosm.build.us_runtime.puf_donor_io import load_puf_tax_unit_donor -from microcosm.build.us_runtime.puf_e01000_reconciliation import ( - PUF_E01000_RECONCILIATION_SCHEMA_VERSION, - build_puf_e01000_reconciliation_basis, - finalize_puf_e01000_reconciliation, - puf_capital_gains_joint_metrics, - puf_processed_capital_gains_stage, - puf_raw_e01000_stage, -) -from microcosm.build.us_runtime.puf_interest_components import ( - US_PUF_E19200_AGI_BANDS, - US_PUF_E19200_ALL_RETURNS_COMPONENTS, - PufE19200AgiBand, - PufE19200InterestComponents, - split_us_puf_e19200_by_agi_band, -) -from microcosm.build.us_runtime.puf_source_agi import ( - PUF_AGGREGATE_DISAGGREGATION_SEED, - PUF_AGGREGATE_RECIDS, - PUF_SOURCE_YEAR, - PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS, - PUF_SYNTHETIC_RECID_START, - source_year_puf_adjusted_gross_income, -) -from microcosm.build.us_runtime.puf_support import ( - BASE_ASEC_SUPPORT_CHANNEL, - PUF_DONOR_SOURCE_ADJUSTED_GROSS_INCOME_COLUMN, - PUF_TAX_DETAIL_CLONE_INDEX, - PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, - PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, - PUF_TAX_DETAIL_SUPPORT_CHANNEL, - US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING, - US_PUF_SUPPORT_FIT_NAME, - US_PUF_SUPPORT_STAGE_NAME, - PufTaxDetailChainInputs, - clone_us_frame_for_puf_support, - finalize_us_puf_tax_detail_predictions, - has_assembled_support_metadata, - has_support_role_metadata, - impute_us_puf_tax_detail_support, - prepare_us_puf_tax_detail_chain_inputs, - puf_tax_detail_clone_mask, - puf_tax_unit_donor_from_arrays, - spine_source_id_column, - support_channel_column, - support_clone_index_column, - support_role_series, - support_source_id_column, -) -from microcosm.build.us_runtime.puma_ladder import ( - PUMA_LADDER_ARTIFACT_SHA256_ATTR, - PUMA_LADDER_VINTAGES_ATTR, - US_PUMA_LADDER_COLUMNS, - US_PUMA_LADDER_DERIVED_LAYERS, - US_PUMA_LADDER_KIND, - US_PUMA_LADDER_SCHEMA_VERSION, - US_PUMA_LADDER_TRACT_COLUMN, - UsPumaLadder, - assign_us_puma_ladder, - load_us_puma_ladder, - us_puma_ladder_assignment_summary, - us_puma_ladder_gate, - with_household_us_puma_ladder, -) -from microcosm.build.us_runtime.puma_ladder_sources import ( - assemble_us_puma_ladder, - parse_tract_to_puma_relationship, -) -from microcosm.build.us_runtime.qbi_inputs import ( - QBI_ARCHIVED_ASSUMPTIONS_URL, - QBI_ARCHIVED_CLONE_URL, - QBI_ARCHIVED_DERIVATION_URL, - QBI_ARCHIVED_EXPORT_URL, - QBI_ARCHIVED_IMPUTATION_URL, - QBI_ARCHIVED_PUF_ARTIFACT_URL, - QBI_ARCHIVED_SIMULATION_URL, - US_QBI_BOOLEAN_OUTPUT_COLUMNS, - US_QBI_NONCONSTANT_PERSON_COLUMNS, - US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, - US_QBI_OUTPUT_COLUMNS, - US_QBI_STAGE_NAME, - us_qbi_inputs_signal_gate, - us_qbi_inputs_stage_spec, - us_qbi_inputs_summary, - with_us_qbi_input_reconciliation, -) -from microcosm.build.us_runtime.reform_coverage_smoke import ( - us_reform_coverage_smoke_gate, -) -from microcosm.build.us_runtime.reform_validation import ( - REFORM_VALIDATION_SCHEMA_VERSION, - ReformValidationSpec, - in_sample_reform_specs, - load_default_reform_specs, - out_of_sample_reform_specs, - reform_validation_payload, - write_reform_validation, -) -from microcosm.build.us_runtime.register_consistency import ( - us_register_consistency_gate, - us_register_contradictions, -) -from microcosm.build.us_runtime.relationship_inputs import ( - US_RELATIONSHIP_INPUTS_NONCONSTANT_PERSON_COLUMNS, - US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS, - US_RELATIONSHIP_INPUTS_REQUIRED_SOURCE_COLUMNS, - US_RELATIONSHIP_INPUTS_STAGE_NAME, - derive_us_relationship_inputs_from_manifest, - us_relationship_inputs_signal_gate, - us_relationship_inputs_stage_spec, - us_relationship_inputs_summary, - with_us_relationship_inputs, -) -from microcosm.build.us_runtime.release_input_coverage import ( - POST_REFERENCE_ECPS_REQUIRED_INPUTS, - SSI_COUNTABLE_RESOURCE_ASSETS, - US_CGD_ROUTE_REQUIRED_INPUTS, - US_RELEASE_INPUT_COVERAGE_RESOURCE, - ReformCoverageProbe, - ReleaseInputColumn, - ReleaseInputCoverageManifest, - assert_release_input_coverage_manifest_current, - load_release_input_coverage_manifest, - us_release_input_coverage_gate, - us_release_input_coverage_required_columns, - us_release_input_coverage_reviewed_exclusions, - us_release_reform_coverage_probes, -) -from microcosm.build.us_runtime.release_target_parity import ( - RED_LINE_COMPILED_FAMILIES, - US_TARGET_PARITY_FEED_FAMILIES_RESOURCE, - US_TARGET_PARITY_MANIFEST_RESOURCE, - TargetFamily, - TargetFence, - TargetParityManifest, - assert_target_parity_manifest_current, - load_target_parity_feed_families, - load_target_parity_manifest, - registry_target_family_ids, - us_release_target_parity_compiled_families, - us_release_target_parity_gate, - us_release_target_parity_reviewed_exclusions, - us_target_family_id, -) -from microcosm.build.us_runtime.retirement_contributions import ( - US_RETIREMENT_CONTRIBUTION_NONCONSTANT_PERSON_COLUMNS, - US_RETIREMENT_CONTRIBUTION_OUTPUT_COLUMNS, - US_RETIREMENT_CONTRIBUTION_REQUIRED_SOURCE_COLUMNS, - US_RETIREMENT_CONTRIBUTION_STAGE_NAME, - derive_us_retirement_contributions_from_manifest, - impute_us_retirement_contributions_to_puf_support_from_manifest, - us_retirement_contributions_signal_gate, - us_retirement_contributions_stage_spec, - us_retirement_contributions_summary, - with_us_retirement_contribution_inputs, -) -from microcosm.build.us_runtime.retirement_distributions import ( - RETIREMENT_DISTRIBUTIONS_ARCHIVED_DERIVATION_URL, - RETIREMENT_DISTRIBUTIONS_ARCHIVED_PARAMETERS_URL, - US_RETIREMENT_DISTRIBUTION_NONCONSTANT_PERSON_COLUMNS, - US_RETIREMENT_DISTRIBUTION_OUTPUT_COLUMNS, - US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS, - US_RETIREMENT_DISTRIBUTION_STAGE_NAME, - derive_us_retirement_distributions_from_manifest, - impute_us_retirement_distributions_to_puf_support_from_manifest, - us_retirement_distributions_signal_gate, - us_retirement_distributions_stage_spec, - us_retirement_distributions_summary, - with_us_retirement_distribution_inputs, -) -from microcosm.build.us_runtime.salt_refund_income import ( - SALT_REFUND_ARCHIVED_DERIVATION_URL, - SALT_REFUND_ARCHIVED_EXPORT_URL, - SALT_REFUND_ARCHIVED_IMPUTATION_URL, - SALT_REFUND_ARCHIVED_PERSON_ALLOCATION_URL, - SALT_REFUND_ARCHIVED_PUF_ARTIFACT_URL, - US_SALT_REFUND_NONCONSTANT_PERSON_COLUMNS, - US_SALT_REFUND_OUTPUT_COLUMNS, - US_SALT_REFUND_STAGE_NAME, - derive_us_salt_refund_income_from_puf, - us_salt_refund_income_signal_gate, - us_salt_refund_income_stage_spec, - us_salt_refund_income_summary, -) -from microcosm.build.us_runtime.scf_auto_loans import ( - QUALIFIED_AUTO_LOAN_ANNUAL_ISSUANCE_TARGET, - SCF_2022_FULL_EXTRACT_MEMBER, - SCF_2022_FULL_EXTRACT_MEMBER_SHA256, - SCF_2022_FULL_EXTRACT_URL, - SCF_2022_FULL_EXTRACT_ZIP_SHA256, - SCF_AUTO_LOAN_AMOUNT_COLUMNS, - SCF_AUTO_LOAN_RATE_COLUMNS, - US_SCF_AUTO_LOAN_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_SCF_AUTO_LOAN_OUTPUT_COLUMNS, - fetch_scf_2022_full_extract, - impute_us_scf_auto_loans, - load_scf_2022_auto_loan_donor, - qualified_auto_loan_interest_proxy, - us_scf_auto_loans_signal_gate, - us_scf_auto_loans_stage_spec, - us_scf_auto_loans_summary, - with_us_scf_auto_loan_inputs, -) -from microcosm.build.us_runtime.scf_wealth import ( - FINANCIAL_ASSET_BLEND_AUDIT_KEY, - FINANCIAL_ASSET_SOURCE_SCF_PROBABILITY, - SCF_FINANCIAL_ASSET_TARGET_COMPONENTS, - SCF_NET_WORTH_TARGET_COMPONENTS, - SCF_WEALTH_PREDICTORS, - US_SCF_FINANCIAL_ASSET_OUTPUT_COLUMNS, - US_SCF_NET_WORTH_OUTPUT_COLUMNS, - US_SCF_WEALTH_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_SCF_WEALTH_NONCONSTANT_PERSON_COLUMNS, - US_SCF_WEALTH_STAGE_NAME, - fetch_scf_2022_summary_extract, - financial_asset_source_is_scf, - impute_us_scf_financial_assets, - impute_us_scf_net_worth, - impute_us_sipp_scf_financial_assets, - load_scf_2022_financial_asset_donor, - us_scf_wealth_signal_gate, - us_scf_wealth_stage_spec, - us_scf_wealth_summary, - with_us_scf_wealth_inputs, -) -from microcosm.build.us_runtime.sipp_financial_assets import ( - SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_ID_PARTS, - SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_TYPE, - SIPP_2023_FINANCIAL_ASSET_DONOR_REVISION, - SIPP_2023_FINANCIAL_ASSET_DONOR_SHA256, - SIPP_2023_FINANCIAL_ASSET_DONOR_SIZE_BYTES, - SIPP_2023_FINANCIAL_ASSET_DONOR_URL, - SIPP_FINANCIAL_ASSET_DONOR_WEIGHT_COLUMN, - SIPP_FINANCIAL_ASSET_MODEL_PREDICTORS, - SIPP_FINANCIAL_ASSET_SOURCE_COLUMNS, - SIPP_FINANCIAL_ASSET_TARGET_ALLOCATION_COLUMNS, - SIPP_FINANCIAL_ASSET_TARGET_SOURCE_COLUMNS, - fetch_sipp_2023_financial_asset_donor, - impute_us_sipp_financial_assets, - load_sipp_2023_financial_asset_donor, -) -from microcosm.build.us_runtime.sipp_head_start import ( - HEAD_START_SIPP_DICTIONARY_URL, - SIPP_2023_HEAD_START_DONOR_REVISION, - SIPP_2023_HEAD_START_DONOR_SHA256, - SIPP_2023_HEAD_START_DONOR_SIZE_BYTES, - SIPP_2023_HEAD_START_DONOR_URL, - SIPP_HEAD_START_FIT_PARAMETERS, - SIPP_HEAD_START_MODEL_PREDICTORS, - SIPP_HEAD_START_READ_PARAMETERS, - SIPP_HEAD_START_SOURCE_COLUMNS, - US_SIPP_HEAD_START_NONCONSTANT_PERSON_COLUMNS, - US_SIPP_HEAD_START_OUTPUT_COLUMNS, - US_SIPP_HEAD_START_REQUIRED_SOURCE_COLUMNS, - US_SIPP_HEAD_START_STAGE_NAME, - fetch_sipp_2023_head_start_donor, - impute_us_sipp_head_start, - load_sipp_2023_head_start_donor, - us_sipp_head_start_signal_gate, - us_sipp_head_start_stage_spec, - us_sipp_head_start_summary, - with_us_sipp_head_start_input, -) -from microcosm.build.us_runtime.sipp_tips import ( - CENSUS_OCCUPATION_CODE_TO_TTOC, - SIPP_2023_TIP_DONOR_REVISION, - SIPP_2023_TIP_DONOR_SHA256, - SIPP_2023_TIP_DONOR_URL, - SIPP_TIP_OUTPUT_COLUMNS, - SIPP_TIP_PREDICTORS, - US_SIPP_TIPS_NONCONSTANT_PERSON_COLUMNS, - US_SIPP_TIPS_OUTPUT_COLUMNS, - US_SIPP_TIPS_REQUIRED_SOURCE_COLUMNS, - US_SIPP_TIPS_STAGE_NAME, - derive_treasury_tipped_occupation_code, - fetch_sipp_2023_tip_donor, - impute_us_sipp_tips, - load_sipp_2023_tip_donor, - us_sipp_tips_signal_gate, - us_sipp_tips_stage_spec, - us_sipp_tips_summary, - with_us_sipp_tip_inputs, -) -from microcosm.build.us_runtime.sipp_vehicles import ( - SIPP_2023_VEHICLE_DONOR_REVISION, - SIPP_2023_VEHICLE_DONOR_SHA256, - SIPP_2023_VEHICLE_DONOR_SIZE_BYTES, - SIPP_2023_VEHICLE_DONOR_URL, - US_SIPP_VEHICLE_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_SIPP_VEHICLE_OUTPUT_COLUMNS, - fetch_sipp_2023_vehicle_donor, - load_sipp_2023_vehicle_donor, - us_sipp_vehicles_signal_gate, - us_sipp_vehicles_stage_spec, - us_sipp_vehicles_summary, - with_us_sipp_vehicle_inputs, -) -from microcosm.build.us_runtime.snap_discretionary_exemption import ( - US_SNAP_DISCRETIONARY_EXEMPTION_NONCONSTANT_PERSON_COLUMNS, - US_SNAP_DISCRETIONARY_EXEMPTION_OUTPUT_COLUMN, - US_SNAP_DISCRETIONARY_EXEMPTION_REQUIRED_SOURCE_COLUMNS, - US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME, - derive_us_snap_discretionary_exemption_from_manifest, - us_snap_discretionary_exemption_signal_gate, - us_snap_discretionary_exemption_stage_spec, - us_snap_discretionary_exemption_summary, - with_us_snap_discretionary_exemption_inputs, -) -from microcosm.build.us_runtime.snap_state_take_up import ( - US_SNAP_CASELOAD_TOLERANCE, - US_SNAP_HOUSEHOLDS_TARGET_TABLE, - US_SNAP_STATE_TAKE_UP_ANCHOR, - US_SNAP_STATE_TAKE_UP_STAGE, - us_snap_state_take_up_diagnostics, - us_snap_state_take_up_gate, - with_us_snap_state_take_up, - write_us_snap_state_take_up_diagnostics, -) -from microcosm.build.us_runtime.snap_take_up import ( - US_SNAP_TAKE_UP_OUTPUT_COLUMN, - US_SNAP_TAKE_UP_RAW_COLUMN, - US_SNAP_TAKE_UP_STAGE_NAME, - derive_us_snap_take_up_from_manifest, - us_snap_take_up_signal_gate, - us_snap_take_up_stage_spec, - us_snap_take_up_summary, - with_us_snap_take_up_inputs, -) -from microcosm.build.us_runtime.source_coverage import ( - CHRONICLE_US_SOURCE_COVERAGE_CONTRACT_COMMIT, - LEDGER_US_SOURCE_COVERAGE_CONTRACT_COMMIT, - US_SOURCE_COVERAGE, - hard_target_package_aliases, - source_gap_family_ids, - us_source_coverage_diagnostics, - us_source_coverage_gate, - validation_only_family_ids, - write_us_source_coverage_diagnostics, -) -from microcosm.build.us_runtime.source_runtime import ( - disaggregate_us_puf_aggregate_records_from_manifest, - us_source_operation_handlers, -) -from microcosm.build.us_runtime.spine_agreement import ( - DEFAULT_CATEGORICAL_TOTAL_VARIATION_TOLERANCE, - DEFAULT_INCIDENCE_RATIO_BOUNDS, - DEFAULT_QUANTILE_ENVELOPE_TOLERANCE, - DEFAULT_SPINE_AGREEMENT_QUANTILES, - US_SPINE_AGREEMENT_REGISTRY, - SpineAgreementSpec, - default_spine_agreement_registry, - normalize_transfer_family_name, - spine_agreement_gate, - validate_spine_agreement_registry, -) -from microcosm.build.us_runtime.spine_assembly import assemble_spines -from microcosm.build.us_runtime.ssi_disability_criteria import ( - SIPP_2023_SSI_DISABILITY_DONOR_REVISION, - SIPP_2023_SSI_DISABILITY_DONOR_SHA256, - SIPP_2023_SSI_DISABILITY_DONOR_SIZE_BYTES, - SIPP_2023_SSI_DISABILITY_DONOR_URL, - SIPP_SSI_DISABILITY_DIFFICULTY_PREDICTORS, - SIPP_SSI_DISABILITY_FIT_PARAMETERS, - SIPP_SSI_DISABILITY_MODEL_PREDICTORS, - SIPP_SSI_DISABILITY_READ_PARAMETERS, - SIPP_SSI_DISABILITY_SOURCE_COLUMNS, - SSI_DISABILITY_ARCHIVED_CPS_URL, - SSI_DISABILITY_ARCHIVED_EXTENDED_CPS_URL, - SSI_DISABILITY_ARCHIVED_SIPP_URL, - SSI_DISABILITY_ARCHIVED_SOURCE_IMPUTE_URL, - SSI_DISABILITY_SIPP_DICTIONARY_URL, - US_SSI_DISABILITY_CRITERIA_NONCONSTANT_PERSON_COLUMNS, - US_SSI_DISABILITY_CRITERIA_OUTPUT_COLUMNS, - US_SSI_DISABILITY_CRITERIA_STAGE_NAME, - fetch_sipp_2023_ssi_disability_donor, - impute_us_ssi_disability_criteria, - load_sipp_2023_ssi_disability_donor, - us_ssi_disability_criteria_signal_gate, - us_ssi_disability_criteria_stage_spec, - us_ssi_disability_criteria_summary, - with_us_ssi_disability_criteria, -) -from microcosm.build.us_runtime.ssi_take_up import ( - SSI_TAKE_UP_ARCHIVED_DERIVATION_URL, - SSI_TAKE_UP_ARCHIVED_EXPORT_URL, - SSI_TAKE_UP_ARCHIVED_RANDOMNESS_URL, - SSI_TAKE_UP_ARCHIVED_REPORTER_URL, - SSI_TAKE_UP_ARCHIVED_TARGETS_URL, - SSI_TAKE_UP_SSA_SOURCE_URL, - US_SSI_TAKE_UP_ANCHOR, - US_SSI_TAKE_UP_BAND_DELIVERY_RELATIVE_TOLERANCE, - US_SSI_TAKE_UP_ENFORCED_BAND_KEYS, - US_SSI_TAKE_UP_NONCONSTANT_PERSON_COLUMNS, - US_SSI_TAKE_UP_OUTPUT_COLUMNS, - US_SSI_TAKE_UP_PHASE_ASSIGNMENT, - US_SSI_TAKE_UP_PHASE_RELEASE_FINAL, - US_SSI_TAKE_UP_PRIOR_BASIS_CURRENT_FRAME, - US_SSI_TAKE_UP_PRIOR_BASIS_RELEASE_ARTIFACT, - US_SSI_TAKE_UP_REQUIRED_SOURCE_COLUMNS, - US_SSI_TAKE_UP_STAGE_NAME, - US_SSI_TAKE_UP_TARGET_TABLE_NAME, - SSITakeUpBandPriorBasis, - SSITakeUpPriorBasis, - ssi_take_up_prior_basis_from_artifact, - ssi_take_up_prior_basis_from_diagnostics, - us_ssi_take_up_delivery_gate, - us_ssi_take_up_diagnostics, - us_ssi_take_up_gate, - us_ssi_take_up_reporter_source_ids, - with_us_ssi_take_up, - write_us_ssi_take_up_diagnostics, -) -from microcosm.build.us_runtime.support_provenance import ( - us_reported_coverage_vintage_signal_gate, -) -from microcosm.build.us_runtime.take_up import ( - US_TAKE_UP_SHARE_BAND, - SeededTakeUpResult, - us_take_up_participation_diagnostics, - us_take_up_signal_gate, - us_take_up_summary, - with_us_take_up_inputs, - write_us_take_up_participation_diagnostics, -) -from microcosm.build.us_runtime.take_up_contract import ( - TakeUpContract, - TakeUpProgram, - assert_take_up_contract_current, - assert_take_up_treatments_consistent, - count_calibrated_take_up_programs, - load_take_up_contract, - seeded_take_up_programs, -) -from microcosm.build.us_runtime.validation_input_coverage import ( - US_VALIDATION_PROVISION_INPUT_LEAVES, - ValidationInputLeaf, - assert_validation_leaf_registry_current, - us_source_stage_outputs, - us_validation_input_coverage_gate, -) -from microcosm.build.us_runtime.voluntary_filing import ( - SIPP_2023_VOLUNTARY_FILING_DONOR_REVISION, - SIPP_2023_VOLUNTARY_FILING_DONOR_SHA256, - SIPP_2023_VOLUNTARY_FILING_DONOR_SIZE_BYTES, - SIPP_2023_VOLUNTARY_FILING_DONOR_URL, - SIPP_VOLUNTARY_FILING_MODEL_PREDICTORS, - SIPP_VOLUNTARY_FILING_SOURCE_COLUMNS, - US_VOLUNTARY_FILING_NONCONSTANT_TAX_UNIT_COLUMNS, - US_VOLUNTARY_FILING_OUTPUT_COLUMNS, - US_VOLUNTARY_FILING_STAGE_NAME, - VOLUNTARY_FILING_ARCHIVED_DERIVATION_URL, - VOLUNTARY_FILING_ARCHIVED_PARAMETERS_URL, - VOLUNTARY_FILING_SIPP_DICTIONARY_URL, - fetch_sipp_2023_voluntary_filing_donor, - impute_us_voluntary_filing, - load_sipp_2023_voluntary_filing_donor, - us_voluntary_filing_signal_gate, - us_voluntary_filing_stage_spec, - us_voluntary_filing_summary, - with_us_voluntary_filing_input, -) -from microcosm.build.us_runtime.weeks_unemployed import ( - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER, - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_CRC32, - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SHA256, - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SIZE_BYTES, - ASEC_2023_WEEKS_UNEMPLOYED_RAW_ROWS, - ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_COLUMNS, - ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256, - ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_YEAR, - ASEC_2023_WEEKS_UNEMPLOYED_UNIQUE_KEYS, - ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_SOURCE_SHARE, - ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_WEEKS, - ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SHA256, - ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SIZE_BYTES, - ASEC_2023_WEEKS_UNEMPLOYED_ZIP_URL, - US_WEEKS_UNEMPLOYED_NONCONSTANT_PERSON_COLUMNS, - US_WEEKS_UNEMPLOYED_OUTPUT_COLUMNS, - US_WEEKS_UNEMPLOYED_REQUIRED_SOURCE_COLUMNS, - US_WEEKS_UNEMPLOYED_STAGE_NAME, - WEEKS_UNEMPLOYED_ARCHIVED_DERIVATION_URL, - WEEKS_UNEMPLOYED_ARCHIVED_PUF_IMPUTATION_URL, - WEEKS_UNEMPLOYED_ARCHIVED_SOURCE_URL, - WEEKS_UNEMPLOYED_DERIVE_PARAMETERS, - WEEKS_UNEMPLOYED_PUF_IMPUTATION_PARAMETERS, - WEEKS_UNEMPLOYED_PUF_PREDICTORS, - WEEKS_UNEMPLOYED_READ_PARAMETERS, - derive_us_weeks_unemployed_from_manifest, - fetch_asec_2023_weeks_unemployed_source, - fill_asec_2022_weeks_unemployed_source, - impute_us_weeks_unemployed_to_puf_support_from_manifest, - load_asec_2023_weeks_unemployed_source, - us_weeks_unemployed_signal_gate, - us_weeks_unemployed_stage_spec, - us_weeks_unemployed_summary, - with_us_weeks_unemployed, -) -from microcosm.build.us_runtime.wic_claim import ( - US_WIC_CLAIM_NONCONSTANT_PERSON_COLUMNS, - US_WIC_CLAIM_OUTPUT_COLUMNS, - US_WIC_CLAIM_REQUIRED_SOURCE_COLUMNS, - US_WIC_CLAIM_STAGE_NAME, - WIC_CLAIM_ARCHIVED_DERIVATION_URL, - WIC_CLAIM_ARCHIVED_PARAMETERS_URL, - WIC_CLAIM_ARCHIVED_RANDOMNESS_URL, - WIC_CLAIM_FNS_SOURCE_URL, - derive_us_wic_claim_from_manifest, - us_wic_claim_signal_gate, - us_wic_claim_stage_spec, - us_wic_claim_summary, - with_us_wic_claim_input, -) -from microcosm.build.us_runtime.workers_compensation import ( - US_WORKERS_COMPENSATION_NONCONSTANT_PERSON_COLUMNS, - US_WORKERS_COMPENSATION_OUTPUT_COLUMNS, - US_WORKERS_COMPENSATION_REQUIRED_SOURCE_COLUMNS, - US_WORKERS_COMPENSATION_STAGE_NAME, - WORKERS_COMPENSATION_ARCHIVED_DERIVATION_URL, - WORKERS_COMPENSATION_ARCHIVED_PUF_IMPUTATION_URL, - WORKERS_COMPENSATION_ARCHIVED_PUF_OUTPUTS_URL, - WORKERS_COMPENSATION_ARCHIVED_SOURCE_COLUMNS_URL, - derive_us_workers_compensation_from_asec, - derive_us_workers_compensation_from_manifest, - impute_us_workers_compensation_to_puf_support_from_manifest, - us_workers_compensation_signal_gate, - us_workers_compensation_stage_spec, - us_workers_compensation_summary, - with_us_workers_compensation, -) from microcosm.frame import Frame __all__ = [ "ASEC_RAW_STAGE_ARTIFACT_KIND", "ASEC_RAW_STAGE_CHECKPOINT_FILENAME", + "ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION", "ASEC_RAW_STAGE_OPERATOR_STATUS", "ASEC_RAW_STAGE_SCHEMA_VERSION", "ASEC_RAW_STAGE_STAGE", @@ -1875,6 +828,7 @@ "load_default_congressional_district_vintage_crosswalk", "load_asec_pre_clone_checkpoint", "load_asec_raw_stage_checkpoint", + "load_asec_raw_stage_checkpoint_v4", "load_puf_tax_unit_donor", "normalize_district_code", "parse_baf_cd_layer", @@ -1990,9 +944,1067 @@ "parse_tract_to_puma_relationship", "us_puma_ladder_assignment_summary", "us_puma_ladder_gate", + "us_puma_ladder_joint_support_gate", "with_household_us_puma_ladder", ] +# Explicit compatibility targets: no discovery or source parsing at runtime. +_EXPORT_MODULES = { + "microcosm.build.us_runtime.adult_care": ( + "US_ADULT_CARE_CHILD_QUALIFYING_AGE_LIMIT", + "US_ADULT_CARE_EARNED_INCOME_SOURCES", + "US_ADULT_CARE_OUTPUT_COLUMNS", + "US_ADULT_CARE_REQUIRED_SOURCE_COLUMNS", + "US_ADULT_CARE_STAGE_NAME", + "derive_us_adult_care_from_manifest", + "us_adult_care_signal_gate", + "us_adult_care_stage_spec", + "us_adult_care_summary", + "with_us_adult_care_inputs", + ), + "microcosm.build.us_runtime.alimony": ( + "ALIMONY_ASEC_ARCHIVED_DERIVATION_URL", + "ALIMONY_PUF_ARCHIVED_DERIVATION_URL", + "STRIKE_BENEFITS_ASEC_ARCHIVED_DERIVATION_URL", + "US_ALIMONY_NONCONSTANT_PERSON_COLUMNS", + "US_ALIMONY_OUTPUT_COLUMNS", + "US_ALIMONY_STAGE_NAME", + "US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS", + "derive_us_alimony_from_asec", + "derive_us_alimony_from_puf", + "us_alimony_signal_gate", + "us_alimony_stage_spec", + "us_alimony_summary", + ), + "microcosm.build.us_runtime.asec_checkpoint": ( + "ASEC_RAW_STAGE_ARTIFACT_KIND", + "ASEC_RAW_STAGE_CHECKPOINT_FILENAME", + "ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION", + "ASEC_RAW_STAGE_OPERATOR_STATUS", + "ASEC_RAW_STAGE_SCHEMA_VERSION", + "ASEC_RAW_STAGE_STAGE", + "load_asec_pre_clone_checkpoint", + "load_asec_raw_stage_checkpoint", + "load_asec_raw_stage_checkpoint_v4", + ), + "microcosm.build.us_runtime.asec_pool": ( + "AsecSource", + "build_pooled_asec_unit_frame", + "load_asec_h5_tables", + "pool_asec_sources", + ), + "microcosm.build.us_runtime.capital_gain_details": ( + "CAPITAL_GAIN_DETAILS_ARCHIVED_DERIVATION_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_EXPORT_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_IMPUTATION_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_PERSON_ALLOCATION_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_PUF_ARTIFACT_URL", + "US_CAPITAL_GAIN_DETAILS_NONCONSTANT_PERSON_COLUMNS", + "US_CAPITAL_GAIN_DETAILS_NONCONSTANT_TAX_UNIT_COLUMNS", + "US_CAPITAL_GAIN_DETAILS_OUTPUT_COLUMNS", + "US_CAPITAL_GAIN_DETAILS_STAGE_NAME", + "derive_us_capital_gain_details_from_puf", + "us_capital_gain_details_signal_gate", + "us_capital_gain_details_stage_spec", + "us_capital_gain_details_summary", + ), + "microcosm.build.us_runtime.casualty_losses": ( + "US_CASUALTY_LOSS_NONCONSTANT_PERSON_COLUMNS", + "US_CASUALTY_LOSS_OUTPUT_COLUMNS", + "US_CASUALTY_LOSS_STAGE_NAME", + "derive_us_casualty_loss_from_puf", + "us_casualty_loss_signal_gate", + "us_casualty_loss_stage_spec", + "us_casualty_loss_summary", + ), + "microcosm.build.us_runtime.child_support": ( + "CHILD_SUPPORT_ARCHIVED_PUF_IMPUTATION_URL", + "CHILD_SUPPORT_ARCHIVED_PUF_OUTPUTS_URL", + "CHILD_SUPPORT_EXPENSE_ARCHIVED_DERIVATION_URL", + "CHILD_SUPPORT_RECEIVED_ARCHIVED_DERIVATION_URL", + "US_CHILD_SUPPORT_NONCONSTANT_PERSON_COLUMNS", + "US_CHILD_SUPPORT_OUTPUT_COLUMNS", + "US_CHILD_SUPPORT_REQUIRED_SOURCE_COLUMNS", + "US_CHILD_SUPPORT_STAGE_NAME", + "derive_us_child_support_from_asec", + "derive_us_child_support_from_manifest", + "impute_us_child_support_to_puf_support_from_manifest", + "us_child_support_signal_gate", + "us_child_support_stage_spec", + "us_child_support_summary", + "with_us_child_support_inputs", + ), + "microcosm.build.us_runtime.childcare": ( + "US_CHILDCARE_OUTPUT_COLUMNS", + "US_CHILDCARE_REQUIRED_SOURCE_COLUMNS", + "US_CHILDCARE_STAGE_NAME", + "derive_us_childcare_from_manifest", + "impute_us_childcare_to_puf_support_from_manifest", + "us_childcare_signal_gate", + "us_childcare_stage_spec", + "us_childcare_summary", + "with_us_childcare_inputs", + ), + "microcosm.build.us_runtime.congressional_district_geography": ( + "CONGRESSIONAL_DISTRICT_GEOID_COLUMN", + "SOI_CONGRESSIONAL_DISTRICT_RECORD_SET_ID", + "assign_congressional_districts_to_households", + "congressional_district_assignment_summary", + "congressional_district_distribution_from_ledger_facts", + "with_household_congressional_districts", + ), + "microcosm.build.us_runtime.congressional_district_vintage": ( + "CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR", + "CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR", + "CURRENT_CONGRESSIONAL_DISTRICT_PREFIX", + "CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE", + "DEFAULT_CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_RESOURCE", + "SOURCE_CONGRESSIONAL_DISTRICT_PREFIX", + "default_congressional_district_vintage_crosswalk_path", + "load_congressional_district_vintage_crosswalk", + "load_default_congressional_district_vintage_crosswalk", + "translate_congressional_district_facts_to_current_vintage", + ), + "microcosm.build.us_runtime.congressional_district_vintage_crosswalk": ( + "CROSSWALK_BASIS_BLOCK_POPULATION", + "build_cd_vintage_crosswalk_rows", + "normalize_district_code", + "parse_baf_cd_layer", + "parse_national_cd_bef_districts", + ), + "microcosm.build.us_runtime.cps_carried": ( + "CPS_CARRIED_FORMULA_OWNED_COLUMNS", + "CPS_CARRIED_PERSON_INPUTS", + "CPS_CARRIED_SPM_UNIT_INPUTS", + "CPS_REPORTED_TANF_AMOUNT_RAW_COLUMN", + "CPS_REPORTED_TANF_TYPE_RAW_COLUMN", + "CPS_REPORTED_WIC_RAW_COLUMN", + "US_REPORTED_COVERAGE_PERSON_INPUTS", + "US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS", + "WIC_CARRIER_ADJUDICATION_URL", + "derive_us_cps_carried_inputs", + "reported_tanf_enrollment_by_spm_unit", + "reported_wic_receipt_carrier", + ), + "microcosm.build.us_runtime.demographics": ( + "AGE_BANDS", + "DEMOGRAPHICS_SCHEMA_VERSION", + "AgeBand", + "compute_age_distribution", + "demographics_payload", + "write_demographics", + ), + "microcosm.build.us_runtime.disability_benefits": ( + "DISABILITY_BENEFITS_ARCHIVED_DERIVATION_URL", + "DISABILITY_BENEFITS_ARCHIVED_PUF_IMPUTATION_URL", + "DISABILITY_BENEFITS_ARCHIVED_PUF_OUTPUTS_URL", + "DISABILITY_BENEFITS_ARCHIVED_SOURCE_COLUMNS_URL", + "US_DISABILITY_BENEFITS_NONCONSTANT_PERSON_COLUMNS", + "US_DISABILITY_BENEFITS_OUTPUT_COLUMNS", + "US_DISABILITY_BENEFITS_REQUIRED_SOURCE_COLUMNS", + "US_DISABILITY_BENEFITS_STAGE_NAME", + "derive_us_disability_benefits_from_asec", + "derive_us_disability_benefits_from_manifest", + "impute_us_disability_benefits_to_puf_support_from_manifest", + "us_disability_benefits_signal_gate", + "us_disability_benefits_stage_spec", + "us_disability_benefits_summary", + "with_us_disability_benefits", + ), + "microcosm.build.us_runtime.domestic_production": ( + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_DERIVATION_URL", + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_EXPORT_URL", + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_IMPUTATION_URL", + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_PUF_ARTIFACT_URL", + "US_DOMESTIC_PRODUCTION_ALD_NONCONSTANT_TAX_UNIT_COLUMNS", + "US_DOMESTIC_PRODUCTION_ALD_OUTPUT_COLUMNS", + "US_DOMESTIC_PRODUCTION_ALD_STAGE_NAME", + "derive_us_domestic_production_ald_from_puf", + "us_domestic_production_ald_signal_gate", + "us_domestic_production_ald_stage_spec", + "us_domestic_production_ald_summary", + ), + "microcosm.build.us_runtime.education_assistance_source": ( + "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", + "ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS", + "fetch_asec_education_assistance_source", + "fill_asec_education_assistance_source", + "load_asec_education_assistance_sources", + ), + "microcosm.build.us_runtime.education_inputs": ( + "US_AOTC_ELIGIBILITY_OUTPUT_COLUMNS", + "US_EDUCATION_INPUTS_NONCONSTANT_PERSON_COLUMNS", + "US_EDUCATION_INPUTS_OUTPUT_COLUMNS", + "US_EDUCATION_INPUTS_OWNED_OUTPUT_COLUMNS", + "US_EDUCATION_INPUTS_REQUIRED_SOURCE_COLUMNS", + "US_EDUCATION_INPUTS_STAGE_NAME", + "derive_us_education_inputs_from_manifest", + "us_education_inputs_signal_gate", + "us_education_inputs_stage_spec", + "us_education_inputs_summary", + "with_us_education_inputs", + ), + "microcosm.build.us_runtime.educator_expenses": ( + "EDUCATOR_EXPENSE_ARCHIVED_ALLOCATION_URL", + "EDUCATOR_EXPENSE_ARCHIVED_DERIVATION_URL", + "EDUCATOR_EXPENSE_ARCHIVED_EXPORT_URL", + "EDUCATOR_EXPENSE_ARCHIVED_PUF_IMPUTATION_URL", + "US_EDUCATOR_EXPENSE_NONCONSTANT_PERSON_COLUMNS", + "US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS", + "US_EDUCATOR_EXPENSE_STAGE_NAME", + "derive_us_educator_expense_from_puf", + "us_educator_expense_signal_gate", + "us_educator_expense_stage_spec", + "us_educator_expense_summary", + ), + "microcosm.build.us_runtime.eligibility_inputs": ( + "US_ELIGIBILITY_INPUTS_NONCONSTANT_PERSON_COLUMNS", + "US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS", + "US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS", + "US_ELIGIBILITY_INPUTS_STAGE_NAME", + "derive_us_eligibility_inputs_from_manifest", + "us_eligibility_inputs_signal_gate", + "us_eligibility_inputs_stage_spec", + "us_eligibility_inputs_summary", + "with_us_eligibility_inputs", + ), + "microcosm.build.us_runtime.energy_subsidy": ( + "ENERGY_SUBSIDY_ARCHIVED_CPS_DERIVATION_URL", + "ENERGY_SUBSIDY_ARCHIVED_PUF_IMPUTATION_URL", + "US_ENERGY_SUBSIDY_OUTPUT_COLUMNS", + "US_ENERGY_SUBSIDY_REQUIRED_SOURCE_COLUMNS", + "US_ENERGY_SUBSIDY_STAGE_NAME", + "derive_us_energy_subsidy_from_manifest", + "impute_us_energy_subsidy_to_puf_support_from_manifest", + "us_energy_subsidy_signal_gate", + "us_energy_subsidy_stage_spec", + "us_energy_subsidy_summary", + "with_us_energy_subsidy_input", + ), + "microcosm.build.us_runtime.farm_business_income": ( + "FARM_BUSINESS_INCOME_ARCHIVED_CPS_FARM_INCOME_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_DERIVATION_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_EXPORT_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_IMPUTATION_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_OVERRIDE_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_PUF_ARTIFACT_URL", + "US_FARM_BUSINESS_INCOME_NONCONSTANT_PERSON_COLUMNS", + "US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS", + "US_FARM_BUSINESS_INCOME_STAGE_NAME", + "derive_us_farm_business_income_from_puf", + "us_farm_business_income_signal_gate", + "us_farm_business_income_stage_spec", + "us_farm_business_income_summary", + ), + "microcosm.build.us_runtime.fiscal_targets": ( + "SOI_VARIABLE_MAP", + "US_FISCAL_LEDGER_PARITY_REGISTRY", + "US_FISCAL_LEDGER_PARITY_REPORT", + "US_FISCAL_MACRO_REALISM_BANDS", + "US_FISCAL_TARGET_COVERAGE_REQUIREMENTS", + "US_FISCAL_TARGET_LEDGER_REFERENCES", + "US_FISCAL_TARGET_REFERENCES", + "US_FISCAL_TARGET_REGISTRY", + "US_FISCAL_TARGET_SPECS", + "US_FISCAL_TARGET_SUPPORT_EXCLUSIONS", + "US_JCT_TAX_EXPENDITURE_REFORMS", + "US_JCT_TAX_EXPENDITURE_TARGET_REFERENCES", + "US_JCT_TAX_EXPENDITURE_TARGET_SPECS", + "US_SOI_FISCAL_TARGET_REFERENCES", + "US_SOI_FISCAL_TARGET_SPECS", + "US_STATE_INCOME_TAX_TARGET_REFERENCES", + "US_STATE_INCOME_TAX_TARGET_SPECS", + "SimpleTaxExpenditureReform", + "compile_us_fiscal_target_registry", + ), + "microcosm.build.us_runtime.form_4952": ( + "FORM_4952_ARCHIVED_DERIVATION_URL", + "FORM_4952_ARCHIVED_EXPORT_URL", + "FORM_4952_ARCHIVED_IMPUTATION_URL", + "FORM_4952_ARCHIVED_PERSON_ALLOCATION_URL", + "FORM_4952_ARCHIVED_PUF_ARTIFACT_URL", + "US_FORM_4952_NONCONSTANT_PERSON_COLUMNS", + "US_FORM_4952_OUTPUT_COLUMNS", + "US_FORM_4952_STAGE_NAME", + "derive_us_form_4952_election_from_puf", + "us_form_4952_election_signal_gate", + "us_form_4952_election_stage_spec", + "us_form_4952_election_summary", + ), + "microcosm.build.us_runtime.geography_ladder": ( + "GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR", + "GEOGRAPHY_LADDER_VINTAGES_ATTR", + "US_BLOCK_LADDER_DERIVED_LAYERS", + "US_BLOCK_LADDER_KIND", + "US_BLOCK_LADDER_SCHEMA_VERSION", + "US_GEOGRAPHY_LADDER_COLUMNS", + "US_NYC_COUNTY_FIPS", + "UsBlockLadder", + "assign_us_geography_ladder", + "load_us_block_ladder", + "us_geography_ladder_assignment_summary", + "us_geography_ladder_gate", + "with_household_us_geography_ladder", + ), + "microcosm.build.us_runtime.hours_worked": ( + "US_HOURS_WORKED_NONCONSTANT_PERSON_COLUMNS", + "US_HOURS_WORKED_OUTPUT_COLUMNS", + "US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS", + "US_HOURS_WORKED_POOL_OUTPUT_COLUMNS", + "US_HOURS_WORKED_REQUIRED_SOURCE_COLUMNS", + "US_HOURS_WORKED_STAGE_NAME", + "derive_us_hours_worked_from_manifest", + "us_hours_worked_signal_gate", + "us_hours_worked_stage_spec", + "us_hours_worked_summary", + "with_us_hours_worked_inputs", + ), + "microcosm.build.us_runtime.housing_inputs": ( + "ACS_2022_RENT_ARTIFACT_SHA256", + "HOUSING_INPUTS_ARCHIVED_ACS_DERIVATION_URL", + "HOUSING_INPUTS_ARCHIVED_CPS_RENT_URL", + "HOUSING_INPUTS_ARCHIVED_CPS_SPM_URL", + "HOUSING_INPUTS_ARCHIVED_PUF_IMPUTATION_URL", + "HOUSING_TAKE_UP_ARCHIVED_DERIVATION_URL", + "HOUSING_TAKE_UP_ARCHIVED_HUD_ETL_URL", + "HOUSING_TAKE_UP_ARCHIVED_PARAMETER_URL", + "US_HOUSING_HOUSEHOLD_OUTPUT_COLUMNS", + "US_HOUSING_INPUTS_OUTPUT_COLUMNS", + "US_HOUSING_INPUTS_STAGE_NAME", + "US_HOUSING_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_HOUSING_NONCONSTANT_PERSON_COLUMNS", + "US_HOUSING_NONCONSTANT_SPM_UNIT_COLUMNS", + "US_HOUSING_PERSON_OUTPUT_COLUMNS", + "US_HOUSING_REQUIRED_HOUSEHOLD_SOURCE_COLUMNS", + "US_HOUSING_REQUIRED_PERSON_SOURCE_COLUMNS", + "US_HOUSING_SPM_UNIT_OUTPUT_COLUMNS", + "derive_us_housing_inputs", + "impute_us_housing_assistance_to_puf_support", + "impute_us_pre_subsidy_rent", + "load_acs_2022_rent_donor", + "us_housing_inputs_signal_gate", + "us_housing_inputs_stage_spec", + "us_housing_inputs_summary", + "with_us_housing_inputs", + ), + "microcosm.build.us_runtime.immigration": ( + "IMMIGRATION_STATUS_VALUES", + "SSN_CARD_TYPE_VALUES", + "US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS", + "US_IMMIGRATION_OUTPUT_COLUMNS", + "US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS", + "US_IMMIGRATION_STAGE_NAME", + "UndocumentedControls", + "derive_us_immigration_status_from_manifest", + "us_immigration_composition_gate", + "us_immigration_composition_summary", + "us_immigration_stage_spec", + "with_us_immigration_inputs", + ), + "microcosm.build.us_runtime.input_mass": ("us_input_mass_totals",), + "microcosm.build.us_runtime.medicaid_take_up": ( + "US_MEDICAID_ENROLLMENT_SUBSTITUTIONS", + "US_MEDICAID_ENROLLMENT_TARGET_ROLE", + "US_MEDICAID_ENROLLMENT_TARGET_TABLE", + "US_MEDICAID_ENROLLMENT_TOLERANCE", + "US_MEDICAID_TAKE_UP_ANCHOR", + "US_MEDICAID_TAKE_UP_STAGE", + "US_MEDICAID_TAKE_UP_VARIABLE", + "MedicaidEnrollmentSubstitution", + "apply_us_medicaid_enrollment_substitutions", + "us_medicaid_source_person_table", + "us_medicaid_take_up_diagnostics", + "us_medicaid_take_up_gate", + "with_us_medicaid_take_up", + "write_us_medicaid_take_up_diagnostics", + ), + "microcosm.build.us_runtime.medicare_take_up": ( + "MEDICARE_TAKE_UP_ARCHIVED_CLONE_URL", + "MEDICARE_TAKE_UP_ARCHIVED_DERIVATION_URL", + "MEDICARE_TAKE_UP_ARCHIVED_EXPORT_URL", + "MEDICARE_TAKE_UP_ARCHIVED_SOURCE_COLUMNS_URL", + "US_MEDICARE_TAKE_UP_NONCONSTANT_PERSON_COLUMNS", + "US_MEDICARE_TAKE_UP_OUTPUT_COLUMNS", + "US_MEDICARE_TAKE_UP_REQUIRED_SOURCE_COLUMNS", + "US_MEDICARE_TAKE_UP_STAGE_NAME", + "derive_us_medicare_take_up_from_manifest", + "us_medicare_take_up_signal_gate", + "us_medicare_take_up_stage_spec", + "us_medicare_take_up_summary", + "with_us_medicare_take_up_input", + ), + "microcosm.build.us_runtime.misc_itemized": ( + "US_MISC_ITEMIZED_NONCONSTANT_PERSON_COLUMNS", + "US_MISC_ITEMIZED_OUTPUT_COLUMNS", + "US_MISC_ITEMIZED_STAGE_NAME", + "derive_us_misc_itemized_from_puf", + "us_misc_itemized_signal_gate", + "us_misc_itemized_stage_spec", + "us_misc_itemized_summary", + ), + "microcosm.build.us_runtime.nonzero_shares": ("nonzero_share", "us_nonzero_shares"), + "microcosm.build.us_runtime.operator_boundary": ( + "FORMULA_OWNED_SOURCE_COLUMNS", + "PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES", + "assert_operator_free_source_frame", + ), + "microcosm.build.us_runtime.org_wages": ( + "BLS_STATE_UNION_REPRESENTATION_RATE_2024", + "FLSA_EXECUTIVE_ADMINISTRATIVE_PROFESSIONAL_OCCUPATION_CODES", + "FLSA_OVERTIME_OCCUPATION_CODES", + "ORG_2024_DONOR_CONTENT_SHA256", + "ORG_2024_DONOR_FILENAME", + "ORG_PREDICTORS", + "US_ORG_WAGES_NONCONSTANT_PERSON_COLUMNS", + "US_ORG_WAGES_OUTPUT_COLUMNS", + "US_ORG_WAGES_REQUIRED_SOURCE_COLUMNS", + "US_ORG_WAGES_STAGE_NAME", + "derive_flsa_overtime_premium", + "derive_us_org_occupation_inputs", + "fetch_org_2024_donor", + "impute_us_org_wages", + "load_org_2024_donor", + "us_org_wages_signal_gate", + "us_org_wages_stage_spec", + "us_org_wages_summary", + "with_us_org_wages_inputs", + ), + "microcosm.build.us_runtime.other_health_insurance": ( + "OTHER_HEALTH_INSURANCE_ARCHIVED_DERIVATION_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_IMPUTATION_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_OUTPUTS_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_PREDICTORS_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_SPLICE_URL", + "US_OTHER_HEALTH_INSURANCE_MODELED_PREMIUM_VARIABLES", + "US_OTHER_HEALTH_INSURANCE_NONCONSTANT_PERSON_COLUMNS", + "US_OTHER_HEALTH_INSURANCE_OUTPUT_COLUMNS", + "US_OTHER_HEALTH_INSURANCE_REQUIRED_SOURCE_COLUMNS", + "US_OTHER_HEALTH_INSURANCE_STAGE_NAME", + "US_OTHER_HEALTH_INSURANCE_STAGE_OUTPUT_COLUMNS", + "US_SE_HEALTH_ATTRIBUTION_OUTPUT_COLUMNS", + "US_SE_HEALTH_MEDICARE_AGE_THRESHOLD", + "US_SE_HEALTH_SELF_EMPLOYMENT_INCOME_SOURCES", + "attribute_us_se_health_premiums", + "attribute_us_se_health_premiums_from_manifest", + "derive_us_other_health_insurance_from_asec", + "derive_us_other_health_insurance_from_manifest", + "impute_us_other_health_insurance_to_puf_support_from_manifest", + "us_other_health_insurance_signal_gate", + "us_other_health_insurance_stage_spec", + "us_other_health_insurance_summary", + "with_us_other_health_insurance_inputs", + ), + "microcosm.build.us_runtime.parity_reference": ( + "ECPS_PARITY_KNOWN_GAPS_RESOURCE", + "ECPS_PARITY_REFERENCE_RESOURCE", + "EcpsParityReference", + "EcpsParitySource", + "ParityKnownGap", + "load_ecps_parity_known_gaps", + "load_ecps_parity_reference", + ), + "microcosm.build.us_runtime.pregnancy": ( + "US_PREGNANCY_NONCONSTANT_PERSON_COLUMNS", + "US_PREGNANCY_OUTPUT_COLUMN", + "US_PREGNANCY_REQUIRED_SOURCE_COLUMNS", + "US_PREGNANCY_STAGE_NAME", + "derive_us_pregnancy_from_manifest", + "us_pregnancy_signal_gate", + "us_pregnancy_stage_spec", + "us_pregnancy_summary", + "with_us_pregnancy_inputs", + ), + "microcosm.build.us_runtime.prior_year_income": ( + "PRIOR_YEAR_INCOME_ARCHIVED_DERIVATION_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_FINALIZER_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_FORMULA_OUTPUT_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_PUF_IMPUTATION_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_PUF_OUTPUTS_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_PUF_SPLICE_URL", + "US_PRIOR_YEAR_INCOME_FORMULA_OWNED_OUTPUT_COLUMNS", + "US_PRIOR_YEAR_INCOME_NONCONSTANT_PERSON_COLUMNS", + "US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS", + "US_PRIOR_YEAR_INCOME_PERSISTED_OUTPUT_COLUMNS", + "US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS", + "US_PRIOR_YEAR_INCOME_STAGE_NAME", + "derive_us_prior_year_income_from_manifest", + "impute_us_prior_year_income_to_puf_support_from_manifest", + "us_prior_year_income_signal_gate", + "us_prior_year_income_source_reconciliation_gate", + "us_prior_year_income_stage_spec", + "us_prior_year_income_summary", + "with_us_prior_year_income_inputs", + ), + "microcosm.build.us_runtime.public_assistance_type_source": ( + "ASEC_PUBLIC_ASSISTANCE_TYPE_AUDIT_PINS", + "ASEC_PUBLIC_ASSISTANCE_TYPE_INCOME_YEARS", + "PAW_TYPE_TANF_CODES", + "PAW_TYPE_VALID_CODES", + "fill_asec_public_assistance_type_source", + "load_asec_public_assistance_type_sources", + ), + "microcosm.build.us_runtime.operator_column_contracts": ( + "PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS", + "PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS", + "PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN", + "PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS", + "PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS", + "US_PUF_SUPPORT_STAGE_NAME", + "US_QBI_BOOLEAN_OUTPUT_COLUMNS", + "US_QBI_NONNEGATIVE_OUTPUT_COLUMNS", + "US_QBI_OUTPUT_COLUMNS", + ), + "microcosm.build.us_runtime.puf_capital_gains_tail": ( + "PUF_CAPITAL_GAINS_TAIL_MANIFEST_SCHEMA_VERSION", + "PUF_CAPITAL_GAINS_TAIL_POSITIVE_MASS_FIVE_X_TARGET", + "PUF_CAPITAL_GAINS_TAIL_QUANTILE", + "PUF_CAPITAL_GAINS_TAIL_STAGE_NAME", + "PUF_CAPITAL_GAINS_TAIL_SUPPORT_CHANNEL", + "PUF_CAPITAL_GAINS_TAIL_SUPPORT_CONTRACT_VERSION", + "assert_puf_capital_gains_tail_survives_selection", + "puf_capital_gains_tail_concentration_gate", + "puf_capital_gains_tail_support_contract_identity", + "puf_capital_gains_tail_terminal_support_receipt", + "select_puf_capital_gains_tail_donors", + "transfer_puf_capital_gains_tail", + "validate_puf_capital_gains_tail_manifest", + "validate_puf_capital_gains_tail_terminal_support_receipt", + "write_puf_capital_gains_tail_manifest", + ), + "microcosm.build.us_runtime.puf_donor_io": ("load_puf_tax_unit_donor",), + "microcosm.build.us_runtime.puf_e01000_reconciliation": ( + "PUF_E01000_RECONCILIATION_SCHEMA_VERSION", + "build_puf_e01000_reconciliation_basis", + "finalize_puf_e01000_reconciliation", + "puf_capital_gains_joint_metrics", + "puf_processed_capital_gains_stage", + "puf_raw_e01000_stage", + ), + "microcosm.build.us_runtime.puf_interest_components": ( + "US_PUF_E19200_AGI_BANDS", + "US_PUF_E19200_ALL_RETURNS_COMPONENTS", + "PufE19200AgiBand", + "PufE19200InterestComponents", + "split_us_puf_e19200_by_agi_band", + ), + "microcosm.build.us_runtime.puf_source_agi": ( + "PUF_AGGREGATE_DISAGGREGATION_SEED", + "PUF_AGGREGATE_RECIDS", + "PUF_SOURCE_YEAR", + "PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS", + "PUF_SYNTHETIC_RECID_START", + "source_year_puf_adjusted_gross_income", + ), + "microcosm.build.us_runtime.puf_support": ( + "BASE_ASEC_SUPPORT_CHANNEL", + "PUF_DONOR_SOURCE_ADJUSTED_GROSS_INCOME_COLUMN", + "PUF_TAX_DETAIL_CLONE_INDEX", + "PUF_TAX_DETAIL_SUPPORT_CHANNEL", + "US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING", + "US_PUF_SUPPORT_FIT_NAME", + "PufTaxDetailChainInputs", + "clone_us_frame_for_puf_support", + "finalize_us_puf_tax_detail_predictions", + "has_assembled_support_metadata", + "has_support_role_metadata", + "impute_us_puf_tax_detail_support", + "prepare_us_puf_tax_detail_chain_inputs", + "puf_tax_detail_clone_mask", + "puf_tax_unit_donor_from_arrays", + "spine_source_id_column", + "support_channel_column", + "support_clone_index_column", + "support_role_series", + "support_source_id_column", + ), + "microcosm.build.us_runtime.puma_ladder": ( + "PUMA_LADDER_ARTIFACT_SHA256_ATTR", + "PUMA_LADDER_VINTAGES_ATTR", + "US_PUMA_LADDER_COLUMNS", + "US_PUMA_LADDER_DERIVED_LAYERS", + "US_PUMA_LADDER_KIND", + "US_PUMA_LADDER_SCHEMA_VERSION", + "US_PUMA_LADDER_TRACT_COLUMN", + "UsPumaLadder", + "assign_us_puma_ladder", + "load_us_puma_ladder", + "us_puma_ladder_assignment_summary", + "us_puma_ladder_gate", + "us_puma_ladder_joint_support_gate", + "with_household_us_puma_ladder", + ), + "microcosm.build.us_runtime.puma_ladder_sources": ( + "assemble_us_puma_ladder", + "parse_tract_to_puma_relationship", + ), + "microcosm.build.us_runtime.qbi_inputs": ( + "QBI_ARCHIVED_ASSUMPTIONS_URL", + "QBI_ARCHIVED_CLONE_URL", + "QBI_ARCHIVED_DERIVATION_URL", + "QBI_ARCHIVED_EXPORT_URL", + "QBI_ARCHIVED_IMPUTATION_URL", + "QBI_ARCHIVED_PUF_ARTIFACT_URL", + "QBI_ARCHIVED_SIMULATION_URL", + "US_QBI_NONCONSTANT_PERSON_COLUMNS", + "US_QBI_STAGE_NAME", + "us_qbi_inputs_signal_gate", + "us_qbi_inputs_stage_spec", + "us_qbi_inputs_summary", + "with_us_qbi_input_reconciliation", + ), + "microcosm.build.us_runtime.reform_coverage_smoke": ( + "us_reform_coverage_smoke_gate", + ), + "microcosm.build.us_runtime.reform_validation": ( + "REFORM_VALIDATION_SCHEMA_VERSION", + "ReformValidationSpec", + "in_sample_reform_specs", + "load_default_reform_specs", + "out_of_sample_reform_specs", + "reform_validation_payload", + "write_reform_validation", + ), + "microcosm.build.us_runtime.register_consistency": ( + "us_register_consistency_gate", + "us_register_contradictions", + ), + "microcosm.build.us_runtime.relationship_inputs": ( + "US_RELATIONSHIP_INPUTS_NONCONSTANT_PERSON_COLUMNS", + "US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS", + "US_RELATIONSHIP_INPUTS_REQUIRED_SOURCE_COLUMNS", + "US_RELATIONSHIP_INPUTS_STAGE_NAME", + "derive_us_relationship_inputs_from_manifest", + "us_relationship_inputs_signal_gate", + "us_relationship_inputs_stage_spec", + "us_relationship_inputs_summary", + "with_us_relationship_inputs", + ), + "microcosm.build.us_runtime.release_input_coverage": ( + "POST_REFERENCE_ECPS_REQUIRED_INPUTS", + "SSI_COUNTABLE_RESOURCE_ASSETS", + "US_CGD_ROUTE_REQUIRED_INPUTS", + "US_RELEASE_INPUT_COVERAGE_RESOURCE", + "ReformCoverageProbe", + "ReleaseInputColumn", + "ReleaseInputCoverageManifest", + "assert_release_input_coverage_manifest_current", + "load_release_input_coverage_manifest", + "us_release_input_coverage_gate", + "us_release_input_coverage_required_columns", + "us_release_input_coverage_reviewed_exclusions", + "us_release_reform_coverage_probes", + ), + "microcosm.build.us_runtime.release_target_parity": ( + "RED_LINE_COMPILED_FAMILIES", + "US_TARGET_PARITY_FEED_FAMILIES_RESOURCE", + "US_TARGET_PARITY_MANIFEST_RESOURCE", + "TargetFamily", + "TargetFence", + "TargetParityManifest", + "assert_target_parity_manifest_current", + "load_target_parity_feed_families", + "load_target_parity_manifest", + "registry_target_family_ids", + "us_release_target_parity_compiled_families", + "us_release_target_parity_gate", + "us_release_target_parity_reviewed_exclusions", + "us_target_family_id", + ), + "microcosm.build.us_runtime.retirement_contributions": ( + "US_RETIREMENT_CONTRIBUTION_NONCONSTANT_PERSON_COLUMNS", + "US_RETIREMENT_CONTRIBUTION_OUTPUT_COLUMNS", + "US_RETIREMENT_CONTRIBUTION_REQUIRED_SOURCE_COLUMNS", + "US_RETIREMENT_CONTRIBUTION_STAGE_NAME", + "derive_us_retirement_contributions_from_manifest", + "impute_us_retirement_contributions_to_puf_support_from_manifest", + "us_retirement_contributions_signal_gate", + "us_retirement_contributions_stage_spec", + "us_retirement_contributions_summary", + "with_us_retirement_contribution_inputs", + ), + "microcosm.build.us_runtime.retirement_distributions": ( + "RETIREMENT_DISTRIBUTIONS_ARCHIVED_DERIVATION_URL", + "RETIREMENT_DISTRIBUTIONS_ARCHIVED_PARAMETERS_URL", + "US_RETIREMENT_DISTRIBUTION_NONCONSTANT_PERSON_COLUMNS", + "US_RETIREMENT_DISTRIBUTION_OUTPUT_COLUMNS", + "US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS", + "US_RETIREMENT_DISTRIBUTION_STAGE_NAME", + "derive_us_retirement_distributions_from_manifest", + "impute_us_retirement_distributions_to_puf_support_from_manifest", + "us_retirement_distributions_signal_gate", + "us_retirement_distributions_stage_spec", + "us_retirement_distributions_summary", + "with_us_retirement_distribution_inputs", + ), + "microcosm.build.us_runtime.salt_refund_income": ( + "SALT_REFUND_ARCHIVED_DERIVATION_URL", + "SALT_REFUND_ARCHIVED_EXPORT_URL", + "SALT_REFUND_ARCHIVED_IMPUTATION_URL", + "SALT_REFUND_ARCHIVED_PERSON_ALLOCATION_URL", + "SALT_REFUND_ARCHIVED_PUF_ARTIFACT_URL", + "US_SALT_REFUND_NONCONSTANT_PERSON_COLUMNS", + "US_SALT_REFUND_OUTPUT_COLUMNS", + "US_SALT_REFUND_STAGE_NAME", + "derive_us_salt_refund_income_from_puf", + "us_salt_refund_income_signal_gate", + "us_salt_refund_income_stage_spec", + "us_salt_refund_income_summary", + ), + "microcosm.build.us_runtime.scf_auto_loans": ( + "QUALIFIED_AUTO_LOAN_ANNUAL_ISSUANCE_TARGET", + "SCF_2022_FULL_EXTRACT_MEMBER", + "SCF_2022_FULL_EXTRACT_MEMBER_SHA256", + "SCF_2022_FULL_EXTRACT_URL", + "SCF_2022_FULL_EXTRACT_ZIP_SHA256", + "SCF_AUTO_LOAN_AMOUNT_COLUMNS", + "SCF_AUTO_LOAN_RATE_COLUMNS", + "US_SCF_AUTO_LOAN_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_SCF_AUTO_LOAN_OUTPUT_COLUMNS", + "fetch_scf_2022_full_extract", + "impute_us_scf_auto_loans", + "load_scf_2022_auto_loan_donor", + "qualified_auto_loan_interest_proxy", + "us_scf_auto_loans_signal_gate", + "us_scf_auto_loans_stage_spec", + "us_scf_auto_loans_summary", + "with_us_scf_auto_loan_inputs", + ), + "microcosm.build.us_runtime.scf_wealth": ( + "FINANCIAL_ASSET_BLEND_AUDIT_KEY", + "FINANCIAL_ASSET_SOURCE_SCF_PROBABILITY", + "SCF_FINANCIAL_ASSET_TARGET_COMPONENTS", + "SCF_NET_WORTH_TARGET_COMPONENTS", + "SCF_WEALTH_PREDICTORS", + "US_SCF_FINANCIAL_ASSET_OUTPUT_COLUMNS", + "US_SCF_NET_WORTH_OUTPUT_COLUMNS", + "US_SCF_WEALTH_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_SCF_WEALTH_NONCONSTANT_PERSON_COLUMNS", + "US_SCF_WEALTH_STAGE_NAME", + "fetch_scf_2022_summary_extract", + "financial_asset_source_is_scf", + "impute_us_scf_financial_assets", + "impute_us_scf_net_worth", + "impute_us_sipp_scf_financial_assets", + "load_scf_2022_financial_asset_donor", + "us_scf_wealth_signal_gate", + "us_scf_wealth_stage_spec", + "us_scf_wealth_summary", + "with_us_scf_wealth_inputs", + ), + "microcosm.build.us_runtime.sipp_financial_assets": ( + "SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_ID_PARTS", + "SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_TYPE", + "SIPP_2023_FINANCIAL_ASSET_DONOR_REVISION", + "SIPP_2023_FINANCIAL_ASSET_DONOR_SHA256", + "SIPP_2023_FINANCIAL_ASSET_DONOR_SIZE_BYTES", + "SIPP_2023_FINANCIAL_ASSET_DONOR_URL", + "SIPP_FINANCIAL_ASSET_DONOR_WEIGHT_COLUMN", + "SIPP_FINANCIAL_ASSET_MODEL_PREDICTORS", + "SIPP_FINANCIAL_ASSET_SOURCE_COLUMNS", + "SIPP_FINANCIAL_ASSET_TARGET_ALLOCATION_COLUMNS", + "SIPP_FINANCIAL_ASSET_TARGET_SOURCE_COLUMNS", + "fetch_sipp_2023_financial_asset_donor", + "impute_us_sipp_financial_assets", + "load_sipp_2023_financial_asset_donor", + ), + "microcosm.build.us_runtime.sipp_head_start": ( + "HEAD_START_SIPP_DICTIONARY_URL", + "SIPP_2023_HEAD_START_DONOR_REVISION", + "SIPP_2023_HEAD_START_DONOR_SHA256", + "SIPP_2023_HEAD_START_DONOR_SIZE_BYTES", + "SIPP_2023_HEAD_START_DONOR_URL", + "SIPP_HEAD_START_FIT_PARAMETERS", + "SIPP_HEAD_START_MODEL_PREDICTORS", + "SIPP_HEAD_START_READ_PARAMETERS", + "SIPP_HEAD_START_SOURCE_COLUMNS", + "US_SIPP_HEAD_START_NONCONSTANT_PERSON_COLUMNS", + "US_SIPP_HEAD_START_OUTPUT_COLUMNS", + "US_SIPP_HEAD_START_REQUIRED_SOURCE_COLUMNS", + "US_SIPP_HEAD_START_STAGE_NAME", + "fetch_sipp_2023_head_start_donor", + "impute_us_sipp_head_start", + "load_sipp_2023_head_start_donor", + "us_sipp_head_start_signal_gate", + "us_sipp_head_start_stage_spec", + "us_sipp_head_start_summary", + "with_us_sipp_head_start_input", + ), + "microcosm.build.us_runtime.sipp_tips": ( + "CENSUS_OCCUPATION_CODE_TO_TTOC", + "SIPP_2023_TIP_DONOR_REVISION", + "SIPP_2023_TIP_DONOR_SHA256", + "SIPP_2023_TIP_DONOR_URL", + "SIPP_TIP_OUTPUT_COLUMNS", + "SIPP_TIP_PREDICTORS", + "US_SIPP_TIPS_NONCONSTANT_PERSON_COLUMNS", + "US_SIPP_TIPS_OUTPUT_COLUMNS", + "US_SIPP_TIPS_REQUIRED_SOURCE_COLUMNS", + "US_SIPP_TIPS_STAGE_NAME", + "derive_treasury_tipped_occupation_code", + "fetch_sipp_2023_tip_donor", + "impute_us_sipp_tips", + "load_sipp_2023_tip_donor", + "us_sipp_tips_signal_gate", + "us_sipp_tips_stage_spec", + "us_sipp_tips_summary", + "with_us_sipp_tip_inputs", + ), + "microcosm.build.us_runtime.sipp_vehicles": ( + "SIPP_2023_VEHICLE_DONOR_REVISION", + "SIPP_2023_VEHICLE_DONOR_SHA256", + "SIPP_2023_VEHICLE_DONOR_SIZE_BYTES", + "SIPP_2023_VEHICLE_DONOR_URL", + "US_SIPP_VEHICLE_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_SIPP_VEHICLE_OUTPUT_COLUMNS", + "fetch_sipp_2023_vehicle_donor", + "load_sipp_2023_vehicle_donor", + "us_sipp_vehicles_signal_gate", + "us_sipp_vehicles_stage_spec", + "us_sipp_vehicles_summary", + "with_us_sipp_vehicle_inputs", + ), + "microcosm.build.us_runtime.snap_discretionary_exemption": ( + "US_SNAP_DISCRETIONARY_EXEMPTION_NONCONSTANT_PERSON_COLUMNS", + "US_SNAP_DISCRETIONARY_EXEMPTION_OUTPUT_COLUMN", + "US_SNAP_DISCRETIONARY_EXEMPTION_REQUIRED_SOURCE_COLUMNS", + "US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME", + "derive_us_snap_discretionary_exemption_from_manifest", + "us_snap_discretionary_exemption_signal_gate", + "us_snap_discretionary_exemption_stage_spec", + "us_snap_discretionary_exemption_summary", + "with_us_snap_discretionary_exemption_inputs", + ), + "microcosm.build.us_runtime.snap_state_take_up": ( + "US_SNAP_CASELOAD_TOLERANCE", + "US_SNAP_HOUSEHOLDS_TARGET_TABLE", + "US_SNAP_STATE_TAKE_UP_ANCHOR", + "US_SNAP_STATE_TAKE_UP_STAGE", + "us_snap_state_take_up_diagnostics", + "us_snap_state_take_up_gate", + "with_us_snap_state_take_up", + "write_us_snap_state_take_up_diagnostics", + ), + "microcosm.build.us_runtime.snap_take_up": ( + "US_SNAP_TAKE_UP_OUTPUT_COLUMN", + "US_SNAP_TAKE_UP_RAW_COLUMN", + "US_SNAP_TAKE_UP_STAGE_NAME", + "derive_us_snap_take_up_from_manifest", + "us_snap_take_up_signal_gate", + "us_snap_take_up_stage_spec", + "us_snap_take_up_summary", + "with_us_snap_take_up_inputs", + ), + "microcosm.build.us_runtime.source_coverage": ( + "CHRONICLE_US_SOURCE_COVERAGE_CONTRACT_COMMIT", + "LEDGER_US_SOURCE_COVERAGE_CONTRACT_COMMIT", + "US_SOURCE_COVERAGE", + "hard_target_package_aliases", + "source_gap_family_ids", + "us_source_coverage_diagnostics", + "us_source_coverage_gate", + "validation_only_family_ids", + "write_us_source_coverage_diagnostics", + ), + "microcosm.build.us_runtime.source_runtime": ( + "disaggregate_us_puf_aggregate_records_from_manifest", + "us_source_operation_handlers", + ), + "microcosm.build.us_runtime.spine_agreement": ( + "DEFAULT_CATEGORICAL_TOTAL_VARIATION_TOLERANCE", + "DEFAULT_INCIDENCE_RATIO_BOUNDS", + "DEFAULT_QUANTILE_ENVELOPE_TOLERANCE", + "DEFAULT_SPINE_AGREEMENT_QUANTILES", + "US_SPINE_AGREEMENT_REGISTRY", + "SpineAgreementSpec", + "default_spine_agreement_registry", + "normalize_transfer_family_name", + "spine_agreement_gate", + "validate_spine_agreement_registry", + ), + "microcosm.build.us_runtime.spine_assembly": ("assemble_spines",), + "microcosm.build.us_runtime.ssi_disability_criteria": ( + "SIPP_2023_SSI_DISABILITY_DONOR_REVISION", + "SIPP_2023_SSI_DISABILITY_DONOR_SHA256", + "SIPP_2023_SSI_DISABILITY_DONOR_SIZE_BYTES", + "SIPP_2023_SSI_DISABILITY_DONOR_URL", + "SIPP_SSI_DISABILITY_DIFFICULTY_PREDICTORS", + "SIPP_SSI_DISABILITY_FIT_PARAMETERS", + "SIPP_SSI_DISABILITY_MODEL_PREDICTORS", + "SIPP_SSI_DISABILITY_READ_PARAMETERS", + "SIPP_SSI_DISABILITY_SOURCE_COLUMNS", + "SSI_DISABILITY_ARCHIVED_CPS_URL", + "SSI_DISABILITY_ARCHIVED_EXTENDED_CPS_URL", + "SSI_DISABILITY_ARCHIVED_SIPP_URL", + "SSI_DISABILITY_ARCHIVED_SOURCE_IMPUTE_URL", + "SSI_DISABILITY_SIPP_DICTIONARY_URL", + "US_SSI_DISABILITY_CRITERIA_NONCONSTANT_PERSON_COLUMNS", + "US_SSI_DISABILITY_CRITERIA_OUTPUT_COLUMNS", + "US_SSI_DISABILITY_CRITERIA_STAGE_NAME", + "fetch_sipp_2023_ssi_disability_donor", + "impute_us_ssi_disability_criteria", + "load_sipp_2023_ssi_disability_donor", + "us_ssi_disability_criteria_signal_gate", + "us_ssi_disability_criteria_stage_spec", + "us_ssi_disability_criteria_summary", + "with_us_ssi_disability_criteria", + ), + "microcosm.build.us_runtime.ssi_take_up": ( + "SSI_TAKE_UP_ARCHIVED_DERIVATION_URL", + "SSI_TAKE_UP_ARCHIVED_EXPORT_URL", + "SSI_TAKE_UP_ARCHIVED_RANDOMNESS_URL", + "SSI_TAKE_UP_ARCHIVED_REPORTER_URL", + "SSI_TAKE_UP_ARCHIVED_TARGETS_URL", + "SSI_TAKE_UP_SSA_SOURCE_URL", + "US_SSI_TAKE_UP_ANCHOR", + "US_SSI_TAKE_UP_BAND_DELIVERY_RELATIVE_TOLERANCE", + "US_SSI_TAKE_UP_ENFORCED_BAND_KEYS", + "US_SSI_TAKE_UP_NONCONSTANT_PERSON_COLUMNS", + "US_SSI_TAKE_UP_OUTPUT_COLUMNS", + "US_SSI_TAKE_UP_PHASE_ASSIGNMENT", + "US_SSI_TAKE_UP_PHASE_RELEASE_FINAL", + "US_SSI_TAKE_UP_PRIOR_BASIS_CURRENT_FRAME", + "US_SSI_TAKE_UP_PRIOR_BASIS_RELEASE_ARTIFACT", + "US_SSI_TAKE_UP_REQUIRED_SOURCE_COLUMNS", + "US_SSI_TAKE_UP_STAGE_NAME", + "US_SSI_TAKE_UP_TARGET_TABLE_NAME", + "SSITakeUpBandPriorBasis", + "SSITakeUpPriorBasis", + "ssi_take_up_prior_basis_from_artifact", + "ssi_take_up_prior_basis_from_diagnostics", + "us_ssi_take_up_delivery_gate", + "us_ssi_take_up_diagnostics", + "us_ssi_take_up_gate", + "us_ssi_take_up_reporter_source_ids", + "with_us_ssi_take_up", + "write_us_ssi_take_up_diagnostics", + ), + "microcosm.build.us_runtime.support_provenance": ( + "us_reported_coverage_vintage_signal_gate", + ), + "microcosm.build.us_runtime.take_up": ( + "US_TAKE_UP_SHARE_BAND", + "SeededTakeUpResult", + "us_take_up_participation_diagnostics", + "us_take_up_signal_gate", + "us_take_up_summary", + "with_us_take_up_inputs", + "write_us_take_up_participation_diagnostics", + ), + "microcosm.build.us_runtime.take_up_contract": ( + "TakeUpContract", + "TakeUpProgram", + "assert_take_up_contract_current", + "assert_take_up_treatments_consistent", + "count_calibrated_take_up_programs", + "load_take_up_contract", + "seeded_take_up_programs", + ), + "microcosm.build.us_runtime.validation_input_coverage": ( + "US_VALIDATION_PROVISION_INPUT_LEAVES", + "ValidationInputLeaf", + "assert_validation_leaf_registry_current", + "us_source_stage_outputs", + "us_validation_input_coverage_gate", + ), + "microcosm.build.us_runtime.voluntary_filing": ( + "SIPP_2023_VOLUNTARY_FILING_DONOR_REVISION", + "SIPP_2023_VOLUNTARY_FILING_DONOR_SHA256", + "SIPP_2023_VOLUNTARY_FILING_DONOR_SIZE_BYTES", + "SIPP_2023_VOLUNTARY_FILING_DONOR_URL", + "SIPP_VOLUNTARY_FILING_MODEL_PREDICTORS", + "SIPP_VOLUNTARY_FILING_SOURCE_COLUMNS", + "US_VOLUNTARY_FILING_NONCONSTANT_TAX_UNIT_COLUMNS", + "US_VOLUNTARY_FILING_OUTPUT_COLUMNS", + "US_VOLUNTARY_FILING_STAGE_NAME", + "VOLUNTARY_FILING_ARCHIVED_DERIVATION_URL", + "VOLUNTARY_FILING_ARCHIVED_PARAMETERS_URL", + "VOLUNTARY_FILING_SIPP_DICTIONARY_URL", + "fetch_sipp_2023_voluntary_filing_donor", + "impute_us_voluntary_filing", + "load_sipp_2023_voluntary_filing_donor", + "us_voluntary_filing_signal_gate", + "us_voluntary_filing_stage_spec", + "us_voluntary_filing_summary", + "with_us_voluntary_filing_input", + ), + "microcosm.build.us_runtime.weeks_unemployed": ( + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER", + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_CRC32", + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SHA256", + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SIZE_BYTES", + "ASEC_2023_WEEKS_UNEMPLOYED_RAW_ROWS", + "ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_COLUMNS", + "ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256", + "ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_YEAR", + "ASEC_2023_WEEKS_UNEMPLOYED_UNIQUE_KEYS", + "ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_SOURCE_SHARE", + "ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_WEEKS", + "ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SHA256", + "ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SIZE_BYTES", + "ASEC_2023_WEEKS_UNEMPLOYED_ZIP_URL", + "US_WEEKS_UNEMPLOYED_NONCONSTANT_PERSON_COLUMNS", + "US_WEEKS_UNEMPLOYED_OUTPUT_COLUMNS", + "US_WEEKS_UNEMPLOYED_REQUIRED_SOURCE_COLUMNS", + "US_WEEKS_UNEMPLOYED_STAGE_NAME", + "WEEKS_UNEMPLOYED_ARCHIVED_DERIVATION_URL", + "WEEKS_UNEMPLOYED_ARCHIVED_PUF_IMPUTATION_URL", + "WEEKS_UNEMPLOYED_ARCHIVED_SOURCE_URL", + "WEEKS_UNEMPLOYED_DERIVE_PARAMETERS", + "WEEKS_UNEMPLOYED_PUF_IMPUTATION_PARAMETERS", + "WEEKS_UNEMPLOYED_PUF_PREDICTORS", + "WEEKS_UNEMPLOYED_READ_PARAMETERS", + "derive_us_weeks_unemployed_from_manifest", + "fetch_asec_2023_weeks_unemployed_source", + "fill_asec_2022_weeks_unemployed_source", + "impute_us_weeks_unemployed_to_puf_support_from_manifest", + "load_asec_2023_weeks_unemployed_source", + "us_weeks_unemployed_signal_gate", + "us_weeks_unemployed_stage_spec", + "us_weeks_unemployed_summary", + "with_us_weeks_unemployed", + ), + "microcosm.build.us_runtime.wic_claim": ( + "US_WIC_CLAIM_NONCONSTANT_PERSON_COLUMNS", + "US_WIC_CLAIM_OUTPUT_COLUMNS", + "US_WIC_CLAIM_REQUIRED_SOURCE_COLUMNS", + "US_WIC_CLAIM_STAGE_NAME", + "WIC_CLAIM_ARCHIVED_DERIVATION_URL", + "WIC_CLAIM_ARCHIVED_PARAMETERS_URL", + "WIC_CLAIM_ARCHIVED_RANDOMNESS_URL", + "WIC_CLAIM_FNS_SOURCE_URL", + "derive_us_wic_claim_from_manifest", + "us_wic_claim_signal_gate", + "us_wic_claim_stage_spec", + "us_wic_claim_summary", + "with_us_wic_claim_input", + ), + "microcosm.build.us_runtime.workers_compensation": ( + "US_WORKERS_COMPENSATION_NONCONSTANT_PERSON_COLUMNS", + "US_WORKERS_COMPENSATION_OUTPUT_COLUMNS", + "US_WORKERS_COMPENSATION_REQUIRED_SOURCE_COLUMNS", + "US_WORKERS_COMPENSATION_STAGE_NAME", + "WORKERS_COMPENSATION_ARCHIVED_DERIVATION_URL", + "WORKERS_COMPENSATION_ARCHIVED_PUF_IMPUTATION_URL", + "WORKERS_COMPENSATION_ARCHIVED_PUF_OUTPUTS_URL", + "WORKERS_COMPENSATION_ARCHIVED_SOURCE_COLUMNS_URL", + "derive_us_workers_compensation_from_asec", + "derive_us_workers_compensation_from_manifest", + "impute_us_workers_compensation_to_puf_support_from_manifest", + "us_workers_compensation_signal_gate", + "us_workers_compensation_stage_spec", + "us_workers_compensation_summary", + "with_us_workers_compensation", + ), +} +_LAZY_EXPORTS = { + name: (module, name) for module, names in _EXPORT_MODULES.items() for name in names +} + @dataclass(frozen=True) class BuildConfig: @@ -2045,428 +2057,560 @@ def to_manifest(self) -> dict[str, object]: } -#: The US donor graph: every imputation stage's primary survey, with -#: citations. This is the single place the build's sources are declared — -#: the observatory's sources diagram and the dataset card derive from it. -US_DONORS: Mapping[str, DonorSpec] = { - "scf_wealth": DonorSpec( - survey="Fed SCF 2022 + Census SIPP 2023", - source="https://www.federalreserve.gov/econres/scfindex.htm", - notes=( - "SCF anchors signed net worth and one half of household liquid-asset " - "vectors; the immutable SIPP 2023 public-use donor supplies the " - "other half and restores low liquid-asset mass. Auto loans use the " - "full SCF separately." - ), - ), - US_SSI_DISABILITY_CRITERIA_STAGE_NAME: DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Latent under-65 SSI disability/blindness criterion from the " - "pinned full 2023 public-use donor. ASEC and PUF-support people " - "are predicted separately, and only direct under-65 ASEC SSI " - "reporters receive the observed-reporter anchor." - ), - ), - US_SIPP_HEAD_START_STAGE_NAME: DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Direct December nursery/preschool federally sponsored-program " - "responses train a weighted Head Start take-up proxy for ages " - "3--5; strict reported structural negatives exclude hot-decked " - "answers and the prediction is shared by support clones." - ), - ), - US_SSI_TAKE_UP_STAGE_NAME: DonorSpec( - survey="CPS ASEC reported SSI + SSA SSI Monthly Statistics December 2024", - source=SSI_TAKE_UP_SSA_SOURCE_URL, - notes=( - "Direct ASEC SSI_VAL reporters anchor person-level take-up; " - "eligible source-person identities are count-calibrated by age to " - "SSA December 2024 Federal-payment recipient counts and fanned to " - "both support clones." - ), - ), - "sipp_tips": DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes="Tip income for tipped occupations.", - ), - "org_wages": DonorSpec( - survey="CPS ORG", - source=("https://www2.census.gov/programs-surveys/cps/datasets/2024/basic/"), - notes=( - "Hourly-wage labor-market inputs. Donor load failures abort the " - "build — the silent zero-fallback this stage once had is " - "structurally impossible under StagePlan." - ), - ), - "meps_esi_premiums": DonorSpec( - survey="MEPS-IC", - source="https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", - notes="Employer-sponsored insurance premium parameters.", - ), - "aca_marketplace_inputs": DonorSpec( - survey="CPS ASEC + CMS Marketplace Open Enrollment PUFs", - source="https://www.cms.gov/marketplace/resources/data/public-use-files", - notes=( - "Marketplace take-up and selected-plan inputs: CPS reported " - "Marketplace coverage and premium reports anchor the records; " - "CMS OEP enrollment, APTC, and metal-level tables provide the " - "calibration targets." - ), - ), - "medicaid_take_up": DonorSpec( - survey="CPS ASEC reported coverage + CMS Medicaid monthly enrollment snapshot", - source="https://data.medicaid.gov/dataset/6165f45b-ca93-5bb5-9d06-db29c692a360", - notes=( - "Medicaid take-up by anchored count-calibration (contract " - "treatment count_calibrated, microcosm #331): CPS-reported " - "Medicaid coverage at interview anchors the flag; the fill is " - "calibrated to CMS December 2024 state enrollment snapshots. " - "Point-in-time semantics per #332; heals the #170 " - "enrollment==eligibility degeneracy." - ), - ), - US_SNAP_STATE_TAKE_UP_STAGE: DonorSpec( - survey=( - "Census CPS ASEC reported receipt + USDA FNS state " - "average-monthly household caseloads" - ), - source="https://www.fns.usda.gov/pd/supplemental-nutrition-assistance-program-snap", - notes=( - "SNAP take-up by anchored count-calibration (contract treatment " - "count_calibrated, microcosm #372): reported ASEC receipt anchors " - "the flag; the fill is calibrated per state to FNS FY2024 " - "average-monthly household counts among eligible non-anchored " - "units, replacing the national snap_take_up fill that bakes in " - "state-dependent CPS underreporting." - ), - ), - US_OTHER_HEALTH_INSURANCE_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured PHIP_VAL is reduced by PolicyEngine-calculated CHIP, " - "Marketplace, and Medicaid premiums after take-up stages; an " - "ASEC-trained weighted QRF replaces both premium leaves on the " - "PUF support half." - ), - ), - US_PRIOR_YEAR_INCOME_STAGE_NAME: DonorSpec( - survey="CPS ASEC (prior year)", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Adjacent-year PERIDNUM join for measured prior-year earnings, " - "with Census allocation flags and sentinels enforced. A joint " - "eight-predictor weighted QRF replaces both earnings leaves on " - "the PUF support half; signed self-employment losses survive." - ), - ), - US_IMMIGRATION_STAGE_NAME: DonorSpec( - survey="CPS ASEC + published unauthorized-population estimates", - source=( - "https://www.pewresearch.org/short-reads/2024/07/22/" - "what-we-know-about-unauthorized-immigrants-living-in-the-us/" - ), - notes=( - "SSN card type and immigration status from ASEC citizenship, " - "entry-year, nativity, and program-participation fields via the " - "ASEC-UA residual method (SSRN 4662801), targeted to published " - "undocumented population/worker/student control totals." - ), - ), - US_HOURS_WORKED_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Hours-worked inputs mapped directly from measured ASEC person " - "variables (HRSWK, A_HRS1, WKSWORK) — nothing imputed. Without " - "them the engine defaults every person to 40 weekly hours and " - "hours-conditioned rules (SNAP work requirements) become no-ops." - ), - ), - US_SNAP_TAKE_UP_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + USDA FNS participation rate estimates", - source="https://www.fns.usda.gov/snap/participation-rates", - notes=( - "SNAP take-up: reported recipients (SPM_SNAPSUB) always take " - "up; non-reporting units drawn to the cited FNS participation " - "rate. Without it the engine defaults every eligible unit to " - "100% take-up." - ), - ), - US_ELIGIBILITY_INPUTS_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "SNAP eligibility/exemption inputs mapped directly from " - "measured ASEC person variables (PEDIS*, A_HSCOL/A_FTPT, " - "PEPAR1/PEPAR2, VET_VAL, SSI_VAL) — nothing imputed. Without " - "them disability, student, parent/child, and veteran " - "exemption channels default to False/0." - ), - ), - US_RELATIONSHIP_INPUTS_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured household-head and marital-status flags mapped exactly " - "from P_SEQ and A_MARITL; nothing is imputed." - ), - ), - US_MEDICARE_TAKE_UP_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured Medicare enrollment mapped exactly from MCARE == 1 and " - "copied onto the PUF support clone; no take-up rate is applied." - ), - ), - US_RETIREMENT_DISTRIBUTION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "ASEC rows map exactly from all four DST_SC*/DST_VAL* pairs; the " - "archived CPS-only QRF replaces the four populated non-IRA leaves " - "on PUF support while preserving IRA channel ownership. Nothing " - "is allocated across accounts." - ), - ), - US_PREGNANCY_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + CDC natality-derived national pregnancy rate", - source="https://www.cdc.gov/nchs/nvss/births.htm", - notes=( - "Pregnancy seeded among women 15-44 at the national " - "point-in-time rate (births x 39/52 over female 15-44 " - "population), matching the retired pipeline's national " - "fallback; state-level rates are follow-up work (#351). " - "The ASEC does not measure pregnancy." - ), - ), - US_WIC_CLAIM_STAGE_NAME: DonorSpec( - survey="USDA FNS WIC Eligibility and Enrollment Estimates + Census CPS ASEC", - source="https://www.fns.usda.gov/research/wic/eligibility-and-coverage-rates-2022", - notes=( - "Stable person-level claim draws use official CY2022 FNS category " - "coverage rates after the pregnancy and parent-input stages. The " - "all-postpartum rate is used because no hermetic source identifies " - "breastfeeding; nutritional risk remains separately excluded." - ), - ), - US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + statutory exemption cap (7 U.S.C. 2015(o)(6))", - source="https://www.law.cornell.edu/uscode/text/7/2015#o_6", - notes=( - "ABAWD discretionary exemptions seeded at the statutory cap " - "(8% from FY2024) across potentially covered adults 18-64; " - "the engine intersects with modeled coverage. Assumes full " - "state usage of the cap (#323)." - ), - ), - US_CHILDCARE_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured replicated SPM_CHILDCAREXPNS is validated and carried " - "to the SPM-unit childcare leaf. After support expansion, an " - "ASEC-trained weighted QRF replaces only the PUF half and the " - "archived first-person reduction places predictions on SPM units." - ), - ), - US_ADULT_CARE_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Section 21 CDCC adult-care inputs (microcosm#451 item 1): the " - "qualifying flag is the measured PEDISDRS self-care difficulty " - "item; the expense leg is a seeded, weight-targeted draw from " - "the measured ASEC childcare-expense distribution (the same " - "section 21 expense class), restricted to tax units where the " - "statute can bind, with the 21(d)(2) spouse deeming honored. " - "Neither ASEC nor the SIPP 2023 PUF releases an in-household " - "adult-care expenditure amount, so the level proxy is declared " - "in the stage manifest." - ), - ), - US_ENERGY_SUBSIDY_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured replicated SPM_ENGVAL is validated and carried to the " - "SPM-unit energy-subsidy leaf. After support expansion, an " - "ASEC-trained weighted QRF replaces only the PUF half and the " - "archived first-person reduction places predictions on SPM units." - ), - ), - US_CHILD_SUPPORT_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured annual CSP_VAL receipts and positive CHSP_VAL expenses " - "are carried directly. After PUF tax-detail imputation, one " - "ASEC-trained weighted QRF jointly replaces both person leaves " - "on the PUF support half using the archived predictor subset." - ), - ), - US_DISABILITY_BENEFITS_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured annual DIS_VAL1/DIS_VAL2 benefits are retained only " - "when their source code is not workers' compensation. After PUF " - "tax-detail imputation, an ASEC-trained weighted QRF replaces the " - "CPS-only person leaf on the PUF support half." - ), - ), - US_WORKERS_COMPENSATION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured annual WC_VAL is carried directly. After PUF tax-detail " - "imputation, an ASEC-trained weighted QRF replaces the CPS-only " - "person leaf on the PUF support half." - ), - ), - US_WEEKS_UNEMPLOYED_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured LKWEEKS is carried directly, including an exact " - "identity-keyed repair of the omitted 2022-income-year column " - "from the pinned official 2023 ASEC archive. An ASEC-trained " - "QRF then replaces only the PUF support half, with the archived " - "unemployment-compensation zero rule." - ), - ), - "puf_tax_detail": DonorSpec( - survey="IRS PUF 2015 (uprated)", - source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", - notes=( - "Itemized-deduction detail, versioned processed-PUF Section 199A " - "simulation leaves (carried without redrawing), partnership SE, " - "source-year-AGI E19200 mortgage/non-mortgage split, direct " - "E00800/E03500 alimony, direct E20500 casualty loss, and the E20400 " - "miscellaneous-expense proxy. The pinned processed PUF uprates its " - "raw TY2015 rows before seeded disclosure-record replacement and " - "includes a Forbes-backed 3,900-record open tail; this runtime " - "reconstructs the bounded-record AGI lineage and anchors every " - "Forbes-tail record in the final published AGI band without rerunning " - "Forbes synthesis. Support is clipped to the PUF's realized ranges." - ), - ), - US_EDUCATION_INPUTS_STAGE_NAME: DonorSpec( - survey="IRS PUF 2015 (uprated) + Census CPS ASEC", - source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", - notes=( - "Qualified tuition comes from the PUF E03230/E87530 maximum; " - "the published retired path drops the reported AOTC output and its " - "five affirmative factual inputs therefore follow positive tuition; " - "educational assistance carries directly from ASEC ED_VAL." - ), - ), - US_RETIREMENT_CONTRIBUTION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + published retirement-contribution shares", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured ASEC RETCB_VAL is allocated across five desired " - "retirement-contribution leaves using archived IRS/BEA/Vanguard/" - "PSCA shares, then CPS-trained QRF predictions replace the PUF " - "support half. PolicyEngine-US owns all statutory caps." - ), - ), - "capital_gain_distributions": DonorSpec( - survey="IRS SOI Sales of Capital Assets (TY2015) + Pub 1304 Table 1.4", - source=( - "https://www.irs.gov/statistics/" - "soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns" - ), - notes=( - "Schedule D line 13 capital gain distributions split out of " - "long-term gains as a memo component at the national SOCA-derived " - "share; the direct-1040 route is already a PUF-stage output and " - "the two routes are mutually exclusive on a real return." - ), - ), - US_HOUSING_INPUTS_STAGE_NAME: DonorSpec( - survey="Census ACS 2022", - source="https://www.census.gov/programs-surveys/acs", - notes=( - "Exact ASEC H_TENURE and SPM housing carries plus annual ACS PUMS " - "rent imputation on household heads; reported housing assistance " - "alone receives the archived second-stage PUF-support QRF." - ), - ), - "vehicle_assets": DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Household vehicle count and value from the pinned full 2023 " - "public-use donor. They remain independent policy inputs until " - "the full mixed-source net-worth reconciliation is restored." - ), - ), - US_VOLUNTARY_FILING_STAGE_NAME: DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Measured 2023 SIPP filing and expected-filing responses replace " - "the retired uncited demographic probability table. Reciprocal " - "spouses form one source unit, reported dependents are excluded, " - "and one weighted-QRF prediction is shared across support clones." - ), - ), -} +# Annotations preserve the public type surface without initializing lazy values. +US_DONORS: Mapping[str, DonorSpec] +US_STAGE_NAMES: tuple[str, ...] +US_SOURCE_MANIFEST: SourceManifest +US_SOURCE_STAGE_SPECS: tuple[SourceStageSpec, ...] +US_NONNEGATIVE_SOURCE_OUTPUTS: frozenset[str] +US_SUPPORT_SPINE_MANIFEST: SupportSpineManifest +US_SUPPORT_SPINE_SPEC: SupportSpineSpec + +_EXPORT_LOCK = RLock() + + +def _resolve_export(name: str): + if name in globals(): + return globals()[name] + module, attribute = _LAZY_EXPORTS[name] + # Never hold our lock across Python's module import locks. + value = getattr(import_module(module), attribute) + with _EXPORT_LOCK: + return globals().setdefault(name, value) + + +def _initialize_source_manifest() -> None: + with _EXPORT_LOCK: + if all(name in globals() for name in _SOURCE_EXPORTS): + return + manifest = globals().get("US_SOURCE_MANIFEST") + if manifest is None: + manifest = _load_us_source_manifest() + values = { + "US_SOURCE_MANIFEST": manifest, + "US_SOURCE_STAGE_SPECS": manifest.stages, + "US_NONNEGATIVE_SOURCE_OUTPUTS": frozenset( + output + for stage in manifest.stages + for output in stage.nonnegative_outputs + ), + } + for name, value in values.items(): + globals().setdefault(name, value) + + +def _initialize_support_manifest() -> None: + with _EXPORT_LOCK: + if all(name in globals() for name in _SUPPORT_EXPORTS): + return + manifest = globals().get("US_SUPPORT_SPINE_MANIFEST") + if manifest is None: + manifest = _load_us_support_spine_manifest() + values = { + "US_SUPPORT_SPINE_MANIFEST": manifest, + "US_SUPPORT_SPINE_SPEC": manifest.support_spine, + } + for name, value in values.items(): + globals().setdefault(name, value) -#: Stage order of the US build. Derivation stages (no donor) interleave with -#: the donor imputations; the export/calibration stages close the plan. -US_STAGE_NAMES: tuple[str, ...] = ( - "asec_load", - "unit_assignment", - "derive_cps_carried", - US_PRIOR_YEAR_INCOME_STAGE_NAME, - US_IMMIGRATION_STAGE_NAME, - US_HOURS_WORKED_STAGE_NAME, - US_SNAP_TAKE_UP_STAGE_NAME, - US_RELATIONSHIP_INPUTS_STAGE_NAME, - US_MEDICARE_TAKE_UP_STAGE_NAME, - US_HOUSING_INPUTS_STAGE_NAME, - US_RETIREMENT_DISTRIBUTION_STAGE_NAME, - US_ELIGIBILITY_INPUTS_STAGE_NAME, - US_PREGNANCY_STAGE_NAME, - US_WIC_CLAIM_STAGE_NAME, - US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME, - US_RETIREMENT_CONTRIBUTION_STAGE_NAME, - US_CHILDCARE_STAGE_NAME, - US_ADULT_CARE_STAGE_NAME, - US_ENERGY_SUBSIDY_STAGE_NAME, - US_PUF_SUPPORT_STAGE_NAME, - "puf_tax_detail", - US_CHILD_SUPPORT_STAGE_NAME, - US_DISABILITY_BENEFITS_STAGE_NAME, - US_WORKERS_COMPENSATION_STAGE_NAME, - US_WEEKS_UNEMPLOYED_STAGE_NAME, - US_EDUCATION_INPUTS_STAGE_NAME, - "capital_gain_distributions", - "scf_wealth", - US_SSI_DISABILITY_CRITERIA_STAGE_NAME, - US_SIPP_HEAD_START_STAGE_NAME, - US_SSI_TAKE_UP_STAGE_NAME, - "sipp_tips", - "org_wages", - "meps_esi_premiums", - "mortgage_conversion", - "vehicle_assets", - US_VOLUNTARY_FILING_STAGE_NAME, - "entity_placement", - "aca_marketplace_inputs", - "medicaid_take_up", - US_SNAP_STATE_TAKE_UP_STAGE, - US_OTHER_HEALTH_INSURANCE_STAGE_NAME, - "export", + +_SOURCE_EXPORTS = ( + "US_SOURCE_MANIFEST", + "US_SOURCE_STAGE_SPECS", + "US_NONNEGATIVE_SOURCE_OUTPUTS", ) +_SUPPORT_EXPORTS = ("US_SUPPORT_SPINE_MANIFEST", "US_SUPPORT_SPINE_SPEC") + + +def __getattr__(name: str): + if name in _LAZY_EXPORTS: + return _resolve_export(name) + if name in ("US_DONORS", "US_STAGE_NAMES"): + _initialize_plan() + elif name in _SOURCE_EXPORTS: + _initialize_source_manifest() + elif name in _SUPPORT_EXPORTS: + _initialize_support_manifest() + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + return globals()[name] + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) + + +def _initialize_plan() -> None: + if "US_DONORS" in globals() and "US_STAGE_NAMES" in globals(): + return + ssi_take_up_ssa_source_url = _resolve_export("SSI_TAKE_UP_SSA_SOURCE_URL") + us_adult_care_stage_name = _resolve_export("US_ADULT_CARE_STAGE_NAME") + us_childcare_stage_name = _resolve_export("US_CHILDCARE_STAGE_NAME") + us_child_support_stage_name = _resolve_export("US_CHILD_SUPPORT_STAGE_NAME") + us_disability_benefits_stage_name = _resolve_export( + "US_DISABILITY_BENEFITS_STAGE_NAME" + ) + us_education_inputs_stage_name = _resolve_export("US_EDUCATION_INPUTS_STAGE_NAME") + us_eligibility_inputs_stage_name = _resolve_export( + "US_ELIGIBILITY_INPUTS_STAGE_NAME" + ) + us_energy_subsidy_stage_name = _resolve_export("US_ENERGY_SUBSIDY_STAGE_NAME") + us_hours_worked_stage_name = _resolve_export("US_HOURS_WORKED_STAGE_NAME") + us_housing_inputs_stage_name = _resolve_export("US_HOUSING_INPUTS_STAGE_NAME") + us_immigration_stage_name = _resolve_export("US_IMMIGRATION_STAGE_NAME") + us_medicare_take_up_stage_name = _resolve_export("US_MEDICARE_TAKE_UP_STAGE_NAME") + us_other_health_insurance_stage_name = _resolve_export( + "US_OTHER_HEALTH_INSURANCE_STAGE_NAME" + ) + us_pregnancy_stage_name = _resolve_export("US_PREGNANCY_STAGE_NAME") + us_prior_year_income_stage_name = _resolve_export("US_PRIOR_YEAR_INCOME_STAGE_NAME") + us_puf_support_stage_name = _resolve_export("US_PUF_SUPPORT_STAGE_NAME") + us_relationship_inputs_stage_name = _resolve_export( + "US_RELATIONSHIP_INPUTS_STAGE_NAME" + ) + us_retirement_contribution_stage_name = _resolve_export( + "US_RETIREMENT_CONTRIBUTION_STAGE_NAME" + ) + us_retirement_distribution_stage_name = _resolve_export( + "US_RETIREMENT_DISTRIBUTION_STAGE_NAME" + ) + us_sipp_head_start_stage_name = _resolve_export("US_SIPP_HEAD_START_STAGE_NAME") + us_snap_discretionary_exemption_stage_name = _resolve_export( + "US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME" + ) + us_snap_state_take_up_stage = _resolve_export("US_SNAP_STATE_TAKE_UP_STAGE") + us_snap_take_up_stage_name = _resolve_export("US_SNAP_TAKE_UP_STAGE_NAME") + us_ssi_disability_criteria_stage_name = _resolve_export( + "US_SSI_DISABILITY_CRITERIA_STAGE_NAME" + ) + us_ssi_take_up_stage_name = _resolve_export("US_SSI_TAKE_UP_STAGE_NAME") + us_voluntary_filing_stage_name = _resolve_export("US_VOLUNTARY_FILING_STAGE_NAME") + us_weeks_unemployed_stage_name = _resolve_export("US_WEEKS_UNEMPLOYED_STAGE_NAME") + us_wic_claim_stage_name = _resolve_export("US_WIC_CLAIM_STAGE_NAME") + us_workers_compensation_stage_name = _resolve_export( + "US_WORKERS_COMPENSATION_STAGE_NAME" + ) + with _EXPORT_LOCK: + us_donors: Mapping[str, DonorSpec] = { + "scf_wealth": DonorSpec( + survey="Fed SCF 2022 + Census SIPP 2023", + source="https://www.federalreserve.gov/econres/scfindex.htm", + notes=( + "SCF anchors signed net worth and one half of household liquid-asset " + "vectors; the immutable SIPP 2023 public-use donor supplies the " + "other half and restores low liquid-asset mass. Auto loans use the " + "full SCF separately." + ), + ), + us_ssi_disability_criteria_stage_name: DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Latent under-65 SSI disability/blindness criterion from the " + "pinned full 2023 public-use donor. ASEC and PUF-support people " + "are predicted separately, and only direct under-65 ASEC SSI " + "reporters receive the observed-reporter anchor." + ), + ), + us_sipp_head_start_stage_name: DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Direct December nursery/preschool federally sponsored-program " + "responses train a weighted Head Start take-up proxy for ages " + "3--5; strict reported structural negatives exclude hot-decked " + "answers and the prediction is shared by support clones." + ), + ), + us_ssi_take_up_stage_name: DonorSpec( + survey="CPS ASEC reported SSI + SSA SSI Monthly Statistics December 2024", + source=ssi_take_up_ssa_source_url, + notes=( + "Direct ASEC SSI_VAL reporters anchor person-level take-up; " + "eligible source-person identities are count-calibrated by age to " + "SSA December 2024 Federal-payment recipient counts and fanned to " + "both support clones." + ), + ), + "sipp_tips": DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes="Tip income for tipped occupations.", + ), + "org_wages": DonorSpec( + survey="CPS ORG", + source=( + "https://www2.census.gov/programs-surveys/cps/datasets/2024/basic/" + ), + notes=( + "Hourly-wage labor-market inputs. Donor load failures abort the " + "build — the silent zero-fallback this stage once had is " + "structurally impossible under StagePlan." + ), + ), + "meps_esi_premiums": DonorSpec( + survey="MEPS-IC", + source="https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", + notes="Employer-sponsored insurance premium parameters.", + ), + "aca_marketplace_inputs": DonorSpec( + survey="CPS ASEC + CMS Marketplace Open Enrollment PUFs", + source="https://www.cms.gov/marketplace/resources/data/public-use-files", + notes=( + "Marketplace take-up and selected-plan inputs: CPS reported " + "Marketplace coverage and premium reports anchor the records; " + "CMS OEP enrollment, APTC, and metal-level tables provide the " + "calibration targets." + ), + ), + "medicaid_take_up": DonorSpec( + survey="CPS ASEC reported coverage + CMS Medicaid monthly enrollment snapshot", + source="https://data.medicaid.gov/dataset/6165f45b-ca93-5bb5-9d06-db29c692a360", + notes=( + "Medicaid take-up by anchored count-calibration (contract " + "treatment count_calibrated, microcosm #331): CPS-reported " + "Medicaid coverage at interview anchors the flag; the fill is " + "calibrated to CMS December 2024 state enrollment snapshots. " + "Point-in-time semantics per #332; heals the #170 " + "enrollment==eligibility degeneracy." + ), + ), + us_snap_state_take_up_stage: DonorSpec( + survey=( + "Census CPS ASEC reported receipt + USDA FNS state " + "average-monthly household caseloads" + ), + source="https://www.fns.usda.gov/pd/supplemental-nutrition-assistance-program-snap", + notes=( + "SNAP take-up by anchored count-calibration (contract treatment " + "count_calibrated, microcosm #372): reported ASEC receipt anchors " + "the flag; the fill is calibrated per state to FNS FY2024 " + "average-monthly household counts among eligible non-anchored " + "units, replacing the national snap_take_up fill that bakes in " + "state-dependent CPS underreporting." + ), + ), + us_other_health_insurance_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured PHIP_VAL is reduced by PolicyEngine-calculated CHIP, " + "Marketplace, and Medicaid premiums after take-up stages; an " + "ASEC-trained weighted QRF replaces both premium leaves on the " + "PUF support half." + ), + ), + us_prior_year_income_stage_name: DonorSpec( + survey="CPS ASEC (prior year)", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Adjacent-year PERIDNUM join for measured prior-year earnings, " + "with Census allocation flags and sentinels enforced. A joint " + "eight-predictor weighted QRF replaces both earnings leaves on " + "the PUF support half; signed self-employment losses survive." + ), + ), + us_immigration_stage_name: DonorSpec( + survey="CPS ASEC + published unauthorized-population estimates", + source=( + "https://www.pewresearch.org/short-reads/2024/07/22/" + "what-we-know-about-unauthorized-immigrants-living-in-the-us/" + ), + notes=( + "SSN card type and immigration status from ASEC citizenship, " + "entry-year, nativity, and program-participation fields via the " + "ASEC-UA residual method (SSRN 4662801), targeted to published " + "undocumented population/worker/student control totals." + ), + ), + us_hours_worked_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Hours-worked inputs mapped directly from measured ASEC person " + "variables (HRSWK, A_HRS1, WKSWORK) — nothing imputed. Without " + "them the engine defaults every person to 40 weekly hours and " + "hours-conditioned rules (SNAP work requirements) become no-ops." + ), + ), + us_snap_take_up_stage_name: DonorSpec( + survey="Census CPS ASEC + USDA FNS participation rate estimates", + source="https://www.fns.usda.gov/snap/participation-rates", + notes=( + "SNAP take-up: reported recipients (SPM_SNAPSUB) always take " + "up; non-reporting units drawn to the cited FNS participation " + "rate. Without it the engine defaults every eligible unit to " + "100% take-up." + ), + ), + us_eligibility_inputs_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "SNAP eligibility/exemption inputs mapped directly from " + "measured ASEC person variables (PEDIS*, A_HSCOL/A_FTPT, " + "PEPAR1/PEPAR2, VET_VAL, SSI_VAL) — nothing imputed. Without " + "them disability, student, parent/child, and veteran " + "exemption channels default to False/0." + ), + ), + us_relationship_inputs_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured household-head and marital-status flags mapped exactly " + "from P_SEQ and A_MARITL; nothing is imputed." + ), + ), + us_medicare_take_up_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured Medicare enrollment mapped exactly from MCARE == 1 and " + "copied onto the PUF support clone; no take-up rate is applied." + ), + ), + us_retirement_distribution_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "ASEC rows map exactly from all four DST_SC*/DST_VAL* pairs; the " + "archived CPS-only QRF replaces the four populated non-IRA leaves " + "on PUF support while preserving IRA channel ownership. Nothing " + "is allocated across accounts." + ), + ), + us_pregnancy_stage_name: DonorSpec( + survey="Census CPS ASEC + CDC natality-derived national pregnancy rate", + source="https://www.cdc.gov/nchs/nvss/births.htm", + notes=( + "Pregnancy seeded among women 15-44 at the national " + "point-in-time rate (births x 39/52 over female 15-44 " + "population), matching the retired pipeline's national " + "fallback; state-level rates are follow-up work (#351). " + "The ASEC does not measure pregnancy." + ), + ), + us_wic_claim_stage_name: DonorSpec( + survey="USDA FNS WIC Eligibility and Enrollment Estimates + Census CPS ASEC", + source="https://www.fns.usda.gov/research/wic/eligibility-and-coverage-rates-2022", + notes=( + "Stable person-level claim draws use official CY2022 FNS category " + "coverage rates after the pregnancy and parent-input stages. The " + "all-postpartum rate is used because no hermetic source identifies " + "breastfeeding; nutritional risk remains separately excluded." + ), + ), + us_snap_discretionary_exemption_stage_name: DonorSpec( + survey="Census CPS ASEC + statutory exemption cap (7 U.S.C. 2015(o)(6))", + source="https://www.law.cornell.edu/uscode/text/7/2015#o_6", + notes=( + "ABAWD discretionary exemptions seeded at the statutory cap " + "(8% from FY2024) across potentially covered adults 18-64; " + "the engine intersects with modeled coverage. Assumes full " + "state usage of the cap (#323)." + ), + ), + us_childcare_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured replicated SPM_CHILDCAREXPNS is validated and carried " + "to the SPM-unit childcare leaf. After support expansion, an " + "ASEC-trained weighted QRF replaces only the PUF half and the " + "archived first-person reduction places predictions on SPM units." + ), + ), + us_adult_care_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Section 21 CDCC adult-care inputs (microcosm#451 item 1): the " + "qualifying flag is the measured PEDISDRS self-care difficulty " + "item; the expense leg is a seeded, weight-targeted draw from " + "the measured ASEC childcare-expense distribution (the same " + "section 21 expense class), restricted to tax units where the " + "statute can bind, with the 21(d)(2) spouse deeming honored. " + "Neither ASEC nor the SIPP 2023 PUF releases an in-household " + "adult-care expenditure amount, so the level proxy is declared " + "in the stage manifest." + ), + ), + us_energy_subsidy_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured replicated SPM_ENGVAL is validated and carried to the " + "SPM-unit energy-subsidy leaf. After support expansion, an " + "ASEC-trained weighted QRF replaces only the PUF half and the " + "archived first-person reduction places predictions on SPM units." + ), + ), + us_child_support_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured annual CSP_VAL receipts and positive CHSP_VAL expenses " + "are carried directly. After PUF tax-detail imputation, one " + "ASEC-trained weighted QRF jointly replaces both person leaves " + "on the PUF support half using the archived predictor subset." + ), + ), + us_disability_benefits_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured annual DIS_VAL1/DIS_VAL2 benefits are retained only " + "when their source code is not workers' compensation. After PUF " + "tax-detail imputation, an ASEC-trained weighted QRF replaces the " + "CPS-only person leaf on the PUF support half." + ), + ), + us_workers_compensation_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured annual WC_VAL is carried directly. After PUF tax-detail " + "imputation, an ASEC-trained weighted QRF replaces the CPS-only " + "person leaf on the PUF support half." + ), + ), + us_weeks_unemployed_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured LKWEEKS is carried directly, including an exact " + "identity-keyed repair of the omitted 2022-income-year column " + "from the pinned official 2023 ASEC archive. An ASEC-trained " + "QRF then replaces only the PUF support half, with the archived " + "unemployment-compensation zero rule." + ), + ), + "puf_tax_detail": DonorSpec( + survey="IRS PUF 2015 (uprated)", + source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", + notes=( + "Itemized-deduction detail, versioned processed-PUF Section 199A " + "simulation leaves (carried without redrawing), partnership SE, " + "source-year-AGI E19200 mortgage/non-mortgage split, direct " + "E00800/E03500 alimony, direct E20500 casualty loss, and the E20400 " + "miscellaneous-expense proxy. The pinned processed PUF uprates its " + "raw TY2015 rows before seeded disclosure-record replacement and " + "includes a Forbes-backed 3,900-record open tail; this runtime " + "reconstructs the bounded-record AGI lineage and anchors every " + "Forbes-tail record in the final published AGI band without rerunning " + "Forbes synthesis. Support is clipped to the PUF's realized ranges." + ), + ), + us_education_inputs_stage_name: DonorSpec( + survey="IRS PUF 2015 (uprated) + Census CPS ASEC", + source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", + notes=( + "Qualified tuition comes from the PUF E03230/E87530 maximum; " + "the published retired path drops the reported AOTC output and its " + "five affirmative factual inputs therefore follow positive tuition; " + "educational assistance carries directly from ASEC ED_VAL." + ), + ), + us_retirement_contribution_stage_name: DonorSpec( + survey="Census CPS ASEC + published retirement-contribution shares", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured ASEC RETCB_VAL is allocated across five desired " + "retirement-contribution leaves using archived IRS/BEA/Vanguard/" + "PSCA shares, then CPS-trained QRF predictions replace the PUF " + "support half. PolicyEngine-US owns all statutory caps." + ), + ), + "capital_gain_distributions": DonorSpec( + survey="IRS SOI Sales of Capital Assets (TY2015) + Pub 1304 Table 1.4", + source=( + "https://www.irs.gov/statistics/" + "soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns" + ), + notes=( + "Schedule D line 13 capital gain distributions split out of " + "long-term gains as a memo component at the national SOCA-derived " + "share; the direct-1040 route is already a PUF-stage output and " + "the two routes are mutually exclusive on a real return." + ), + ), + us_housing_inputs_stage_name: DonorSpec( + survey="Census ACS 2022", + source="https://www.census.gov/programs-surveys/acs", + notes=( + "Exact ASEC H_TENURE and SPM housing carries plus annual ACS PUMS " + "rent imputation on household heads; reported housing assistance " + "alone receives the archived second-stage PUF-support QRF." + ), + ), + "vehicle_assets": DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Household vehicle count and value from the pinned full 2023 " + "public-use donor. They remain independent policy inputs until " + "the full mixed-source net-worth reconciliation is restored." + ), + ), + us_voluntary_filing_stage_name: DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Measured 2023 SIPP filing and expected-filing responses replace " + "the retired uncited demographic probability table. Reciprocal " + "spouses form one source unit, reported dependents are excluded, " + "and one weighted-QRF prediction is shared across support clones." + ), + ), + } + us_stage_names: tuple[str, ...] = ( + "asec_load", + "unit_assignment", + "derive_cps_carried", + us_prior_year_income_stage_name, + us_immigration_stage_name, + us_hours_worked_stage_name, + us_snap_take_up_stage_name, + us_relationship_inputs_stage_name, + us_medicare_take_up_stage_name, + us_housing_inputs_stage_name, + us_retirement_distribution_stage_name, + us_eligibility_inputs_stage_name, + us_pregnancy_stage_name, + us_wic_claim_stage_name, + us_snap_discretionary_exemption_stage_name, + us_retirement_contribution_stage_name, + us_childcare_stage_name, + us_adult_care_stage_name, + us_energy_subsidy_stage_name, + us_puf_support_stage_name, + "puf_tax_detail", + us_child_support_stage_name, + us_disability_benefits_stage_name, + us_workers_compensation_stage_name, + us_weeks_unemployed_stage_name, + us_education_inputs_stage_name, + "capital_gain_distributions", + "scf_wealth", + us_ssi_disability_criteria_stage_name, + us_sipp_head_start_stage_name, + us_ssi_take_up_stage_name, + "sipp_tips", + "org_wages", + "meps_esi_premiums", + "mortgage_conversion", + "vehicle_assets", + us_voluntary_filing_stage_name, + "entity_placement", + "aca_marketplace_inputs", + "medicaid_take_up", + us_snap_state_take_up_stage, + us_other_health_insurance_stage_name, + "export", + ) + globals().setdefault("US_DONORS", us_donors) + globals().setdefault("US_STAGE_NAMES", us_stage_names) def _load_us_source_manifest() -> SourceManifest: @@ -2481,15 +2625,6 @@ def _load_us_support_spine_manifest() -> SupportSpineManifest: ) -US_SOURCE_MANIFEST = _load_us_source_manifest() -US_SUPPORT_SPINE_MANIFEST = _load_us_support_spine_manifest() -US_SUPPORT_SPINE_SPEC: SupportSpineSpec = US_SUPPORT_SPINE_MANIFEST.support_spine -US_SOURCE_STAGE_SPECS: tuple[SourceStageSpec, ...] = US_SOURCE_MANIFEST.stages -US_NONNEGATIVE_SOURCE_OUTPUTS: frozenset[str] = frozenset( - output for stage in US_SOURCE_STAGE_SPECS for output in stage.nonnegative_outputs -) - - def us_plan( implementations: Mapping[str, Callable[[Frame], Frame]], ) -> StagePlan: @@ -2510,23 +2645,26 @@ def us_plan( ValueError: If any declared stage lacks an implementation, or an implementation is supplied for an undeclared stage. """ - missing = [name for name in US_STAGE_NAMES if name not in implementations] + _initialize_plan() + stage_names = globals()["US_STAGE_NAMES"] + donors = globals()["US_DONORS"] + missing = [name for name in stage_names if name not in implementations] if missing: raise ValueError( f"us_plan needs an implementation for every declared stage; " f"missing {missing}. There are no stubs or fallbacks by design." ) - unknown = sorted(set(implementations) - set(US_STAGE_NAMES)) + unknown = sorted(set(implementations) - set(stage_names)) if unknown: raise ValueError( f"Unknown stage implementation(s) {unknown}; declared stages " - f"are {list(US_STAGE_NAMES)}." + f"are {list(stage_names)}." ) return StagePlan( Stage( name=name, transform=implementations[name], - donor=US_DONORS.get(name), + donor=donors.get(name), ) - for name in US_STAGE_NAMES + for name in stage_names ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py b/packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py new file mode 100644 index 000000000..dca569472 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py @@ -0,0 +1,143 @@ +"""Bounded, checksummed codec preserving target-only money source authority.""" + +import struct +from hashlib import sha256 + +from .asec_current_money import ( + _STATE_TOKEN, + HEADER_MAX_BYTES, + MAX_HOUSEHOLDS, + MAX_PERSONS, + AuthenticatedAsecSource, + MoneyBindings, + MoneyField, + ReadyCurrentMoney, + RestatedAsecMoney, + SyntheticReadyCurrentMoney, + _parse, + _require, + _validate_money, + require_complete_current_money, +) + +MAGIC = b"MCAM2024\x01" +_CONTENT_DOMAIN = b"microcosm/asec-current-money-content/1\0" +# 32 person fields, one household observation, 8+1+1+1 bytes per field/row. +PAYLOAD_MAX_BYTES = ( + len(MAGIC) + 4 + HEADER_MAX_BYTES + 11 * (32 * MAX_PERSONS + MAX_HOUSEHOLDS) + 32 +) + + +def _ready_parts( + ready: SyntheticReadyCurrentMoney | ReadyCurrentMoney, +) -> tuple[bytes, ...]: + """One canonical serialization shared by transport and content identity.""" + _require( + type(ready) in (SyntheticReadyCurrentMoney, ReadyCurrentMoney), + "SYNTHETIC_READY_REQUIRED", + ) + _validate_money(ready, ready.bindings.spec, nominal=False) + # Recheck completeness even for a directly constructed or replaced value. + require_complete_current_money( + RestatedAsecMoney(ready.bindings, ready.fields, _token=_STATE_TOKEN), + ready.bindings.spec, + production=type(ready) is ReadyCurrentMoney, + ) + parts = [MAGIC, struct.pack(" str: + """Bind the full canonical header and field body, apart from transport checksum.""" + digest = sha256(_CONTENT_DOMAIN) + for part in _ready_parts(ready): + digest.update(part) + return digest.hexdigest() + + +def encode_current_money( + ready: SyntheticReadyCurrentMoney | ReadyCurrentMoney, +) -> bytes: + """Encode immutable target amounts and evidence, including field-scoped origins. + + Schema 1 retains legacy zero provenance. Schema 2 binds the restored-source + PTOTVAL origin policy through the exact header/spec; the byte widths and + checksum framing are unchanged. Per-field validation rejects swapped codes. + """ + parts = _ready_parts(ready) + checksum = sha256() + for part in parts: + checksum.update(part) + return b"".join((*parts, checksum.digest())) + + +def decode_current_money( + payload: bytes, + expected_bindings: MoneyBindings, + *, + expected: ReadyCurrentMoney | None = None, +) -> SyntheticReadyCurrentMoney | ReadyCurrentMoney: + """Authenticate replay content against issued readiness before state creation. + + Verified source bindings alone cannot authorize a restated amount body. + Synthetic replay retains its binding-only API and refuses an expected object. + """ + _require( + type(payload) is bytes + and len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + _require(type(expected_bindings) is MoneyBindings, "EXPECTED_BINDINGS_REQUIRED") + expected_bindings.__post_init__() + authenticated = type(expected_bindings.spec.source) is AuthenticatedAsecSource + if authenticated: + _require(type(expected) is ReadyCurrentMoney, "EXPECTED_CONTENT_REQUIRED") + _require(expected.bindings == expected_bindings, "EXPECTED_CONTENT_BINDING") + _validate_money(expected, expected_bindings.spec, nominal=False) + else: + _require(expected is None, "EXPECTED_CONTENT_SYNTHETIC") + view = memoryview(payload) + _require(payload.startswith(MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" None: + """Require genuine-summary fields, numeric domains and registered bands. + + This validates evidence shape and the fixed decision policy, not the source + identity or truth of the supplied measurements. Genuine summary functions + emit Python integers/floats and list-valued bands; immutable tuple bands are + also accepted for callers that freeze the same data in memory. + """ + prefix = f"Malformed {family}-input summary" + fields = { + "unique_counts", + *share_bands, + *(name for name, _ in share_bands.values()), + *invariants, + } + if not isinstance(summary, Mapping) or set(summary) != fields: + raise ValueError(f"{prefix}: expected the exact registered field inventory.") + counts = summary["unique_counts"] + if not isinstance(counts, Mapping) or set(counts) != set(outputs): + raise ValueError(f"{prefix}: expected a count for every registered output.") + for name, value in (*counts.items(), *((key, summary[key]) for key in invariants)): + if type(value) is not int or value < 0: + raise ValueError(f"{prefix}: {name!r} must be a nonnegative integer.") + for share_name, (band_name, registered) in share_bands.items(): + share = summary[share_name] + # The closed interval also refuses both infinities and NaN without + # coercing strings, bools, or oversized integer evidence to float64. + if type(share) not in (int, float) or not 0.0 <= share <= 1.0: + raise ValueError(f"{prefix}: {share_name!r} must be finite in [0, 1].") + band = summary[band_name] + if ( + not isinstance(band, (list, tuple)) + or len(band) != 2 + or any(type(value) not in (int, float) for value in band) + or tuple(band) != registered + ): + raise ValueError( + f"{prefix}: {band_name!r} differs from its registered band." + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_housing_universe.json b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_housing_universe.json new file mode 100644 index 000000000..272b560dd --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_housing_universe.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "source_kind": "microcosm.acs_housing_universe_source.v1", + "vintage": 2024, + "dictionary": { + "url": "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf", + "sha256": "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff", + "pages": [1, 2, 3, 8, 32, 33] + }, + "vacancy_definition": "acs_2024_source_vacancy_v1", + "vacancy": "TYPEHUGQ=1, NP=0, no linked persons, WGTP=1..9999, blank TEN; separate source vacancy evidence, not a change to the pure classifier", + "gq_definition": "acs_2024_gq_person_placeholder_design_v1", + "gq": "TYPEHUGQ=2/3, NP=1, WGTP=0, blank TEN, one linked person with PWGTP=1..9999; one source person placeholder at DESIGN weight, not a physical facility or calibrated descendant", + "occupied_unknown_tenure": "Retained in occupied HU total with explicitly unclassified tenure; never infer from model tenure_type", + "origin": "CD source household origin: ACS arm, original household member name and SHA256, vintage integer2024, raw SERIALNO string; person additionally retains original person member and normalized SPORDER", + "parser": "UTF8/noBOM, strict comma QUOTE_MINIMAL, lexical projected fields, no trimming/NA inference; same ordered full role headers; every physical record full width", + "encoding": "acs-housing-lexical-projection/1", + "classifier_columns": ["interview_scope", "physical_unit", "household_kind", "tenure_subtype", "occupied_hu", "hu_tenure_class", "unresolved_reasons", "TEN_valid"], + "us_archive_scope_assumption": "50 US states and DC; observed Puerto Rico refuses rather than silently changing scope", + "release_eligible": false, + "calibrated_descendants_approved": false, + "B19001_B25003_activation": false +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py new file mode 100644 index 000000000..fc4928883 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py @@ -0,0 +1,203 @@ +"""Pure, unbound ACS housing-unit semantics; no source authentication. + +ACS TYPEHUGQ/NP observations do not supply the CPS interview-scope or +household-kind observations. Those output axes therefore remain source-specific +not-observed codes, even for a known occupied ACS housing unit. Invalid TEN +payloads are uninterpreted placeholders, never observed zeros. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +__all__ = ["ACSHousingUniverseRefusalError", "classify_acs_housing_universe"] + + +class ACSHousingUniverseRefusalError(ValueError): + """Refuse an invalid input contract without reporting source values or keys.""" + + +def _owned_index(index: pd.Index) -> pd.Index: + """Copy axis storage, reconstructing indexes with shared deep-copy caches.""" + if isinstance(index, pd.RangeIndex): + return pd.RangeIndex(index.start, index.stop, index.step, name=index.name) + if isinstance(index, pd.MultiIndex): + return pd.MultiIndex( + levels=[_owned_index(level) for level in index.levels], + codes=[code.copy() for code in index.codes], + names=list(index.names), + sortorder=index.sortorder, + verify_integrity=False, + ) + if isinstance(index, pd.CategoricalIndex): + return pd.CategoricalIndex( + pd.Categorical.from_codes( + index.codes.copy(), + categories=_owned_index(index.categories), + ordered=index.ordered, + ), + name=index.name, + ) + return index.copy(deep=True) + + +def _header() -> dict: + """Fresh semantic metadata only: no source rows, keys, counts, or pins.""" + return { + "schema_version": 1, + "artifact_kind": "microcosm.acs_housing_universe_unbound.v1", + "source_authentication": "unbound_semantic_classifier", + "release_eligible": False, + "source_mapping": { + "TYPEHUGQ": { + "1": "housing_unit", + "2": "institutional_group_quarters", + "3": "noninstitutional_group_quarters", + }, + "NP": "nonnegative int64; no additional upper bound", + "TEN": "native ACS codes; see tenure_subtype codebook", + "occupied_hu": ( + "TYPEHUGQ=1 and NP>0 establishes occupancy; TYPEHUGQ=2/3 " + "is outside the HU universe; TYPEHUGQ=1 and NP=0 leaves " + "occupancy unresolved and does not assert vacancy" + ), + "hu_tenure_class": ( + "only known occupied HUs: valid TEN=1/2 owner, valid TEN=3/4 " + "renter, invalid TEN unclassified but retained in the HU total" + ), + "interview_scope": "CPS interview observation not supplied by ACS inputs", + "household_kind": "CPS household-kind observation not supplied by ACS inputs", + "ignored_columns": "all other columns, including model tenure_type", + "row_policy": "retain every row in original order, including all GQ rows", + }, + "codebooks": { + "interview_scope": {"0": "not_observed_in_these_ACS_inputs"}, + "physical_unit": {"1": "housing_unit", "2": "group_quarters"}, + "household_kind": {"0": "not_observed_in_these_ACS_inputs"}, + "tenure_subtype": { + "0": "unavailable", + "1": "owned_with_mortgage_or_loan", + "2": "owned_outright", + "3": "paying_rent", + "4": "no_rent_paid", + }, + "occupied_hu": { + "0": "occupancy_not_established", + "1": "occupied_housing_unit", + "2": "outside_housing_unit_universe", + }, + "hu_tenure_class": { + "0": "outside_or_unresolved_occupied_HU_universe", + "1": "owner", + "2": "renter_including_no_rent", + "3": "occupied_HU_tenure_unclassified", + }, + "unresolved_reasons": { + "0": "no_unresolved_reason", + "1": "housing_unit_occupancy_not_established", + "2": "occupied_housing_unit_tenure_unavailable", + }, + "TEN_valid": {"0": "unavailable", "1": "observed_native_TEN"}, + }, + "validity_placeholder_semantics": { + "input": "exact numpy int64 TYPEHUGQ/NP/TEN and positional bool ndarray mask", + "invalid_TEN": ( + "any int64 payload is an uninterpreted placeholder; never an " + "observed zero and never checked against the observed TEN domain" + ), + "output": "uint8 codes; tenure_subtype=0 iff TEN_valid=0 on every row", + "GQ": "unavailable TEN, including a blank, adds no conflict or reason", + "unresolved_reasons": "bit mask: 1 occupancy not established; 2 occupied HU TEN unavailable", + }, + "limits": [ + "no_authenticated_source_or_source_pins", + "no_CPS_interview_or_household_kind_observations", + "no_B19001_income_concept_bridge", + "no_scientific_certification", + "no_stacked_population_or_denominator_certification", + ], + } + + +def classify_acs_housing_universe( + households: pd.DataFrame, *, tenure_valid: np.ndarray +) -> tuple[pd.DataFrame, dict]: + """Classify supplied ACS observations without authenticating their source. + + Require an exact DataFrame with unique columns and native numpy int64 + TYPEHUGQ, NP, and TEN. TYPEHUGQ must be 1/2/3 and NP nonnegative. The + positional validity argument must be an exact one-dimensional bool ndarray + of matching length. Only valid TEN cells are observed and require codes + 1/2/3/4; invalid payloads can be any int64. No coercion or nullable path is + provided. Other columns, including tenure_type, are ignored. + + Return an owned uint8 DataFrame preserving every row and its index, plus a + fresh, data-independent semantic header. Both CPS-specific axes stay zero + (not observed). NP=0 does not establish vacancy. Unknown occupied-HU tenure + stays explicitly unclassified within the total; GQ tenure never changes + the HU universe. The classifier performs no I/O or source authentication. + """ + if type(households) is not pd.DataFrame: + raise ACSHousingUniverseRefusalError("households_must_be_exact_dataframe") + if not households.columns.is_unique: + raise ACSHousingUniverseRefusalError("household_columns_must_be_unique") + required = ("TYPEHUGQ", "NP", "TEN") + if not set(required).issubset(set(households.columns)): + raise ACSHousingUniverseRefusalError("required_ACS_columns_missing") + for column in required: + dtype = households[column].dtype + if not isinstance(dtype, np.dtype) or dtype != np.dtype(np.int64): + raise ACSHousingUniverseRefusalError("ACS_columns_must_be_numpy_int64") + if ( + type(tenure_valid) is not np.ndarray + or tenure_valid.dtype != np.dtype(np.bool_) + or tenure_valid.ndim != 1 + or len(tenure_valid) != len(households) + ): + raise ACSHousingUniverseRefusalError( + "TEN_validity_must_be_matching_bool_vector" + ) + + unit_type = households["TYPEHUGQ"].to_numpy(copy=False) + persons = households["NP"].to_numpy(copy=False) + tenure = households["TEN"].to_numpy(copy=False) + if np.any((unit_type < 1) | (unit_type > 3)): + raise ACSHousingUniverseRefusalError("TYPEHUGQ_outside_domain") + if np.any(persons < 0): + raise ACSHousingUniverseRefusalError("NP_must_be_nonnegative") + observed_tenure = tenure[tenure_valid] + if np.any((observed_tenure < 1) | (observed_tenure > 4)): + raise ACSHousingUniverseRefusalError("valid_TEN_outside_domain") + + housing_unit = unit_type == 1 + occupied = housing_unit & (persons > 0) + occupancy_unknown = housing_unit & (persons == 0) + tenure_subtype = np.zeros(len(households), dtype=np.uint8) + tenure_subtype[tenure_valid] = observed_tenure + occupied_hu = np.zeros(len(households), dtype=np.uint8) + occupied_hu[occupied] = 1 + occupied_hu[~housing_unit] = 2 + hu_tenure_class = np.zeros(len(households), dtype=np.uint8) + hu_tenure_class[occupied & tenure_valid & (tenure_subtype <= 2)] = 1 + hu_tenure_class[occupied & tenure_valid & (tenure_subtype >= 3)] = 2 + hu_tenure_class[occupied & ~tenure_valid] = 3 + reasons = np.zeros(len(households), dtype=np.uint8) + reasons[occupancy_unknown] |= 1 + reasons[occupied & ~tenure_valid] |= 2 + + output = pd.DataFrame( + { + "interview_scope": np.zeros(len(households), dtype=np.uint8), + "physical_unit": np.where(housing_unit, 1, 2).astype(np.uint8), + "household_kind": np.zeros(len(households), dtype=np.uint8), + "tenure_subtype": tenure_subtype, + "occupied_hu": occupied_hu, + "hu_tenure_class": hu_tenure_class, + "unresolved_reasons": reasons, + "TEN_valid": tenure_valid.astype(np.uint8), + }, + index=_owned_index(households.index), + copy=True, + ) + return output, _header() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py new file mode 100644 index 000000000..104c15936 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py @@ -0,0 +1,933 @@ +"""Closed ACS source observations, with a same-snapshot native Frame bridge. + +Source projections retain vacancies, both GQ classes and unknown occupied tenure. +Their graph transport is separate; source authentication requires reconstruction +from the two fixed Census archives. No source or model download occurs here. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import shutil +import stat +import tempfile +import zipfile +from contextlib import contextmanager, suppress +from dataclasses import InitVar, dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import dtype_for_token +from microcosm.graph.store import ContentStore + +from .acs_housing_universe import classify_acs_housing_universe +from .acs_inputs import map_acs_native_inputs +from .acs_pums import AcsPumsSource, _validate_person_counts, build_acs_pums_unit_frame +from .acs_sources import load_acs_source_manifest +from .graph_implementation import implementation_hash +from .operator_boundary import assert_operator_free_source_frame +from .source_csv_builtin import capture_csv_reader + +ACS_HU_STAGE = "acs_housing_universe_2024" +ACS_HU_CODEC = "us-acs-housing-universe-2024-v1" +ACS_HU_SOURCE_MAX_BYTES = 8 * 1024**3 +ACS_HU_RECEIPT_MAX_BYTES = 1024**2 +_CHUNK = 1024**2 +_MEMBER_MAX = 8 * 1024**3 +_EXPANDED_MAX = 16 * 1024**3 +_MEMBER_COUNT_MAX = 64 +_RECORD_MAX = 1024**2 +_FIELD_MAX = 64 * 1024 +_STATES = frozenset( + "01 02 04 05 06 08 09 10 11 12 13 15 16 17 18 19 20 21 22 23 24 25 " + "26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 44 45 46 47 48 " + "49 50 51 53 54 55 56".split() +) +_H_FIELDS = ("SERIALNO", "TYPEHUGQ", "NP", "TEN", "WGTP", "PUMA") +_P_FIELDS = ("SERIALNO", "SPORDER", "PWGTP") +_LINEAGE = ("source_member", "source_row_ordinal") +# No source/resource I/O merely to register the callable beside legacy codecs. +# Invented tests replace this private slot; public callers have no pin argument. +_ARCHIVE_PINS = None +_TOKEN = object() + + +class ACSHousingSourceError(ValueError): + """Sanitized source refusal; never print source keys or cells.""" + + +def _require(condition, code): + if not condition: + raise ACSHousingSourceError(code) + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _json(value) -> bytes: + return canonical_json(value) + + +def _parse_json(payload: bytes, cap: int) -> dict: + _require(type(payload) is bytes and len(payload) <= cap, "JSON_SIZE") + try: + value = json.loads(payload) + _require(type(value) is dict and _json(value) == payload, "CANONICAL_JSON") + except RecursionError: + raise ACSHousingSourceError("JSON_DEPTH") from None + return value + + +def _definition() -> tuple[dict, str]: + raw = Path(__file__).with_name("acs_2024_housing_universe.json").read_bytes() + return json.loads(raw), _sha(raw) + + +def _implementation() -> str: + _require(capture_csv_reader(csv) is not None, "SOURCE_CSV_READER_CHANGED") + return implementation_hash(ACS_HU_STAGE) + + +def _pins(): + if _ARCHIVE_PINS is not None: + return _ARCHIVE_PINS + return tuple( + (a.role, a.filename, a.sha256, a.size_bytes) + for a in load_acs_source_manifest().artifacts + ) + + +def _identity(value: os.stat_result): + return tuple( + getattr(value, key) + for key in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + ) + + +def _path_components(path: Path): + path = path.absolute() + for parent in reversed((path, *path.parents)): + value = parent.lstat() + _require(not stat.S_ISLNK(value.st_mode), "SYMLINK_PATH") + return Path(os.path.abspath(path)) + + +def _directory(path: Path): + path = _path_components(path) + _require(stat.S_ISDIR(path.lstat().st_mode), "DIRECTORY_REQUIRED") + return path + + +def _copy(source: Path, destination: Path, cap: int, *, exact_size=None): + source = _path_components(source) + fd = os.open(source, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + _require(stat.S_ISREG(before.st_mode), "REGULAR_FILE_REQUIRED") + _require(before.st_size <= cap, "FILE_TOO_LARGE") + if exact_size is not None: + _require(before.st_size == exact_size, "SOURCE_SIZE") + digest, count = hashlib.sha256(), 0 + with destination.open("xb") as output: + while chunk := stream.read(min(_CHUNK, cap - count + 1)): + count += len(chunk) + _require(count <= cap, "FILE_GREW") + digest.update(chunk) + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + _require( + count == before.st_size + and _identity(before) == _identity(os.fstat(stream.fileno())) + and _identity(before) == _identity(source.lstat()), + "FILE_CHANGED", + ) + _path_components(source) + destination.chmod(0o400) + return digest.hexdigest() + + +def _persisted_sha(path, size): + digest, count = hashlib.sha256(), 0 + fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + _require( + stat.S_ISREG(before.st_mode) and before.st_size == size, "SNAPSHOT_SIZE" + ) + while chunk := stream.read(min(_CHUNK, size - count + 1)): + count += len(chunk) + _require(count <= size, "SNAPSHOT_GREW") + digest.update(chunk) + _require( + count == size and _identity(before) == _identity(os.fstat(stream.fileno())), + "SNAPSHOT_CHANGED", + ) + return digest.hexdigest() + + +def _write(path: Path, data: bytes, cap: int): + _require(len(data) <= cap, "OUTPUT_SIZE") + with path.open("xb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + path.chmod(0o400) + _require(_persisted_sha(path, len(data)) == _sha(data), "PERSISTED_OUTPUT") + + +@contextmanager +def _capture(source_dir, snapshot_root): + root = _directory(Path(snapshot_root)) + source = _directory(Path(source_dir)) + _require(not root.is_relative_to(source), "SNAPSHOT_SOURCE_OVERLAP") + pins = _pins() + expected = {p[1] for p in pins} + _require(expected == {"csv_hus.zip", "csv_pus.zip"}, "SOURCE_ROSTER") + _require({p.name for p in source.iterdir()} == expected, "SOURCE_ROSTER") + # Full private projection and selected publication/candidate capture can + # coexist. Existing captures already reduce free space and are not deducted + # a second time. This bound is operational, not a source-size prediction. + required = ( + sum(p[3] for p in pins) + + 2 * ACS_HU_SOURCE_MAX_BYTES + + 2 * ACS_HU_RECEIPT_MAX_BYTES + + 1024**3 + ) + _require(shutil.disk_usage(root).free >= required, "INSUFFICIENT_DISK") + private = Path(tempfile.mkdtemp(prefix="acs-hu-capture-", dir=root)) + paths = {} + try: + for role, name, digest, size in pins: + target = private / name + actual = _copy(source / name, target, size, exact_size=size) + _require(actual == digest, "SOURCE_SHA256") + _require(_persisted_sha(target, size) == digest, "PERSISTED_SOURCE_SHA256") + paths[role] = target + yield private, paths, pins + _require(pins == _pins(), "SOURCE_AUTHORITY_CHANGED") + except Exception as error: + # Captures belong exclusively to this invocation. Keep failed evidence; + # no rename can replace an earlier artifact or caller input. + code = ( + str(error) + if isinstance(error, ACSHousingSourceError) + else "SOURCE_PREPARATION_REFUSED" + ) + # Recording is best effort and strictly subordinate: if the record + # cannot be written, the refusal it was recording still propagates. + with suppress(Exception): + failure = _json({"status": "failed", "reason": code}) + if not (private / "failure.json").exists(): + _write(private / "failure.json", failure, ACS_HU_RECEIPT_MAX_BYTES) + raise + + +def _csv_record(raw: bytes) -> list[str]: + csv_reader = capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + _require(0 < len(raw) <= _RECORD_MAX, "CSV_RECORD_SIZE") + _require(not raw.startswith(b"\xef\xbb\xbf"), "CSV_BOM") + if raw.endswith(b"\n"): + raw = raw[:-1] + if raw.endswith(b"\r"): + raw = raw[:-1] + _require( + bool(raw) and b"\x00" not in raw and b"\r" not in raw and b"\n" not in raw, + "CSV_PHYSICAL_RECORD", + ) + values = next( + csv_reader( + [raw.decode("utf-8", errors="strict")], + strict=True, + quoting=csv.QUOTE_MINIMAL, + ) + ) + _require( + all(len(v.encode("utf-8")) <= _FIELD_MAX for v in values), "CSV_FIELD_SIZE" + ) + return values + + +def _archive(path: Path, role: str): + """Validate the complete central directory before resolving any member name.""" + prefix = "psam_hus" if role == "household" else "psam_pus" + required = _H_FIELDS if role == "household" else _P_FIELDS + inventory, rows, role_header, fields, total_bytes = [], [], None, None, 0 + with zipfile.ZipFile(path) as archive: + members = archive.infolist() + _require(0 < len(members) <= _MEMBER_COUNT_MAX, "ZIP_MEMBER_COUNT") + names = [m.filename for m in members] + _require( + len({n.casefold() for n in names}) == len(names), "ZIP_DUPLICATE_MEMBER" + ) + for member in members: + name = member.filename + mode = member.external_attr >> 16 + _require( + name not in ("", ".", "..") + and "/" not in name + and "\\" not in name + and "\x00" not in name, + "ZIP_MEMBER_PATH", + ) + _require( + not member.is_dir() and (stat.S_IFMT(mode) in (0, stat.S_IFREG)), + "ZIP_MEMBER_TYPE", + ) + _require( + not member.flag_bits & 1 + and member.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED), + "ZIP_MEMBER_ENCODING", + ) + _require(0 <= member.file_size <= _MEMBER_MAX, "ZIP_MEMBER_SIZE") + total_bytes += member.file_size + _require(total_bytes <= _EXPANDED_MAX, "ZIP_EXPANDED_SIZE") + _require( + not name.casefold().startswith(prefix) + or name.casefold().endswith(".csv"), + "ZIP_PREFIX_NONCSV", + ) + selected_bytes = 0 + for member in sorted(members, key=lambda m: m.filename): + applicable = member.filename.casefold().startswith(prefix) + digest, count, header_raw, header, record_count = ( + hashlib.sha256(), + 0, + b"", + None, + 0, + ) + with archive.open(member) as stream: + while True: + raw = ( + stream.readline( + min(_RECORD_MAX + 1, member.file_size - count + 1) + ) + if applicable + else stream.read(min(_CHUNK, member.file_size - count + 1)) + ) + if not raw: + break + count += len(raw) + _require(count <= member.file_size, "ZIP_MEMBER_GREW") + digest.update(raw) + if not applicable: + continue + tokens = _csv_record(raw) + if header is None: + header, header_raw = tokens, raw + _require( + all(header) and len(set(header)) == len(header), + "CSV_HEADER", + ) + _require(set(required) <= set(header), "CSV_REQUIRED_FIELDS") + if role == "household": + _require( + bool({"ST", "STATE"} & set(header)), + "CSV_STATE_REQUIRED", + ) + if role_header is None: + role_header = header + fields = ( + ( + *required, + *(n for n in ("ST", "STATE") if n in header), + ) + if role == "household" + else required + ) + _require(header == role_header, "CSV_ROLE_HEADER_MISMATCH") + positions = [header.index(f) for f in fields] + continue + _require(len(tokens) == len(header), "CSV_ROW_WIDTH") + record_count += 1 + row = [ + *(tokens[p] for p in positions), + member.filename, + record_count, + ] + selected_bytes += len(_json(row)) + _require( + selected_bytes <= ACS_HU_SOURCE_MAX_BYTES, + "PROJECTION_TOO_LARGE", + ) + rows.append(row) + _require(count == member.file_size, "ZIP_MEMBER_SIZE_MISMATCH") + _require(not applicable or header is not None, "CSV_HEADER_MISSING") + inventory.append( + { + "name": member.filename, + "compressed_bytes": member.compress_size, + "bytes": count, + "crc32": member.CRC, + "sha256": digest.hexdigest(), + "is_data": applicable, + "header_hex": header_raw.hex(), + "header_sha256": _sha(header_raw), + "header": header, + "rows": record_count, + } + ) + _require(role_header is not None, "ZIP_ROLE_MISSING") + return list((*fields, *_LINEAGE)), rows, inventory + + +def _integer(token, low, high, label): + _require( + type(token) is str + and re.fullmatch(r"[0-9]+", token, flags=re.ASCII) is not None + and len(token) <= 5, + label, + ) + value = int(token) + _require(low <= value <= high, label) + return value + + +def _tables(projection): + _require( + set(projection) + == {"format", "household_columns", "person_columns", "households", "persons"} + and projection["format"] == "acs-housing-lexical-projection/1", + "PROJECTION_SCHEMA", + ) + hcols, pcols = projection["household_columns"], projection["person_columns"] + _require( + hcols + in [ + list((*_H_FIELDS, *state, *_LINEAGE)) + for state in (("ST",), ("STATE",), ("ST", "STATE")) + ] + and pcols == list((*_P_FIELDS, *_LINEAGE)), + "PROJECTION_COLUMNS", + ) + for cols, name in ((hcols, "households"), (pcols, "persons")): + _require(type(projection[name]) is list, "PROJECTION_ROWS") + for row in projection[name]: + _require( + type(row) is list + and len(row) == len(cols) + and all(type(v) is str for v in row[:-1]) + and type(row[-1]) is int + and row[-1] > 0, + "PROJECTION_ROW", + ) + household = pd.DataFrame(projection["households"], columns=hcols) + person = pd.DataFrame(projection["persons"], columns=pcols) + _require(not household.empty, "EMPTY_HOUSEHOLD_SOURCE") + for table in (household, person): + _require( + all( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", value, flags=re.ASCII) + and int(value[-7:]) > 0 + for value in table.SERIALNO + ), + "SERIALNO", + ) + _require(not household.SERIALNO.duplicated().any(), "DUPLICATE_HOUSEHOLD") + _require(set(person.SERIALNO) <= set(household.SERIALNO), "ORPHAN_PERSON") + typed = pd.DataFrame(index=household.index) + for name, low, high in (("TYPEHUGQ", 1, 3), ("NP", 0, 20), ("WGTP", 0, 9999)): + typed[name] = np.asarray( + [_integer(v, low, high, name) for v in household[name]], dtype="int64" + ) + valid = household.TEN.to_numpy() != "" + typed["TEN"] = np.asarray( + [ + _integer(v, 1, 4, "TEN") if present else 0 + for v, present in zip(household.TEN, valid, strict=True) + ], + dtype="int64", + ) + lines = np.asarray( + [ + _integer(v, 1, 20, "SPORDER") + if len(v) <= 2 + else _integer("", 1, 20, "SPORDER") + for v in person.SPORDER + ], + dtype="int64", + ) + pw = np.asarray( + [_integer(v, 1, 9999, "PWGTP") for v in person.PWGTP], dtype="int64" + ) + _require( + not pd.MultiIndex.from_arrays([person.SERIALNO, lines]).duplicated().any(), + "DUPLICATE_PERSON", + ) + state = "ST" if "ST" in household else "STATE" + _require(set(household[state]) <= _STATES, "STATE_SCOPE") + if "ST" in household and "STATE" in household: + _require(household.ST.equals(household.STATE), "STATE_CONFLICT") + for v in household.PUMA: + _require(len(v) == 5, "PUMA") + _integer(v, 100, 81003, "PUMA") + gq = typed.TYPEHUGQ.to_numpy() != 1 + vacant = ~gq & (typed.NP.to_numpy() == 0) + _require( + np.array_equal(household.SERIALNO.str.startswith("2024GQ").to_numpy(), gq), + "SERIALNO_TYPE", + ) + _require(bool((typed.NP.to_numpy()[gq] == 1).all()), "GQ_NP") + _require( + bool((typed.WGTP.to_numpy()[gq] == 0).all()) + and bool((typed.WGTP.to_numpy()[~gq] > 0).all()), + "WGTP_SCOPE", + ) + _require(not bool(valid[gq | vacant].any()), "TEN_NIU_SCOPE") + full_h = typed.assign(SERIALNO=household.SERIALNO) + _validate_person_counts(full_h, person) + codes, _header = classify_acs_housing_universe( + typed.loc[:, ["TYPEHUGQ", "NP", "TEN"]], tenure_valid=valid.astype(bool) + ) + return household, person, typed, lines, pw, codes + + +def _select(projection, serialnos): + household, person, typed, lines, _pw, _codes = _tables(projection) + if serialnos is None: + chosen = tuple(sorted(household.SERIALNO)) + else: + _require( + type(serialnos) is tuple + and bool(serialnos) + and all(type(v) is str for v in serialnos) + and len(set(serialnos)) == len(serialnos), + "SELECTION_KEYS", + ) + _require(set(serialnos) <= set(household.SERIALNO), "SELECTION_UNKNOWN") + chosen = tuple(sorted(serialnos)) + horder = household.sort_values("SERIALNO", kind="stable").index + porder = ( + person.assign(_line=lines) + .sort_values(["SERIALNO", "_line"], kind="stable") + .index + ) + chosen_set = set(chosen) + selected = { + **projection, + "households": [ + projection["households"][i] + for i in horder + if household.SERIALNO[i] in chosen_set + ], + "persons": [ + projection["persons"][i] for i in porder if person.SERIALNO[i] in chosen_set + ], + } + full_counts = { + "households": len(household), + "persons": len(person), + "vacant": int(((typed.TYPEHUGQ == 1) & (typed.NP == 0)).sum()), + "institutional_gq": int((typed.TYPEHUGQ == 2).sum()), + "noninstitutional_gq": int((typed.TYPEHUGQ == 3).sum()), + } + return selected, chosen, full_counts + + +@dataclass(frozen=True) +class AuthenticatedACSHousingSource: + """Owned immutable source bytes; DataFrame access returns independent copies.""" + + projection_json: bytes + receipt_json: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "AUTHENTICATED_SOURCE_CONSTRUCTOR") + + @property + def households(self): + return _tables(json.loads(self.projection_json))[0] + + @property + def persons(self): + return _tables(json.loads(self.projection_json))[1] + + @property + def codes(self): + return _tables(json.loads(self.projection_json))[5] + + +def _reconstruct(private, paths, pins, serialnos, implementation): + hc, h, hi = _archive(paths["household"], "household") + pc, p, pi = _archive(paths["person"], "person") + full = { + "format": "acs-housing-lexical-projection/1", + "household_columns": hc, + "person_columns": pc, + "households": h, + "persons": p, + } + full_bytes = _json(full) + _write(private / "full-projection.json", full_bytes, ACS_HU_SOURCE_MAX_BYTES) + selected, chosen, counts = _select(full, serialnos) + projection = _json(selected) + _require(len(projection) <= ACS_HU_SOURCE_MAX_BYTES, "PROJECTION_TOO_LARGE") + definition, definition_sha = _definition() + receipt = { + "format": "microcosm.acs_housing_universe_source.v1", + "release_eligible": False, + "vintage": 2024, + "definition_sha256": definition_sha, + "source_definitions": [ + definition["vacancy_definition"], + definition["gq_definition"], + ], + "implementation_sha256": implementation, + "parser_profile": { + "engine": "stdlib_csv_reader", + "encoding": "utf-8", + "quoting": "QUOTE_MINIMAL", + "strict": True, + "csv.field_size_limit": csv.field_size_limit(), + "field_max_bytes": _FIELD_MAX, + "record_max_bytes": _RECORD_MAX, + }, + "archives": [ + {"role": r, "filename": n, "sha256": d, "bytes": s} for r, n, d, s in pins + ], + "members": {"household": hi, "person": pi}, + "full_projection_sha256": _sha(full_bytes), + "full_counts": counts, + "selection": "all" if serialnos is None else "exact_serialnos", + "selected_serialnos_sha256": _sha(_json(chosen)), + "selected_counts": { + "households": len(selected["households"]), + "persons": len(selected["persons"]), + }, + "projection_sha256": _sha(projection), + "projection_bytes": len(projection), + "source_weight_kind": "design", + "calibrated_descendants_approved": False, + } + receipt_bytes = _json(receipt) + _require(len(receipt_bytes) <= ACS_HU_RECEIPT_MAX_BYTES, "RECEIPT_TOO_LARGE") + return AuthenticatedACSHousingSource(projection, receipt_bytes, _token=_TOKEN) + + +def _run_source(source_dir, snapshot_root, serialnos, output_dir, readback): + try: + original = _directory(Path(source_dir)) + destination = Path(os.path.abspath(output_dir)) + _require(not destination.is_relative_to(original), "OUTPUT_SOURCE_OVERLAP") + if readback: + _directory(destination) + else: + _directory(destination.parent) + _require(not destination.exists(), "OUTPUT_EXISTS") + implementation = _implementation() + with _capture(source_dir, snapshot_root) as (private, paths, pins): + result = _reconstruct(private, paths, pins, serialnos, implementation) + _require(_implementation() == implementation, "IMPLEMENTATION_CHANGED") + if readback: + _directory(destination) + _require( + {p.name for p in destination.iterdir()} + == {"projection.json", "receipt.json"}, + "OUTPUT_ROSTER", + ) + for name, expected, cap in ( + ( + "projection.json", + result.projection_json, + ACS_HU_SOURCE_MAX_BYTES, + ), + ("receipt.json", result.receipt_json, ACS_HU_RECEIPT_MAX_BYTES), + ): + target = private / ("candidate-" + name) + # SOURCE_SIZE and FILE_TOO_LARGE are the pinned-archive + # codes. A candidate of the wrong length is a different + # fact, and an operator has to be able to tell them apart, + # so only this call's size refusals are renamed. + try: + digest = _copy( + destination / name, target, cap, exact_size=len(expected) + ) + except ACSHousingSourceError as error: + if str(error) not in ("SOURCE_SIZE", "FILE_TOO_LARGE"): + raise + raise ACSHousingSourceError("RECONSTRUCTION_SIZE") from None + _require( + digest == _sha(expected) + and _persisted_sha(target, len(expected)) == digest, + "RECONSTRUCTION_MISMATCH", + ) + else: + _directory(destination.parent) + destination.mkdir(mode=0o700, exist_ok=False) + _write( + destination / "projection.json", + result.projection_json, + ACS_HU_SOURCE_MAX_BYTES, + ) + _write( + destination / "receipt.json", + result.receipt_json, + ACS_HU_RECEIPT_MAX_BYTES, + ) + return result + except ACSHousingSourceError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + zipfile.BadZipFile, + UnicodeError, + csv.Error, + ): + raise ACSHousingSourceError("SOURCE_PREPARATION_REFUSED") from None + + +def produce_acs_housing_source( + source_dir, *, snapshot_root, output_dir, serialnos=None +): + """Authenticate full parents, retain whole selected keys, publish exclusively.""" + return _run_source(source_dir, snapshot_root, serialnos, output_dir, False) + + +def load_acs_housing_source(source_dir, output_dir, *, snapshot_root, serialnos=None): + """Fresh full-parent reconstruction; candidate files never supply authority.""" + return _run_source(source_dir, snapshot_root, serialnos, output_dir, True) + + +def verify_acs_frame_projection(frame: Frame, projection: dict): + """Check a Frame against source observations, without granting source authority.""" + household, person, typed, lines, pw, _codes = _tables(projection) + h = frame.table("household") + p = frame.person + expected_h = household.loc[typed.NP > 0].reset_index(drop=True) + _require( + list(h.SERIALNO.astype(str)) == list(expected_h.SERIALNO), + "FRAME_HOUSEHOLD_ORIGIN", + ) + indexed = household.set_index("SERIALNO") + observed = indexed.loc[h.SERIALNO.astype(str)] + # The unchanged Frame builder retains TYPEHUGQ/NP/TEN but not raw WGTP. + # WGTP stays lexical in the source artifact and binds the typed design + # weights below, including the distinct GQ PWGTP substitution. + for field in ("TYPEHUGQ", "NP"): + _require( + np.array_equal(h[field].to_numpy(), observed[field].map(int).to_numpy()), + "FRAME_RAW_CONTROLS", + ) + valid = observed.TEN.to_numpy() != "" + _require(np.array_equal(h.TEN.notna().to_numpy(), valid), "FRAME_TEN_VALIDITY") + _require( + np.array_equal( + h.TEN.to_numpy()[valid], observed.TEN[valid].map(int).to_numpy() + ), + "FRAME_TEN_VALUES", + ) + state = "ST" if "ST" in observed else "STATE" + _require( + list(h.ST.astype(str)) == list(observed[state]) + and list(h.PUMA.astype(str)) == list(observed.PUMA), + "FRAME_GEOGRAPHY", + ) + ids = dict(zip(h.SERIALNO.astype(str), h.household_id, strict=True)) + _require(len(p) == len(person), "FRAME_PERSON_COUNT") + expected_ids = np.asarray([ids[s] for s in person.SERIALNO], dtype="int64") + _require( + np.array_equal(p.person_household_id.to_numpy(), expected_ids) + and np.array_equal(p.source_household_id.to_numpy(), expected_ids) + and np.array_equal(p.SPORDER.to_numpy(), lines) + and list(p.source_person_id.astype(str)) == [str(v) for v in lines], + "FRAME_PERSON_ORIGIN", + ) + _require( + np.array_equal(p.PWGTP.to_numpy(), pw) + and bool((p.source_year.to_numpy() == 2024).all()), + "FRAME_PERSON_SOURCE", + ) + _require( + frame.weighted_entities == ("household",) + and frame.weights_for("household").kind.value == "design", + "FRAME_WEIGHT_KIND", + ) + by_serial = dict(zip(person.SERIALNO, pw, strict=True)) + expected_weights = np.asarray( + [ + by_serial[serial] if int(weight) == 0 else int(weight) + for serial, weight in zip(observed.index, observed.WGTP, strict=True) + ], + dtype="float64", + ) + _require( + frame.weights_for("household").values.tobytes() == expected_weights.tobytes(), + "FRAME_DESIGN_WEIGHT", + ) + + +def frame_content_sha256(frame: Frame): + """Typed storage identity of every declared cell, axis, stratum and weight.""" + digest = hashlib.sha256(b"microcosm.acs-frame-storage/1\0") + + def series(values): + digest.update(_json({"dtype": str(values.dtype), "name": values.name})) + mask = values.isna().to_numpy(dtype=bool) + digest.update(mask.tobytes()) + array = values.to_numpy(copy=False) + if isinstance(values.dtype, np.dtype) and not values.dtype.hasobject: + digest.update(np.ascontiguousarray(array).tobytes()) + else: + for value, missing in zip(array, mask, strict=True): + item = ( + None + if missing + else value.item() + if isinstance(value, np.generic) + else value + ) + raw = _json(item) + digest.update(len(raw).to_bytes(8, "little")) + digest.update(raw) + + _require(frame.schema == US_SCHEMA, "FRAME_SCHEMA") + digest.update(_json(dict(frame.metadata))) + _require(not frame.mass_log, "FRAME_MASS_LOG") + for entity in frame.entities: + table = frame.table(entity) + digest.update(_json([entity, list(table.columns)])) + series(pd.Series(table.index.to_numpy(), name=table.index.name)) + for column in table: + series(table[column]) + series(frame.strata) + series(pd.Series(frame.strata.index.to_numpy(), name=frame.strata.index.name)) + for entity in frame.weighted_entities: + digest.update(_json([entity, frame.weights_for(entity).kind.value])) + digest.update(frame.weights_for(entity).values.tobytes()) + return digest.hexdigest() + + +@dataclass(frozen=True) +class PreparedACSHousingPopulation: + frame: Frame + source: AuthenticatedACSHousingSource + receipt_json: bytes + + @property + def receipt(self): + return json.loads(self.receipt_json) + + +def prepare_acs_housing_population(source_dir, *, snapshot_root, serialnos=None): + """Validate the full lexical source; construct only exact selected native rows.""" + try: + serialnos = AcsPumsSource.snapshot_serialnos(serialnos) + implementation = _implementation() + with _capture(source_dir, snapshot_root) as (private, paths, pins): + source = _reconstruct(private, paths, pins, serialnos, implementation) + projection = json.loads(source.projection_json) + _require(bool(projection["persons"]), "EMPTY_GRAPH_POPULATION") + raw, _builder_receipt = build_acs_pums_unit_frame( + AcsPumsSource( + paths["household"], + paths["person"], + vintage=2024, + max_households=None, + ), + serialnos=serialnos, + ) + mapped = map_acs_native_inputs(raw) + frame = mapped.frame + assert_operator_free_source_frame( + frame, + label="ACS HU authenticated source", + native_inputs=mapped.native_inputs, + ) + verify_acs_frame_projection(frame, projection) + before = frame_content_sha256(frame) + transitions = [] + original_dtypes = { + (e, c): repr(frame.table(e)[c].dtype) + for e in frame.entities + for c in frame.table(e) + } + canonicalize_frame_string_dtypes( + frame, boundary="ACS HU graph storage", in_place=True + ) + for entity in frame.entities: + table = frame.table(entity) + for column in table: + original = table[column] + if isinstance( + original.dtype, pd.StringDtype + ) and original.dtype != dtype_for_token("string"): + promoted = original.astype(dtype_for_token("string")) + _require( + original.isna().equals(promoted.isna()) + and np.array_equal( + original[original.notna()].to_numpy(), + promoted[promoted.notna()].to_numpy(), + ), + "GRAPH_STRING_PROMOTION", + ) + table[column] = promoted + if original_dtypes[(entity, column)] != repr(table[column].dtype): + transitions.append( + { + "entity": entity, + "column": column, + "from": original_dtypes[(entity, column)], + "to": repr(table[column].dtype), + } + ) + verify_acs_frame_projection(frame, projection) + _require(_implementation() == implementation, "IMPLEMENTATION_CHANGED") + receipt = { + "format": "microcosm.acs_housing_preparation.v2", + "release_eligible": False, + "source_receipt_sha256": _sha(source.receipt_json), + "projection_sha256": _sha(source.projection_json), + "implementation_sha256": implementation, + "same_snapshot_frame_and_observations": True, + "full_source_frame_before_selection": serialnos is None, + "full_source_lexical_projection": True, + "native_selection_before_person_accumulation": serialnos is not None, + "selection_kind": "all" + if serialnos is None + else "engineering_exact_keys", + "requested_serialnos": serialnos, + "full_source_inclusion_probability": None, + "pre_promotion_frame_sha256": before, + "frame_sha256": frame_content_sha256(frame), + "dtype_transitions": transitions, + "entity_rows": {e: frame.n(e) for e in frame.entities}, + "weight_kind": "design", + "HU_columns": "artifact_only", + } + return PreparedACSHousingPopulation(frame, source, _json(receipt)) + except ACSHousingSourceError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + zipfile.BadZipFile, + UnicodeError, + csv.Error, + ): + raise ACSHousingSourceError("FRAME_PREPARATION_REFUSED") from None + + +def load_graph_acs_housing_universe(path, *, store=None): + """Frame codec with an explicit store and a dedicated private capture child.""" + _require(type(store) is ContentStore, "EXPLICIT_CONTENT_STORE_REQUIRED") + root = _directory(store.root) / "acs-hu-source-captures-v1" + root.mkdir(mode=0o700, exist_ok=True) + return prepare_acs_housing_population(path, snapshot_root=root).frame diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py index 88b29412f..2d8e8e99e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py @@ -247,6 +247,14 @@ def _map_tenure( transformation="ACS TEN enum recode through SPM membership", register=register, ) + # These enum outputs are strings even when the selected source households + # all have unreported tenure. Declare that type without inventing a value; + # generic serialization cannot infer a type from an all-missing object axis. + string_dtype = pd.StringDtype(storage="python", na_value=np.nan) + household["tenure_type"] = household["tenure_type"].astype(string_dtype) + spm_unit["spm_unit_tenure_type"] = spm_unit["spm_unit_tenure_type"].astype( + string_dtype + ) def _map_housing_amounts( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py new file mode 100644 index 000000000..770fdc9bb --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py @@ -0,0 +1,711 @@ +"""Closed real ACS preparation plus literal coverage issuance, without admission. + +Private process ownership is required in addition to immutable evidence bytes. +This is a native population successor, not a decoder or a graph attachment. +""" + +from __future__ import annotations + +import ast +import builtins +import importlib.util +import json +import sys +import tempfile +from _thread import RLock +from dataclasses import InitVar, dataclass +from pathlib import Path +from types import BuiltinFunctionType, CodeType, FunctionType +from weakref import WeakKeyDictionary + +from microcosm.frame import Frame + +from . import acs_housing_universe_source as housing +from . import acs_person_coverage_authentication as coverage +from . import graph_implementation as implementation + +PROTOCOL = "microcosm.acs-native-coverage-binding.v2" +MAX_ARCHIVE_BYTES = 8 * 1024**3 # combined compressed bytes, before capture +MAX_EXPANDED_BYTES = 16 * 1024**3 # combined, before opening any member +MAX_SOURCE_ROWS = 6_000_000 # per role, before full-source construction +MAX_EVIDENCE_BYTES = 2 * 1024**2 +# Exact accepted direct AGEP -> A_AGE and AGEP -> age implementation, and owners. +# A new transform/owner version requires explicit review of this successor. +_ACCEPTED = { + "acs_pums.py": "6ecf79f0dfb0c0bc0ad0af6be5fa65bd8c2d1e1968009402e4c8dee347de70dc", + "acs_inputs.py": "aa4a8aeaba63dfef2f3e04fb89de59766deb088ed7f4d290aeba0425739916da", + "acs_housing_universe_source.py": "beb46a4a05a13580a868be423809a77946441dcafc3a0f57160561157e93e9a3", + "acs_person_coverage_authentication.py": "475aa795c8a5b49a0dd3405a0877866dddd03012a1f2b9743447fab1fe85bcff", +} +_TOKEN = object() +_ISSUED = WeakKeyDictionary() + +# Only bytecode compilation is reused across producers. Fresh source reads, +# AST checks, local code indexes and loaded-function checks remain mandatory. +_COMPILE_CACHE_MAX_ENTRIES = 128 +_COMPILE_CACHE_MAX_SOURCE_BYTES = 16 * 1024**2 +_COMPILE_CACHE_MAX_ENTRY_BYTES = 1024**2 +_COMPILE_CACHE_COMPILER = compile +_COMPILE_CACHE_DEFAULT_OPTIMIZE = sys.flags.optimize +_COMPILE_CACHE = {} +_COMPILE_CACHE_LOCK = RLock() + + +class ACSNativeCoverageBindingError(ValueError): + """Static refusal without source values, paths or exception chains.""" + + +def _require(condition, code): + if not condition: + raise ACSNativeCoverageBindingError(code) + + +def _clear_compile_cache(): + """Clear only compiled-source outputs, for process-local test isolation.""" + with _COMPILE_CACHE_LOCK: + _COMPILE_CACHE.clear() + + +def _compile_source( + source, filename, mode="exec", *, flags=0, dont_inherit=True, optimize=-1 +): + """Reuse bounded immutable bytecode; never reuse loaded-function validity. + + Non-original compilers and context-dependent or non-exact inputs bypass the + cache. Compiler warning/audit events occur on misses; AST parsing still runs + on every _live_code call. This shares the trusted-process scope of that check. + """ + compiler = compile + eligible = ( + compiler is _COMPILE_CACHE_COMPILER + and type(compiler) is BuiltinFunctionType + and compiler.__module__ == "builtins" + and compiler.__name__ == "compile" + and compiler.__self__ is builtins + and type(source) is bytes + and type(filename) is str + and type(mode) is str + and mode in ("exec", "eval", "single") + and type(flags) is int + and flags >= 0 + and dont_inherit is True + and type(optimize) is int + and optimize in (-1, 0, 1, 2) + and len(source) <= _COMPILE_CACHE_MAX_ENTRY_BYTES + and len(source) <= _COMPILE_CACHE_MAX_SOURCE_BYTES + and _COMPILE_CACHE_MAX_ENTRIES > 0 + ) + key = None + if eligible: + effective_optimize = ( + _COMPILE_CACHE_DEFAULT_OPTIMIZE if optimize == -1 else optimize + ) + key = ( + source, + filename, + mode, + flags, + dont_inherit, + effective_optimize, + id(compiler), + ) + with _COMPILE_CACHE_LOCK: + entry = _COMPILE_CACHE.get(key) + if ( + type(entry) is tuple + and len(entry) == 3 + and type(entry[0]) is tuple + and entry[0] == key + and entry[1] is compiler + and type(entry[2]) is CodeType + ): + return entry[2] + _COMPILE_CACHE.pop(key, None) + + # Compile outside the lock: instrumentation/audit hooks may reenter, and a + # concurrent duplicate miss is harmless. Never retain a failed compilation. + code = compiler( + source, + filename, + mode, + flags=flags, + dont_inherit=dont_inherit, + optimize=optimize, + ) + if key is not None and type(code) is CodeType: + with _COMPILE_CACHE_LOCK: + # A nested/concurrent call may already have filled this key. FIFO + # eviction bounds retained source keys; it is not a hard RSS cap. + if key not in _COMPILE_CACHE: + while _COMPILE_CACHE and ( + len(_COMPILE_CACHE) >= _COMPILE_CACHE_MAX_ENTRIES + or sum(len(item[0]) for item in _COMPILE_CACHE) + len(source) + > _COMPILE_CACHE_MAX_SOURCE_BYTES + ): + del _COMPILE_CACHE[next(iter(_COMPILE_CACHE))] + _COMPILE_CACHE[key] = (key, compiler, code) + return code + + +def _live_code(module, compiled): + """Check loaded Python implementations against their current source bytes. + + Covers module functions, imported function aliases and source class methods; + generated dataclass methods have no source code object. This is a drift + check in a trusted process, not a sandbox against arbitrary Python execution. + """ + path = getattr(module, "__file__", None) + if path is None: + return + tree = ast.parse(Path(path).read_bytes()) + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + value = getattr(module, node.name, None) + _require( + value is not None and value.__module__ == module.__name__, + "LOADED_PRODUCER", + ) + elif isinstance(node, ast.ImportFrom) and node.module != "__future__": + origin_name = ( + importlib.util.resolve_name( + "." * node.level + (node.module or ""), module.__package__ + ) + if node.level + else node.module + ) + origin = sys.modules.get(origin_name) + for alias in node.names: + if alias.name != "*" and origin is not None: + _require( + getattr(module, alias.asname or alias.name, None) + is getattr(origin, alias.name, None), + "LOADED_PRODUCER", + ) + + def check(function): + if not isinstance(function, FunctionType): + return + origin = sys.modules.get(function.__globals__.get("__name__")) + path = getattr(origin, "__file__", None) + _require(path is not None, "LOADED_PRODUCER") + if function.__code__.co_filename == "": + return # dataclass-generated methods + if path not in compiled: + codes = {} + + def visit(code): + codes[code.co_qualname] = code + for value in code.co_consts: + if isinstance(value, CodeType): + visit(value) + + visit( + _compile_source( + Path(path).read_bytes(), path, "exec", dont_inherit=True + ) + ) + compiled[path] = codes + _require( + function.__globals__ is vars(origin) + and function.__code__ == compiled[path].get(function.__code__.co_qualname), + "LOADED_PRODUCER", + ) + + for cell in function.__closure__ or (): + if isinstance(cell.cell_contents, FunctionType): + check(cell.cell_contents) + + for value in vars(module).values(): + if isinstance(value, FunctionType): + check(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + for method in vars(value).values(): + if isinstance(method, (classmethod, staticmethod)): + method = method.__func__ + if isinstance(method, property): + check(method.fget) + check(method.fset) + else: + check(method) + + +def _producer(): + _require(PROTOCOL == "microcosm.acs-native-coverage-binding.v2", "NATIVE_PROTOCOL") + for name, expected in _ACCEPTED.items(): + _require( + coverage._sha(Path(__file__).with_name(name).read_bytes()) == expected, + "UNREVIEWED_PREPARATION", + ) + manifest = implementation.implementation_manifest(housing.ACS_HU_STAGE) + # Reuse the reviewed complete preparation closure, without changing its + # graph inventory or historical hashes. Include source-only coverage's own + # closure, C _csv provider (libpython where builtin), and this successor. + names = { + "microcosm.build.us_runtime.acs_person_coverage_authentication", + "microcosm.build.us_runtime.acs_person_coverage_columns", + __name__, + } + for item in manifest["modules"]: + package, relative = item.split("/", 1) + name = package + "." + relative.removesuffix(".py").replace("/", ".") + names.add(name.removesuffix(".__init__")) + compiled = {} + for name in sorted(names): + module = sys.modules.get(name) + if module is not None: + _live_code(module, compiled) + return { + "preparation": manifest, + "coverage": coverage._producer(), + "issuer_sha256": coverage._sha(Path(__file__).read_bytes()), + "limits": [ + MAX_ARCHIVE_BYTES, + MAX_EXPANDED_BYTES, + MAX_SOURCE_ROWS, + MAX_EVIDENCE_BYTES, + sys.modules[housing.AcsPumsSource.__module__].MAX_EXACT_HOUSEHOLDS, + sys.modules[housing.AcsPumsSource.__module__].MAX_EXACT_PERSON_ROWS, + ], + "accepted_age_transform": "literal_numeric_identity_AGEP_to_AGEP_A_AGE_age", + } + + +def _archives(pins): + _require( + len(pins) == 2 and {p[0] for p in pins} == {"household", "person"}, + "SOURCE_ROLES", + ) + return [ + {"role": role, "filename": name, "sha256": digest, "bytes": size} + for role, name, digest, size in pins + ] + + +def _members_equal(left, right): + # Owners intentionally have distinct parser profiles and inventory shapes. + # Compare common captured byte/header/row identities, never parser claims. + fields = ("name", "bytes", "sha256", "compressed_bytes", "crc32", "rows") + for role in ("household", "person"): + a, b = left[role], right[role] + _require(len(a) == len(b), "SOURCE_MEMBER_IDENTITY") + for x, y in zip(a, b, strict=True): + _require(all(x[k] == y[k] for k in fields), "SOURCE_MEMBER_IDENTITY") + if y["applicable"]: + _require( + x["header_sha256"] == y["header_sha256"], "SOURCE_MEMBER_IDENTITY" + ) + + +def _preflight(paths, serialnos=None): + # Directory/expanded-byte gates precede member opening. Streaming lexical + # gates then precede preparation's full projection and pandas construction. + expanded = 0 + for role, path in paths.items(): + with coverage.zipfile.ZipFile(path) as archive: + members, _prefix = coverage._members(archive, role) + expanded += sum(member.file_size for member in members) + _require(expanded <= MAX_EXPANDED_BYTES, "EXPANDED_BUDGET") + inventories = {} + if serialnos is None: + # Preserve the whole-source route and its pre-construction row ceiling. + for role, path in paths.items(): + inventories[role], _empty = coverage._inventory(path, role, frozenset()) + _require( + sum(m["rows"] for m in inventories[role]) <= MAX_SOURCE_ROWS, + "SOURCE_ROW_BUDGET", + ) + if role == "person": + _require( + sum(m["rows"] for m in inventories[role]) + <= coverage.literal.MAX_SELECTED_ROWS, + "NATIVE_ROW_BUDGET", + ) + return inventories, None + selected = frozenset(serialnos) + inventories["household"], households = coverage._inventory( + paths["household"], "household", selected + ) + _require( + sum(m["rows"] for m in inventories["household"]) <= MAX_SOURCE_ROWS, + "SOURCE_ROW_BUDGET", + ) + _require(set(households) == selected, "SELECTION_UNKNOWN") + _require(all(0 <= n <= 20 for n in households.values()), "SELECTED_NP") + expected_rows = sum(households.values()) + _require(expected_rows > 0, "EMPTY_SELECTED_POPULATION") + _require(expected_rows <= coverage.literal.MAX_SELECTED_ROWS, "NATIVE_ROW_BUDGET") + # _inventory bounds the actual selected row/body accumulation, independently + # of reported NP. Every source member is still streamed and hashed. + inventories["person"], roster = coverage._inventory( + paths["person"], "person", selected + ) + _require( + sum(m["rows"] for m in inventories["person"]) <= MAX_SOURCE_ROWS, + "SOURCE_ROW_BUDGET", + ) + counts = dict.fromkeys(selected, 0) + for serial, _line in roster: + counts[serial] += 1 + _require(counts == households, "SELECTED_ROSTER") + _require(len(roster) == expected_rows, "SELECTED_ROSTER") + return inventories, (households, roster) + + +@dataclass(frozen=True, slots=True) +class _Owned: + frame: Frame + payload: bytes + prepared: housing.PreparedACSHousingPopulation + literal: coverage.AuthenticatedACSPersonCoverage + snapshots: tuple + pins: tuple + + +@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False) +class AuthenticatedACSNativeCoverage: + """Process-owned immutable evidence; the live Frame is checked on every borrow.""" + + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "ISSUANCE_CONSTRUCTOR") + + @property + def receipt(self): + return json.loads(_owned(self).payload) + + @property + def frame(self): + verify_acs_native_coverage(self) + return _owned(self).frame + + @property + def coverage_table(self): + return _owned(self).literal.table + + @property + def source_coverage(self): + """The unchanged predecessor still has an ungranted native_binding.""" + return _owned(self).literal + + +def _owned(issuance): + _require(type(issuance) is AuthenticatedACSNativeCoverage, "ISSUANCE_TYPE") + owned = _ISSUED.get(issuance) + _require( + owned is not None + and type(issuance.payload) is bytes + and issuance.payload == owned.payload, + "ISSUANCE_NOT_OWNED", + ) + return owned + + +def _verify_sources(snapshots, pins): + for paths in snapshots: + for role, _name, digest, size in pins: + path = paths[role] + housing._path_components(path) + _require(housing._persisted_sha(path, size) == digest, "SOURCE_CHANGED") + _require(housing._pins() == pins, "SOURCE_AUTHORITY_CHANGED") + + +def _frame_sha256(frame): + # Supplement the unchanged owner's typed-cell identity with pandas axis + # descriptors/attrs and every entity's resolved original DESIGN weights. + details = {"storage": housing.frame_content_sha256(frame), "entities": {}} + for entity in frame.entities: + table = frame.table(entity) + weights = frame.resolve_weights(entity) + details["entities"][entity] = { + "index_type": type(table.index).__name__, + "index_dtype": repr(table.index.dtype), + "column_axis_name": table.columns.name, + "column_axis_dtype": repr(table.columns.dtype), + "dtypes": [repr(dtype) for dtype in table.dtypes], + "attrs": table.attrs, + "allows_duplicate_labels": table.flags.allows_duplicate_labels, + "weight_kind": weights.kind.value, + "weight_dtype": str(weights.values.dtype), + "weight_shape": list(weights.values.shape), + "weight_sha256": coverage._sha(weights.values.tobytes()), + } + details["strata_attrs"] = frame.strata.attrs + details["strata_index_type"] = type(frame.strata.index).__name__ + details["strata_allows_duplicate_labels"] = ( + frame.strata.flags.allows_duplicate_labels + ) + _require(not frame._link_tables, "UNEXPECTED_LINK_TABLES") + return coverage._sha(coverage._json(details, MAX_EVIDENCE_BYTES)) + + +def _verify_prepared_frame(prepared): + recorded = json.loads(prepared.receipt_json)["frame_sha256"] + _require( + housing.frame_content_sha256(prepared.frame) == recorded, + "PREPARATION_FRAME_CHANGED", + ) + return recorded + + +def _verify_frame(owned): + receipt = json.loads(owned.payload) + _require( + housing.frame_content_sha256(owned.frame) == receipt["frame_sha256"] + and _frame_sha256(owned.frame) == receipt["exact_frame_sha256"], + "FRAME_CHANGED", + ) + _require( + _verify_prepared_frame(owned.prepared) == receipt["frame_sha256"], + "PREPARATION_FRAME_CHANGED", + ) + + +def _verify(owned, producer): + receipt = json.loads(owned.payload) + frame, prepared, literal = owned.frame, owned.prepared, owned.literal + _require(receipt["protocol"] == PROTOCOL, "NATIVE_PROTOCOL") + _require(receipt["producer"] == producer, "PRODUCER_CHANGED") + _require( + prepared.frame is frame + and coverage._sha(prepared.receipt_json) + == receipt["preparation_receipt_sha256"] + and coverage._sha(prepared.source.receipt_json) + == receipt["housing_receipt_sha256"] + and coverage._sha(prepared.source.projection_json) + == receipt["projection_sha256"] + and coverage._sha(literal.payload) == receipt["coverage_payload_sha256"], + "EVIDENCE_CHANGED", + ) + _verify_frame(owned) + housing.verify_acs_frame_projection( + frame, json.loads(prepared.source.projection_json) + ) + consistency = coverage.verify_acs_coverage_native_consistency( + literal, frame + ).receipt + _require( + consistency["original_age"]["relation"] == "numeric_identity" + and consistency["original_age"]["matching_rows"] == frame.n("person"), + "ORIGINAL_AGE_IDENTITY", + ) + _verify_sources(owned.snapshots, owned.pins) + _verify_frame(owned) + + +def verify_acs_native_coverage(issuance, frame=None): + """Authenticate this issuance and exact live Frame, sources and producer. + + A separately built equal Frame is not this issuance. Any borrowed mutable + Frame must be verified again at consumption; retaining a borrow bypasses no + checks here but Python cannot intercept arbitrary downstream table reads. + Callers must hold exclusive access to mutable tables during verification + and consumption. Final checks detect changes during archive/producer work; + they do not make pandas tables an atomic concurrent snapshot. + """ + try: + owned = _owned(issuance) + _require(frame is None or frame is owned.frame, "FRAME_NOT_ISSUED") + producer = _producer() + _verify(owned, producer) + _require(_producer() == producer, "PRODUCER_CHANGED") + _verify_frame(owned) + return issuance + except ACSNativeCoverageBindingError: + raise + except Exception: + raise ACSNativeCoverageBindingError("NATIVE_VERIFICATION_REFUSED") from None + + +def issue_acs_native_coverage( + source_dir, *, snapshot_root, serialnos=None, candidate_path=None +): + """Run real closed owners with optional exact engineering household selection. + + The complete selected roster is bounded before native construction. Full + source validation and lexical projection remain. Candidate bytes, if given, + are compared only after independent real reconstruction and never decoded. + """ + try: + serialnos = housing.AcsPumsSource.snapshot_serialnos(serialnos) + # Snapshot path-like inputs once too, before producer/capture work. + source_dir = Path(source_dir).absolute() + snapshot_root = Path(snapshot_root).absolute() + candidate_path = ( + None if candidate_path is None else Path(candidate_path).absolute() + ) + coverage._json( + serialnos, min(MAX_EVIDENCE_BYTES, housing.ACS_HU_RECEIPT_MAX_BYTES) + ) + pins = housing._pins() + archives = _archives(pins) + _require(sum(p[3] for p in pins) <= MAX_ARCHIVE_BYTES, "ARCHIVE_BUDGET") + producer = _producer() + with housing._capture(source_dir, snapshot_root) as (private, paths, captured): + _require(captured == pins, "SOURCE_AUTHORITY_CHANGED") + inventory, selected_roster = _preflight(paths, serialnos) + # These roots are outside the archive-only captured source directory. + roots = [ + Path(tempfile.mkdtemp(prefix=prefix, dir=snapshot_root)) + for prefix in ("acs-native-preparation-", "acs-native-coverage-") + ] + prepared = housing.prepare_acs_housing_population( + private, snapshot_root=roots[0], serialnos=serialnos + ) + _verify_prepared_frame(prepared) + _require( + prepared.receipt["format"] == "microcosm.acs_housing_preparation.v2", + "PREPARATION_PROTOCOL", + ) + _require( + prepared.receipt["requested_serialnos"] + == (None if serialnos is None else list(serialnos)), + "PREPARATION_SELECTION", + ) + keys, _roster_sha = coverage._native_roster(prepared.frame) + if selected_roster is not None: + households, roster = selected_roster + _require( + {s: n for s, n in households.items() if n > 0} + == keys.groupby("SERIALNO").size().to_dict() + and set(roster) + == set(zip(keys.SERIALNO, keys.SPORDER, strict=True)), + "SELECTED_NATIVE_ROSTER", + ) + literal = coverage.load_authenticated_acs_person_coverage( + private, snapshot_root=roots[1], frame=prepared.frame + ) + hreceipt, creceipt = ( + json.loads(prepared.source.receipt_json), + literal.receipt, + ) + _require( + hreceipt["archives"] == creceipt["archives"] == archives, + "SOURCE_ARCHIVE_IDENTITY", + ) + _members_equal(hreceipt["members"], creceipt["members"]) + _require(creceipt["members"] == inventory, "SOURCE_MEMBER_IDENTITY") + relation = literal.native_binding.receipt["original_age"] + _require( + relation["relation"] == "numeric_identity", "ORIGINAL_AGE_IDENTITY" + ) + snapshots = [ + { + role: Path(source_dir).absolute() / name + for role, name, _d, _s in pins + }, + paths, + ] + for root in roots: + children = tuple(root.iterdir()) + _require(len(children) == 1 and children[0].is_dir(), "CAPTURE_ROSTER") + snapshots.append( + {role: children[0] / name for role, name, _d, _s in pins} + ) + receipt = { + "protocol": PROTOCOL, + "producer": producer, + "archives": archives, + "vintage": 2024, + "selection": { + "kind": "all" if serialnos is None else "engineering_exact_keys", + "requested_serialnos": serialnos, + "complete_selected_roster": True, + "person_rows": prepared.frame.n("person"), + "raw_person_keys_sha256": coverage._sha( + coverage._json( + sorted( + zip(keys.SERIALNO, map(int, keys.SPORDER), strict=True) + ), + coverage.MAX_BODY_BYTES, + ) + ), + "vacant_serialnos": None + if selected_roster is None + else sorted(s for s, n in selected_roster[0].items() if n == 0), + "vacancy_status": "source_record_without_population_rows", + "before_person_accumulation_and_unit_assignment": serialnos + is not None, + "full_source_inclusion_probability": None, + "representative_sample": False, + }, + "original_literal_authentication": { + "source_authenticated": True, + "fields": ["AGEP", "MIL", "ESR"], + "raw_literals_preserved": True, + "predecessor_native_binding_authenticated": False, + }, + "native_preparation_issuance": { + "population_binding_authenticated": True, + "producer_executed_by_issuer": True, + "complete_original_nonvacant_roster": serialnos is None, + "complete_selected_original_roster": True, + "source_aliases_and_vintage_checked": True, + "age_relationship": "literal_AGEP_to_native_AGEP_A_AGE_to_model_age_identity", + "matching_age_rows": relation["matching_rows"], + "native_roster_sha256": creceipt["native_consistency"][ + "native_roster_sha256" + ], + }, + "design_anchors": { + "authenticated": True, + "kind": "design", + "source": "WGTP; single-person PWGTP for WGTP=0 GQ placeholders", + "renormalized": False, + "original_weight_literals_projection_sha256": coverage._sha( + prepared.source.projection_json + ), + "weight_sha256": coverage._sha( + prepared.frame.weights_for("household").values.tobytes() + ), + }, + "assumptions": { + "coverage_status": "literal_source_fields_only", + "cross_survey_coverage_equivalence_established": False, + "domain_assignment_authenticated": False, + "period_harmonized": False, + "release_eligible": False, + "graph_attachment": "requires_reviewed_typed_successor", + }, + "capture": { + "owners_share_physical_snapshot": False, + "owners_separately_capture_same_pins": True, + "issuer_preflight_capture_count": 1, + "owner_capture_count": 2, + "full_source_construction_peak_remains": True, + "selected_only_memory": False, + }, + "frame_sha256": housing.frame_content_sha256(prepared.frame), + "exact_frame_sha256": _frame_sha256(prepared.frame), + "preparation_receipt_sha256": coverage._sha(prepared.receipt_json), + "housing_receipt_sha256": coverage._sha(prepared.source.receipt_json), + "projection_sha256": coverage._sha(prepared.source.projection_json), + "coverage_payload_sha256": coverage._sha(literal.payload), + } + payload = coverage._json(receipt, MAX_EVIDENCE_BYTES) + owned = _Owned( + prepared.frame, payload, prepared, literal, tuple(snapshots), pins + ) + if candidate_path is not None: + target = private / "candidate.native-coverage" + digest = housing._copy( + Path(candidate_path), target, len(payload), exact_size=len(payload) + ) + _require( + digest == coverage._sha(payload) + and housing._persisted_sha(target, len(payload)) == digest, + "CANDIDATE_MISMATCH", + ) + _verify(owned, producer) + _require(_producer() == producer, "PRODUCER_CHANGED") + _verify_frame(owned) + # Capture's exit authority check must complete before minting the token. + _verify_frame(owned) + result = AuthenticatedACSNativeCoverage(payload, _token=_TOKEN) + _ISSUED[result] = owned + return result + except ACSNativeCoverageBindingError: + raise + except Exception: + raise ACSNativeCoverageBindingError("NATIVE_ISSUANCE_REFUSED") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py new file mode 100644 index 000000000..fe0346500 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py @@ -0,0 +1,667 @@ +"""Closed ACS coverage source issuance; native consistency grants no identity. + +The existing prepared ACS container is publicly constructible. Until a closed +prepared/native authority exists, even a consistent Frame receives only an +UngrantedACSNativeBinding. Nothing here attaches columns or prepares a population. +""" + +from __future__ import annotations + +import _csv +import csv +import hashlib +import io +import json +import platform +import re +import stat +import sys +import sysconfig +import zipfile +from dataclasses import InitVar, dataclass +from pathlib import Path +from types import BuiltinFunctionType + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import acs_housing_universe_source as custody +from . import acs_person_coverage_columns as literal +from .acs_pums import AcsPumsSource +from .source_csv_builtin import csv_reader_bound + +PROTOCOL = "microcosm.acs-person-coverage-authentication.v1" +MAX_HEADER_BYTES = 1024**2 +MAX_BODY_BYTES = 64 * 1024**2 +MAX_RECORD_BYTES = 400_000 +MAX_CSV_HEADER_BYTES = 64 * 1024 +MAX_TOKEN_BYTES = 64 * 1024 +_MAGIC = b"ACS-COVERAGE/1\n" +_TOKEN = object() +_CSV_READER = _csv.reader +_COLUMNS = (*literal.READ_COLUMNS, "MIL_state", "ESR_state") +_IMPLEMENTATION_FILES = ( + "acs_person_coverage_authentication.py", + "acs_person_coverage_columns.py", + "acs_housing_universe_source.py", + "acs_sources.py", + "acs_pums.py", + "source_csv_builtin.py", + "acs_2024_1yr_sources.json", +) + + +class ACSCoverageAuthenticationError(ValueError): + """Static refusal code, without source paths, keys, tokens or cause chains.""" + + +def _require(condition, code): + if not condition: + raise ACSCoverageAuthenticationError(code) + + +def _sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def _json(value, cap=MAX_HEADER_BYTES): + # ASCII JSON escapes preserve CR/LF/HT and distinguish null from "". Values + # originate in byte-bounded records or the fixed, bounded header inventory. + # Count exact escaped bytes before an encoder can allocate a whole token. + count = 0 + + def charge(size): + nonlocal count + count += size + _require(count <= cap, "CANONICAL_SIZE") + + def visit(item): + if isinstance(item, str): + charge(2) + for char in item: + code = ord(char) + charge( + 2 + if char in '\\"\b\f\n\r\t' + else 1 + if 32 <= code <= 126 + else 6 + if code <= 0xFFFF + else 12 + ) + elif item is None: + charge(4) + elif type(item) is bool: + charge(4 if item else 5) + elif type(item) is int: + charge(len(str(item))) + elif isinstance(item, (list, tuple)): + charge(2 + max(0, len(item) - 1)) + for part in item: + visit(part) + elif type(item) is dict: + charge(2 + max(0, len(item) - 1) + len(item)) + for key, part in item.items(): + _require(type(key) is str, "CANONICAL_KEY") + visit(key) + visit(part) + else: + _require(False, "CANONICAL_TYPE") + + visit(value) + parts, count = [], 0 + for part in json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ).iterencode(value): + count += len(part) + _require(count <= cap, "CANONICAL_SIZE") + parts.append(part.encode("ascii")) + return b"".join(parts) + + +def _producer(): + # Provider bytes alone cannot attest the callable used by either parsing + # pass. Preserve the real C binding across issuance/verification, including + # refusal when both public aliases were replaced before this owner imported. + _require( + type(_CSV_READER) is BuiltinFunctionType + and _CSV_READER.__module__ == "_csv" + and _CSV_READER.__name__ == "reader" + and _CSV_READER.__self__ is _csv + and csv.reader is _csv.reader is _CSV_READER + and literal.csv is custody.csv is csv + and csv_reader_bound(csv), + "CSV_READER_CHANGED", + ) + parser_binary = getattr(_csv, "__file__", None) + if parser_binary is None: + # This runtime builds _csv into libpython. Other builds ship an + # extension or link the parser into the executable itself. + parser_binary = ( + Path(sysconfig.get_config_var("LIBDIR")) + / sysconfig.get_config_var("LDLIBRARY") + if sysconfig.get_config_var("Py_ENABLE_SHARED") + else Path(sys.executable) + ) + return { + "protocol": PROTOCOL, + "files": { + name: _sha(Path(__file__).with_name(name).read_bytes()) + for name in _IMPLEMENTATION_FILES + }, + "python": platform.python_version(), + "pandas": pd.__version__, + "numpy": np.__version__, + "csv_binary_sha256": _sha(Path(parser_binary).read_bytes()), + "stdlib": { + name: _sha(Path(module.__file__).read_bytes()) + for name, module in ( + ("csv", csv), + ("zipfile", zipfile), + ("json", json), + ) + }, + "parser": { + "reader": "_csv.reader:verified_builtin_identity", + "encoding": "utf-8-sig", + "newline": "", + "csv_field_size_limit": csv.field_size_limit(), + "literal_record_chars": literal.MAX_CSV_RECORD_CHARS, + "source_rows": literal.MAX_ROWS, + "selected_rows": literal.MAX_SELECTED_ROWS, + }, + "ceilings": { + "header": MAX_HEADER_BYTES, + "body": MAX_BODY_BYTES, + "record": MAX_RECORD_BYTES, + "csv_header": MAX_CSV_HEADER_BYTES, + "token": MAX_TOKEN_BYTES, + "member": custody._MEMBER_MAX, + "expanded_archive": custody._EXPANDED_MAX, + "member_count": custody._MEMBER_COUNT_MAX, + }, + } + + +def _records(stream): + """Fence raw logical records/tokens before UTF-8 decoding or CSV allocation. + + Quote parity only locates boundaries; the unchanged strict literal parser + subsequently owns CSV validity. CR, LF and CRLF are preserved, including + inside quotes. The conservative token ceiling includes raw CSV quoting. + """ + record = bytearray() + quoted, pending_cr, first, token_bytes = False, False, True, 0 + cap = MAX_CSV_HEADER_BYTES + while block := stream.read(4096): + for byte in block: + if pending_cr: + if byte == 10: + _require(len(record) < cap, "CSV_RECORD_BYTES") + record.append(byte) + yield bytes(record) + record.clear() + first, pending_cr, token_bytes = False, False, 0 + if byte == 10: + continue + cap = MAX_CSV_HEADER_BYTES if first else MAX_RECORD_BYTES + _require(len(record) < cap, "CSV_RECORD_BYTES") + if byte == 44 and not quoted: + token_bytes = 0 + else: + token_bytes += 1 + _require(token_bytes <= MAX_TOKEN_BYTES, "CSV_TOKEN_BYTES") + record.append(byte) + if byte == 34: + quoted = not quoted + if not quoted and byte in (10, 13): + if byte == 13: + pending_cr = True + else: + yield bytes(record) + record.clear() + first, token_bytes = False, 0 + if record: + yield bytes(record) + + +def _decode_record(raw, *, first): + # BOM is only consumed at member start, exactly as in the literal reader. + with io.TextIOWrapper( + io.BytesIO(raw), encoding="utf-8-sig" if first else "utf-8", newline="" + ) as text: + reader = literal._literal_csv_records(text) + result = next(reader, None) + _require(result is not None and next(reader, None) is None, "CSV_RECORD") + return result + + +def _members(archive, role): + # These are the closed housing owner's complete central-directory checks, + # kept local because _archive also invokes its physical-record parser. + # Editing/extracting that owner would change historical custody producers. + prefix = "psam_hus" if role == "household" else "psam_pus" + members = archive.infolist() + _require(0 < len(members) <= custody._MEMBER_COUNT_MAX, "ZIP_MEMBER_COUNT") + names = [m.filename for m in members] + _require(len({n.casefold() for n in names}) == len(names), "ZIP_DUPLICATE_MEMBER") + expanded = 0 + for member in members: + name, mode = member.filename, member.external_attr >> 16 + _require( + name not in ("", ".", "..") + and not any(c in name for c in ("/", "\\", "\x00")), + "ZIP_MEMBER_PATH", + ) + _require( + not member.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG), + "ZIP_MEMBER_TYPE", + ) + _require( + not member.flag_bits & 1 + and member.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED), + "ZIP_MEMBER_ENCODING", + ) + _require(0 <= member.file_size <= custody._MEMBER_MAX, "ZIP_MEMBER_SIZE") + expanded += member.file_size + _require(expanded <= custody._EXPANDED_MAX, "ZIP_EXPANDED_SIZE") + _require( + not name.casefold().startswith(prefix) or name.casefold().endswith(".csv"), + "ZIP_PREFIX_NONCSV", + ) + _require(any(n.casefold().startswith(prefix) for n in names), "ZIP_ROLE_MISSING") + return sorted(members, key=lambda m: m.filename), prefix + + +def _inventory(path, role, serialnos): + inventory, selected, rows, selected_budget = [], {}, 0, 0 + required = ("SERIALNO", "NP") if role == "household" else literal.READ_COLUMNS + with zipfile.ZipFile(path) as archive: + members, prefix = _members(archive, role) + for member in members: + applicable = member.filename.casefold().startswith(prefix) + digest, count, row_count, header, header_sha = ( + hashlib.sha256(), + 0, + 0, + None, + None, + ) + with archive.open(member) as stream: + if not applicable: + while block := stream.read(min(4096, member.file_size - count + 1)): + count += len(block) + _require(count <= member.file_size, "ZIP_MEMBER_GREW") + digest.update(block) + else: + for raw in _records(stream): + count += len(raw) + _require(count <= member.file_size, "ZIP_MEMBER_GREW") + digest.update(raw) + values = _decode_record(raw, first=header is None) + if header is None: + header, header_sha = values, _sha(raw) + _require( + all(header) + and len(set(header)) == len(header) + and set(required) <= set(header), + "CSV_HEADER", + ) + positions = [header.index(c) for c in required] + continue + _require(len(values) == len(header), "CSV_ROW_WIDTH") + rows += 1 + row_count += 1 + _require(rows <= literal.MAX_ROWS, "SOURCE_ROWS") + cells = [values[p] for p in positions] + if cells[0] not in serialnos: + continue + # Upper bound on escaped row + statuses + lineage before + # the low-level reader allocates its selected DataFrame. + selected_budget += 6 * len(raw) + 1024 + _require( + selected_budget <= MAX_BODY_BYTES, "SELECTED_BODY_BUDGET" + ) + if role == "household": + key = cells[0] + _require(re.fullmatch(r"[0-9]{1,2}", cells[1]), "SOURCE_NP") + value = int(cells[1]) + else: + _require( + re.fullmatch(r"[0-9]{1,2}", cells[1]), "SOURCE_SPORDER" + ) + key = (cells[0], int(cells[1])) + _require(1 <= key[1] <= 20, "SOURCE_SPORDER") + value = [member.filename, row_count] + _require(key not in selected, "SOURCE_DUPLICATE_KEY") + _require( + len(selected) < literal.MAX_SELECTED_ROWS, "SELECTED_ROWS" + ) + selected[key] = value + _require(count == member.file_size, "ZIP_MEMBER_SIZE_MISMATCH") + _require(not applicable or header is not None, "CSV_HEADER_MISSING") + inventory.append( + { + "name": member.filename, + "bytes": count, + "sha256": digest.hexdigest(), + "compressed_bytes": member.compress_size, + "crc32": member.CRC, + "applicable": applicable, + "header_sha256": header_sha, + "rows": row_count, + } + ) + return inventory, selected + + +def _native_roster(frame): + _require(type(frame) is Frame, "NATIVE_FRAME_REQUIRED") + p, h = frame.person, frame.table("household") + _require(0 < len(h) <= len(p) <= literal.MAX_SELECTED_ROWS, "NATIVE_ROWS") + _require(p.columns.is_unique and h.columns.is_unique, "NATIVE_COLUMNS") + _require( + { + "person_id", + "person_household_id", + "SPORDER", + "A_LINENO", + "source_year", + "source_household_id", + "source_person_id", + "source_row_id", + "AGEP", + "A_AGE", + "age", + } + <= set(p) + and {"household_id", "SERIALNO", "NP"} <= set(h), + "NATIVE_COLUMNS", + ) + for table, names in ( + ( + p, + ( + "person_id", + "person_household_id", + "SPORDER", + "A_LINENO", + "source_year", + "source_household_id", + "source_row_id", + ), + ), + (h, ("household_id", "NP")), + ): + for name in names: + _require( + pd.api.types.is_integer_dtype(table[name].dtype) + and not table[name].isna().any(), + "NATIVE_INTEGER", + ) + _require( + not p.person_id.duplicated().any() + and not h.household_id.duplicated().any() + and not p.source_row_id.duplicated().any() + and bool((p.source_row_id >= 0).all()), + "NATIVE_IDS", + ) + _require( + not h.SERIALNO.isna().any() and not h.SERIALNO.duplicated().any(), + "NATIVE_SERIALNO", + ) + _require(set(p.person_household_id) == set(h.household_id), "NATIVE_MEMBERSHIP") + household = h.set_index("household_id") + serials = p.person_household_id.map(household.SERIALNO) + _require(bool((p.source_year == 2024).all()), "NATIVE_VINTAGE") + _require( + np.array_equal(p.SPORDER, p.A_LINENO) + and np.array_equal(p.person_household_id, p.source_household_id) + and list(p.source_person_id) == [str(v) for v in p.SPORDER], + "NATIVE_ALIASES", + ) + keys = literal._keys( + pd.DataFrame({"SERIALNO": serials, "SPORDER": p.SPORDER}) + ).reset_index(drop=True) + counts = p.groupby("person_household_id", sort=False).size() + _require(np.array_equal(counts.loc[h.household_id], h.NP), "NATIVE_HOUSEHOLD_COUNT") + roster = ( + [int(pid), int(hid), serial, int(order), int(row)] + for pid, hid, serial, order, row in zip( + p.person_id, + p.person_household_id, + keys.SERIALNO, + keys.SPORDER, + p.source_row_id, + strict=True, + ) + ) + digest = hashlib.sha256() + for row in roster: + digest.update(_json(row, MAX_RECORD_BYTES - 1) + b"\n") + return keys, digest.hexdigest() + + +def _age_relation(frame, table): + matched, unresolved, mismatch = 0, 0, 0 + for raw, ag, aa, age in zip( + table.AGEP, frame.person.AGEP, frame.person.A_AGE, frame.person.age, strict=True + ): + if re.fullmatch(r"[0-9]{1,2}", raw) is None: + unresolved += 1 + elif all( + isinstance(v, (int, float, np.integer, np.floating)) + and not isinstance(v, (bool, np.bool_)) + and np.isfinite(v) + and v == int(raw) + for v in (ag, aa, age) + ): + matched += 1 + else: + mismatch += 1 + return { + "matching_rows": matched, + "unresolved_literal_rows": unresolved, + "mismatching_rows": mismatch, + "relation": "numeric_identity" + if not unresolved and not mismatch + else "unproven", + "preparation_provenance_authenticated": False, + } + + +@dataclass(frozen=True, slots=True) +class UngrantedACSNativeBinding: + """Owned consistency evidence, explicitly lacking prepared population proof.""" + + receipt_json: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "BINDING_CONSTRUCTOR") + + @property + def receipt(self): + return json.loads(self.receipt_json) + + +@dataclass(frozen=True, slots=True) +class AuthenticatedACSPersonCoverage: + """Immutable source envelope; candidate bytes cannot construct this type.""" + + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + + def _parts(self): + start = len(_MAGIC) + 4 + size = int.from_bytes(self.payload[len(_MAGIC) : start], "big") + return self.payload[start : start + size], self.payload[start + size :] + + @property + def receipt(self): + return json.loads(self._parts()[0]) + + @property + def table(self): + rows = [json.loads(row) for row in self._parts()[1].splitlines()] + result = pd.DataFrame([r[: len(_COLUMNS)] for r in rows], columns=_COLUMNS) + for column in _COLUMNS: + result[column] = result[column].astype( + "int64" if column == "SPORDER" else "string" + ) + return result + + @property + def native_binding(self): + return UngrantedACSNativeBinding( + _json( + { + **self.receipt["native_consistency"], + "source_payload_sha256": _sha(self.payload), + "population_binding_authenticated": False, + "missing_authority": "closed_prepared_acs_native_population", + }, + MAX_HEADER_BYTES, + ), + _token=_TOKEN, + ) + + +def verify_acs_coverage_native_consistency(coverage, frame): + """Check the exact live parent; never upgrade consistency to population proof.""" + try: + _require(type(coverage) is AuthenticatedACSPersonCoverage, "SOURCE_TYPE") + _require(coverage.receipt["producer"] == _producer(), "PRODUCER_CHANGED") + _keys, roster = _native_roster(frame) + evidence = coverage.native_binding.receipt + _require( + roster == evidence["native_roster_sha256"] + and custody.frame_content_sha256(frame) == evidence["frame_sha256"], + "NATIVE_FRAME_CHANGED", + ) + _require( + _age_relation(frame, coverage.table) == evidence["original_age"], + "NATIVE_AGE_CHANGED", + ) + return coverage.native_binding + except ACSCoverageAuthenticationError: + raise + except Exception: + raise ACSCoverageAuthenticationError("NATIVE_CONSISTENCY_REFUSED") from None + + +def load_authenticated_acs_person_coverage( + source_dir, *, snapshot_root, frame, candidate_path=None +): + """Reconstruct from closed default archive pins, then compare optional bytes. + + ``frame`` supplies a checked selection, never source or population authority. + No manifest/pin/preparation-receipt argument and no publishing API exists. + Successful and failed private captures retain the housing owner's policy. + """ + try: + _require(type(frame) is Frame, "NATIVE_FRAME_REQUIRED") + before = custody.frame_content_sha256(frame) + keys, roster_sha = _native_roster(frame) + producer = _producer() + with custody._capture(source_dir, snapshot_root) as (private, paths, pins): + _require( + {r for r, _n, _d, _s in pins} == {"household", "person"} + and len(pins) == 2, + "SOURCE_ROLES", + ) + members = {} + members["household"], households = _inventory( + paths["household"], "household", set(keys.SERIALNO) + ) + members["person"], lineage = _inventory( + paths["person"], "person", set(keys.SERIALNO) + ) + _require( + households == keys.groupby("SERIALNO").size().to_dict(), + "SOURCE_HOUSEHOLD_ROSTER", + ) + _require( + set(lineage) == set(zip(keys.SERIALNO, keys.SPORDER, strict=True)), + "SOURCE_PERSON_ROSTER", + ) + table, original_receipt = literal.read_acs_person_coverage_columns( + AcsPumsSource(paths["household"], paths["person"], vintage=2024), + person_keys=keys, + chunksize=min(1000, len(keys)), + ) + body, size = [], 0 + for row in table.itertuples(index=False, name=None): + raw = ( + _json([*row, *lineage[(row[0], row[1])]], MAX_RECORD_BYTES - 1) + + b"\n" + ) + size += len(raw) + _require(size <= MAX_BODY_BYTES, "BODY_SIZE") + body.append(raw) + body = b"".join(body) + header = { + "protocol": PROTOCOL, + "encoding": "ascii-escaped-json-header-and-ndjson-v1", + "producer": producer, + "source_authenticated": True, + "vintage": 2024, + "population_binding_authenticated": False, + "coverage_status": "literal_source_fields_only", + "cross_survey_coverage_equivalence_established": False, + "domain_assignment_authenticated": False, + "period_harmonized": False, + "release_eligible": False, + "archives": [ + {"role": r, "filename": n, "sha256": d, "bytes": s} + for r, n, d, s in pins + ], + "members": members, + "field_contract": literal.coverage_field_contract(), + "original_literal_receipt": original_receipt, + "columns": [*_COLUMNS, "source_member", "source_row_ordinal"], + "body_bytes": len(body), + "body_sha256": _sha(body), + "rows": len(table), + "native_consistency": { + "frame_sha256": before, + "native_roster_sha256": roster_sha, + "complete_households_checked": True, + "source_aliases_checked": True, + "original_age": _age_relation(frame, table), + }, + } + raw_header = _json(header, MAX_HEADER_BYTES) + payload = _MAGIC + len(raw_header).to_bytes(4, "big") + raw_header + body + # No candidate header, digest, DataFrame or claimed producer is parsed. + if candidate_path is not None: + target = private / "candidate.coverage" + actual = custody._copy( + Path(candidate_path), target, len(payload), exact_size=len(payload) + ) + _require( + actual == _sha(payload) + and custody._persisted_sha(target, len(payload)) == actual, + "CANDIDATE_MISMATCH", + ) + for role, _name, digest, size in pins: + _require( + custody._persisted_sha(paths[role], size) == digest, + "SNAPSHOT_CHANGED", + ) + _require(_producer() == producer, "PRODUCER_CHANGED") + _require(_native_roster(frame)[1] == roster_sha, "NATIVE_FRAME_CHANGED") + _require( + custody.frame_content_sha256(frame) == before, "NATIVE_FRAME_CHANGED" + ) + # _capture's authority-stability check must complete before issuance. + return AuthenticatedACSPersonCoverage(payload, _token=_TOKEN) + except ACSCoverageAuthenticationError: + raise + except Exception: + raise ACSCoverageAuthenticationError("SOURCE_RECONSTRUCTION_REFUSED") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py new file mode 100644 index 000000000..53b692dad --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py @@ -0,0 +1,295 @@ +"""Additive, literal ACS person coverage fields for an exact native roster. + +This reader preserves MIL and ESR separately, including their different age +universes. It performs no population-domain allocation and authenticates no +source file. A source owner must bind the archive and requested native keys +before these observations may support a genuine graph. The existing ACS +loader, its columns and its implementation identity are unchanged. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import re +from pathlib import Path +from zipfile import ZipFile + +import pandas as pd + +from .acs_pums import AcsPumsSource +from .source_csv_builtin import capture_csv_reader + +PROTOCOL = "microcosm.acs-person-coverage-columns.v1" +DICTIONARY_URL = ( + "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/" + "PUMS_Data_Dictionary_2024.pdf" +) +DICTIONARY_SHA256 = "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff" +KEYS = ("SERIALNO", "SPORDER") +READ_COLUMNS = (*KEYS, "AGEP", "MIL", "ESR") +MAX_ROWS = 6_000_000 +MAX_SELECTED_ROWS = 1_000_000 +MAX_CSV_RECORD_CHARS = 100_000 +_NON_CSV_CONTROLS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]") + + +def coverage_field_contract() -> dict: + """A fresh source-definition document, suitable for provenance consumers.""" + return { + "protocol": PROTOCOL, + "survey": "ACS", + "survey_year": 2024, + "record_grain": "person", + "read_columns": list(READ_COLUMNS), + "dictionary": {"url": DICTIONARY_URL, "sha256": DICTIONARY_SHA256}, + "fields": { + "MIL": { + "pdf_page": 41, + "minimum_age": 17, + "blank_meaning": "outside age universe", + "codes": { + "1": "active_duty", + "2": "past_active_duty", + "3": "training_only", + "4": "never_served", + }, + }, + "ESR": { + "pdf_page": 56, + "minimum_age": 16, + "blank_meaning": "outside age universe", + "codes": { + "1": "civilian_working", + "2": "civilian_job_absent", + "3": "unemployed", + "4": "armed_forces_working", + "5": "armed_forces_job_absent", + "6": "outside_labor_force", + }, + }, + }, + "cross_survey_residence_equivalence_established": False, + } + + +def _json(value) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _keys(table: pd.DataFrame) -> pd.DataFrame: + """Canonical native identities; no row-position join or key imputation.""" + if ( + not isinstance(table, pd.DataFrame) + or not table.columns.is_unique + or not set(KEYS).issubset(table) + ): + raise ValueError("ACS coverage requires native person keys") + result = table.loc[:, list(KEYS)].copy() + if result.isna().any().any(): + raise ValueError("ACS coverage native keys are missing") + serial = result.SERIALNO + if not serial.map( + lambda x: ( + isinstance(x, str) and re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", x) is not None + ) + ).all(): + raise ValueError("ACS coverage native serial is not a 2024 HU/GQ identity") + order = result.SPORDER.astype("string") + if not order.str.fullmatch(r"[0-9]{1,2}").all(): + raise ValueError("ACS coverage native person order is not an integer") + result["SPORDER"] = order.astype("int64") + if not result.SPORDER.between(1, 20).all(): + raise ValueError("ACS coverage native person order is outside 1..20") + if result.duplicated(list(KEYS)).any(): + raise ValueError("ACS coverage native person keys repeat") + return result + + +def _field_state(age: str, raw: str, *, minimum_age: int, codes: dict) -> str: + if not re.fullmatch(r"[0-9]{1,2}", age): + return "age_unresolved" + if int(age) < minimum_age: + return "outside_age_universe" if raw == "" else "value_below_age_universe" + if raw == "": + return "missing_in_universe" + return "observed_code" if raw in codes else "unlabelled_code" + + +def _literal_csv_records(stream): + """Check controls before CSV parsing, bounding each complete logical record.""" + csv_reader = capture_csv_reader(csv) + if csv_reader is None: + raise ValueError("SOURCE_CSV_READER_CHANGED") + record_chars = 0 + + def lines(): + nonlocal record_chars + while line := stream.readline(MAX_CSV_RECORD_CHARS - record_chars + 1): + record_chars += len(line) + if record_chars > MAX_CSV_RECORD_CHARS: + raise ValueError("ACS coverage CSV record character bound exceeded") + # HT is literal whitespace; CR/LF delimit records or remain literal + # inside quotes. newline="" prevents universal-newline rewriting. + if _NON_CSV_CONTROLS.search(line): + raise ValueError("ACS coverage CSV contains a forbidden control") + yield line + + reader = csv_reader(lines(), strict=True) + try: + while True: + record_chars = 0 + record = next(reader, None) + if record is None: + return + yield record + except (csv.Error, UnicodeError) as exc: + raise ValueError("ACS coverage invalid literal CSV") from exc + + +def _scan_acs_person_coverage(source, consume_row): + """Exhaust the literal source once before returning its member/row counts. + + The private consumer receives only exact READ_COLUMNS strings. It cannot + signal successful early completion: all members, records and widths are + checked unless an exception refuses the scan. Source custody and requested + roster checks remain the caller's responsibility. + """ + if type(source) is not AcsPumsSource or source.vintage != 2024: + raise ValueError("ACS coverage projection requires the 2024 source type") + rows = 0 + with ZipFile(source.person_zip) as archive: + members = sorted( + name + for name in archive.namelist() + if Path(name).name.lower().startswith("psam_pus") + and name.lower().endswith(".csv") + ) + if not members or len(members) != len(set(members)): + raise ValueError( + "ACS coverage person member roster is missing or duplicated" + ) + for name in members: + with ( + archive.open(name) as member, + io.TextIOWrapper(member, encoding="utf-8-sig", newline="") as stream, + ): + reader = _literal_csv_records(stream) + header = next(reader, []) + if len(header) != len(set(header)) or not set(READ_COLUMNS).issubset( + header + ): + raise ValueError( + "ACS coverage source columns are missing or duplicated" + ) + positions = [header.index(column) for column in READ_COLUMNS] + for record in reader: + rows += 1 + if rows > MAX_ROWS: + raise ValueError("ACS coverage source row bound exceeded") + if len(record) != len(header): + raise ValueError( + "ACS coverage CSV record width differs from header" + ) + consume_row(tuple(record[position] for position in positions)) + return members, rows + + +def read_acs_person_coverage_columns( + source: AcsPumsSource, + *, + person_keys: pd.DataFrame, + chunksize: int = 100_000, +) -> tuple[pd.DataFrame, dict]: + """Stream an additive person projection and align exact selected households. + + ``person_keys`` must include every person in each requested source + household. The read refuses missing, duplicate or extra native person keys + in those households, including differences discovered in later members. + Other households are streamed past. The separate max_households option on + the legacy source is irrelevant here: the exact native roster is authority. + + AGEP, MIL and ESR remain literal strings, including empty Census cells. + Each member is parsed once, with full record widths checked before column + or household selection. Non-CSV C0/C1 controls (including NUL) refuse; + tabs and quoted CR/LF remain literal. Records, including their physical + line endings, are bounded to MAX_CSV_RECORD_CHARS decoded characters. + State columns distinguish printed codes, out-of-universe blanks, missing + in-universe values, unexpected codes and age conflicts. They are evidence + states, not a combined military predicate or survey membership decision. + + Only invented archives have been executed in development. Genuine use + requires source authentication and a separately bounded source-read plan. + """ + if type(source) is not AcsPumsSource or source.vintage != 2024: + raise ValueError("ACS coverage projection requires the 2024 source type") + if type(chunksize) is not int or not 0 < chunksize <= 100_000: + raise ValueError("ACS coverage chunksize must be in 1..100000") + expected = _keys(person_keys).reset_index(drop=True) + if not 0 < len(expected) <= MAX_SELECTED_ROWS: + raise ValueError("ACS coverage selected person count is outside the bound") + retained = frozenset(expected.SERIALNO) + pieces, batch, selected_rows = [], [], 0 + + def consume_row(cells): + nonlocal batch, selected_rows + if cells[0] not in retained: + return + selected_rows += 1 + if selected_rows > len(expected): + raise ValueError("ACS coverage source has extra selected-household people") + batch.append(cells) + if len(batch) == chunksize: + pieces.append(pd.DataFrame(batch, columns=READ_COLUMNS, dtype="string")) + batch = [] + + members, rows = _scan_acs_person_coverage(source, consume_row) + if batch: + pieces.append(pd.DataFrame(batch, columns=READ_COLUMNS, dtype="string")) + if not pieces: + raise ValueError("ACS coverage requested persons are missing") + observed = pd.concat(pieces, ignore_index=True) + observed_keys = _keys(observed) + observed["SPORDER"] = observed_keys.SPORDER + wanted = pd.MultiIndex.from_frame(expected) + actual = pd.MultiIndex.from_frame(observed_keys) + if set(wanted) != set(actual): + raise ValueError("ACS coverage source and requested native roster differ") + result = observed.set_index(list(KEYS)).loc[wanted].reset_index() + contract = coverage_field_contract() + for field, definition in contract["fields"].items(): + result[field + "_state"] = pd.Series( + [ + _field_state( + age, + raw, + minimum_age=definition["minimum_age"], + codes=definition["codes"], + ) + for age, raw in zip(result.AGEP, result[field], strict=True) + ], + dtype="string", + ) + receipt = { + "protocol": PROTOCOL, + "contract_sha256": hashlib.sha256(_json(contract)).hexdigest(), + "source_authenticated": False, + "coverage_status": "literal_source_fields_only", + "release_eligible": False, + "member_names": members, + "source_rows_streamed": rows, + "selected_person_rows": len(result), + "native_roster_sha256": hashlib.sha256( + _json(expected.to_dict("records")) + ).hexdigest(), + "projection_sha256": hashlib.sha256( + _json(result.to_dict("records")) + ).hexdigest(), + "legacy_max_households_applied": False, + "read_columns": list(READ_COLUMNS), + } + return result, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py new file mode 100644 index 000000000..44d97c96f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py @@ -0,0 +1,579 @@ +"""Closed raw ACS catalogue before native population or unit construction. + +The full housing lexical projection and raw catalogue remain in memory. Literal +persons are scanned once and records are assembled in whole-household batches; +this does not make national preparation memory-bounded. No domain is classified +here. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +import weakref +from dataclasses import InitVar, dataclass +from pathlib import Path +from types import FunctionType + +from . import acs_housing_universe_source as housing +from . import acs_native_coverage_binding as native +from . import acs_person_coverage_columns as literal +from . import survey_population_domains as domains + +PROTOCOL = "microcosm.acs-source-catalogue.v1" +_ARCHIVE_BYTES = 8 * 1024**3 +_EXPANDED_BYTES = 16 * 1024**3 +_MEMBERS = 64 +_SOURCE_ROWS = 6_000_000 +_BATCH_PEOPLE = 100_000 +_CATALOGUE_BYTES = 2 * 1024**3 +_RECEIPT_BYTES = 1024**2 +_TOKEN = object() +_ISSUED = {} + + +class ACSSourceCatalogueError(ValueError): + """Static refusal, without source identities, paths or cause chains.""" + + +def _require(condition, code): + if not condition: + raise ACSSourceCatalogueError(code) + + +def _local_bytes(): + return { + module.__name__: housing._sha(Path(module.__file__).read_bytes()) + for module in (sys.modules[__name__], domains) + } + + +def _local_runtime(): + """Capture exact loaded code, including generated raw dataclass methods.""" + result = {} + for module in (sys.modules[__name__], domains): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[(module.__name__, name)] = value.__code__ + elif isinstance(value, type) and value.__module__ == module.__name__: + result[(module.__name__, name)] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[(module.__name__, name, method)] = function.__code__ + return result + + +def _immutable_pins(pins): + _require( + type(pins) is tuple + and len(pins) == 2 + and all( + type(pin) is tuple + and len(pin) == 4 + and all(type(value) is str for value in pin[:3]) + and type(pin[3]) is int + for pin in pins + ), + "SOURCE_AUTHORITY_TYPE", + ) + return pins + + +def _final_seals(pins, pin_authority): + # Pure comparisons after the last producer/source I/O, including capture + # exit. Another read would reopen the same interval this seal closes. + # _pins() loads the code manifest when the private override is None. Bind + # that exact authority mode and any override here without reopening I/O. + current = housing._ARCHIVE_PINS + if current is not None: + _immutable_pins(current) + _require( + current == pin_authority and (current is None or current == pins), + "SOURCE_AUTHORITY_CHANGED", + ) + _require(_local_runtime() == _RUNTIME_AUTHORITY, "PRODUCER_CODE_CHANGED") + + +def _producer(): + _require(PROTOCOL == "microcosm.acs-source-catalogue.v1", "PROTOCOL_CHANGED") + _require( + literal.KEYS == ("SERIALNO", "SPORDER") + and literal.READ_COLUMNS == ("SERIALNO", "SPORDER", "AGEP", "MIL", "ESR"), + "LITERAL_CONTRACT_CHANGED", + ) + limits = ( + _ARCHIVE_BYTES, + _EXPANDED_BYTES, + _MEMBERS, + _SOURCE_ROWS, + _BATCH_PEOPLE, + _CATALOGUE_BYTES, + _RECEIPT_BYTES, + ) + ceilings = (8 * 1024**3, 16 * 1024**3, 64, 6_000_000, 100_000, 2 * 1024**3, 1024**2) + _require( + all( + type(v) is int and 0 < v <= cap + for v, cap in zip(limits, ceilings, strict=True) + ), + "LIMITS", + ) + checked = native._producer() + compiled = {} + for module in (domains, sys.modules[__name__]): + native._live_code(module, compiled) + code = _local_bytes() + _require(code == _BYTE_AUTHORITY, "PRODUCER_CODE_CHANGED") + _require(_local_runtime() == _RUNTIME_AUTHORITY, "PRODUCER_CODE_CHANGED") + return { + "native_source_closure": checked, + "catalogue_sha256": code[__name__], + "raw_domain_types_sha256": code[domains.__name__], + "limits": list(limits), + } + + +def _preflight(paths): + """Keep full-source gates separate from native's 1M selected-row gate.""" + expanded = 0 + for role, path in paths.items(): + with native.coverage.zipfile.ZipFile(path) as archive: + members, _prefix = native.coverage._members(archive, role) + _require(len(members) <= _MEMBERS, "MEMBER_LIMIT") + expanded += sum(member.file_size for member in members) + _require(expanded <= _EXPANDED_BYTES, "EXPANDED_LIMIT") + inventories = {} + for role, path in paths.items(): + inventory, empty = native.coverage._inventory(path, role, frozenset()) + _require(not empty, "UNEXPECTED_SELECTED_ROWS") + _require( + sum(member["rows"] for member in inventory) <= _SOURCE_ROWS, + "SOURCE_ROW_LIMIT", + ) + inventories[role] = inventory + return inventories + + +def _source_checks(source_dir, paths, pins): + _require(housing._pins() == pins, "SOURCE_AUTHORITY_CHANGED") + for role, name, digest, size in pins: + for path in (source_dir / name, paths[role]): + housing._path_components(path) + _require(housing._persisted_sha(path, size) == digest, "SOURCE_CHANGED") + + +def _charge(record, digest, size): + raw = native.coverage._json(record, max(0, _CATALOGUE_BYTES - size - 1)) + b"\n" + size += len(raw) + _require(size <= _CATALOGUE_BYTES, "CATALOGUE_BYTE_LIMIT") + digest.update(raw) + return size + + +def _collect(projection, paths): + # Validate the actual persisted full projection, independently of the + # publicly constructible returned housing capsule's mutable attributes. + housing._tables(projection) + hcols = {name: i for i, name in enumerate(projection["household_columns"])} + pcols = {name: i for i, name in enumerate(projection["person_columns"])} + by_household = {} + positions = {} + people = [None] * len(projection["persons"]) + for index, row in enumerate(projection["persons"]): + serial, raw_order = row[pcols["SERIALNO"]], row[pcols["SPORDER"]] + key = (serial, int(raw_order)) + _require(key not in positions, "DUPLICATE_PERSON") + positions[key] = index + by_household.setdefault(serial, []).append(index) + household_keys, started_households = set(), set() + prospective_size = 0 + + def reserve(value, punctuation): + # An empty household's [] gains exactly the encoded person lengths and + # one comma between people. Charge before retaining literal fields. + nonlocal prospective_size + raw = native.coverage._json( + value, max(0, _CATALOGUE_BYTES - prospective_size - punctuation) + ) + prospective_size += len(raw) + punctuation + _require(prospective_size <= _CATALOGUE_BYTES, "CATALOGUE_BYTE_LIMIT") + + def household_record(row, members): + return ( + row[hcols["SERIALNO"]], + row[hcols["TYPEHUGQ"]], + row[hcols["NP"]], + row[hcols["WGTP"]], + row[hcols["source_member"]], + row[hcols["source_row_ordinal"]], + members, + ) + + for row in projection["households"]: + serial, count = row[hcols["SERIALNO"]], int(row[hcols["NP"]]) + _require(serial not in household_keys, "DUPLICATE_HOUSEHOLD") + household_keys.add(serial) + _require(count <= _BATCH_PEOPLE, "HOUSEHOLD_EXCEEDS_BATCH") + reserve(household_record(row, ()), 1) + _require(set(by_household) <= household_keys, "GLOBAL_PERSON_KEYS") + contract = literal.coverage_field_contract() + + def consume_row(cells): + serial, order, age, mil, esr = cells + _require( + all(type(v) is str for v in cells) + and literal.re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", serial) is not None + and literal.re.fullmatch(r"[0-9]{1,2}", order) is not None + and 1 <= int(order) <= 20, + "LITERAL_PERSON_KEY", + ) + key = (serial, int(order)) + _require(key in positions, "UNEXPECTED_LITERAL_PERSON") + index = positions[key] + _require(people[index] is None, "DUPLICATE_LITERAL_PERSON") + states = { + field: literal._field_state( + age, + raw, + minimum_age=contract["fields"][field]["minimum_age"], + codes=contract["fields"][field]["codes"], + ) + for field, raw in (("MIL", mil), ("ESR", esr)) + } + _require(all(type(v) is str for v in states.values()), "LITERAL_TYPES") + original = projection["persons"][index] + person = ( + original[pcols["SPORDER"]], + age, + esr, + states["ESR"], + mil, + states["MIL"], + original[pcols["PWGTP"]], + original[pcols["source_member"]], + original[pcols["source_row_ordinal"]], + ) + reserve(person, int(serial in started_households)) + people[index] = person + started_households.add(serial) + + if positions: + _members, rows = literal._scan_acs_person_coverage( + housing.AcsPumsSource(paths["household"], paths["person"], vintage=2024), + consume_row, + ) + _require(rows == len(positions), "GLOBAL_PERSON_KEYS") + _require(all(person is not None for person in people), "GLOBAL_PERSON_KEYS") + records, vacancies, batch = [], [], [] + digest, size, batches, batch_people = hashlib.sha256(), 0, 0, 0 + + def consume(): + nonlocal size, batches + count = sum(len(by_household.get(row[hcols["SERIALNO"]], ())) for row in batch) + if count: + # Preserve the prior selected-reader ceiling on each assembly + # batch, even though no selected DataFrame is allocated here. + _require( + count <= min(_BATCH_PEOPLE, literal.MAX_SELECTED_ROWS), + "BATCH_PERSON_LIMIT", + ) + batches += 1 + for row in batch: + serial = row[hcols["SERIALNO"]] + members = tuple(people[index] for index in by_household.get(serial, ())) + record = household_record(row, members) + _require(len(members) == int(record[2]), "HOUSEHOLD_COMPLETENESS") + size = _charge(record, digest, size) + ( + vacancies if int(record[1]) == 1 and int(record[2]) == 0 else records + ).append(record) + + for row in projection["households"]: + count = int(row[hcols["NP"]]) + if batch and batch_people + count > _BATCH_PEOPLE: + consume() + batch, batch_people = [], 0 + batch.append(row) + batch_people += count + if batch: + consume() + _require(size == prospective_size, "CATALOGUE_BYTE_ACCOUNTING") + return ( + tuple(records), + tuple(vacancies), + { + "households": len(household_keys), + "people": len(positions), + "occupied_hu": sum(int(r[1]) == 1 for r in records), + "institutional_gq": sum(int(r[1]) == 2 for r in records), + "noninstitutional_gq": sum(int(r[1]) == 3 for r in records), + "vacancies": len(vacancies), + "literal_batches": batches, + "canonical_record_bytes": size, + "canonical_record_sha256": digest.hexdigest(), + }, + ) + + +def _household(record): + serial, kind, count, weight, _member, _ordinal, people = record + key = domains.HouseholdKey(domains.Source.ACS, 2024, 2024, serial) + return domains.AcsHousehold( + key, + kind, + count, + weight, + tuple( + domains.AcsPerson(order, age, esr, esr_state, mil, mil_state, pwgtp, key) + for order, age, esr, esr_state, mil, mil_state, pwgtp, _pmember, _pordinal in people + ), + ) + + +@dataclass(frozen=True, slots=True) +class _Owned: + receipt: bytes + records: tuple + vacancies: tuple + source_dir: Path + paths: tuple + pins: tuple + pin_authority: tuple | None + producer: bytes + projection_path: Path + projection_bytes: int + projection_sha256: str + + +@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False) +class AuthenticatedACSSourceCatalogue: + """Process-issued compact receipt; raw views are independent frozen values.""" + + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "ISSUANCE_CONSTRUCTOR") + + def validate(self): + return verify_acs_source_catalogue(self) + + def to_bytes(self): + return _checked(self).receipt + + @property + def receipt(self): + return json.loads(self.to_bytes()) + + @property + def households(self): + return tuple(_household(record) for record in _checked(self).records) + + @property + def exclusion_ledger(self): + return tuple(_household(record) for record in _checked(self).vacancies) + + @property + def lineage(self): + owned = _checked(self) + return { + r[0]: { + "household": (r[4], r[5]), + "persons": tuple((p[0], p[7], p[8]) for p in r[6]), + } + for group in (owned.records, owned.vacancies) + for r in group + } + + +def _lookup(value): + _require(type(value) is AuthenticatedACSSourceCatalogue, "ISSUANCE_TYPE") + entry = _ISSUED.get(id(value)) + _require(entry is not None and entry[0]() is value, "ISSUANCE_NOT_OWNED") + owned = entry[1] + _require( + type(value.payload) is bytes and value.payload == owned.receipt, + "ISSUANCE_CHANGED", + ) + return owned + + +def _checked(value): + try: + owned = _lookup(value) + before = native.coverage._json(_producer(), _RECEIPT_BYTES) + _require(before == owned.producer, "PRODUCER_CHANGED") + _source_checks(owned.source_dir, dict(owned.paths), owned.pins) + _require( + housing._persisted_sha(owned.projection_path, owned.projection_bytes) + == owned.projection_sha256, + "PROJECTION_CHANGED", + ) + _require( + native.coverage._json(_producer(), _RECEIPT_BYTES) == before, + "PRODUCER_CHANGED", + ) + _final_seals(owned.pins, owned.pin_authority) + _require(_lookup(value) is owned, "ISSUANCE_CHANGED") + return owned + except ACSSourceCatalogueError: + raise + except Exception: + raise ACSSourceCatalogueError("CATALOGUE_VERIFICATION_REFUSED") from None + + +def verify_acs_source_catalogue(value): + _checked(value) + return value + + +def issue_acs_source_catalogue(source_dir, *, snapshot_root, candidate=None): + """Execute closed real owners; candidate bytes never supply source authority.""" + try: + source_dir = Path(source_dir).absolute() + snapshot_root = Path(snapshot_root).absolute() + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _RECEIPT_BYTES), + "CANDIDATE_TYPE_SIZE", + ) + pin_authority = housing._ARCHIVE_PINS + if pin_authority is not None: + _immutable_pins(pin_authority) + pins = _immutable_pins(housing._pins()) + archives = native._archives(pins) + _require(sum(p[3] for p in pins) <= _ARCHIVE_BYTES, "ARCHIVE_LIMIT") + producer = _producer() + producer_bytes = native.coverage._json(producer, _RECEIPT_BYTES) + with housing._capture(source_dir, snapshot_root) as (private, paths, captured): + _require(captured == pins, "SOURCE_AUTHORITY_CHANGED") + inventories = _preflight(paths) + source = housing._reconstruct( + private, paths, pins, None, housing._implementation() + ) + source_receipt = json.loads(source.receipt_json) + native._members_equal(source_receipt["members"], inventories) + projection_path = private / "full-projection.json" + with projection_path.open("rb") as stream: + projection_raw = stream.read(housing.ACS_HU_SOURCE_MAX_BYTES + 1) + _require( + len(projection_raw) <= housing.ACS_HU_SOURCE_MAX_BYTES, + "PROJECTION_SIZE", + ) + projection_sha256 = housing._sha(projection_raw) + _require( + projection_sha256 == source_receipt["full_projection_sha256"], + "PROJECTION_CHANGED", + ) + projection_bytes = len(projection_raw) + projection = json.loads(projection_raw) + del projection_raw + records, vacancies, counts = _collect(projection, paths) + _require( + counts["households"] == sum(m["rows"] for m in inventories["household"]) + and counts["people"] == sum(m["rows"] for m in inventories["person"]), + "GLOBAL_SOURCE_COMPLETENESS", + ) + receipt = native.coverage._json( + { + "protocol": PROTOCOL, + "producer": producer, + "archives": archives, + "members": inventories, + "source_authenticated": True, + "population_binding_authenticated": False, + "release_eligible": False, + "domain_assignment_authenticated": False, + "selection_performed": False, + "source_year": 2024, + "survey_year": 2024, + "scope": "all_occupied_hu_and_both_gq_with_separate_vacancies", + "field_contract": literal.coverage_field_contract(), + "complete_original_membership": True, + "original_anchors_preserved": True, + "housing_projection_sha256": projection_sha256, + "counts": counts, + "canonical_record_schema": [ + "SERIALNO", + "TYPEHUGQ", + "NP", + "WGTP", + "household_member", + "household_row", + [ + "SPORDER", + "AGEP", + "ESR", + "ESR_state", + "MIL", + "MIL_state", + "PWGTP", + "person_member", + "person_row", + ], + ], + "canonical_record_order": "housing_source_member_then_row_ordinal", + "execution": { + "native_frame_constructed": False, + "unit_assignment_executed": False, + "full_housing_projection_retained": True, + "full_person_scan_per_literal_batch": False, + "literal_full_scans": int(bool(counts["people"])), + "national_efficiency_claimed": False, + }, + }, + _RECEIPT_BYTES, + ) + if candidate is not None: + _require(candidate == receipt, "CANDIDATE_MISMATCH") + _source_checks(source_dir, paths, pins) + _require( + housing._persisted_sha(projection_path, projection_bytes) + == projection_sha256, + "PROJECTION_CHANGED", + ) + owned = _Owned( + receipt, + records, + vacancies, + source_dir, + tuple(paths.items()), + pins, + pin_authority, + producer_bytes, + projection_path, + projection_bytes, + projection_sha256, + ) + # Capture exit can read the default source manifest. Recheck producer + # bytes/live code after that I/O, then finish with pure final seals. + _require( + native.coverage._json(_producer(), _RECEIPT_BYTES) == producer_bytes, + "PRODUCER_CHANGED", + ) + _final_seals(pins, pin_authority) + result = AuthenticatedACSSourceCatalogue(receipt, _token=_TOKEN) + identity = id(result) + + def cleanup(reference): + entry = _ISSUED.get(identity) + if entry is not None and entry[0] is reference: + del _ISSUED[identity] + + _ISSUED[identity] = (weakref.ref(result, cleanup), owned) + return result + except ACSSourceCatalogueError: + raise + except Exception: + raise ACSSourceCatalogueError("CATALOGUE_ISSUANCE_REFUSED") from None + + +# Authority begins at this import, rather than accepting replacement source and +# matching live code as a newly approved implementation on first issuance. +_BYTE_AUTHORITY = _local_bytes() +_RUNTIME_AUTHORITY = _local_runtime() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py index dd0456fca..f4431a860 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py @@ -46,6 +46,8 @@ ACS_2024_1YR_SPINE = "acs_2024_1yr" ACS_2024_1YR_VINTAGE = 2024 DEFAULT_CHUNKSIZE = 100_000 +MAX_EXACT_HOUSEHOLDS = 1_000_000 +MAX_EXACT_PERSON_ROWS = 1_000_000 _HOUSEHOLD_REQUIRED = ( "SERIALNO", @@ -69,6 +71,14 @@ "TAXAMT", "TYPEHUGQ", ) +# 2024 ACS PUMS dictionary p.3: TYPEHUGQ 1 is a housing-unit record, 2 an +# institutional and 3 a noninstitutional group-quarters person record. Only +# occupied code-1 rows survive the vacancy drop in _occupied_households. +_HOUSEHOLD_AXIS_KINDS = { + 1: "occupied_housing_unit", + 2: "institutional_gq_person", + 3: "noninstitutional_gq_person", +} _PERSON_REQUIRED = ( "SERIALNO", "SPORDER", @@ -168,11 +178,36 @@ def __post_init__(self) -> None: if self.max_households is not None and self.max_households <= 0: raise ValueError("max_households must be positive when provided.") + @staticmethod + def snapshot_serialnos(serialnos): + """Freeze exact raw native keys; no sampling probability is implied.""" + if serialnos is None: + return None + if ( + type(serialnos) is not tuple + or not 0 < len(serialnos) <= MAX_EXACT_HOUSEHOLDS + or any( + type(key) is not str + or len(key) != 13 + or key[:6] not in {"2024HU", "2024GQ"} + or not key[6:].isascii() + or not key[6:].isdigit() + or int(key[6:]) == 0 + for key in serialnos + ) + or len(set(serialnos)) != len(serialnos) + ): + raise ValueError( + "ACS exact selection requires bounded unique raw native keys." + ) + return serialnos + def load_acs_pums_tables( source: AcsPumsSource, *, chunksize: int = DEFAULT_CHUNKSIZE, + serialnos: tuple[str, ...] | None = None, ) -> tuple[dict[str, pd.DataFrame], dict[str, Any]]: """Read and key-align the household and person PUMS tables. @@ -180,6 +215,11 @@ def load_acs_pums_tables( missing; this stage never converts an out-of-universe blank to zero. """ + serialnos = AcsPumsSource.snapshot_serialnos(serialnos) + if serialnos is not None and source.max_households is not None: + raise ValueError( + "ACS exact serialnos and max_households are ambiguous together." + ) if chunksize <= 0: raise ValueError("chunksize must be positive.") household, household_members = _read_archive( @@ -207,7 +247,24 @@ def load_acs_pums_tables( raise ValueError(f"ACS duplicate household SERIALNO value(s): {examples}.") household = household.sort_values("SERIALNO", kind="stable").reset_index(drop=True) all_household_serials = frozenset(household["SERIALNO"].tolist()) + full_household = household household, vacant_count = _occupied_households(household) + if serialnos is not None: + if not set(serialnos) <= all_household_serials: + raise ValueError("ACS exact selection contains absent household keys.") + weights = pd.to_numeric(household.WGTP, errors="coerce").to_numpy(dtype=float) + if not np.isfinite(weights).all() or (weights < 0).any(): + raise ValueError("ACS WGTP must be finite and nonnegative.") + _validate_amount_columns( + (household.RNTP, household.GRNTP, household.TAXAMT), household.ADJHSG + ) + household = household.loc[household.SERIALNO.isin(serialnos)].reset_index( + drop=True + ) + if household.NP.sum() > MAX_EXACT_PERSON_ROWS: + raise ValueError( + "ACS selected complete roster exceeds native person budget." + ) if source.max_households is not None and len(household) > source.max_households: household = _smoke_household_selection(household, source.max_households) selected_serials = frozenset(household["SERIALNO"].tolist()) @@ -220,6 +277,7 @@ def load_acs_pums_tables( chunksize=chunksize, valid_serials=all_household_serials, retained_serials=selected_serials, + validate_households=full_household if serialnos is not None else None, ) duplicate_people = person.duplicated(["SERIALNO", "SPORDER"], keep=False) if duplicate_people.any(): @@ -247,6 +305,14 @@ def load_acs_pums_tables( "vacant_household_rows_dropped": vacant_count, "max_households": source.max_households, } + if serialnos is not None: + metadata["exact_selection"] = { + "requested_serialnos": serialnos, + "populated_serialnos": tuple(household.SERIALNO), + "vacant_serialnos": tuple(sorted(set(serialnos) - selected_serials)), + "selection_kind": "engineering_exact_keys", + "full_source_inclusion_probability": None, + } return {"household": household, "person": person}, metadata @@ -254,10 +320,13 @@ def build_acs_pums_unit_frame( source: AcsPumsSource, *, chunksize: int = DEFAULT_CHUNKSIZE, + serialnos: tuple[str, ...] | None = None, ) -> tuple[Frame, dict[str, Any]]: """Construct the ACS 2024 1-year US entity frame.""" - tables, metadata = load_acs_pums_tables(source, chunksize=chunksize) + tables, metadata = load_acs_pums_tables( + source, chunksize=chunksize, serialnos=serialnos + ) household = tables["household"].copy() person = tables["person"].copy() @@ -299,6 +368,11 @@ def build_acs_pums_unit_frame( metadata.update( { "weighted_household_population": frame.weights_for("household").total, + # That total mixes housing-unit and GQ-person design mass. + "household_axis_composition": _household_axis_composition( + household, + household_weights, + ), "relationship_pointer_policy": ( "RELSHIPP reference/spouse pairing; own/adopted/stepchildren " "point to reference person and present spouse; all other " @@ -330,10 +404,12 @@ def _read_archive( chunksize: int, valid_serials: frozenset[str] | None = None, retained_serials: frozenset[str] | None = None, + validate_households: pd.DataFrame | None = None, ) -> tuple[pd.DataFrame, list[str]]: if not path.is_file(): raise FileNotFoundError(f"ACS PUMS archive not found: {path}") pieces: list[pd.DataFrame] = [] + source_roster = {} with ZipFile(path) as archive: members = sorted( name @@ -357,7 +433,7 @@ def _read_archive( usecols = [column for column in (*required, *optional) if column in columns] string_columns = { column: "string" - for column in ("SERIALNO", "ST", "STATE", "PUMA") + for column in ("SERIALNO", "ST", "STATE", "PUMA", "AGEP") if column in usecols } with archive.open(member_name) as member: @@ -369,6 +445,11 @@ def _read_archive( low_memory=False, ) for chunk in reader: + if "AGEP" in chunk: + # Validate original literals before numeric inference or + # selection can hide an unresolved source age. The native + # coverage contract accepts only one/two ASCII digits. + chunk["AGEP"] = _original_source_ages(chunk["AGEP"]) if valid_serials is not None: orphan = ~chunk["SERIALNO"].isin(valid_serials) if orphan.any(): @@ -382,15 +463,88 @@ def _read_archive( "ACS person SERIALNO value(s) missing from the " f"household archive: {examples}." ) + if validate_households is not None: + _validate_source_chunk(chunk, source_roster) if retained_serials is not None: chunk = chunk.loc[chunk["SERIALNO"].isin(retained_serials)] if not chunk.empty: pieces.append(chunk) + if validate_households is not None: + _validate_source_roster(validate_households, source_roster) if not pieces: return pd.DataFrame(columns=[*required, *optional]), members return pd.concat(pieces, ignore_index=True), members +def _original_source_ages(ages: pd.Series) -> np.ndarray: + """Preserve the native coverage owner's literal 0–99 age identity contract.""" + if not ages.str.fullmatch(r"[0-9]{1,2}", na=False).all(): + raise ValueError("ACS original AGEP must contain one or two ASCII digits.") + return ages.to_numpy(dtype=np.int64) + + +def _validate_amount_columns(amounts, factor): + """Keep the native mapper's finite observed-amount/factor gates before filtering.""" + adjustment = pd.to_numeric(factor, errors="coerce").to_numpy(dtype=float) + for values in amounts: + amount = pd.to_numeric(values, errors="coerce").to_numpy(dtype=float) + observed = ~np.isnan(amount) + if (observed & ~np.isfinite(amount)).any(): + raise ValueError("ACS source amount must be finite.") + if (observed & (~np.isfinite(adjustment) | (adjustment <= 0))).any(): + raise ValueError("ACS source adjustment must be finite and positive.") + + +def _validate_source_chunk(person, roster): + # Only global key sets and small household structural summaries survive a + # chunk. Unselected observation rows never enter the accumulated person table. + lines = _required_integer(person, "SPORDER") + marital = _required_integer(person, "MAR") + relations = _required_integer(person, "RELSHIPP") + sexes = _required_integer(person, "SEX") + if not set(marital) <= set(_ACS_TO_CPS_MARITAL_STATUS): + raise ValueError("ACS source MAR contains unsupported codes.") + if not set(relations) <= {20, *_ACS_SPOUSE_CODES, *_ACS_TO_CPS_RELATIONSHIP}: + raise ValueError("ACS source RELSHIPP contains unsupported codes.") + if not set(sexes) <= {1, 2}: + raise ValueError("ACS source SEX contains unsupported codes.") + _validate_amount_columns( + (person.WAGP, person.SEMP, person.INTP, person.RETP, person.SSP, person.SSIP), + person.ADJINC, + ) + weights = pd.to_numeric(person.PWGTP, errors="coerce").to_numpy(dtype=float) + for serial, line, relation, mar, weight in zip( + person.SERIALNO, lines, relations, marital, weights, strict=True + ): + state = roster.setdefault(serial, [set(), 0, 0, True, True, True]) + if line in state[0]: + raise ValueError("ACS duplicate person key in complete source.") + state[0].add(int(line)) + state[1] += int(relation == 20) + state[2] += int(relation in _ACS_SPOUSE_CODES) + state[3] &= relation not in {20, *_ACS_SPOUSE_CODES} or mar == 1 + state[4] &= relation in {37, 38} + state[5] &= bool(np.isfinite(weight) and weight > 0) + + +def _validate_source_roster(household, roster): + for serial, count, kind in household[["SERIALNO", "NP", "TYPEHUGQ"]].itertuples( + index=False, name=None + ): + state = dict.get(roster, serial) + if (len(state[0]) if state else 0) != int(count): + raise ValueError("ACS NP/person row-count mismatch in complete source.") + if not state: + continue + if int(kind) != 1 and not state[5]: + raise ValueError("ACS GQ PWGTP must be finite and positive.") + if not state[4]: + if state[1] != 1 or state[2] > 1: + raise ValueError("ACS source reference/spouse roster is invalid.") + if state[2] and not state[3]: + raise ValueError("ACS source spouse pair must have MAR=1.") + + def _occupied_households(household: pd.DataFrame) -> tuple[pd.DataFrame, int]: people = pd.to_numeric(household["NP"], errors="coerce") if people.isna().any() or (people < 0).any() or (people % 1 != 0).any(): @@ -616,6 +770,52 @@ def _household_weights(household: pd.DataFrame, person: pd.DataFrame) -> Weights return Weights(raw, WeightKind.DESIGN) +def _household_axis_composition( + household: pd.DataFrame, + household_weights: Weights, +) -> dict[str, int | float]: + """Split the loaded household-axis rows and DESIGN mass by TYPEHUGQ. + + The axis mixes occupied physical housing units with one-person group- + quarters person placeholders. Each part retains its own statistical unit; + their mixed sum is not a housing-unit count or a person count. These are + source design weights, with no common-survey or calibrated-population + claim. The rows described are the ones this build + loaded, after the vacancy drop and any ``max_households`` selection. + + Admissibility of the codes and their weights is decided upstream by + ``_occupied_households``; this only partitions what that check accepted, + and refuses if the accepted rows do not partition. + """ + + if "TYPEHUGQ" not in household.columns: + raise ValueError("ACS household-axis composition requires the TYPEHUGQ column.") + design = np.asarray(household_weights.values, dtype=np.float64) + if design.shape[0] != len(household): + raise ValueError( + "ACS household-axis composition needs one DESIGN weight per loaded " + f"household row; got {design.shape[0]} for {len(household)} row(s)." + ) + kind = pd.to_numeric(household["TYPEHUGQ"], errors="coerce").to_numpy( + dtype=np.float64, + na_value=np.nan, + ) + composition: dict[str, int | float] = {} + partitioned = 0 + for code, label in _HOUSEHOLD_AXIS_KINDS.items(): + selected = kind == code + rows = int(selected.sum()) + composition[f"{label}_rows"] = rows + composition[f"{label}_design_weight_total"] = float(design[selected].sum()) + partitioned += rows + if partitioned != len(household): + raise ValueError( + "ACS household-axis composition does not partition the loaded rows: " + f"{partitioned} of {len(household)} carry a described TYPEHUGQ code." + ) + return composition + + def _attach_household_source_columns( frame: Frame, source_household: pd.DataFrame, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py index a3fb00a6b..e6cf40c8c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py @@ -48,6 +48,10 @@ from microcosm.fit.qrf import detect_regime from microcosm.frame import EntitySchema, Frame, Weights +from .operator_column_contracts import ( + ACS_DERIVED_TRANSFER_INPUTS as ACS_DERIVED_TRANSFER_INPUTS, +) + QRF: Any | None = None __all__ = [ @@ -182,6 +186,7 @@ def _pregnancy_structural_policy_identity(*, enabled: bool) -> dict[str, object] ).hexdigest() return payload + _IMMIGRATION_STATUS_TARGETS = ( "ssn_card_type", "immigration_status_str", @@ -635,14 +640,6 @@ def required_acs_transfer_inputs() -> frozenset[str]: ) -#: Person columns the default transfer DERIVES deterministically after the -#: QRF fits (never fitted themselves). Coverage checks require them on the -#: recipient exactly like declared plan targets. -ACS_DERIVED_TRANSFER_INPUTS: tuple[str, ...] = ( - "schedule_d_capital_gain_distributions", -) - - def acs_derived_transfer_expectations( target_families: TargetFamilies, ) -> dict[str, str]: @@ -1138,8 +1135,8 @@ def _prepare_pregnancy_structural_plan( "through 44." ) - source_codes, group_count, key_column, representatives = ( - _pregnancy_source_groups(table) + source_codes, group_count, key_column, representatives = _pregnancy_source_groups( + table ) eligible_min = np.ones(group_count, dtype=np.int8) eligible_max = np.zeros(group_count, dtype=np.int8) @@ -1449,9 +1446,7 @@ def transfer_acs_inputs( person, pregnancy_plan, ) - pregnancy_entity, pregnancy_family, pregnancy_targets = ( - pregnancy_request - ) + pregnancy_entity, pregnancy_family, pregnancy_targets = pregnancy_request if pregnancy_targets != (_PREGNANCY_TARGET,): # pragma: no cover raise AssertionError( "Pregnancy structural target was not isolated before receipt." diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py new file mode 100644 index 000000000..7cad3394d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py @@ -0,0 +1,652 @@ +"""Issue selected 2024 ASEC observations with original household DESIGN anchors. + +The closed parent and person-coverage owners still authenticate all three source +cohorts. Only the selected 2024 observations become this new population. Exact +key selection is an engineering operation with no inclusion-probability claim. +No pooling coefficient, age adjustment, coverage classification or allocation is +performed here. Source evidence stays outside the selected Frame's metadata. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import sys +import weakref +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import NamedTuple + +import numpy as np +import pandas as pd + +from microcosm.build.cd_benchmark import origin +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.frame import bundle as frame_owner +from microcosm.frame import weights as weight_owner + +from . import asec_coverage_authentication as coverage_owner +from . import asec_current_money_source as parent_owner +from . import asec_household_coverage_fields as field_owner +from . import asec_original_household_weights as anchor_owner +from . import asec_person_income_source as restoration +from . import graph_context + +ARTIFACT_KIND = "microcosm.us.asec-2024-native-population.v1" +_MAX_HOUSEHOLDS = 100_000 +_MAX_PERSONS = 600_000 +_MAX_EVIDENCE_BYTES = 128 * 1024**2 +_MAX_CHECKPOINT_BYTES = 8 * 1024**3 +_ISSUED: dict[int, tuple[weakref.ReferenceType, bytes, object]] = {} + + +class AsecNativePopulationError(ValueError): + """Static refusal codes; no source paths, native keys or source values.""" + + +def _require(condition, code): + if not condition: + raise AsecNativePopulationError(code) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _encode(value): + result = bytearray() + for part in json.JSONEncoder( + ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ).iterencode(value): + part = part.encode("ascii") + _require(len(result) + len(part) <= _MAX_EVIDENCE_BYTES, "EVIDENCE_SIZE") + result.extend(part) + return bytes(result) + + +def _modules(): + return ( + sys.modules[__name__], + parent_owner, + restoration, + coverage_owner, + anchor_owner, + field_owner, + graph_context, + origin, + frame_owner, + weight_owner, + ) + + +def _implementation(): + runtime = {m.__name__: coverage_owner._runtime_code(m) for m in _modules()} + _require(runtime == _RUNTIME_AUTHORITY, "PRODUCER_CODE_CHANGED") + return { + "kind": ARTIFACT_KIND, + "runtime": runtime, + "code": {m.__name__: _sha(Path(m.__file__).read_bytes()) for m in _modules()}, + "parent": parent_owner._verification_identity(), + "restoration": restoration._implementation(), + "coverage": coverage_owner._implementation(), + "anchors": anchor_owner._implementation(), + "household_fields": field_owner._producer(), + "limits": [ + _MAX_HOUSEHOLDS, + _MAX_PERSONS, + _MAX_EVIDENCE_BYTES, + _MAX_CHECKPOINT_BYTES, + ], + "periods": [2024, 2025, 2024], + } + + +def _path(value): + _require(type(value) is str or isinstance(value, Path), "SOURCE_PATH") + return Path(value).absolute() + + +def _inputs(parent, household, restored, persons, member, selected, candidate): + # Snapshot every caller-owned lookup before fingerprinting or source I/O. + _require(isinstance(persons, Mapping) and len(persons) == 3, "PERSON_PATHS") + paths = {} + for year in persons: + _require( + type(year) is int + and year in (2022, 2023, 2024) + and year not in paths + and len(paths) < 3, + "PERSON_PATHS", + ) + paths[year] = _path(persons[year]) + _require(set(paths) == {2022, 2023, 2024} and len(persons) == 3, "PERSON_PATHS") + if selected is not None: + _require( + type(selected) is tuple and 0 < len(selected) <= _MAX_HOUSEHOLDS, + "SELECTION", + ) + for key in selected: + _require( + type(key) is tuple + and len(key) == 2 + and all(type(v) is int for v in key) + and key[0] == 2024 + and 1 <= key[1] <= 99999, + "SELECTION", + ) + _require(len(set(selected)) == len(selected), "SELECTION") + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _MAX_EVIDENCE_BYTES), + "CANDIDATE", + ) + return ( + _path(parent), + _path(household), + _path(restored), + paths, + _path(member), + selected, + candidate, + ) + + +def _file_identity(path, expected, maximum, size=None): + """Check original input bytes on every borrow, with bounded regular-file I/O.""" + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + before = os.fstat(descriptor) + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= maximum, + "SOURCE_FILE_BOUNDS", + ) + _require(size is None or before.st_size == size, "SOURCE_FILE_SIZE") + digest, count = hashlib.sha256(), 0 + while block := os.read(descriptor, min(1024**2, before.st_size - count + 1)): + count += len(block) + _require(count <= before.st_size, "SOURCE_FILE_CHANGED") + digest.update(block) + after = os.fstat(descriptor) + keys = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + _require( + count == before.st_size + and digest.hexdigest() == expected + and all(getattr(before, k) == getattr(after, k) for k in keys), + "SOURCE_FILE_CHANGED", + ) + finally: + os.close(descriptor) + + +def _frame_identity(frame): + _require( + type(frame) is Frame and frame.schema == US_SCHEMA and not frame.links, + "FRAME_SHAPE", + ) + _require( + set(frame._tables) == set(US_SCHEMA.entities) + and not frame._link_tables + and set(frame._weights) == set(frame.weighted_entities), + "FRAME_STORAGE", + ) + # The parent signature binds typed cells, ordered axes, metadata, mass log, + # schema and stored weights. Bind additional mutable pandas/ndarray state. + decorations = [] + for name in frame.entities: + table = frame.table(name) + _require(type(table) is pd.DataFrame, "FRAME_TABLE_TYPE") + decorations.append([name, table.attrs, table.flags.allows_duplicate_labels]) + decorations.append( + ["strata", frame.strata.attrs, frame.strata.flags.allows_duplicate_labels] + ) + weights = [] + for entity in frame.weighted_entities: + vector = frame.weights_for(entity) + _require( + type(vector) is Weights and type(vector.values) is np.ndarray, + "FRAME_WEIGHT_TYPE", + ) + weights.append( + [ + entity, + vector.kind.value, + str(vector.values.dtype), + list(vector.values.shape), + vector.values.flags.writeable, + _sha(vector.values.tobytes()), + ] + ) + return _sha( + _encode( + { + "base": parent_owner._frame_signature(frame), + "decorations": decorations, + "weights": weights, + } + ) + ) + + +def _roster(parent, coverage, anchors, fields, selected): + projection = coverage.table() + people = parent.frame.person + _require(len(projection) == len(people), "PARENT_ROSTER") + for column in coverage_owner._PARENT_COLUMNS: + _require( + projection[column].tolist() == people[column].tolist(), "PARENT_ROSTER" + ) + anchor_doc, field_doc = anchors.document, fields.document + _require( + anchor_doc["members"] == field_doc["members"] + and len(anchor_doc["members"]) == 1, + "HOUSEHOLD_MEMBER_BINDING", + ) + member = anchor_doc["members"][0] + source = next(s for s in coverage.receipt["sources"] if s["source_year"] == 2024) + _require( + member["income_year"] == 2024 + and member["survey_year"] == 2025 + and source["survey_year"] == 2025 + and member["archive_sha256"] == source["archive_sha256"], + "COHORT_ARCHIVE_BINDING", + ) + anchor_rows = {r["native_household_id"]: r for r in anchor_doc["records"]} + field_rows = {r["native_household_id"]: r for r in field_doc["records"]} + _require( + len(anchor_rows) == len(anchor_doc["records"]) + and len(field_rows) == len(field_doc["records"]) + and anchor_rows.keys() == field_rows.keys(), + "HOUSEHOLD_ROSTER", + ) + native_to_receiving, receiving_to_native, person_keys = {}, {}, set() + counts = {} + for row in projection.itertuples(index=False): + key = (int(row.source_year), int(row.source_household_id)) + receiving = int(row.person_household_id) + _require( + native_to_receiving.setdefault(key, receiving) == receiving + and receiving_to_native.setdefault(receiving, key) == key, + "HOUSEHOLD_PARTITION", + ) + person_key = (key[0], row.PERIDNUM, int(row.A_LINENO)) + _require(person_key not in person_keys, "PERSON_ROSTER") + person_keys.add(person_key) + counts[key] = counts.get(key, 0) + 1 + all_keys = {key for key in native_to_receiving if key[0] == 2024} + _require(bool(all_keys), "EMPTY_COHORT") + for native, row in field_rows.items(): + anchor = anchor_rows[native] + for column in ( + "income_year", + "survey_year", + "member_id", + "member_sha256", + "member_row_1based", + "origin_key", + "H_SEQ", + "H_HHTYPE", + ): + _require(row[column] == anchor[column], "HOUSEHOLD_PROJECTION_BINDING") + _require( + row["income_year"] == 2024 and row["survey_year"] == 2025, + "HOUSEHOLD_COHORT", + ) + key = (2024, native) + if row["H_HHTYPE"] == "1": + _require( + key in all_keys and row["reported_person_count"] == counts[key], + "HOUSEHOLD_MEMBER_COUNT", + ) + else: + _require(key not in all_keys, "HOUSEHOLD_INTERVIEW_STATUS") + _require(all(k[1] in field_rows for k in all_keys), "MISSING_HOUSEHOLD") + chosen = all_keys if selected is None else set(selected) + _require(chosen <= all_keys and len(chosen) <= _MAX_HOUSEHOLDS, "SELECTION_UNKNOWN") + mask = np.array( + [receiving_to_native[int(h)] in chosen for h in people.person_household_id], + dtype=bool, + ) + _require(0 < int(mask.sum()) <= _MAX_PERSONS, "SELECTED_PERSON_BOUNDS") + selected_ids = set(people.loc[mask, "person_household_id"].tolist()) + ordered_ids = [ + int(h) + for h in parent.frame.table("household").household_id + if h in selected_ids + ] + ordered_keys = tuple(receiving_to_native[h] for h in ordered_ids) + _require(set(ordered_keys) == chosen, "RECEIVING_HOUSEHOLDS") + exact = anchors.original_weight_fractions(ordered_keys) + rows = [] + for receiving, key, fraction in zip(ordered_ids, ordered_keys, exact, strict=True): + row = anchor_rows[key[1]] + expected_origin = origin.origin_key( + origin.AsecHouseholdOrigin( + origin.SourceMember( + member["canonical_member_id"], member["member_sha256"] + ), + 2024, + key[1], + ) + ) + _require(row["origin_key"] == expected_origin, "ORIGIN_BINDING") + rows.append( + { + "household_id": receiving, + "native_key": list(key), + "origin_key": expected_origin, + "HSUP_WGT": row["HSUP_WGT"], + "numerator": row["weight_integer_units"], + "denominator": 100, + "fraction": [fraction.numerator, fraction.denominator], + "source": row, + "household_fields": field_rows[key[1]], + } + ) + roster = projection.loc[mask].to_dict(orient="records") + return mask, np.array([float(f) for f in exact], dtype=np.float64), rows, roster + + +def _descendant(parent, mask, anchors): + """Select observations without using legacy weight slices as source anchors.""" + frame = parent.frame + _require(frame.schema == US_SCHEMA and not frame.links, "PARENT_SCHEMA") + person = frame.person.loc[mask].copy(deep=True) + tables = {"person": person} + for entity in US_SCHEMA.group_entities: + column = US_SCHEMA.membership_column(entity) + ids = set(person[column].tolist()) + # Referenced groups cannot silently retain only part of their members. + _require(not frame.person.loc[~mask, column].isin(ids).any(), "SPLIT_GROUP") + table = frame.table(entity) + tables[entity] = table.loc[ + table[US_SCHEMA.id_column(entity)].isin(ids) + ].reset_index(drop=True) + descendant = Frame( + tables, + US_SCHEMA, + {"household": Weights(anchors, WeightKind.DESIGN)}, + frame.strata.loc[mask].copy(deep=True), + metadata={}, + mass_log=(), + ) + parent_owner._detach_frame_axes(descendant) + return descendant + + +class _State(NamedTuple): + """Tuple storage keeps the external seal immutable even to object.__setattr__.""" + + frame: Frame + parent: object + coverage: object + anchors: object + fields: object + source_files: tuple + producer: bytes + parent_identity: str + frame_identity: str + context: bytes + attached_evidence: tuple + + +def _attached_evidence(parent, coverage, anchors, fields): + """Capture immutable issued bytes; the parent authority also has identity.""" + values = ( + parent.source._validate(), + coverage._header, + coverage._body, + anchor_owner._checked_capsule_payload(anchors), + anchor_owner._checked_capsule_payload(fields), + ) + _require(all(type(value) is bytes for value in values), "ATTACHED_EVIDENCE_CHANGED") + return (parent.source, *values) + + +def _validate_state(state): + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require(_frame_identity(state.frame) == state.frame_identity, "FRAME_CHANGED") + _require( + _frame_identity(state.parent.frame) == state.parent_identity, "PARENT_CHANGED" + ) + state.parent.validate() + coverage_owner.verify_asec_coverage_parent(state.coverage, state.parent) + anchor_owner.verify_asec_household_weights_source(state.anchors) + field_owner.verify_asec_household_coverage_fields(state.fields) + for path, expected, maximum, size in state.source_files: + _file_identity(path, expected, maximum, size) + # File checks/foreign owner calls may yield. Recheck the actual receiving + # frame and producer after them, not only before final source verification. + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require( + _frame_identity(state.parent.frame) == state.parent_identity, "PARENT_CHANGED" + ) + _require( + _frame_identity(state.frame) == state.frame_identity + and graph_context.encode_us_frame_context(state.frame) == state.context, + "FRAME_CHANGED", + ) + # These final seal checks perform no source/resource I/O. The immutable + # tuple belongs to the external population seal, not mutable capsule state. + current = _attached_evidence( + state.parent, state.coverage, state.anchors, state.fields + ) + _require( + current[0] is state.attached_evidence[0] + and current[1:] == state.attached_evidence[1:], + "ATTACHED_EVIDENCE_CHANGED", + ) + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsec2024NativePopulation: + """Process-issued population; every accessor verifies its full live binding.""" + + payload: bytes + + def _checked(self): + try: + entry = _ISSUED.get(id(self)) + _require( + type(self) is AuthenticatedAsec2024NativePopulation + and entry is not None + and entry[0]() is self + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + _validate_state(entry[2]) + _require( + _ISSUED.get(id(self)) is entry + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + except AsecNativePopulationError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + OverflowError, + ): + raise AsecNativePopulationError("NATIVE_BINDING_REFUSAL") from None + + def _state(self): + return self._checked()[2] + + def validate(self): + self._state() + + @property + def frame(self): + return self._state().frame + + @property + def context(self): + return self._state().context + + @property + def receipt(self): + return json.loads(self._checked()[1]) + + def to_bytes(self): + return self._checked()[1] + + +def load_authenticated_asec_2024_native_population( + parent_path, + household_attachment_path, + person_income_attachment_path, + *, + person_member_paths, + household_member_path, + selected_households=None, + candidate=None, +): + """Reconstruct from closed source paths; candidate bytes never grant authority. + + ``selected_households`` is a nonempty unique tuple of ``(2024, H_SEQ)`` + integer keys, or None for the complete 2024 source cohort. This is not a + probability sample. The authenticated parent/coverage still read all cohorts. + """ + try: + ( + parent_path, + household_path, + restored_path, + persons, + member, + selection, + candidate, + ) = _inputs( + parent_path, + household_attachment_path, + person_income_attachment_path, + person_member_paths, + household_member_path, + selected_households, + candidate, + ) + producer = _encode(_implementation()) + parent = restoration.load_authenticated_restored_current_money_source( + parent_path, household_path, restored_path, member_paths=persons + ) + parent_identity = _frame_identity(parent.frame) + coverage = coverage_owner.authenticate_asec_coverage( + parent, member_paths=persons + ) + anchors = anchor_owner.load_authenticated_asec_household_weights({2024: member}) + fields = field_owner.load_authenticated_asec_household_coverage_fields( + {2024: member} + ) + mask, weights, rows, roster = _roster( + parent, coverage, anchors, fields, selection + ) + frame = _descendant(parent, mask, weights) + context = graph_context.encode_us_frame_context(frame) + identity = json.loads(parent.source.identity) + source_files = [ + (parent_path, identity["parent_sha256"], _MAX_CHECKPOINT_BYTES, None), + ( + household_path, + identity["attachment_sha256"], + _MAX_CHECKPOINT_BYTES, + None, + ), + ( + restored_path, + identity["person_income_attachment_sha256"], + _MAX_CHECKPOINT_BYTES, + None, + ), + ] + for source in coverage.receipt["sources"]: + source_files.append( + ( + persons[source["source_year"]], + source["member_sha256"], + source["member_bytes"], + source["member_bytes"], + ) + ) + pin = anchors.document["members"][0] + source_files.append( + (member, pin["member_sha256"], pin["size_bytes"], pin["size_bytes"]) + ) + state = _State( + frame, + parent, + coverage, + anchors, + fields, + tuple(source_files), + producer, + parent_identity, + _frame_identity(frame), + context, + _attached_evidence(parent, coverage, anchors, fields), + ) + payload = _encode( + { + "kind": ARTIFACT_KIND, + "source_year": 2024, + "income_year": 2024, + "survey_year": 2025, + "analysis_year": 2024, + "selection": { + "kind": "complete_2024_cohort" + if selection is None + else "exact_engineering_keys", + "keys": [row["native_key"] for row in rows], + "inclusion_probabilities_known": False, + }, + "households": rows, + "persons": roster, + "raw_age_relation": "original_A_AGE_identity_no_adjustment", + "original_weight_conversion": "float64(Fraction(HSUP_WGT_integer,100))", + "weight_kind": "design", + "frame_sha256": state.frame_identity, + "context_sha256": _sha(context), + "producer_sha256": _sha(producer), + "parent_custody": { + "cohorts": [2022, 2023, 2024], + "population_cohorts": [2024], + "parent_identity": identity, + "frame_sha256": parent_identity, + }, + "person_coverage": coverage.receipt, + "household_weights_sha256": _sha(anchors.payload), + "household_fields_sha256": _sha(fields.payload), + "coverage_classification_performed": False, + "source_share_allocation_performed": False, + "release_eligible": False, + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + _validate_state(state) + result = AuthenticatedAsec2024NativePopulation(payload) + key = id(result) + + def discard(reference): + entry = _ISSUED.get(key) + if entry is not None and entry[0] is reference: + del _ISSUED[key] + + _ISSUED[key] = (weakref.ref(result, discard), payload, state) + result.validate() + return result + except AsecNativePopulationError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError, OverflowError): + raise AsecNativePopulationError("NATIVE_SOURCE_REFUSAL") from None + + +# Pins can be substituted explicitly by invented tests. Callable implementation +# changes cannot silently establish a new producer merely by rebuilding receipts. +_RUNTIME_AUTHORITY = {m.__name__: coverage_owner._runtime_code(m) for m in _modules()} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py index 0c6ebe0fa..6299a4762 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py @@ -12,6 +12,7 @@ import re from collections.abc import Mapping +from dataclasses import dataclass from pathlib import Path import numpy as np @@ -24,9 +25,15 @@ frame_identity, ) from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.build.us_runtime.education_assistance_source import ( + ASEC_EDUCATION_ASSISTANCE_ARCHIVES, +) from microcosm.build.us_runtime.operator_boundary import ( assert_operator_free_source_frame, ) +from microcosm.build.us_runtime.reported_coverage_source import ( + ASEC_REPORTED_COVERAGE_RAW_COLUMNS, +) from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights __all__ = [ @@ -34,9 +41,11 @@ "ASEC_RAW_STAGE_CHECKPOINT_FILENAME", "ASEC_RAW_STAGE_OPERATOR_STATUS", "ASEC_RAW_STAGE_SCHEMA_VERSION", + "ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION", "ASEC_RAW_STAGE_STAGE", "load_asec_pre_clone_checkpoint", "load_asec_raw_stage_checkpoint", + "load_asec_raw_stage_checkpoint_v4", ] _OUTER_STAGE_ARTIFACT_KIND = "populace_outer_stage_frame" @@ -61,6 +70,9 @@ # artifacts lack the gate column and must fail loudly rather than let # PAW_VAL-only conflation back in (microcosm#591). ASEC_RAW_STAGE_SCHEMA_VERSION = 3 +# V4 is a distinct measured-coverage contract. Never reinterpret a v3 file +# under this schema merely because its columns happen to look compatible. +ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION = 4 ASEC_RAW_STAGE_STAGE = "raw_source_mapping" _RAW_STAGE_BINDING_KEYS = frozenset( { @@ -100,6 +112,23 @@ ) +@dataclass(frozen=True) +class _RawStagePolicy: + version: int + coverage_columns: frozenset[str] = frozenset() + + @property + def mapping_columns(self) -> frozenset[str]: + return _RAW_SOURCE_MAPPING_COLUMNS | self.coverage_columns + + +_RAW_STAGE_V3 = _RawStagePolicy(ASEC_RAW_STAGE_SCHEMA_VERSION) +_RAW_STAGE_V4 = _RawStagePolicy( + ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION, + frozenset(ASEC_REPORTED_COVERAGE_RAW_COLUMNS), +) + + def load_asec_pre_clone_checkpoint( path: str | Path, ) -> tuple[Frame, dict[str, object]]: @@ -152,11 +181,33 @@ def load_asec_raw_stage_checkpoint( contract even if it happens to carry a structurally valid US ``Frame``. """ + return _load_asec_raw_stage_checkpoint(path, policy=_RAW_STAGE_V3) + + +def load_asec_raw_stage_checkpoint_v4( + path: str | Path, +) -> tuple[Frame, dict[str, object]]: + """Load only v4 measured-coverage sources with registered per-year pins. + + This validates a declared source artifact, not model accuracy or release + eligibility. The caller must independently authenticate the whole file's + digest; shape and registered source pins alone do not authenticate values. + """ + return _load_asec_raw_stage_checkpoint(path, policy=_RAW_STAGE_V4) + + +def _load_asec_raw_stage_checkpoint( + path: str | Path, + *, + policy: _RawStagePolicy, +) -> tuple[Frame, dict[str, object]]: + checkpoint_path = Path(path) loaded = load_frame_checkpoint(checkpoint_path) metadata = _validate_raw_stage_binding( loaded.metadata, path=checkpoint_path, + policy=policy, ) stored_identity = FrameIdentity.from_payload( metadata["identity"], @@ -176,7 +227,11 @@ def load_asec_raw_stage_checkpoint( loaded.frame, label=f"ASEC raw-stage checkpoint {checkpoint_path}", ) - _validate_raw_stage_source_columns(loaded.frame, path=checkpoint_path) + _validate_raw_stage_source_columns( + loaded.frame, path=checkpoint_path, policy=policy + ) + if policy.coverage_columns: + _validate_coverage_v4_binding(loaded.frame, metadata, path=checkpoint_path) source_construction_identity = FrameIdentity.from_payload( metadata["source_construction_identity"], label="ASEC raw-stage source-construction identity", @@ -250,6 +305,7 @@ def _validate_raw_stage_binding( metadata: dict[str, object], *, path: Path, + policy: _RawStagePolicy = _RAW_STAGE_V3, ) -> dict[str, object]: actual_keys = frozenset(metadata) if actual_keys != _RAW_STAGE_BINDING_KEYS: @@ -265,8 +321,10 @@ def _validate_raw_stage_binding( "ASEC artifact." ) schema_version = metadata["schema_version"] - if schema_version != ASEC_RAW_STAGE_SCHEMA_VERSION or isinstance( - schema_version, bool + if ( + schema_version != policy.version + or isinstance(schema_version, bool) + or (policy.coverage_columns and type(schema_version) is not int) ): raise ValueError( f"ASEC raw-stage checkpoint {path} has an unsupported raw-stage " @@ -292,7 +350,9 @@ def _validate_raw_stage_binding( "lowercase SHA-256 digest." ) _validate_source_receipt(metadata["source_receipt"], path=path) - _validate_raw_source_mappings(metadata["raw_source_mappings"], path=path) + _validate_raw_source_mappings( + metadata["raw_source_mappings"], path=path, policy=policy + ) return dict(metadata) @@ -338,17 +398,19 @@ def _validate_source_receipt(receipt: object, *, path: Path) -> None: years.add(year) -def _validate_raw_source_mappings(mappings: object, *, path: Path) -> None: +def _validate_raw_source_mappings( + mappings: object, *, path: Path, policy: _RawStagePolicy = _RAW_STAGE_V3 +) -> None: if not isinstance(mappings, Mapping): raise ValueError( f"ASEC raw-stage checkpoint {path} raw_source_mappings must be an object." ) - if frozenset(mappings) != _RAW_SOURCE_MAPPING_COLUMNS: + if frozenset(mappings) != policy.mapping_columns: raise ValueError( f"ASEC raw-stage checkpoint {path} raw_source_mappings must bind " - f"exactly {sorted(_RAW_SOURCE_MAPPING_COLUMNS)}." + f"exactly {sorted(policy.mapping_columns)}." ) - for column in sorted(_RAW_SOURCE_MAPPING_COLUMNS): + for column in sorted(policy.mapping_columns): mapping = mappings[column] if not isinstance(mapping, Mapping) or frozenset(mapping) != ( _RAW_SOURCE_MAPPING_KEYS @@ -400,9 +462,13 @@ def _validate_raw_source_mappings(mappings: object, *, path: Path) -> None: ) -def _validate_raw_stage_source_columns(frame: Frame, *, path: Path) -> None: +def _validate_raw_stage_source_columns( + frame: Frame, *, path: Path, policy: _RawStagePolicy = _RAW_STAGE_V3 +) -> None: person = frame.table("person") - missing = sorted(_RAW_STAGE_REQUIRED_PERSON_COLUMNS - set(person)) + missing = sorted( + (_RAW_STAGE_REQUIRED_PERSON_COLUMNS | policy.coverage_columns) - set(person) + ) if missing: raise ValueError( f"ASEC raw-stage checkpoint {path} is not input-complete; missing " @@ -458,6 +524,99 @@ def _validate_raw_stage_source_columns(frame: Frame, *, path: Path) -> None: "in {0, 1, 2, 3}." ) + for column in sorted(policy.coverage_columns): + values = pd.to_numeric(person[column], errors="coerce").to_numpy( + dtype=np.float64 + ) + boolean = ( + person[column] + .map(lambda value: isinstance(value, (bool, np.bool_))) + .to_numpy() + ) + if not (np.isfinite(values) & np.isin(values, (1.0, 2.0)) & ~boolean).all(): + raise ValueError( + f"ASEC v4 raw-stage checkpoint {path} {column} must be complete " + "integer recodes in {1, 2}, not booleans." + ) + + +def _validate_coverage_v4_binding( + frame: Frame, metadata: Mapping, *, path: Path +) -> None: + """Cross-check source vintages and actual registered coverage source pins.""" + person = frame.table("person") + normalized_years = pd.to_numeric(person["source_year"]).astype(np.int64) + frame_years = set(normalized_years) + source_years = {source["year"] for source in metadata["source_receipt"]["sources"]} + if frame_years != source_years: + raise ValueError( + f"ASEC v4 checkpoint {path} frame/source receipt year coverage differs." + ) + if frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError( + f"ASEC v4 checkpoint {path} requires household design weights." + ) + peridnum = person["PERIDNUM"].map( + lambda value: value.decode() if isinstance(value, bytes) else value + ) + if not peridnum.map( + lambda value: ( + isinstance(value, str) and re.fullmatch(r"[0-9]{22}", value) is not None + ) + ).all(): + raise ValueError( + f"ASEC v4 checkpoint {path} PERIDNUM must be exact 22-digit strings." + ) + keys = pd.DataFrame({"source_year": normalized_years, "PERIDNUM": peridnum}) + if keys.duplicated().any(): + raise ValueError( + f"ASEC v4 checkpoint {path} repeats a source-year/PERIDNUM identity." + ) + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + mapping = metadata["raw_source_mappings"][column] + pins = mapping["source_pins"] + years = [pin["income_year"] for pin in pins] + if len(years) != len(set(years)) or set(years) != source_years: + raise ValueError( + f"ASEC v4 {column} source pin year coverage differs or repeats." + ) + for pin in pins: + registered = ASEC_EDUCATION_ASSISTANCE_ARCHIVES.get(pin["income_year"]) + if registered is None or pin != { + "income_year": registered.income_year, + "locator": registered.zip_url, + "member": registered.member, + "member_sha256": registered.member_sha256, + "sha256": registered.zip_sha256, + }: + raise ValueError( + f"ASEC v4 {column} source pin differs from the registered archive." + ) + audit = mapping["audit"] + if set(audit) != {str(year) for year in source_years}: + raise ValueError(f"ASEC v4 {column} audit year coverage differs.") + for year in source_years: + row = audit[str(year)] + if not isinstance(row, Mapping) or set(row) != { + "rows", + "yes_rows", + "no_rows", + "weighted_yes_share", + }: + raise ValueError(f"ASEC v4 {column} audit is malformed.") + counts = [row[key] for key in ("rows", "yes_rows", "no_rows")] + share = row["weighted_yes_share"] + if ( + any(type(count) is not int or count < 0 for count in counts) + or counts[0] != ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year].rows + or counts[1] + counts[2] != counts[0] + or isinstance(share, bool) + or not isinstance(share, (int, float)) + or not np.isfinite(share) + or not 0 <= share <= 1 + ): + raise ValueError(f"ASEC v4 {column} audit counts/share are invalid.") + def _validate_asec_frame( frame: Frame, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py new file mode 100644 index 000000000..5641051e3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py @@ -0,0 +1,754 @@ +"""Closed, source-only PRPERTYP evidence for an exact current-money parent. + +No attachment, public pin override, population preparation or publication API. +The low-level reader remains non-authoritative inside this owned envelope. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import stat +import struct +import sys +import tempfile +import weakref +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path +from types import CodeType, FunctionType + +import pandas as pd + +from . import asec_current_money_source as money +from . import asec_person_coverage_source as literal +from . import source_csv_builtin as _csv_builtin +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +ARTIFACT_KIND = "microcosm.asec_coverage_authentication.v1" +MAGIC = b"MCASCOV\x01" +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +_HEADER_MAX = 262_144 +_BODY_MAX = 268_435_456 +_ROW_MAX = 1_048_576 +_TOKEN_MAX = 65_536 +_CSV_HEADER_MAX = 65_536 +_MAX_PERSONS = 600_000 +_CHUNK = 65_536 +_INT_COLUMNS = ( + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", +) +_STRING_COLUMNS = ("PERIDNUM", "PRPERTYP", "PRPERTYP_state") +COLUMNS = ( + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", + "PRPERTYP", + "PRPERTYP_state", +) +_PARENT_COLUMNS = ( + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", +) +_NUMBERS = struct.Struct("<6q") +_LENGTH = struct.Struct("= 0, "COVERAGE_BODY_BYTES") + if self.quoted: + if byte == 34: + self.quoted, self.after_quote = False, True + elif self.after_quote and byte == 34: + self.quoted, self.after_quote = True, False + elif byte in (10, 13): + if self.header: + self._end_header() + self.row = self.field = 0 + self.column = 0 + self.header = self.after_quote = False + self.start = True + self.after_cr = byte == 13 + elif byte == 44: + self.field = 0 + self.column += 1 + self.start, self.after_quote = True, False + else: + self.quoted = self.start and byte == 34 + self.start = self.after_quote = False + + +def _capture(path, destination, *, size, digest, budget): + """The student-controls exact-size, nonblocking private-capture pattern. + + Add raw record/token limits during that same copy, plus a private inode + identity retained until parsing finishes. No second coordinate parse. + """ + _require( + type(size) is int and 0 < size <= literal.MAX_MEMBER_BYTES, + "COVERAGE_MEMBER_BYTES", + ) + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "COVERAGE_SOURCE_FILE_KIND") + _require(before.st_size == size, "COVERAGE_SOURCE_SIZE") + count, hashed, guard = 0, hashlib.sha256(), _CsvBounds(budget) + with ( + os.fdopen(descriptor, "rb", buffering=0, closefd=False) as src, + destination.open("xb") as dst, + ): + while chunk := src.read(min(_CHUNK, size - count + 1)): + count += len(chunk) + _require(count <= size, "COVERAGE_SOURCE_SIZE") + guard.feed(chunk) + hashed.update(chunk) + dst.write(chunk) + dst.flush() + captured = _identity(os.fstat(dst.fileno())) + _require( + count == size and _identity(before) == _identity(os.fstat(descriptor)), + "COVERAGE_SOURCE_CHANGED", + ) + _require(hashed.hexdigest() == digest, "COVERAGE_SOURCE_DIGEST") + return captured + finally: + os.close(descriptor) + + +def _parent_binding(source): + _require( + type(source) is money.AuthenticatedCurrentMoneySource, + "COVERAGE_AUTHENTICATED_PARENT", + ) + source.validate() + return { + "authority_type": "AuthenticatedCurrentMoneySource", + "source_identity_sha256": _sha(source.source.identity), + "frame_sha256": money._frame_signature(source.frame), + "scope_sha256": _sha(source.scope.identity), + } + + +def _partitions(person): + published_to_parent, parent_to_published = {}, {} + for year, native, parent in person[ + ["source_year", "source_household_id", "person_household_id"] + ].itertuples(index=False, name=None): + published = (year, native) + _require(parent > 0, "COVERAGE_HOUSEHOLD_COORDINATE") + _require( + published_to_parent.setdefault(published, parent) == parent, + "COVERAGE_HOUSEHOLD_SPLIT", + ) + _require( + parent_to_published.setdefault(parent, published) == published, + "COVERAGE_HOUSEHOLD_MERGE", + ) + return { + "relation": "bidirectional_partition_bijection", + "published_key": ["source_year", "source_household_id"], + "parent_key": "person_household_id", + "households": len(parent_to_published), + "rows": len(person), + } + + +def _utf8_size(token): + _require(type(token) is str and len(token) <= _TOKEN_MAX, "COVERAGE_TOKEN_BYTES") + size = sum( + 1 if ord(c) < 128 else 2 if ord(c) < 2048 else 3 if ord(c) < 65536 else 4 + for c in token + ) + _require(size <= _TOKEN_MAX, "COVERAGE_TOKEN_BYTES") + return size + + +def _body(table): + _require(0 < len(table) <= _MAX_PERSONS, "COVERAGE_ROWS") + result = bytearray() + for row in table[ + [ + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", + "PRPERTYP", + "PRPERTYP_state", + ] + ].itertuples(index=False, name=None): + sizes = [_utf8_size(token) for token in row[6:]] + size = _NUMBERS.size + 3 * _LENGTH.size + sum(sizes) + _require(size <= _ROW_MAX, "COVERAGE_ROW_BYTES") + _require(len(result) + _LENGTH.size + size <= _BODY_MAX, "COVERAGE_BODY_BYTES") + result.extend(_LENGTH.pack(size)) + result.extend(_NUMBERS.pack(*row[:6])) + for token, length in zip(row[6:], sizes, strict=True): + result.extend(_LENGTH.pack(length)) + result.extend(token.encode("utf-8")) + return bytes(result) + + +def _decode_rows(body, rows): + _require(type(body) is bytes and len(body) <= _BODY_MAX, "COVERAGE_BODY_BYTES") + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "COVERAGE_ROWS") + view, offset = memoryview(body), 0 + for _ in range(rows): + _require(offset + 4 <= len(view), "COVERAGE_ROW_BYTES") + size = _LENGTH.unpack_from(view, offset)[0] + offset += 4 + end = offset + size + _require( + _NUMBERS.size + 12 <= size <= _ROW_MAX and end <= len(view), + "COVERAGE_ROW_BYTES", + ) + values = list(_NUMBERS.unpack_from(view, offset)) + offset += _NUMBERS.size + for _ in _STRING_COLUMNS: + _require(offset + 4 <= end, "COVERAGE_TOKEN_BYTES") + length = _LENGTH.unpack_from(view, offset)[0] + offset += 4 + _require( + length <= _TOKEN_MAX and offset + length <= end, "COVERAGE_TOKEN_BYTES" + ) + values.append(bytes(view[offset : offset + length]).decode("utf-8")) + offset += length + _require(offset == end, "COVERAGE_ROW_BYTES") + yield values + _require(offset == len(view), "COVERAGE_BODY_BYTES") + + +def _bounded_header(header): + # Count ASCII JSON bytes without first allocating escaped strings or a + # serialized header. This vocabulary is owned metadata, never row tokens. + remaining = _HEADER_MAX + + def charge(size): + nonlocal remaining + remaining -= size + _require(remaining >= 0, "COVERAGE_HEADER_BYTES") + + def visit(value, depth=0): + _require(depth <= 16, "COVERAGE_HEADER_DEPTH") + if isinstance(value, str): + charge(2) + for c in value: + charge( + 2 + if c in '\\"\b\f\n\r\t' + else 6 + if ord(c) < 32 or 127 <= ord(c) <= 65535 + else 12 + if ord(c) > 65535 + else 1 + ) + elif isinstance(value, (list, tuple, dict)): + charge(2 + max(0, len(value) - 1)) + if isinstance(value, dict): + for key, item in value.items(): + _require(type(key) is str, "COVERAGE_HEADER_TYPE") + visit(key, depth + 1) + charge(1) + visit(item, depth + 1) + else: + for item in value: + visit(item, depth + 1) + else: + _require( + value is None + or type(value) is bool + or (type(value) is int and -(2**63) <= value < 2**63), + "COVERAGE_HEADER_TYPE", + ) + charge(len(_json(value))) + + visit(header) + return _json(header) + + +@dataclass(frozen=True, eq=False, slots=True, weakref_slot=True) +class AuthenticatedAsecCoverage: + """Issued immutable bytes, with fresh receipt and table views on demand.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "COVERAGE_CONSTRUCTOR_UNAVAILABLE") + _ISSUED[self] = (_sha(self._header), _sha(self._body)) + + def validate(self): + try: + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "COVERAGE_HEADER_BYTES", + ) + _require( + type(self._body) is bytes and len(self._body) <= _BODY_MAX, + "COVERAGE_BODY_BYTES", + ) + _require( + _ISSUED.get(self) == (_sha(self._header), _sha(self._body)), + "COVERAGE_ISSUED_CONTENT_CHANGED", + ) + header = json.loads(self._header) + _require( + _json(header["implementation"]) == _json(_implementation()), + "COVERAGE_IMPLEMENTATION_CHANGED", + ) + _require(0 < header["rows"] <= _MAX_PERSONS, "COVERAGE_ROWS") + except AsecCoverageAuthenticationError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError): + raise AsecCoverageAuthenticationError("COVERAGE_CONTENT_REFUSAL") from None + + @property + def receipt(self): + self.validate() + return json.loads(self._header) + + @property + def content_sha256(self): + self.validate() + return _sha(self._header + self._body) + + def to_bytes(self): + self.validate() + payload = MAGIC + _LENGTH.pack(len(self._header)) + self._header + self._body + return payload + hashlib.sha256(payload).digest() + + def table(self): + self.validate() + table = pd.DataFrame( + _decode_rows(self._body, self.receipt["rows"]), columns=COLUMNS + ) + for name in _STRING_COLUMNS: + table[name] = pd.array(table[name], dtype="string") + return table + + +def verify_asec_coverage_parent(coverage, source): + """Verify the closed artifact against the exact named live receiving parent.""" + try: + _require( + type(coverage) is AuthenticatedAsecCoverage, + "COVERAGE_AUTHENTICATED_ARTIFACT", + ) + coverage.validate() + _require( + coverage.receipt["parent"] == _parent_binding(source), + "COVERAGE_PARENT_CHANGED", + ) + except AsecCoverageAuthenticationError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError): + raise AsecCoverageAuthenticationError("COVERAGE_PARENT_REFUSAL") from None + + +def _reconstruct(source, member_paths: Mapping[int, str | Path]): + before = _implementation() + parent = _parent_binding(source) + _require( + isinstance(member_paths, Mapping) + and set(member_paths) == {2022, 2023, 2024} + and all(type(year) is int for year in member_paths) + and all(isinstance(p, (str, Path)) for p in member_paths.values()), + "COVERAGE_MEMBER_PATHS", + ) + _require( + tuple(p[0] for p in _MEMBER_PINS) == (2022, 2023, 2024) + and len({p[1] for p in _MEMBER_PINS}) == 3, + "COVERAGE_PIN_COHORTS", + ) + person = source.frame.person + _require(0 < len(person) <= _MAX_PERSONS, "COVERAGE_ROWS") + _require( + set(_INT_COLUMNS + ("PERIDNUM",)) <= set(person), "COVERAGE_PARENT_COLUMNS" + ) + _require( + all(str(person[name].dtype) == "int64" for name in _INT_COLUMNS), + "COVERAGE_PARENT_DTYPE", + ) + # Build only from the validated parent's owned scope/native columns; never + # from model age or a caller's DataFrame/hash/receipt. + person = person[ + [ + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", + ] + ].copy(deep=True) + _require( + tuple(person.person_id) == source.scope.person_ids + and tuple(person.person_household_id) == source.scope.person_household_ids + and tuple(person.source_year) == source.scope.person_years, + "COVERAGE_PARENT_COORDINATES", + ) + _require(_parent_binding(source) == parent, "COVERAGE_PARENT_CHANGED") + roster = person[list(literal.ROSTER_COLUMNS)].copy().reset_index(drop=True) + _require( + tuple(roster.PERIDNUM) == source.scope.person_native_keys, + "COVERAGE_PARENT_NATIVE_KEYS", + ) + membership = _partitions(person) + with tempfile.TemporaryDirectory(prefix="microcosm-asec-coverage-") as directory: + paths, identities, sources = {}, {}, [] + budget = [_BODY_MAX] + pins = {pin[0]: pin for pin in _MEMBER_PINS} + for year in (2022, 2023, 2024): + _, member, archive, digest, rows, size = pins[year] + _require( + type(rows) is int + and 0 < rows <= _MAX_PERSONS + and int((roster.source_year == year).sum()) == rows, + "COVERAGE_COHORT_ROWS", + ) + path = Path(directory) / member + identities[year] = _capture( + member_paths[year], path, size=size, digest=digest, budget=budget + ) + paths[year] = path + sources.append( + { + "source_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive, + "member_sha256": digest, + "member_bytes": size, + "rows": rows, + } + ) + projection, receipt = literal.read_asec_person_coverage_source( + paths, person_roster=roster + ) + for year, path in paths.items(): + _require( + _identity(path.stat(follow_symlinks=False)) == identities[year], + "COVERAGE_PRIVATE_SOURCE_CHANGED", + ) + for name in _INT_COLUMNS[:2]: + projection[name] = person[name].to_numpy(copy=True) + body = _body(projection) + _require(_parent_binding(source) == parent, "COVERAGE_PARENT_CHANGED") + _require( + _json(before) == _json(_implementation()), "COVERAGE_IMPLEMENTATION_CHANGED" + ) + header = { + "artifact_kind": ARTIFACT_KIND, + "encoding": "length_prefixed_utf8_fixed_int64_v1", + "columns": COLUMNS, + "rows": len(projection), + "body_sha256": _sha(body), + "parent": parent, + "sources": sources, + "household_partitions": membership, + "source_authenticated": True, + "named_parent_binding_authenticated": True, + "archive_bytes_read": False, + "source_scope": "all_three_complete_original_person_cohorts", + "age_relation": { + "relation": "original_csv_A_AGE_equals_parent_A_AGE", + "comparison": "strict_unsigned_decimal_int64_identity", + "evidence": "authenticated_private_csv_read_in_this_reconstruction", + "compared_rows": len(projection), + "conflicts": 0, + "model_age_used": False, + }, + "coverage_status": "literal_source_fields_only", + "period_harmonized": False, + "cross_survey_coverage_equivalence_established": False, + "domain_authority": False, + "original_design_weight_semantics_established": False, + "release_eligible": False, + "literal_reader_receipt": receipt, + "implementation": before, + } + return AuthenticatedAsecCoverage(_bounded_header(header), body, _token=_TOKEN) + + +def _compare_candidate(path, payload): + # No candidate header, token, checksum or self-declared source is decoded. + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "COVERAGE_CANDIDATE_FILE_KIND") + _require(before.st_size == len(payload), "COVERAGE_CANDIDATE_BYTES") + offset = 0 + with os.fdopen(descriptor, "rb", buffering=0, closefd=False) as handle: + while chunk := handle.read(min(_CHUNK, len(payload) - offset + 1)): + _require( + chunk == memoryview(payload)[offset : offset + len(chunk)], + "COVERAGE_CANONICAL_BYTES", + ) + offset += len(chunk) + _require( + offset == len(payload) + and _identity(before) == _identity(os.fstat(descriptor)), + "COVERAGE_CANDIDATE_CHANGED", + ) + finally: + os.close(descriptor) + + +def authenticate_asec_coverage(source, *, member_paths, candidate_path=None): + """Independently reconstruct before comparing optional candidate bytes.""" + try: + expected = _reconstruct(source, member_paths) + if candidate_path is not None: + _compare_candidate(candidate_path, expected.to_bytes()) + verify_asec_coverage_parent(expected, source) + return expected + except AsecCoverageAuthenticationError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + OverflowError, + csv.Error, + ): + raise AsecCoverageAuthenticationError("COVERAGE_SOURCE_REFUSAL") from None + + +# The imported literal parser is code authority, not a caller-selectable +# decoder. Refuse monkeypatched parsing/status/field code even before issuance. +_LITERAL_AUTHORITY = _runtime_code(literal) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py new file mode 100644 index 000000000..e1b4c99bc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py @@ -0,0 +1,1099 @@ +"""Pure ASEC current-money decoding and price restatement. + +No resource discovery or source authentication happens here. Only the separate +resource loader reads installed code. Authenticated authority can only come from +the separate verified checkpoint loader; synthetic identities remain distinct. +""" + +import json +import struct +import weakref +from dataclasses import InitVar, dataclass +from dataclasses import fields as dataclass_fields +from enum import IntEnum +from hashlib import sha256 +from types import MappingProxyType + +import numpy as np +import pandas as pd + +RECIPE = "us_asec_current_money_ccpiu_2024_v1" +ZERO_POLICY = "frozen_fillna_zero_origin_unresolved" +RESTORED_SOURCE_KIND = "asec_v4_with_household_and_person_income_observations_v1" +ENCODED_ZERO_POLICY = "authenticated_census_csv_encoded_zero_v1" +RESTORED_FIELD_COLUMNS = MappingProxyType({"PTOTVAL": "asec_PTOTVAL"}) +RESTORED_FIELD_ZERO_POLICY = MappingProxyType({"PTOTVAL": ENCODED_ZERO_POLICY}) + + +class ZeroOrigin(IntEnum): + """Source-encoding provenance; no code establishes a respondent answer.""" + + NOT_ZERO = 0 + FROZEN_FILLNA_UNRESOLVED = 1 + AUTHENTICATED_CENSUS_ENCODED = 2 + + +FIELDS = tuple( + "ANN_VAL CAP_VAL CHSP_VAL CSP_VAL DIS_VAL1 DIS_VAL2 DIV_VAL DST_VAL1 DST_VAL1_YNG DST_VAL2 DST_VAL2_YNG ED_VAL FRSE_VAL HTOTVAL INT_VAL OI_VAL PHIP_VAL PMED_VAL PNSN_VAL POTC_VAL PTOTVAL RETCB_VAL RNT_VAL SEMP_VAL SPM_CAPHOUSESUB SPM_CHILDCAREXPNS SPM_ENGVAL SSI_VAL SS_VAL UC_VAL VET_VAL WC_VAL WSAL_VAL".split() +) +RESOURCE_PINS = ( + "09cefd4d08968cf1dc4dcdc787deef88c77b84549dfb0baa6bf9cbd9e3ce7352", + "88cec207a3c73e5de93d677acf16e8db9ddae9761c644c06da92cacf0feb3f63", + "c56058eaf08b68b2add4a5d2b366559ea6cc5238b039031be7094ef1f6457ff7", +) +MICROUNIT_VERSION = "0.1.0" +MICROUNIT_SHA256 = "79007065113f9600601433dc2633f4392c675e28b87a7383ab8a1f7c6279d47e" +RESOURCE_MAX_BYTES = 512 * 1024 +HEADER_MAX_BYTES = 64 * 1024 +MAX_PERSONS = 1_000_000 +MAX_HOUSEHOLDS = 400_000 +_STATE_TOKEN = object() +_SOURCE_TOKEN = object() +# Keep issuance outside publicly writable evidence objects. Keys use identity, +# not dataclass value equality: independently reconstructed evidence stays equal +# without lending its authority to an unissued copy. Weak references retire +# entries with their objects, and retained bytes are bounded by HEADER_MAX_BYTES. +_SOURCE_ISSUANCE: dict[int, tuple[weakref.ReferenceType, bytes]] = {} + + +class MoneyRefusalError(ValueError): + """Sanitized field/reason refusal; never includes row identifiers or values.""" + + def __init__(self, reason: str, field: str = "contract"): + self.reason = reason + self.field = field if field in FIELDS else "contract" + super().__init__(f"{self.field}: {reason}") + + +def _require(condition: bool, reason: str, field: str = "contract") -> None: + if not condition: + raise MoneyRefusalError(reason, field) + + +def _sha(value: bytes) -> str: + return sha256(value).hexdigest() + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +def _parse(value: bytes, limit: int = RESOURCE_MAX_BYTES) -> dict: + _require(type(value) is bytes and 0 < len(value) <= limit, "JSON_SIZE_OR_TYPE") + + def pairs(items): + result = {} + for key, item in items: + _require(key not in result, "DUPLICATE_JSON_KEY") + result[key] = item + return result + + def constant(_): + raise MoneyRefusalError("NONFINITE_JSON") + + try: + result = json.loads(value, object_pairs_hook=pairs, parse_constant=constant) + except MoneyRefusalError: + raise + except (ValueError, RecursionError) as exc: + raise MoneyRefusalError("MALFORMED_JSON") from exc + _require(type(result) is dict, "JSON_OBJECT_REQUIRED") + return result + + +def _digest(value: str) -> bool: + return ( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +@dataclass(frozen=True) +class AuthenticatedAsecSource: + """Immutable evidence issued only by the actual checkpoint verification loader.""" + + evidence: bytes = b"" + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _SOURCE_TOKEN, "AUTHENTICATED_SOURCE_UNAVAILABLE") + self._validate_content() + key = id(self) + + def discard(reference): + issued = _SOURCE_ISSUANCE.get(key) + if issued is not None and issued[0] is reference: + del _SOURCE_ISSUANCE[key] + + _SOURCE_ISSUANCE[key] = (weakref.ref(self, discard), self.evidence) + + def _validate(self): + issued = _SOURCE_ISSUANCE.get(id(self)) + evidence = self.evidence + _require( + type(self) is AuthenticatedAsecSource + and issued is not None + and issued[0]() is self + and type(evidence) is bytes + and evidence == issued[1], + "AUTHENTICATED_SOURCE_ISSUANCE", + ) + return evidence + + def _validate_content(self): + data = _parse(self.evidence, HEADER_MAX_BYTES) + restored = data.get("source_kind") == RESTORED_SOURCE_KIND + extra = ( + { + "person_income_attachment_sha256", + "field_source_columns", + "field_zero_origin_policy", + } + if restored + else set() + ) + _require( + set(data) + == extra + | { + "schema_version", + "source_kind", + "source_authentication", + "parent_sha256", + "attachment_sha256", + "cohorts", + "sidecars", + "field_roster", + "zero_origin_policy", + "scope_sha256", + "input_sha256", + "frame_sha256", + "verification_sha256", + }, + "AUTHENTICATED_SOURCE_SCHEMA", + ) + _require( + data["schema_version"] == (2 if restored else 1) + and data["source_kind"] + == ( + RESTORED_SOURCE_KIND + if restored + else "asec_v4_with_household_observations_v1" + ) + and data["source_authentication"] == "checkpoint_bytes_verified" + and data["field_roster"] == list(FIELDS) + and data["zero_origin_policy"] == ZERO_POLICY, + "AUTHENTICATED_SOURCE_SCHEMA", + ) + _require( + all( + _digest(data[name]) + for name in ( + "parent_sha256", + "attachment_sha256", + "scope_sha256", + "input_sha256", + "frame_sha256", + "verification_sha256", + ) + ), + "SOURCE_PIN", + ) + if restored: + _require( + _digest(data["person_income_attachment_sha256"]) + and data["field_source_columns"] == RESTORED_FIELD_COLUMNS + and data["field_zero_origin_policy"] == RESTORED_FIELD_ZERO_POLICY, + "RESTORED_SOURCE_SCHEMA", + ) + _require(self.evidence == _json(data), "AUTHENTICATED_SOURCE_SCHEMA") + + @property + def identity(self) -> bytes: + # Return the same immutable bytes checked, not a second mutable lookup. + return self._validate() + + +def _restored_source(source) -> bool: + return ( + type(source) is AuthenticatedAsecSource + and _parse(source.identity)["source_kind"] == RESTORED_SOURCE_KIND + ) + + +def _origin_code(spec, field: str) -> ZeroOrigin: + return ( + ZeroOrigin.AUTHENTICATED_CENSUS_ENCODED + if field == "PTOTVAL" and _restored_source(spec.source) + else ZeroOrigin.FROZEN_FILLNA_UNRESOLVED + ) + + +def _header_provenance(spec) -> dict: + if _restored_source(spec.source): + return { + "schema_version": 2, + "field_zero_origin_policy": dict(RESTORED_FIELD_ZERO_POLICY), + } + return {"schema_version": 1} + + +def _authentication(source) -> str: + return ( + "checkpoint_bytes_verified" + if type(source) is AuthenticatedAsecSource + else "synthetic_unverified" + ) + + +@dataclass(frozen=True) +class SyntheticAsecSource: + """Explicit invented pins, never promoted to authenticated source authority.""" + + parent_sha256: str + attachment_sha256: str + cohort_sha256: tuple[tuple[int, str], ...] + sidecar_sha256: tuple[tuple[str, str], ...] + selection_sha256: str + source_kind: str = "asec_v4_with_household_observations_v1" + field_roster: tuple[str, ...] = FIELDS + zero_origin_policy: str = ZERO_POLICY + + def __post_init__(self): + _require( + all( + _digest(v) + for v in ( + self.parent_sha256, + self.attachment_sha256, + self.selection_sha256, + ) + ), + "SOURCE_PIN", + ) + _require( + type(self.cohort_sha256) is tuple and len(self.cohort_sha256) == 3, + "COHORT_PINS", + ) + _require( + all( + type(p) is tuple and len(p) == 2 and type(p[0]) is int and _digest(p[1]) + for p in self.cohort_sha256 + ), + "COHORT_PINS", + ) + _require( + tuple(p[0] for p in self.cohort_sha256) == (2022, 2023, 2024), + "SOURCE_YEAR_MAPPING", + ) + _require( + type(self.sidecar_sha256) is tuple and len(self.sidecar_sha256) == 2, + "SIDECAR_PINS", + ) + _require( + all( + type(p) is tuple and len(p) == 2 and _digest(p[1]) + for p in self.sidecar_sha256 + ), + "SIDECAR_PINS", + ) + _require( + tuple(p[0] for p in self.sidecar_sha256) + == ("ED_VAL.archive", "ED_VAL.member"), + "SIDECAR_PINS", + ) + _require( + self.source_kind == "asec_v4_with_household_observations_v1", + "UNSUPPORTED_PARENT_KIND", + ) + _require( + type(self.field_roster) is tuple and self.field_roster == FIELDS, + "FIELD_ROSTER", + ) + _require(self.zero_origin_policy == ZERO_POLICY, "ZERO_ORIGIN_POLICY") + + @property + def identity(self) -> bytes: + return _json( + { + **{f.name: getattr(self, f.name) for f in dataclass_fields(self)}, + "source_authentication": "synthetic_unverified", + } + ) + + +@dataclass(frozen=True) +class CurrentMoneyResources: + """Explicit immutable resource bytes and inspected-code/runtime claims. + + Pure compilation checks these claims, not installed files. Use the separate + loader for installed verification. Source authentication belongs to the + separate closed checkpoint loader. + """ + + domains: bytes + price: bytes + consumers: bytes + execution_identity: bytes + inspected_version: str + inspected_module_sha256: str + + def __post_init__(self): + _require( + all( + type(x) is bytes + for x in ( + self.domains, + self.price, + self.consumers, + self.execution_identity, + ) + ), + "RESOURCE_BYTES_REQUIRED", + ) + + +@dataclass(frozen=True) +class MoneyDomain: + name: str + entity: str + column: str + grain: str + minimum: int + maximum: int + zero_semantics: str + + +@dataclass(frozen=True) +class CurrentMoneySpec: + """Closed v1 resources and issued source evidence; no mutable user mappings.""" + + resources: CurrentMoneyResources + source: SyntheticAsecSource | AuthenticatedAsecSource + fields: tuple[MoneyDomain, ...] + factors: tuple[tuple[int, str, str], ...] + identity: bytes + + @property + def sha256(self) -> str: + return _sha(self.identity) + + +def compile_asec_current_money_spec( + resources: CurrentMoneyResources, + source_descriptor: SyntheticAsecSource | AuthenticatedAsecSource, +) -> CurrentMoneySpec: + """Compile pinned declarations; this does not change live nominal consumers.""" + _require( + type(resources) is CurrentMoneyResources + and type(source_descriptor) in (SyntheticAsecSource, AuthenticatedAsecSource), + "SYNTHETIC_SOURCE_REQUIRED", + ) + if type(source_descriptor) is AuthenticatedAsecSource: + source_descriptor._validate() + else: + source_descriptor.__post_init__() + resources.__post_init__() + documents = tuple( + _parse(v) for v in (resources.domains, resources.price, resources.consumers) + ) + # Immutable v1 pins close every nested key, domain, writer, year and address. + # A changed convention requires a reviewed resource/recipe revision. + _require( + tuple( + _sha(v) for v in (resources.domains, resources.price, resources.consumers) + ) + == RESOURCE_PINS, + "RESOURCE_FINGERPRINT", + ) + _require( + resources.inspected_version == MICROUNIT_VERSION + and resources.inspected_module_sha256 == MICROUNIT_SHA256, + "MICROUNIT_CONTRACT_MISMATCH", + ) + execution = _parse(resources.execution_identity, HEADER_MAX_BYTES) + _require( + set(execution) == {"modules", "dependencies", "platform"}, "EXECUTION_IDENTITY" + ) + _require( + type(execution["modules"]) is dict + and set(execution["modules"]) + == { + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_resources.py", + }, + "EXECUTION_MODULES", + ) + _require( + all(_digest(v) for v in execution["modules"].values()), "EXECUTION_MODULES" + ) + _require( + type(execution["dependencies"]) is dict + and set(execution["dependencies"]) == {"python", "numpy", "pandas"}, + "EXECUTION_DEPENDENCIES", + ) + _require( + type(execution["platform"]) is dict + and set(execution["platform"]) + == {"system", "machine", "byteorder", "python_implementation"}, + "EXECUTION_PLATFORM", + ) + _require( + all( + type(v) is str and 0 < len(v) <= 128 + for group in (execution["dependencies"], execution["platform"]) + for v in group.values() + ), + "EXECUTION_IDENTITY", + ) + domains, price, _ = documents + fields = tuple( + MoneyDomain( + **{ + k: f[k] + for k in ( + "name", + "entity", + "column", + "grain", + "minimum", + "maximum", + "zero_semantics", + ) + } + ) + for f in domains["fields"] + ) + factors = tuple( + ( + y, + f"174.4/{price['cells'][str(y)]}", + struct.pack(" None: + _require(type(spec) is CurrentMoneySpec, "SPEC_TYPE") + _require( + spec == compile_asec_current_money_spec(spec.resources, spec.source), + "SPEC_BINDING", + ) + + +@dataclass(frozen=True) +class AsecMoneyScope: + """Ordered source coordinates supplied explicitly with projected tables. + + Pandas labels are representation only. IDs/native keys here belong to each + positional row, and must be projected in the same permutation as the table. + No original source authentication is inferred from this synthetic contract. + """ + + person_ids: tuple[int, ...] + household_ids: tuple[int, ...] + person_household_ids: tuple[int, ...] + person_spm_ids: tuple[int, ...] + person_years: tuple[int, ...] + household_years: tuple[int, ...] + person_native_keys: tuple[str, ...] + household_native_keys: tuple[str, ...] + + def __post_init__(self): + p, h = len(self.person_ids), len(self.household_ids) + _require(0 < p <= MAX_PERSONS and 0 < h <= MAX_HOUSEHOLDS, "SOURCE_SIZE") + for name in ( + "person_ids", + "household_ids", + "person_household_ids", + "person_spm_ids", + "person_years", + "household_years", + ): + vector = getattr(self, name) + _require( + type(vector) is tuple + and all(type(x) is int and -(2**63) <= x < 2**63 for x in vector), + "SCOPE_INTEGER_VECTOR", + ) + _require( + len(vector) == (h if name.startswith("household") else p), + "SCOPE_LENGTH", + ) + _require(len(set(self.person_ids)) == p, "DUPLICATE_PERSON_ID") + _require(len(set(self.household_ids)) == h, "DUPLICATE_HOUSEHOLD_ID") + _require( + set(self.person_household_ids) == set(self.household_ids), + "HOUSEHOLD_MEMBERSHIP", + ) + _require( + set(self.person_years) <= {2022, 2023, 2024} + and set(self.household_years) <= {2022, 2023, 2024}, + "SOURCE_YEAR_MAPPING", + ) + expected = dict(zip(self.household_ids, self.household_years, strict=True)) + _require( + all( + expected[g] == y + for g, y in zip( + self.person_household_ids, self.person_years, strict=True + ) + ), + "MIXED_HOUSEHOLD_YEAR", + ) + spm_years = {} + for spm, year in zip(self.person_spm_ids, self.person_years, strict=True): + _require(spm not in spm_years or spm_years[spm] == year, "MIXED_SPM_YEAR") + spm_years[spm] = year + for name, years, count in ( + ("person_native_keys", self.person_years, p), + ("household_native_keys", self.household_years, h), + ): + keys = getattr(self, name) + _require( + type(keys) is tuple + and len(keys) == count + and all(type(k) is str and 0 < len(k) <= 256 for k in keys), + "NATIVE_KEYS", + ) + _require( + len(set(zip(years, keys, strict=True))) == count, "DUPLICATE_NATIVE_KEY" + ) + + @property + def identity(self) -> bytes: + # Digests bound the header, while the active scope retains real coordinates. + return _json( + {f.name: _sha(_json(getattr(self, f.name))) for f in dataclass_fields(self)} + ) + + +@dataclass(frozen=True) +class AsecMoneyViews: + """Explicit raw projections plus their ordered coordinate association.""" + + person: pd.DataFrame + household: pd.DataFrame + scope: AsecMoneyScope + + +class CodebookStatus(IntEnum): + MISSING_NULL = 0 + AMOUNT_NONZERO = 1 + ZERO_NONE_OR_NIU = 2 + ZERO_DOLLARS_AS_CODED = 3 + DECLARED_NIU = 4 + + +_ZERO_STATUS = { + "none_or_niu_not_distinguishable_from_amount_alone": CodebookStatus.ZERO_NONE_OR_NIU, + "valid_zero_dollars": CodebookStatus.ZERO_DOLLARS_AS_CODED, + "valid_zero_dollars_or_none_as_described": CodebookStatus.ZERO_DOLLARS_AS_CODED, + "niu": CodebookStatus.DECLARED_NIU, +} +_DTYPES = frozenset( + ( + "int8", + "uint8", + "int16", + "uint16", + "int32", + "uint32", + "int64", + "uint64", + "float64", + "Int8", + "UInt8", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "Float64", + ) +) + + +@dataclass(frozen=True) +class MoneyField: + """Immutable canonical little-endian amounts and three uint8 evidence axes.""" + + name: str + amount_bytes: bytes + status_bytes: bytes + validity_bytes: bytes + zero_origin_bytes: bytes + + @property + def amounts(self) -> np.ndarray: + return np.frombuffer(self.amount_bytes, dtype=" np.ndarray: + return np.frombuffer(self.status_bytes, dtype="u1") + + @property + def validity(self) -> np.ndarray: + return np.frombuffer(self.validity_bytes, dtype="u1") + + @property + def zero_origin(self) -> np.ndarray: + return np.frombuffer(self.zero_origin_bytes, dtype="u1") + + +def _validate_field( + field: MoneyField, + domain: MoneyDomain, + count: int, + *, + nominal: bool, + zero_origin_code: ZeroOrigin, +) -> None: + _require(type(field) is MoneyField and field.name == domain.name, "FIELD_IDENTITY") + buffers = ( + field.amount_bytes, + field.status_bytes, + field.validity_bytes, + field.zero_origin_bytes, + ) + _require( + all(type(v) is bytes for v in buffers) + and tuple(map(len, buffers)) == (8 * count, count, count, count), + "FIELD_BUFFER_SHAPE", + domain.name, + ) + a, s, v, z = field.amounts, field.statuses, field.validity, field.zero_origin + _require( + np.isfinite(a).all() + and np.isin(s, [0, 1, 2, 3, 4]).all() + and np.isin(v, [0, 1]).all() + and np.isin(z, [0, 1, 2]).all(), + "FIELD_ENCODING", + domain.name, + ) + _require( + np.array_equal(v == 0, s == CodebookStatus.MISSING_NULL), + "INVALID_SLOT", + domain.name, + ) + _require( + np.array_equal(a != 0, s == CodebookStatus.AMOUNT_NONZERO) + and not np.signbit(a[a == 0]).any(), + "AMOUNT_STATUS_ENCODING", + domain.name, + ) + expected_zero = _ZERO_STATUS[domain.zero_semantics] + allowed = { + CodebookStatus.MISSING_NULL, + CodebookStatus.AMOUNT_NONZERO, + expected_zero, + } + if domain.name == "ANN_VAL": + allowed.add(CodebookStatus.DECLARED_NIU) + _require(np.isin(s, list(allowed)).all(), "FIELD_STATUS", domain.name) + _require( + type(zero_origin_code) is ZeroOrigin + and zero_origin_code != ZeroOrigin.NOT_ZERO, + "ZERO_ORIGIN_POLICY", + domain.name, + ) + expected_origin = (s == expected_zero).astype("u1") * int(zero_origin_code) + _require(np.array_equal(z, expected_origin), "ZERO_ORIGIN_ENCODING", domain.name) + factor_max = 1.0 if nominal else float("174.4") / float("163.6") + lo = min(domain.minimum, domain.minimum * factor_max) + hi = max(domain.maximum, domain.maximum * factor_max) + _require(((a >= lo) & (a <= hi)).all(), "AMOUNT_DOMAIN", domain.name) + if domain.name == "ANN_VAL": + _require((a >= 0).all(), "AMOUNT_DOMAIN", domain.name) + if nominal: + _require((a == np.floor(a)).all(), "FRACTIONAL_SOURCE", domain.name) + + +@dataclass(frozen=True) +class MoneyBindings: + """Expected replay identity, including the actual raw input fingerprint.""" + + spec: CurrentMoneySpec + header: bytes + + def __post_init__(self): + _spec(self.spec) + data = _parse(self.header, HEADER_MAX_BYTES) + keys = { + "schema_version", + "recipe", + "country", + "semantic", + "target_year", + "source_authentication", + "state", + "spec_sha256", + "scope_sha256", + "input_sha256", + "person_rows", + "household_rows", + "fields", + "dtype", + "evidence_dtype", + "person_year_sha256", + "household_year_sha256", + } + provenance = _header_provenance(self.spec) + _require( + set(data) == keys | set(provenance) and self.header == _json(data), + "HEADER_SCHEMA", + ) + fixed = { + **provenance, + "recipe": RECIPE, + "country": "us", + "semantic": "annual_current_money", + "target_year": 2024, + "source_authentication": _authentication(self.spec.source), + "state": "target_current", + "spec_sha256": self.spec.sha256, + "fields": list(FIELDS), + "dtype": " MoneyField: + return _field(self.fields, name) + + +@dataclass(frozen=True) +class RestatedAsecMoney: + bindings: MoneyBindings + fields: tuple[MoneyField, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _STATE_TOKEN, "STATE_CONSTRUCTOR_UNAVAILABLE") + + @property + def header(self) -> bytes: + return self.bindings.header + + def field(self, name: str) -> MoneyField: + return _field(self.fields, name) + + +@dataclass(frozen=True) +class SyntheticReadyCurrentMoney(RestatedAsecMoney): + """Complete invented fixture amounts. This is NOT production authority.""" + + +class ReadyCurrentMoney(RestatedAsecMoney): + """Complete amounts bound to verified source bytes; not release certification.""" + + def __new__(cls, *args, **kwargs): + _require( + kwargs.get("_token") is _STATE_TOKEN, "PRODUCTION_READINESS_UNAVAILABLE" + ) + return super().__new__(cls) + + +def _field(fields: tuple[MoneyField, ...], name: str) -> MoneyField: + _require(name in FIELDS, "UNKNOWN_FIELD") + return fields[FIELDS.index(name)] + + +def _validate_money(value, spec: CurrentMoneySpec, *, nominal: bool) -> None: + _spec(spec) + _require( + type(value.bindings) is MoneyBindings and value.bindings.spec == spec, + "SPEC_MISMATCH", + ) + value.bindings.__post_init__() + header = _parse(value.bindings.header, HEADER_MAX_BYTES) + _require( + type(value.fields) is tuple and len(value.fields) == len(FIELDS), "FIELD_ROSTER" + ) + for domain, field in zip(spec.fields, value.fields, strict=True): + _validate_field( + field, + domain, + header[domain.entity + "_rows"], + nominal=nominal, + zero_origin_code=_origin_code(spec, domain.name), + ) + + +def _index_identity(index: pd.Index) -> bytes: + # An intentionally narrow, honest representation contract; duplicate/nondefault + # signed-int64 labels are supported positionally. No label-based assignment. + _require( + type(index) in (pd.Index, pd.RangeIndex) and str(index.dtype) == "int64", + "UNSUPPORTED_INDEX", + ) + _require(index.name is None or type(index.name) is str, "UNSUPPORTED_INDEX_NAME") + descriptor = { + "type": type(index).__name__, + "name": index.name, + "values_sha256": _sha(index.to_numpy(dtype=" DecodedAsecMoney: + """Decode explicit raw columns positionally, preserving codebook evidence.""" + _require(type(raw_views) is AsecMoneyViews, "TYPED_STATE") + _spec(spec) + _require( + type(source_scope) is AsecMoneyScope + and type(raw_views.scope) is AsecMoneyScope, + "SCOPE_TYPE", + ) + source_scope.__post_init__() + _require(raw_views.scope == source_scope, "SCOPE_MISMATCH") + p, h = len(source_scope.person_ids), len(source_scope.household_ids) + for table, count in ((raw_views.person, p), (raw_views.household, h)): + _require( + type(table) is pd.DataFrame + and len(table) == count + and table.columns.is_unique, + "VIEW_SHAPE", + ) + scope_identity = _json( + { + "coordinates": _parse(source_scope.identity), + "person_index": _parse(_index_identity(raw_views.person.index)), + "household_index": _parse(_index_identity(raw_views.household.index)), + } + ) + result = [] + inputs = [] + # Sorting native group IDs is only a temporary positional validation index. + spm = np.asarray(source_scope.person_spm_ids, dtype="= 10001) & (values <= 99999) & valid).any(), + "DISPUTED_CODEBOOK_RANGE", + domain.name, + ) + _require( + not (((values < domain.minimum) | (values > domain.maximum)) & valid).any(), + "OUTSIDE_CODEBOOK_RANGE", + domain.name, + ) + # Bind received numeric bytes (including -0) and validity before normalization. + received = values.copy() + received[~valid] = 0.0 + inputs.append( + { + "field": domain.name, + "entity": domain.entity, + "column": domain.column, + "dtype": str(series.dtype), + "values_sha256": _sha(received.tobytes()), + "validity_sha256": _sha(valid.astype("u1").tobytes()), + } + ) + status = np.full(len(values), CodebookStatus.AMOUNT_NONZERO, dtype="u1") + zeros = valid & (values == 0) + status[zeros] = _ZERO_STATUS[domain.zero_semantics] + if domain.name == "ANN_VAL": + niu = valid & (values == -1) + status[niu] = CodebookStatus.DECLARED_NIU + values[niu] = 0.0 + status[~valid] = CodebookStatus.MISSING_NULL + values[~valid | (values == 0)] = 0.0 + field = MoneyField( + domain.name, + values.tobytes(), + status.tobytes(), + valid.astype("u1").tobytes(), + (zeros.astype("u1") * int(_origin_code(spec, domain.name))).tobytes(), + ) + _validate_field( + field, + domain, + len(table), + nominal=True, + zero_origin_code=_origin_code(spec, domain.name), + ) + if domain.grain == "spm_unit_repeated_on_person": + for vector in (values, status, valid, zeros): + sorted_values = vector[order] + _require( + not ((sorted_values[1:] != sorted_values[:-1]) & same).any(), + "INCONSISTENT_SPM_AMOUNT", + domain.name, + ) + result.append(field) + header = _json( + { + **_header_provenance(spec), + "recipe": RECIPE, + "country": "us", + "semantic": "annual_current_money", + "target_year": 2024, + "source_authentication": _authentication(spec.source), + "state": "target_current", + "spec_sha256": spec.sha256, + "scope_sha256": _sha(scope_identity), + "input_sha256": _sha(_json(inputs)), + "person_year_sha256": _sha( + np.asarray(source_scope.person_years, dtype=" RestatedAsecMoney: + """Restate once; target-year bytes and normalized zero bypass arithmetic.""" + _require(type(decoded) is DecodedAsecMoney, "TYPED_STATE") + _validate_money(decoded, spec, nominal=True) + header = _parse(decoded.bindings.header, HEADER_MAX_BYTES) + years = {} + for entity in ("person", "household"): + data = getattr(decoded, entity + "_year_bytes") + _require( + type(data) is bytes and len(data) == header[entity + "_rows"] * 8, + "YEAR_BUFFER", + ) + _require(_sha(data) == header[entity + "_year_sha256"], "YEAR_BINDING") + vector = np.frombuffer(data, dtype=" SyntheticReadyCurrentMoney | ReadyCurrentMoney: + """Require every amount; synthetic evidence cannot request verified readiness.""" + _require(type(production) is bool, "PRODUCTION_FLAG") + verified = type(spec.source) is AuthenticatedAsecSource + _require(not production or verified, "SYNTHETIC_NOT_PRODUCTION") + _require(not verified or production, "AUTHENTICATED_READINESS_REQUIRED") + _require(type(restated) is RestatedAsecMoney, "TYPED_STATE") + _validate_money(restated, spec, nominal=False) + for field in restated.fields: + _require(field.validity.all(), "MISSING_REQUIRED_AMOUNT", field.name) + ready_type = ReadyCurrentMoney if verified else SyntheticReadyCurrentMoney + return ready_type(restated.bindings, restated.fields, _token=_STATE_TOKEN) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_consumers_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_consumers_v1.json new file mode 100644 index 000000000..5ac33188e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_consumers_v1.json @@ -0,0 +1,628 @@ +{ + "addresses": [ + { + "column": "ANN_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "ANN_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_pension_income", + "tax_exempt_pension_income" + ], + "role": "cps_source" + }, + { + "column": "CAP_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "CAP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "long_term_capital_gains", + "short_term_capital_gains" + ], + "role": "cps_source" + }, + { + "column": "CHSP_VAL", + "consumers": [ + "child_support" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "CHSP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "child_support_paid" + ], + "role": "cps_source" + }, + { + "column": "CSP_VAL", + "consumers": [ + "child_support" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "CSP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "child_support_received" + ], + "role": "cps_source" + }, + { + "column": "DIS_VAL1", + "consumers": [ + "disability_benefits" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DIS_VAL1", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "disability_benefits" + ], + "role": "cps_source" + }, + { + "column": "DIS_VAL2", + "consumers": [ + "disability_benefits" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DIS_VAL2", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "disability_benefits" + ], + "role": "cps_source" + }, + { + "column": "DIV_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DIV_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "qualified_dividend_income", + "non_qualified_dividend_income" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL1", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL1", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL1_YNG", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL1_YNG", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL2", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL2", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL2_YNG", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL2_YNG", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "ED_VAL", + "consumers": [ + "education_inputs" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "ED_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "educational_assistance" + ], + "role": "cps_source" + }, + { + "column": "FRSE_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "FRSE_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "farm_operations_income" + ], + "role": "cps_source" + }, + { + "column": "asec_HTOTVAL", + "consumers": [ + "asec_household_observations" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "household", + "field": "HTOTVAL", + "grain": "household", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "asec_current_household_total_income" + ], + "role": "cps_source" + }, + { + "column": "INT_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "INT_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_interest_income" + ], + "role": "cps_source" + }, + { + "column": "OI_VAL", + "consumers": [ + "asec_other_income_split", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "OI_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "alimony_income", + "strike_benefits", + "miscellaneous_income" + ], + "role": "cps_source" + }, + { + "column": "PHIP_VAL", + "consumers": [ + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PHIP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "health_insurance_premiums_without_medicare_part_b" + ], + "role": "cps_source" + }, + { + "column": "PMED_VAL", + "consumers": [ + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PMED_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "other_medical_expenses" + ], + "role": "cps_source" + }, + { + "column": "PNSN_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PNSN_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_pension_income", + "tax_exempt_pension_income" + ], + "role": "cps_source" + }, + { + "column": "POTC_VAL", + "consumers": [ + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "POTC_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "over_the_counter_health_expenses" + ], + "role": "cps_source" + }, + { + "column": "PTOTVAL", + "consumers": [ + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PTOTVAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "microunit_current_total_income" + ], + "role": "cps_source" + }, + { + "column": "RETCB_VAL", + "consumers": [ + "retirement_contributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "RETCB_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "retirement_contributions" + ], + "role": "cps_source" + }, + { + "column": "RNT_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "RNT_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "rental_income" + ], + "role": "cps_source" + }, + { + "column": "SEMP_VAL", + "consumers": [ + "cps_carried", + "immigration", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SEMP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "self_employment_income_before_lsr" + ], + "role": "cps_source" + }, + { + "column": "SPM_CAPHOUSESUB", + "consumers": [ + "immigration" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SPM_CAPHOUSESUB", + "grain": "spm_unit_repeated_on_person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "immigration_amount_predictor" + ], + "role": "cps_source" + }, + { + "column": "SPM_CHILDCAREXPNS", + "consumers": [ + "childcare", + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SPM_CHILDCAREXPNS", + "grain": "spm_unit_repeated_on_person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "spm_unit_pre_subsidy_childcare_expenses" + ], + "role": "cps_source" + }, + { + "column": "SPM_ENGVAL", + "consumers": [ + "energy_subsidy" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SPM_ENGVAL", + "grain": "spm_unit_repeated_on_person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "spm_unit_energy_subsidy" + ], + "role": "cps_source" + }, + { + "column": "SSI_VAL", + "consumers": [ + "acs_release_predictors_reported_ssi_anchor" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SSI_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "asec_current_reported_ssi_predictor" + ], + "role": "cps_source" + }, + { + "column": "SS_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SS_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "social_security_retirement", + "social_security_disability", + "social_security_survivors", + "social_security_dependents" + ], + "role": "cps_source" + }, + { + "column": "UC_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "UC_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "unemployment_compensation" + ], + "role": "cps_source" + }, + { + "column": "VET_VAL", + "consumers": [ + "eligibility_inputs" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "VET_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "veterans_benefits" + ], + "role": "cps_source" + }, + { + "column": "WC_VAL", + "consumers": [ + "workers_compensation" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "WC_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "workers_compensation" + ], + "role": "cps_source" + }, + { + "column": "WSAL_VAL", + "consumers": [ + "cps_carried", + "immigration", + "microunit", + "voluntary_filing_current_leaf" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "WSAL_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "employment_income_before_lsr" + ], + "role": "cps_source" + } + ], + "implementation_status": "declarations_only_live_nominal_consumers_unchanged", + "microunit": { + "amount_fields": [ + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PNSN_VAL", + "PTOTVAL", + "RNT_VAL", + "SEMP_VAL", + "SS_VAL", + "UC_VAL", + "WSAL_VAL" + ], + "distribution": "microunit", + "module": "microunit/tax_unit_construction.py", + "module_sha256": "79007065113f9600601433dc2633f4392c675e28b87a7383ab8a1f7c6279d47e", + "version": "0.1.0" + }, + "ownership_policy": "restater_owns_cpi_only_derived_outputs_keep_registered_phase_role_owners", + "routing_policy": "unchanged_source_codes_no_new_DST_universe_exclusion", + "schema_version": 1, + "scope": "current_money_only_no_prior_wages", + "unscaled_fields": [ + "PAW_VAL", + "SPM_SNAPSUB" + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_domains_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_domains_v1.json new file mode 100644 index 000000000..7f09e933e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_domains_v1.json @@ -0,0 +1,3418 @@ +{ + "fields": [ + { + "column": "ANN_VAL", + "declared_niu_codes": [ + -1 + ], + "domain": { + "declared_negative_nonmoney_codes": [ + -1 + ], + "declared_niu_codes": [ + -1 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": -1 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": -1, + "name": "ANN_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 430, + "dictionary_spelling": "ANN_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": -1 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ANN_YN = 1", + "values_description": "-1 = niu\n0-999999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 430, + "dictionary_spelling": "ANN_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": -1 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ANN_YN = 1", + "values_description": "-1 = niu\n0-999999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 438, + "dictionary_spelling": "ANN_VAL", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": -1 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ANN_YN = 1", + "values_description": "-1 = niu\n0-999999 = dollar amount" + } + ], + "zero_semantics": "valid_zero_dollars" + }, + { + "column": "CAP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "CAP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 437, + "dictionary_spelling": "CAP_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "CAP_YN = 1", + "values_description": "0 = none or niu\n1-999999 = captial gains amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 437, + "dictionary_spelling": "CAP_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "CAP_YN = 1", + "values_description": "0 = none or niu\n1-999999 = captial gains amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 445, + "dictionary_spelling": "CAP_VAL", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "CAP_YN = 1", + "values_description": "0 = none or niu\n1-999999 = captial gains amount" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "CHSP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 1 + }, + "zero_semantics": "niu" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "CHSP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 704, + "dictionary_spelling": "CHSP_VAL", + "income_year": 2022, + "pdf_page_1based": 49, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "CHSP_YN = 1", + "values_description": "0 = NIU\n1:99999 = amount paid in child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 704, + "dictionary_spelling": "CHSP_VAL", + "income_year": 2023, + "pdf_page_1based": 50, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "CHSP_YN = 1", + "values_description": "0 = NIU\n1:99999 = amount paid in child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 712, + "dictionary_spelling": "CHSP_VAL", + "income_year": 2024, + "pdf_page_1based": 52, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "CHSP_YN = 1", + "values_description": "0 = NIU\n1:99999 = amount paid in child support" + } + ], + "zero_semantics": "niu" + }, + { + "column": "CSP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "CSP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 710, + "dictionary_spelling": "CSP_VAL", + "income_year": 2022, + "pdf_page_1based": 50, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "CSP_YN = 1", + "values_description": "0 = none or niu\n1-99999 = child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 710, + "dictionary_spelling": "CSP_VAL", + "income_year": 2023, + "pdf_page_1based": 51, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "CSP_YN = 1", + "values_description": "0 = none or niu\n1-99999 = child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 718, + "dictionary_spelling": "CSP_VAL", + "income_year": 2024, + "pdf_page_1based": 52, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "CSP_YN = 1", + "values_description": "0 = none or niu\n1-99999 = child support" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DIS_VAL1", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DIS_VAL1", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 457, + "dictionary_spelling": "DIS_VAL1", + "income_year": 2022, + "pdf_page_1based": 42, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DIS_SC1>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 457, + "dictionary_spelling": "DIS_VAL1", + "income_year": 2023, + "pdf_page_1based": 43, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DIS_SC1>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 465, + "dictionary_spelling": "DIS_VAL1", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DIS_SC1>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DIS_VAL2", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DIS_VAL2", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 463, + "dictionary_spelling": "DIS_VAL2", + "income_year": 2022, + "pdf_page_1based": 42, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DIS_SC2>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 463, + "dictionary_spelling": "DIS_VAL2", + "income_year": 2023, + "pdf_page_1based": 43, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DIS_SC2>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 471, + "dictionary_spelling": "DIS_VAL2", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DIS_SC2>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DIV_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DIV_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 470, + "dictionary_spelling": "DIV_VAL", + "income_year": 2022, + "pdf_page_1based": 42, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DIV_YN = 1", + "values_description": "0 = none or niu\n1-999999 = dividends" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 470, + "dictionary_spelling": "DIV_VAL", + "income_year": 2023, + "pdf_page_1based": 43, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DIV_YN = 1", + "values_description": "0 = none or niu\n1-999999 = dividends" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 478, + "dictionary_spelling": "DIV_VAL", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DIV_YN = 1", + "values_description": "0 = none or niu\n1-999999 = dividends" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL1", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL1", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 487, + "dictionary_spelling": "DST_VAL1", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC1 = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 487, + "dictionary_spelling": "DST_VAL1", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC1 = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 495, + "dictionary_spelling": "DST_VAL1", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC1 = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL1_YNG", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL1_YNG", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 493, + "dictionary_spelling": "DST_VAL1_YNG", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC1_YNG = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 493, + "dictionary_spelling": "DST_VAL1_YNG", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC1_YNG = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 501, + "dictionary_spelling": "DST_VAL1_YNG", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC1_YNG = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL2", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL2", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 499, + "dictionary_spelling": "DST_VAL2", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC2 = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 499, + "dictionary_spelling": "DST_VAL2", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC2 = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 507, + "dictionary_spelling": "DST_VAL2", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC2 = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL2_YNG", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL2_YNG", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 505, + "dictionary_spelling": "DST_VAL2_YNG", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC2_YNG = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 505, + "dictionary_spelling": "DST_VAL2_YNG", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC2_YNG = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 513, + "dictionary_spelling": "DST_VAL2_YNG", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC2_YNG = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "ED_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "ED_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 513, + "dictionary_spelling": "ED_VAL", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ED_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 513, + "dictionary_spelling": "ED_VAL", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ED_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 521, + "dictionary_spelling": "ED_VAL", + "income_year": 2024, + "pdf_page_1based": 46, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ED_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "FRSE_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": -9999999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": -9999999 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": -9999999, + "name": "FRSE_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 382, + "dictionary_spelling": "FRSE_VAL", + "income_year": 2022, + "pdf_page_1based": 40, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-21", + "range_header": { + "maximum": 9999999, + "minimum": -9999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ERN_YN=1 or FRMOTR=1", + "values_description": "0 = none or niu;\n-9999999-9999999 = farm self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 382, + "dictionary_spelling": "FRSE_VAL", + "income_year": 2023, + "pdf_page_1based": 41, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-21", + "range_header": { + "maximum": 9999999, + "minimum": -9999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ERN_YN=1 or FRMOTR=1", + "values_description": "0 = none or niu;\n-9999999-9999999 = farm self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 390, + "dictionary_spelling": "FRSE_VAL", + "income_year": 2024, + "pdf_page_1based": 43, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -9999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ERN_YN=1 or FRMOTR=1", + "values_description": "0 = none or niu;\n-9999999-9999999 = farm self employment" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "asec_HTOTVAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999999, + "minimum": -999999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999999, + "minimum": -999999 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "household", + "grain": "household", + "maximum": 99999999, + "minimum": -999999, + "name": "HTOTVAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 106, + "dictionary_spelling": "HTOTVAL", + "income_year": 2022, + "pdf_page_1based": 5, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6A-5", + "range_header": { + "maximum": 99999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Households", + "values_description": "0 = none\nnegative dollar amount\npositive dollar amount" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 106, + "dictionary_spelling": "HTOTVAL", + "income_year": 2023, + "pdf_page_1based": 5, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6A-5", + "range_header": { + "maximum": 99999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Households", + "values_description": "0 = none\nnegative dollar amount\npositive dollar amount" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 108, + "dictionary_spelling": "HTOTVAL", + "income_year": 2024, + "pdf_page_1based": 12, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6A-5", + "range_header": { + "maximum": 99999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Households", + "values_description": "0 = none\nnegative dollar amount\npositive dollar amount" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "INT_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "INT_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 529, + "dictionary_spelling": "INT_VAL", + "income_year": 2022, + "pdf_page_1based": 44, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "INT_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 529, + "dictionary_spelling": "INT_VAL", + "income_year": 2023, + "pdf_page_1based": 45, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "INT_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 537, + "dictionary_spelling": "INT_VAL", + "income_year": 2024, + "pdf_page_1based": 46, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "INT_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "OI_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "OI_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 541, + "dictionary_spelling": "OI_VAL", + "income_year": 2022, + "pdf_page_1based": 44, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "OI_YN = 1", + "values_description": "0 = none or niu\n1-999999 = other income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 541, + "dictionary_spelling": "OI_VAL", + "income_year": 2023, + "pdf_page_1based": 45, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "OI_YN = 1", + "values_description": "0 = none or niu\n1-999999 = other income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 549, + "dictionary_spelling": "OI_VAL", + "income_year": 2024, + "pdf_page_1based": 47, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-26", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "OI_YN = 1", + "values_description": "0 = none or niu\n1-999999 = other income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "PHIP_VAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "PHIP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 289 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1292, + "dictionary_spelling": "PHIP_VAL", + "income_year": 2022, + "pdf_page_1based": 78, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-59", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1288, + "dictionary_spelling": "PHIP_VAL", + "income_year": 2023, + "pdf_page_1based": 79, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-59", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1455, + "dictionary_spelling": "PHIP_VAL", + "income_year": 2024, + "pdf_page_1based": 83, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-62", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "PMED_VAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "PMED_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 289 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1304, + "dictionary_spelling": "PMED_VAL", + "income_year": 2022, + "pdf_page_1based": 79, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-60", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1300, + "dictionary_spelling": "PMED_VAL", + "income_year": 2023, + "pdf_page_1based": 79, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-59", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1467, + "dictionary_spelling": "PMED_VAL", + "income_year": 2024, + "pdf_page_1based": 83, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-62", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "PNSN_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": 0, + "name": "PNSN_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 563, + "dictionary_spelling": "PNSN_VAL", + "income_year": 2022, + "pdf_page_1based": 45, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-26", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "PEN_YN = 1", + "values_description": "0 = none or niu\n1- 9,999,999 = retirement income" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 563, + "dictionary_spelling": "PNSN_VAL", + "income_year": 2023, + "pdf_page_1based": 46, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-26", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "PEN_YN = 1", + "values_description": "0 = none or niu\n1- 9,999,999 = retirement income" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 571, + "dictionary_spelling": "PNSN_VAL", + "income_year": 2024, + "pdf_page_1based": 47, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-26", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "PEN_YN = 1", + "values_description": "0 = none or niu\n1- 9,999,999 = retirement income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "POTC_VAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "POTC_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 289, + 290 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1310, + "dictionary_spelling": "POTC_VAL", + "income_year": 2022, + "pdf_page_1based": 79, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-60", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "0 - 99999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1306, + "dictionary_spelling": "POTC_VAL", + "income_year": 2023, + "pdf_page_1based": 79, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-59", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "0 - 99999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1473, + "dictionary_spelling": "POTC_VAL", + "income_year": 2024, + "pdf_page_1based": 83, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-62", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "0 - 99999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "PTOTVAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999999, + "minimum": -99999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999999, + "minimum": -99999 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999999, + "minimum": -99999, + "name": "PTOTVAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 580, + "dictionary_spelling": "PTOTVAL", + "income_year": 2022, + "pdf_page_1based": 45, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-26", + "range_header": { + "maximum": 99999999, + "minimum": -99999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons aged 15+", + "values_description": "0 = none\nnegative amt = income (loss)\npositive amt = income" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 580, + "dictionary_spelling": "PTOTVAL", + "income_year": 2023, + "pdf_page_1based": 46, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-26", + "range_header": { + "maximum": 99999999, + "minimum": -99999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons aged 15+", + "values_description": "0 = none\nnegative amt = income (loss)\npositive amt = income" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 588, + "dictionary_spelling": "PTOTVAL", + "income_year": 2024, + "pdf_page_1based": 48, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999999, + "minimum": -99999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons aged 15+", + "values_description": "0 = none\nnegative amt = income (loss)\npositive amt = income" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "RETCB_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "RETCB_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 274 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 592, + "dictionary_spelling": "RETCB_VAL", + "income_year": 2022, + "pdf_page_1based": 46, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "RETCB_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = amount contributed" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 592, + "dictionary_spelling": "RETCB_VAL", + "income_year": 2023, + "pdf_page_1based": 47, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "RETCB_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = amount contributed" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 600, + "dictionary_spelling": "RETCB_VAL", + "income_year": 2024, + "pdf_page_1based": 48, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "RETCB_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = amount contributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "RNT_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": -9999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": -9999 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": -9999, + "name": "RNT_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 613, + "dictionary_spelling": "RNT_VAL", + "income_year": 2022, + "pdf_page_1based": 47, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-28", + "range_header": { + "maximum": 999999, + "minimum": -9999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "RNT_YN = 1", + "values_description": "0 = none or niu;\n-9999-999999 = rental income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 613, + "dictionary_spelling": "RNT_VAL", + "income_year": 2023, + "pdf_page_1based": 48, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-28", + "range_header": { + "maximum": 999999, + "minimum": -9999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "RNT_YN = 1", + "values_description": "0 = none or niu;\n-9999-999999 = rental income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 621, + "dictionary_spelling": "RNT_VAL", + "income_year": 2024, + "pdf_page_1based": 49, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-28", + "range_header": { + "maximum": 999999, + "minimum": -9999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "RNT_YN = 1", + "values_description": "0 = none or niu;\n-9999-999999 = rental income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "SEMP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": -999999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": -999999 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": -999999, + "name": "SEMP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 405, + "dictionary_spelling": "SEMP_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ERN_YN=1 or SEOTR=1", + "values_description": "0 = none or niu;\n-999999-9999999 = own business self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 405, + "dictionary_spelling": "SEMP_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ERN_YN=1 or SEOTR=1", + "values_description": "0 = none or niu;\n-999999-9999999 = own business self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 413, + "dictionary_spelling": "SEMP_VAL", + "income_year": 2024, + "pdf_page_1based": 43, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ERN_YN=1 or SEOTR=1", + "values_description": "0 = none or niu;\n-999999-9999999 = own business self employment" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "SPM_CAPHOUSESUB", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "spm_unit_repeated_on_person", + "maximum": 99999, + "minimum": 0, + "name": "SPM_CAPHOUSESUB", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1436, + "dictionary_spelling": "SPM_CapHouseSub", + "income_year": 2022, + "pdf_page_1based": 85, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-66", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1432, + "dictionary_spelling": "SPM_CapHouseSub", + "income_year": 2023, + "pdf_page_1based": 85, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-65", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 936, + "dictionary_spelling": "SPM_CapHouseSub", + "income_year": 2024, + "pdf_page_1based": 61, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-40", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "SPM_CHILDCAREXPNS", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "spm_unit_repeated_on_person", + "maximum": 999999, + "minimum": 0, + "name": "SPM_CHILDCAREXPNS", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 304 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1447, + "dictionary_spelling": "SPM_ChildcareXpns", + "income_year": 2022, + "pdf_page_1based": 85, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-66", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "$0 to $999,999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1443, + "dictionary_spelling": "SPM_ChildcareXpns", + "income_year": 2023, + "pdf_page_1based": 85, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-65", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "$0 to $999,999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 947, + "dictionary_spelling": "SPM_ChildcareXpns", + "income_year": 2024, + "pdf_page_1based": 61, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-40", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "$0 to $999,999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "SPM_ENGVAL", + "declared_niu_codes": [], + "domain": { + "authority_range_alternatives": [ + { + "location": "Range header", + "maximum": 10000, + "minimum": 0 + }, + { + "location": "Values description", + "maximum": 99999, + "minimum": 0 + } + ], + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "definitely_outside_both_declared_ranges": { + "above": 99999, + "below": 0 + }, + "encoded_range_inclusive": null, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "uncontested_valid_dollar_range_inclusive": { + "maximum": 10000, + "minimum": 0 + }, + "unresolved_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 10001 + }, + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": null, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "partial_conflicting_upper_bound_all_three_vintages", + "entity": "person", + "grain": "spm_unit_repeated_on_person", + "maximum": 10000, + "minimum": 0, + "name": "SPM_ENGVAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 293 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1463, + "dictionary_spelling": "SPM_EngVal", + "income_year": 2022, + "pdf_page_1based": 85, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-66", + "range_header": { + "maximum": 10000, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1459, + "dictionary_spelling": "SPM_EngVal", + "income_year": 2023, + "pdf_page_1based": 86, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-66", + "range_header": { + "maximum": 10000, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 963, + "dictionary_spelling": "SPM_EngVal", + "income_year": 2024, + "pdf_page_1based": 61, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-40", + "range_header": { + "maximum": 10000, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "SSI_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "SSI_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 632, + "dictionary_spelling": "SSI_VAL", + "income_year": 2022, + "pdf_page_1based": 47, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "SSI_YN = 1", + "values_description": "0 = none or niu\n1-99999 = supplemental security income" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 632, + "dictionary_spelling": "SSI_VAL", + "income_year": 2023, + "pdf_page_1based": 48, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "SSI_YN = 1", + "values_description": "0 = none or niu\n1-99999 = supplemental security income" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 640, + "dictionary_spelling": "SSI_VAL", + "income_year": 2024, + "pdf_page_1based": 49, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "SSI_YN = 1", + "values_description": "0 = none or niu\n1-99999 = supplemental security income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "SS_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "SS_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 626, + "dictionary_spelling": "SS_VAL", + "income_year": 2022, + "pdf_page_1based": 47, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "SS_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = social security" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 626, + "dictionary_spelling": "SS_VAL", + "income_year": 2023, + "pdf_page_1based": 48, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "SS_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = social security" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 634, + "dictionary_spelling": "SS_VAL", + "income_year": 2024, + "pdf_page_1based": 49, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "SS_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = social security" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "UC_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "UC_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 664, + "dictionary_spelling": "UC_VAL", + "income_year": 2022, + "pdf_page_1based": 48, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-29", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "UC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = unemployment compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 664, + "dictionary_spelling": "UC_VAL", + "income_year": 2023, + "pdf_page_1based": 49, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-29", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "UC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = unemployment compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 672, + "dictionary_spelling": "UC_VAL", + "income_year": 2024, + "pdf_page_1based": 50, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-29", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "UC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = unemployment compensation" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "VET_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "VET_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 676, + "dictionary_spelling": "VET_VAL", + "income_year": 2022, + "pdf_page_1based": 48, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-29", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "VET_YN = 1", + "values_description": "0 = none or niu\n1-999999 = veterans' payments" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 676, + "dictionary_spelling": "VET_VAL", + "income_year": 2023, + "pdf_page_1based": 49, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-29", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "VET_YN = 1", + "values_description": "0 = none or niu\n1-999999 = veterans' payments" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 684, + "dictionary_spelling": "VET_VAL", + "income_year": 2024, + "pdf_page_1based": 51, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-30", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "VET_YN = 1", + "values_description": "0 = none or niu\n1-999999 = veterans' payments" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "WC_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "WC_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 684, + "dictionary_spelling": "WC_VAL", + "income_year": 2022, + "pdf_page_1based": 49, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "WC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = worker's compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 684, + "dictionary_spelling": "WC_VAL", + "income_year": 2023, + "pdf_page_1based": 50, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "WC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = worker's compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 692, + "dictionary_spelling": "WC_VAL", + "income_year": 2024, + "pdf_page_1based": 51, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "WC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = worker's compensation" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "WSAL_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": 0, + "name": "WSAL_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 422, + "dictionary_spelling": "WSAL_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ERN_YN=1 or WAGEOTR=1", + "values_description": "0 = none or niu;\n1-9999999 = wage and salary" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 422, + "dictionary_spelling": "WSAL_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ERN_YN=1 or WAGEOTR=1", + "values_description": "0 = none or niu;\n1-9999999 = wage and salary" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 430, + "dictionary_spelling": "WSAL_VAL", + "income_year": 2024, + "pdf_page_1based": 43, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ERN_YN=1 or WAGEOTR=1", + "values_description": "0 = none or niu;\n1-9999999 = wage and salary" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + } + ], + "max_households": 400000, + "max_persons": 1000000, + "recipe": "us_asec_current_money_ccpiu_2024_v1", + "registry_sha256": "bdeb4d2fa1c37c6064530d560a62b12c515118608845b30d01b06c2eeeda8c6e", + "schema_version": 1, + "source_years": [ + [ + 2022, + 2023 + ], + [ + 2023, + 2024 + ], + [ + 2024, + 2025 + ] + ], + "zero_origin_policy": "frozen_fillna_zero_origin_unresolved" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_engine_defaults_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_engine_defaults_v1.json new file mode 100644 index 000000000..583c3684d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_engine_defaults_v1.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "artifact_kind": "microcosm.us.asec_current_money_engine_defaults.v1", + "engine_package": "policyengine-us", + "engine_version": "1.819.0", + "allowlist": [], + "allowlist_policy": "no engine default is an approved input to an admitted output; a closure leaf this graph does not produce blocks its root", + "admitted_roots": [ + "capital_gains", + "dividend_income", + "employment_income", + "social_security" + ], + "blocked_roots": { + "interest_income": "needs tax_exempt_interest_income, which this slice does not map", + "pension_income": "needs the public pension leaves, which this slice does not map", + "self_employment_income": "needs sstb_self_employment_income_before_lsr, which this slice does not map" + }, + "release_eligible": false, + "scoring_claim": "none; these are engine outputs on an engineering sample, not a tax or benefit score", + "baseline_runtime": { + "policyengine-us": { + "version": "1.819.0", + "files": 12313, + "source_and_parameters_sha256": "12c474205741021bb45ae618f85ddd6d15e8cbb51dece8334f2d52c04500c674" + }, + "policyengine-core": { + "version": "3.31.0", + "files": 175, + "source_and_parameters_sha256": "fbcc6f3d8e86c023c29e0782d95103155828cfa4ef6a80ed6f785f619be6439e" + } + }, + "baseline_runtime_file_policy": "all package Python source and parameters YAML/JSON; sorted relative paths and SHA256; no population data" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_consumers_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_consumers_v1.json new file mode 100644 index 000000000..cd0270dee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_consumers_v1.json @@ -0,0 +1,127 @@ +{ + "all_current_money_consumers_wired": false, + "artifact_kind": "microcosm.us.asec_current_money_graph_consumers.v1", + "blocked_engine_outputs": { + "interest_income": [ + "tax_exempt_interest_income" + ], + "pension_income": [ + "tax_exempt_public_pension_income", + "taxable_public_pension_income" + ], + "self_employment_income": [ + "sstb_self_employment_income_before_lsr" + ] + }, + "engine_defaults_allowlist": [], + "implementation_status": "prepared_current_money_graph_and_microunit_wired_other_nominal_routes_pending", + "pending_nominal_consumer_routes": [ + "acs_release_predictors_reported_ssi_anchor", + "asec_household_observations", + "asec_other_income_split", + "child_support", + "childcare", + "cps_carried", + "disability_benefits", + "education_inputs", + "eligibility_inputs", + "energy_subsidy", + "immigration", + "retirement_contributions", + "retirement_distributions", + "voluntary_filing_current_leaf", + "workers_compensation" + ], + "release_eligible": false, + "schema_version": 1, + "scope": "current_money_only_no_prior_wages", + "source_declarations": { + "binding": "unchanged historical source recipe; this additive record supersedes implementation status only for the explicitly wired routes below; source address roster and T1/money identities are unchanged", + "resource": "asec_current_money_consumers_v1.json", + "sha256": "c56058eaf08b68b2add4a5d2b366559ea6cc5238b039031be7094ef1f6457ff7" + }, + "wired_routes": { + "asec_prepared_current_money_graph": { + "engine_output_storage": "evaluation_artifact_only", + "engine_outputs": [ + "capital_gains", + "dividend_income", + "employment_income", + "social_security" + ], + "module": "microcosm.build.us_runtime.graph_asec_prepared", + "money_fields": [ + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PHIP_VAL", + "PMED_VAL", + "PNSN_VAL", + "POTC_VAL", + "RNT_VAL", + "SEMP_VAL", + "SPM_CHILDCAREXPNS", + "SS_VAL", + "UC_VAL", + "WSAL_VAL" + ], + "money_input": "typed selected current-money body; never standalone ReadyCurrentMoney", + "person_leaves": [ + "age", + "alimony_income", + "employment_income_before_lsr", + "farm_operations_income", + "health_insurance_premiums_without_medicare_part_b", + "long_term_capital_gains_before_response", + "miscellaneous_income", + "non_qualified_dividend_income", + "other_medical_expenses", + "over_the_counter_health_expenses", + "qualified_dividend_income", + "rental_income", + "self_employment_income_before_lsr", + "short_term_capital_gains", + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + "strike_benefits", + "tax_exempt_private_pension_income", + "taxable_interest_income", + "taxable_ira_distributions", + "taxable_private_pension_income", + "unemployment_compensation" + ], + "spm_unit_leaves": [ + "spm_unit_pre_subsidy_childcare_expenses" + ] + }, + "microunit": { + "module": "microcosm.build.us_runtime.asec_current_money_units", + "money_fields": [ + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PNSN_VAL", + "PTOTVAL", + "RNT_VAL", + "SEMP_VAL", + "SS_VAL", + "UC_VAL", + "WSAL_VAL" + ], + "money_input": "authenticated full-source ReadyCurrentMoney", + "student_controls": "separately authenticated S attached before tax construction; interview-week proxy only" + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py new file mode 100644 index 000000000..bab0c7b02 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py @@ -0,0 +1,34 @@ +"""Pinned additive consumer status, separate from the immutable money recipe. + +The source recipe's declarations and execution identity stay byte-for-byte +unchanged. This graph record reports the routes actually wired by this slice; +it grants no source authority and changes no source address or monetary rule. +""" + +from hashlib import sha256 +from importlib import resources + +from .asec_current_money import RESOURCE_PINS, _parse + +GRAPH_CONSUMERS_RESOURCE = "asec_current_money_graph_consumers_v1.json" +GRAPH_CONSUMERS_SHA256 = ( + "67432a10d05e11dd4b97a80cd4f04ed0eb6e56c4c8e21d5dffe87c5f67e27c66" +) + + +def load_graph_current_money_consumers() -> dict: + """Verify graph status and its unchanged source-declaration parent.""" + package = resources.files(__package__) + payload = package.joinpath(GRAPH_CONSUMERS_RESOURCE).read_bytes() + if sha256(payload).hexdigest() != GRAPH_CONSUMERS_SHA256: + raise ValueError("GRAPH_CONSUMERS_FINGERPRINT") + document = _parse(payload) + parent = document["source_declarations"] + if ( + parent["resource"] != "asec_current_money_consumers_v1.json" + or parent["sha256"] != RESOURCE_PINS[2] + or sha256(package.joinpath(parent["resource"]).read_bytes()).hexdigest() + != parent["sha256"] + ): + raise ValueError("GRAPH_CONSUMERS_PARENT_FINGERPRINT") + return document diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_price_basis_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_price_basis_v1.json new file mode 100644 index 000000000..445650d70 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_price_basis_v1.json @@ -0,0 +1,21 @@ +{ + "algorithm": "parse_float64_target_divide_source_then_multiply_nonzero_v1", + "cells": { + "2022": "163.6", + "2023": "170.0", + "2024": "174.4" + }, + "currency": "USD", + "identity_year": "copy_normalized_bytes", + "numeric_scope": "platform_bitwise", + "price_basis": "census_ccpiu_annual_2024", + "schema_version": 1, + "source_finality": "not_stated_in_source", + "target_year": 2024, + "workbook": { + "sha256": "8a5ef078b02c48c9e6414f3ca0cbdfe4a013ce6acf1a3fda42e6bd9c7da49e48", + "size_bytes": 41472, + "url": "https://www2.census.gov/programs-surveys/demo/tables/p60/286/annual-index-value_annual-percent-change.xls" + }, + "zero": "positive_zero_no_arithmetic" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py new file mode 100644 index 000000000..66d8e6d10 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py @@ -0,0 +1,114 @@ +"""Explicit packaged-resource and installed microunit byte verification. + +This module never imports microunit or runs units. The lower-level packaged +function permits explicit inspected-contract injection for pure synthetic tests; +it does not certify an installed distribution or authenticate source data. +""" + +import platform +import sys +from importlib import metadata, resources + +import numpy as np +import pandas as pd + +from .asec_current_money import ( + MICROUNIT_SHA256, + MICROUNIT_VERSION, + CurrentMoneyResources, + MoneyRefusalError, + _json, + _require, + _sha, +) + +RESOURCE_NAMES = ( + "asec_current_money_domains_v1.json", + "asec_current_money_price_basis_v1.json", + "asec_current_money_consumers_v1.json", +) +MODULE_NAMES = ( + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_resources.py", +) + + +def packaged_current_money_resources( + *, inspected_version: str, inspected_module_sha256: str +) -> CurrentMoneyResources: + """Read package resources with explicit claims, for synthetic fixture injection. + + Use load_current_money_resources to verify the actual installed reader. + Neither resource-reading function creates authenticated source authority. + """ + package = resources.files(__package__) + identity = _json( + { + "modules": { + name: _sha(package.joinpath(name).read_bytes()) for name in MODULE_NAMES + }, + "dependencies": { + "python": platform.python_version(), + "numpy": np.__version__, + "pandas": pd.__version__, + }, + "platform": { + "system": platform.system(), + "machine": platform.machine(), + "byteorder": sys.byteorder, + "python_implementation": platform.python_implementation(), + }, + } + ) + return CurrentMoneyResources( + *(package.joinpath(name).read_bytes() for name in RESOURCE_NAMES), + identity, + inspected_version, + inspected_module_sha256, + ) + + +def load_current_money_resources( + *, microunit_search_path: tuple[str, ...] | None = None +) -> CurrentMoneyResources: + """Verify pinned installed version/module bytes without importing microunit. + + An explicit site-packages search path supports a separately pinned local + environment. Paths are operational and never enter the normative identity. + """ + try: + if microunit_search_path is None: + distribution = metadata.distribution("microunit") + else: + _require( + type(microunit_search_path) is tuple + and all(type(p) is str for p in microunit_search_path), + "MICROUNIT_SEARCH_PATH", + ) + matches = [ + d + for d in metadata.distributions(path=microunit_search_path) + if d.metadata.get("Name", "").lower() == "microunit" + ] + _require(len(matches) == 1, "MICROUNIT_DISTRIBUTION_COUNT") + distribution = matches[0] + _require(distribution.version == MICROUNIT_VERSION, "MICROUNIT_VERSION") + relative = "microunit/tax_unit_construction.py" + _require( + distribution.files is not None + and any(str(f) == relative for f in distribution.files), + "MICROUNIT_MODULE_INVENTORY", + ) + path = distribution.locate_file(relative) + _require(path.stat().st_size <= 1024 * 1024, "MICROUNIT_MODULE_SIZE") + module = path.read_bytes() + _require( + len(module) <= 1024 * 1024 and _sha(module) == MICROUNIT_SHA256, + "MICROUNIT_MODULE_SHA256", + ) + except (metadata.PackageNotFoundError, OSError) as exc: + raise MoneyRefusalError("MICROUNIT_UNAVAILABLE") from exc + return packaged_current_money_resources( + inspected_version=distribution.version, inspected_module_sha256=_sha(module) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py new file mode 100644 index 000000000..71e4010d7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py @@ -0,0 +1,549 @@ +"""Typed subset of an authenticated ASEC current-money body. + +The authenticated ``ReadyCurrentMoney`` object never leaves the preparation +process. Its canonical encoded bytes travel as a typed graph artifact, and this +module parses those bytes into a *body* that carries amounts and evidence +without carrying source readiness. Selecting whole households from a body +produces a second typed artifact whose header names its parent and the exact +selected coordinates. + +Neither type subclasses ``RestatedAsecMoney``, so every existing +``type(x) is ReadyCurrentMoney`` check refuses them: a subset can never be +minted as full-source readiness, and full readiness can only originate in the +reviewed source loaders. +""" + +from __future__ import annotations + +import struct +from dataclasses import InitVar, dataclass +from hashlib import sha256 + +import numpy as np + +from microcosm.graph import ArtifactType + +from ._asec_current_money_codec import _CONTENT_DOMAIN +from ._asec_current_money_codec import MAGIC as BODY_MAGIC +from ._asec_current_money_codec import PAYLOAD_MAX_BYTES as BODY_PAYLOAD_MAX_BYTES +from .asec_current_money import ( + FIELDS, + HEADER_MAX_BYTES, + MAX_HOUSEHOLDS, + MAX_PERSONS, + RECIPE, + MoneyField, + MoneyRefusalError, + _digest, + _json, + _parse, + _require, + _sha, +) + +US_ASEC_CURRENT_MONEY_BODY_TYPE = ArtifactType( + "microcosm.us.asec_current_money_body", 1 +) +US_ASEC_PREPARED_RECEIPT_TYPE = ArtifactType("microcosm.us.asec_prepared_receipt", 3) +US_ASEC_SELECTION_TYPE = ArtifactType("microcosm.us.asec_household_selection", 2) +US_ASEC_SELECTED_MONEY_TYPE = ArtifactType( + "microcosm.us.asec_selected_current_money", 1 +) + +SELECTED_MAGIC = b"MCASELM1\x01" +SELECTED_ARTIFACT_KIND = "microcosm.us.asec_selected_current_money.v1" +_ENTITIES = ("person", "household") +_BODY_TOKEN = object() +_SELECTED_TOKEN = object() +_SELECTED_HEADER_KEYS = frozenset( + { + "schema_version", + "artifact_kind", + "release_eligible", + "recipe", + "target_year", + "state", + "semantic", + "source_authentication", + "parent_header_sha256", + "parent_content_sha256", + "prepared_receipt_sha256", + "selection_sha256", + "person_rows", + "household_rows", + "person_identity_sha256", + "household_identity_sha256", + "fields", + "field_entities", + "dtype", + "evidence_dtype", + } +) +# 32 person fields plus one household observation, four evidence axes each. +SELECTED_PAYLOAD_MAX_BYTES = ( + len(SELECTED_MAGIC) + + 4 + + HEADER_MAX_BYTES + + 11 * (32 * MAX_PERSONS + MAX_HOUSEHOLDS) + + 32 +) + + +def _entities(field_entities) -> tuple[str, ...]: + """Validate a producer-declared field/entity roster against the closed fields.""" + _require( + type(field_entities) is tuple and len(field_entities) == len(FIELDS), + "FIELD_ENTITY_ROSTER", + ) + names = [] + for item in field_entities: + _require( + type(item) is tuple + and len(item) == 2 + and type(item[0]) is str + and item[1] in _ENTITIES, + "FIELD_ENTITY_ROSTER", + ) + names.append(item[0]) + _require(tuple(names) == FIELDS, "FIELD_ENTITY_ROSTER") + return tuple(item[1] for item in field_entities) + + +def _decoded_entities(data: dict) -> tuple[str, ...]: + roster = data.get("field_entities") + _require( + type(roster) is list + and all(type(item) is list and len(item) == 2 for item in roster), + "FIELD_ENTITY_ROSTER", + ) + return _entities(tuple(tuple(item) for item in roster)) + + +def _positions(values: np.ndarray, *, rows: int, reason: str) -> np.ndarray: + _require( + type(values) is np.ndarray + and values.dtype == np.dtype("int64") + and values.ndim == 1, + reason, + ) + _require(0 < len(values) <= rows, reason) + _require((values[:-1] < values[1:]).all() if len(values) > 1 else True, reason) + _require(int(values[0]) >= 0 and int(values[-1]) < rows, reason) + return values + + +def _split( + payload: bytes, cursor: int, entities, counts: dict[str, int] +) -> tuple[MoneyField, ...]: + fields = [] + for name, entity in zip(FIELDS, entities, strict=True): + count = counts[entity] + buffers = [] + for width in (8, 1, 1, 1): + buffers.append(payload[cursor : cursor + width * count]) + cursor += width * count + fields.append(MoneyField(name, *buffers)) + return tuple(fields) + + +def _field_shapes(fields, entities, counts: dict[str, int]) -> None: + for field, entity in zip(fields, entities, strict=True): + count = counts[entity] + widths = ( + len(field.amount_bytes), + len(field.status_bytes), + len(field.validity_bytes), + len(field.zero_origin_bytes), + ) + _require(widths == (8 * count, count, count, count), "FIELD_BUFFER_SHAPE") + amounts = field.amounts + _require(np.isfinite(amounts).all(), "FIELD_ENCODING", field.name) + _require( + np.isin(field.statuses, [0, 1, 2, 3, 4]).all() + and np.isin(field.validity, [0, 1]).all() + and np.isin(field.zero_origin, [0, 1, 2]).all(), + "FIELD_ENCODING", + field.name, + ) + _require(field.validity.all(), "MISSING_REQUIRED_AMOUNT", field.name) + + +@dataclass(frozen=True) +class BoundCurrentMoneyBody: + """Amount/evidence buffers bound to a producer-declared body identity. + + This is emphatically not readiness: it carries no spec, no source evidence + and no authenticated authority, only bytes that a named producer already + authenticated inside its own process. + """ + + header: bytes + content_sha256: str + field_entities: tuple[tuple[str, str], ...] + fields: tuple[MoneyField, ...] + person_rows: int + household_rows: int + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _BODY_TOKEN, "BODY_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + return _parse(self.header, HEADER_MAX_BYTES) + + @property + def header_sha256(self) -> str: + return _sha(self.header) + + def field(self, name: str) -> MoneyField: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.fields[FIELDS.index(name)] + + def entity_of(self, name: str) -> str: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.field_entities[FIELDS.index(name)][1] + + +@dataclass(frozen=True) +class SelectedCurrentMoney: + """Whole-household subset of a body, bound to its exact selected coordinates.""" + + header: bytes + fields: tuple[MoneyField, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _SELECTED_TOKEN, "SELECTED_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + return _parse(self.header, HEADER_MAX_BYTES) + + @property + def header_sha256(self) -> str: + return _sha(self.header) + + @property + def person_rows(self) -> int: + return self.header_data["person_rows"] + + @property + def household_rows(self) -> int: + return self.header_data["household_rows"] + + def field(self, name: str) -> MoneyField: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.fields[FIELDS.index(name)] + + def entity_of(self, name: str) -> str: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.header_data["field_entities"][FIELDS.index(name)][1] + + def amounts(self, name: str) -> np.ndarray: + """Return a writable copy of one field's selected target-year amounts.""" + return self.field(name).amounts.copy() + + +def parse_current_money_body( + payload: bytes, + *, + expected_header_sha256: str, + expected_content_sha256: str, + field_entities: tuple[tuple[str, str], ...], +) -> BoundCurrentMoneyBody: + """Parse canonical money bytes without reconstructing replayable readiness. + + The caller supplies the producer's header and content digests; both must + match exactly. Digests, not header bytes, are what a producer receipt + carries across a graph edge, so binding on them keeps the consumer honest + without re-transporting the header. This deliberately cannot mint a + ``ReadyCurrentMoney``: the authenticated replay contract stays with + ``decode_current_money``. + """ + _require( + type(payload) is bytes + and len(BODY_MAGIC) + 4 + 32 < len(payload) <= BODY_PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + _require(_digest(expected_header_sha256), "EXPECTED_HEADER_DIGEST") + _require(_digest(expected_content_sha256), "EXPECTED_CONTENT_DIGEST") + entities = _entities(field_entities) + _require(payload.startswith(BODY_MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" SelectedCurrentMoney: + """Slice a body to selected positions and bind the selected coordinates.""" + _require(type(body) is BoundCurrentMoneyBody, "TYPED_BODY_REQUIRED") + _require( + all( + _digest(value) + for value in ( + prepared_receipt_sha256, + selection_sha256, + person_identity_sha256, + household_identity_sha256, + ) + ), + "SELECTION_BINDING_DIGEST", + ) + entities = _entities(body.field_entities) + positions = { + "person": _positions( + person_positions, rows=body.person_rows, reason="PERSON_POSITIONS" + ), + "household": _positions( + household_positions, + rows=body.household_rows, + reason="HOUSEHOLD_POSITIONS", + ), + } + selected = [] + for field, entity in zip(body.fields, entities, strict=True): + index = positions[entity] + selected.append( + MoneyField( + field.name, + field.amounts[index].tobytes(), + field.statuses[index].tobytes(), + field.validity[index].tobytes(), + field.zero_origin[index].tobytes(), + ) + ) + parent = body.header_data + header = _json( + { + "schema_version": 1, + "artifact_kind": SELECTED_ARTIFACT_KIND, + "release_eligible": False, + "recipe": RECIPE, + "target_year": 2024, + "state": "target_current", + "semantic": "annual_current_money", + "source_authentication": parent["source_authentication"], + "parent_header_sha256": body.header_sha256, + "parent_content_sha256": body.content_sha256, + "prepared_receipt_sha256": prepared_receipt_sha256, + "selection_sha256": selection_sha256, + "person_rows": len(positions["person"]), + "household_rows": len(positions["household"]), + "person_identity_sha256": person_identity_sha256, + "household_identity_sha256": household_identity_sha256, + "fields": list(FIELDS), + "field_entities": [list(item) for item in body.field_entities], + "dtype": " None: + data = _parse(selected.header, HEADER_MAX_BYTES) + _require(set(data) == _SELECTED_HEADER_KEYS, "SELECTED_HEADER_SCHEMA") + _require(selected.header == _json(data), "SELECTED_HEADER_SCHEMA") + _require( + data["schema_version"] == 1 + and data["artifact_kind"] == SELECTED_ARTIFACT_KIND + and data["release_eligible"] is False + and data["recipe"] == RECIPE + and data["target_year"] == 2024 + and data["state"] == "target_current" + and data["semantic"] == "annual_current_money" + and data["dtype"] == " tuple[bytes, ...]: + _require(type(selected) is SelectedCurrentMoney, "TYPED_SELECTED_REQUIRED") + _validate_selected(selected) + parts = [ + SELECTED_MAGIC, + struct.pack(" bytes: + """Encode the selected subset with a transport checksum over its bytes.""" + parts = _selected_parts(selected) + checksum = sha256() + for part in parts: + checksum.update(part) + return b"".join((*parts, checksum.digest())) + + +def decode_selected_current_money( + payload: bytes, + *, + expected_parent_header_sha256: str, + expected_parent_content_sha256: str, + expected_prepared_receipt_sha256: str, + expected_selection_sha256: str, +) -> SelectedCurrentMoney: + """Decode a selected subset only against its declared parent and selection.""" + _require( + type(payload) is bytes + and len(SELECTED_MAGIC) + 4 + 32 < len(payload) <= SELECTED_PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + _require( + all( + _digest(value) + for value in ( + expected_parent_header_sha256, + expected_parent_content_sha256, + expected_prepared_receipt_sha256, + expected_selection_sha256, + ) + ), + "SELECTION_BINDING_DIGEST", + ) + _require(payload.startswith(SELECTED_MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" 0 for value in counts.values()), + "SELECTED_ROW_BOUND", + ) + expected_size = start + size + 11 * sum(counts[entity] for entity in entities) + 32 + _require(len(payload) == expected_size, "PAYLOAD_LENGTH") + _require( + sha256(memoryview(payload)[:-32]).digest() == payload[-32:], "PAYLOAD_CHECKSUM" + ) + fields = _split(payload, start + size, entities, counts) + value = SelectedCurrentMoney(header, fields, _token=_SELECTED_TOKEN) + _validate_selected(value) + return value + + +__all__ = [ + "BoundCurrentMoneyBody", + "MoneyRefusalError", + "SELECTED_ARTIFACT_KIND", + "SELECTED_MAGIC", + "SELECTED_PAYLOAD_MAX_BYTES", + "SelectedCurrentMoney", + "US_ASEC_CURRENT_MONEY_BODY_TYPE", + "US_ASEC_PREPARED_RECEIPT_TYPE", + "US_ASEC_SELECTED_MONEY_TYPE", + "US_ASEC_SELECTION_TYPE", + "decode_selected_current_money", + "encode_selected_current_money", + "parse_current_money_body", + "select_current_money", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py new file mode 100644 index 000000000..93e846b82 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py @@ -0,0 +1,607 @@ +"""Authenticate the reviewed full ASEC v4 plus household-observation attachment. + +No public pin override, file discovery, download, source selection, or unit fit. +Verification is source authority, not evidence that all downstream consumers are +corrected or that a population is eligible for release. +""" + +import hashlib +import os +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build import frame_checkpoint as checkpoint +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.frame import Frame + +from . import asec_checkpoint, asec_household_observations, operator_boundary +from .asec_current_money import ( + _DTYPES, + _SOURCE_TOKEN, + FIELDS, + ZERO_POLICY, + AsecMoneyScope, + AsecMoneyViews, + AuthenticatedAsecSource, + CurrentMoneySpec, + MoneyRefusalError, + _index_identity, + _json, + _parse, + _require, + _sha, + classify_asec_money, + compile_asec_current_money_spec, + require_complete_current_money, + restate_asec_current_money, +) +from .asec_current_money_resources import load_current_money_resources +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +# Exact reviewed bytes from the accepted current-money source contract and the +# completed household-observation receipt. Tests monkeypatch this PRIVATE tuple +# only for invented files; production has no override/constructor for other pins. +_SOURCE_PINS = ( + "e2f2b7495bfcf1448dfb0acb0a17e93f86a6ab8bef8a70ec0a2981983028cbb5", + "f7f086e262d9a1d9ec6fc1a0b7c32e577128578a4fc5b3f14e13fb8711f7ec2d", + ( + (2022, "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e"), + (2023, "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88"), + (2024, "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d"), + ), +) +_LOAD_TOKEN = object() +_ATTACHMENT_COLUMNS = tuple( + "asec_" + c for c in asec_household_observations.ASEC_HOUSEHOLD_OBSERVATION_COLUMNS +) + + +def _stage_verified(path, expected, staging): + """Load only a verified private copy, closing same-path replacement windows.""" + digest = hashlib.sha256() + try: + with Path(path).open("rb") as src, staging.open("xb") as dst: + before = os.fstat(src.fileno()) + while chunk := src.read(1024 * 1024): + digest.update(chunk) + dst.write(chunk) + after = os.fstat(src.fileno()) + _require( + ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + == ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ), + "SOURCE_BYTES_CHANGED", + ) + _require(digest.hexdigest() == expected, "SOURCE_BYTES_MISMATCH") + except OSError: + raise MoneyRefusalError("SOURCE_BYTES_UNAVAILABLE") from None + + +def _series_digest(digest, series): + """Use the existing checkpoint's closed scalar vocabulary, one column at a time.""" + spec = checkpoint._series_spec(series, label="authenticated source") + digest.update(_json(spec)) + extension = series.array + data, mask = getattr(extension, "_data", None), getattr(extension, "_mask", None) + if isinstance(data, np.ndarray) and isinstance(mask, np.ndarray): + for array in (data, mask): + value = np.ascontiguousarray(array).tobytes() + digest.update(len(value).to_bytes(8, "little")) + digest.update(value) + elif isinstance(series.dtype, np.dtype) and not series.dtype.hasobject: + value = np.ascontiguousarray(series.to_numpy(copy=False)).tobytes() + digest.update(len(value).to_bytes(8, "little")) + digest.update(value) + else: + # Repeated literal strings are common in full survey source frames. + # Cache their exact length-prefixed bytes within this column only; every + # value is still visited and every later seal rereads the current data. + # Both entry and payload bounds keep distinct or large strings bounded. + string_tokens = {} + token_bytes = 0 + for value in series.to_numpy(dtype=object, copy=False): + if type(value) is str: + token = string_tokens.get(value) + if token is None: + encoded = checkpoint._encode_object_scalar(value) + token = len(encoded).to_bytes(8, "little") + encoded + if len(string_tokens) < 1024 and token_bytes + len(token) <= 262144: + string_tokens[value] = token + token_bytes += len(token) + digest.update(token) + continue + encoded = checkpoint._encode_object_scalar(value) + digest.update(len(encoded).to_bytes(8, "little")) + digest.update(encoded) + + +def _frame_signature(frame): + digest = hashlib.sha256(b"microcosm/asec-current-money-source-frame/1\0") + digest.update(_json(checkpoint._checkpoint_metadata(frame, {}))) + digest.update(_json(_plain_metadata(frame.metadata))) + for _, table in checkpoint._frame_tables(frame): + digest.update( + _json( + { + "columns_dtype": repr(table.columns.dtype), + "columns_type": type(table.columns).__name__, + } + ) + ) + _series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + _series_digest(digest, table[column]) + _series_digest(digest, frame.strata) + digest.update(_json(frame.strata.name)) + digest.update(_json(checkpoint._index_spec(frame.strata.index, label="strata"))) + _series_digest( + digest, + pd.Series( + frame.strata.index.to_numpy(copy=False), dtype=frame.strata.index.dtype + ), + ) + for entity in frame.weighted_entities: + digest.update(frame.weights_for(entity).values.tobytes()) + return digest.hexdigest() + + +def _owned_index(index): + """Detach axes, including pandas' materialized RangeIndex cache.""" + if isinstance(index, pd.RangeIndex): + # RangeIndex.copy(deep=True) shares its cached ndarray. Reconstruct the + # same logical range so a child cannot write through to a sealed parent. + return pd.RangeIndex(index.start, index.stop, index.step, name=index.name) + return index.copy(deep=True) + + +def _detach_frame_axes(frame): + """Detach axes of an already copied Frame; preserve its data and axis types.""" + for _, table in checkpoint._frame_tables(frame): + table.index = _owned_index(table.index) + table.columns = _owned_index(table.columns) + frame.strata.index = _owned_index(frame.strata.index) + + +def _plain_metadata(value): + if isinstance(value, Mapping): + return [ + "mapping", + [[key, _plain_metadata(item)] for key, item in sorted(value.items())], + ] + if isinstance(value, tuple): + return ["tuple", [_plain_metadata(item) for item in value]] + if isinstance(value, frozenset): + return [ + "frozenset", + sorted((_plain_metadata(item) for item in value), key=_json), + ] + return ["scalar", value] + + +def _verification_identity(): + package = resources.files(__package__) + names = ( + "asec_current_money.py", + "asec_current_money_source.py", + "asec_checkpoint.py", + "asec_household_observations.py", + "education_assistance_source.py", + "reported_coverage_source.py", + "operator_boundary.py", + ) + shared = resources.files("microcosm.build") + frame_package = resources.files("microcosm.frame") + files = {name: _sha(package.joinpath(name).read_bytes()) for name in names} + files.update( + { + "build/" + name: _sha(shared.joinpath(name).read_bytes()) + for name in ( + "frame_checkpoint.py", + "outer_stage_runtime.py", + "serialization_dtypes.py", + ) + } + ) + files.update( + { + "frame/" + path.name: _sha(path.read_bytes()) + for path in frame_package.iterdir() + if path.name.endswith(".py") + } + ) + # This exact live projection is consumed by the existing source-only guard. + projection = { + family: {entity: sorted(columns) for entity, columns in by_entity.items()} + for family, by_entity in operator_boundary.PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES.items() + } + return _sha( + _json( + { + "modules": files, + "operator_owned": projection, + "pins": _SOURCE_PINS, + "dependencies": { + n: metadata.version(n) for n in ("numpy", "pandas", "h5py") + }, + } + ) + ) + + +def _scope(frame): + person = frame.person + keys = asec_household_observations._household_keys(frame) + + def ints(column): + values = person[column] + _require( + values.dtype == np.dtype("int64") and not values.isna().any(), + "SOURCE_COORDINATE_DTYPE", + ) + return tuple(int(v) for v in values.to_numpy()) + + years = ints("source_year") + _require(set(years) == {2022, 2023, 2024}, "SOURCE_YEAR_MAPPING") + return AsecMoneyScope( + ints("person_id"), + tuple(int(v) for v in keys.household_id), + ints("person_household_id"), + ints("person_spm_unit_id"), + years, + tuple(int(v) for v in keys.source_year), + tuple( + value.decode("utf-8") if isinstance(value, bytes) else value + for value in person.PERIDNUM.tolist() + ), + tuple(str(int(v)) for v in keys.H_SEQ), + ) + + +def _views(frame, scope, *, restored=False): + person = frame.person.loc[:, [f for f in FIELDS if f != "HTOTVAL"]].copy() + if restored: + person["PTOTVAL"] = frame.person["asec_PTOTVAL"].to_numpy(copy=True) + return AsecMoneyViews( + person, + frame.table("household").loc[:, ["asec_HTOTVAL"]].copy(), + scope, + ) + + +def _input_binding(views): + scope_sha = _sha( + _json( + { + "coordinates": _parse(views.scope.identity), + "person_index": _parse(_index_identity(views.person.index)), + "household_index": _parse(_index_identity(views.household.index)), + } + ) + ) + inputs = [] + for entity, table in (("person", views.person), ("household", views.household)): + for column, series in table.items(): + field = "HTOTVAL" if entity == "household" else column + _require(str(series.dtype) in _DTYPES, "UNSUPPORTED_DTYPE", field) + values = series.to_numpy(dtype="= 0 + and row["joined_rows"] == int((keys.source_year == year).sum()) + and row["unreferenced_source_rows"] + == row["source_rows"] - row["joined_rows"] + >= 0, + "ATTACHMENT_SOURCE_COVERAGE", + ) + return cohorts + + +def _source_verification_identity(evidence): + from .asec_current_money import RESTORED_SOURCE_KIND + + if evidence["source_kind"] == RESTORED_SOURCE_KIND: + from .asec_person_income_source import _implementation + + return _implementation() + return _verification_identity() + + +@dataclass(frozen=True) +class AuthenticatedCurrentMoneySource: + frame: Frame + scope: AsecMoneyScope + source: AuthenticatedAsecSource + spec: CurrentMoneySpec + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _LOAD_TOKEN, "SOURCE_CONSTRUCTOR_UNAVAILABLE") + + def validate(self): + _require( + type(self.source) is AuthenticatedAsecSource, + "AUTHENTICATED_SOURCE_ISSUANCE", + ) + self.source._validate() + evidence = _parse(self.source.identity) + _require( + _source_verification_identity(evidence) == evidence["verification_sha256"], + "SOURCE_IMPLEMENTATION_CHANGED", + ) + try: + _require( + _frame_signature(self.frame) == evidence["frame_sha256"] + and _scope(self.frame) == self.scope, + "SOURCE_CHANGED", + ) + except MoneyRefusalError: + raise + except (ValueError, TypeError, KeyError): + raise MoneyRefusalError("SOURCE_CHANGED") from None + + def views(self): + self.validate() + from .asec_current_money import _restored_source + + return _views(self.frame, self.scope, restored=_restored_source(self.source)) + + def ready(self): + views = self.views() + decoded = classify_asec_money(views, self.spec, self.scope) + return require_complete_current_money( + restate_asec_current_money(decoded, self.spec), self.spec, production=True + ) + + +def load_authenticated_current_money_source(parent_path, attachment_path): + """Issue authority only for the two reviewed full-source files and exact scope.""" + try: + return _load(parent_path, attachment_path) + except MoneyRefusalError: + raise + except (ValueError, TypeError, KeyError, OSError): + raise MoneyRefusalError("SOURCE_CONTRACT_REFUSAL") from None + + +def _load(parent_path, attachment_path): + before = _verification_identity() + with tempfile.TemporaryDirectory(prefix="microcosm-money-source-") as directory: + directory = Path(directory) + parent_copy = directory / "parent.h5" + _stage_verified(parent_path, _SOURCE_PINS[0], parent_copy) + parent, metadata = asec_checkpoint.load_asec_raw_stage_checkpoint_v4( + parent_copy + ) + parent_copy.unlink() + attachment_copy = directory / "attachment.h5" + _stage_verified(attachment_path, _SOURCE_PINS[1], attachment_copy) + loaded = checkpoint.load_frame_checkpoint(attachment_copy) + canonicalize_frame_string_dtypes( + loaded.frame, boundary="ASEC money attachment", in_place=True + ) + cohorts = _validate_attachment(parent, metadata, loaded, _SOURCE_PINS[0]) + sidecars = [] + actual_pins = metadata["raw_source_mappings"]["ED_VAL"]["source_pins"] + _require(len(actual_pins) == 3, "ED_VAL_SOURCE_COVERAGE") + for year in (2022, 2023, 2024): + registered = ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year] + expected = { + "income_year": year, + "locator": registered.zip_url, + "member": registered.member, + "member_sha256": registered.member_sha256, + "sha256": registered.zip_sha256, + } + _require(actual_pins.count(expected) == 1, "ED_VAL_REGISTERED_PIN") + sidecars.append( + { + "income_year": year, + "survey_year": registered.survey_year, + "archive_sha256": registered.zip_sha256, + "member": registered.member, + "member_sha256": registered.member_sha256, + } + ) + frame = loaded.frame + scope = _scope(frame) + views = _views(frame, scope) + scope_sha, input_sha = _input_binding(views) + evidence = { + "schema_version": 1, + "source_kind": "asec_v4_with_household_observations_v1", + "source_authentication": "checkpoint_bytes_verified", + "parent_sha256": _SOURCE_PINS[0], + "attachment_sha256": _SOURCE_PINS[1], + "cohorts": cohorts, + "sidecars": sidecars, + "field_roster": FIELDS, + "zero_origin_policy": ZERO_POLICY, + "scope_sha256": scope_sha, + "input_sha256": input_sha, + "frame_sha256": _frame_signature(frame), + "verification_sha256": before, + } + _require(_verification_identity() == before, "SOURCE_IMPLEMENTATION_CHANGED") + authority = AuthenticatedAsecSource(_json(evidence), _token=_SOURCE_TOKEN) + spec = compile_asec_current_money_spec(load_current_money_resources(), authority) + return AuthenticatedCurrentMoneySource( + frame, scope, authority, spec, _token=_LOAD_TOKEN + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py new file mode 100644 index 000000000..3dd275752 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py @@ -0,0 +1,411 @@ +"""Actual tax-only reconstruction from verified target-current ASEC money. + +The transient view has a closed read set. All other observed source columns and +US entities remain untouched. This is not a policy impact or release evaluator. +""" + +import inspect +import sys +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes +from microcosm.frame import Frame, Weights +from microcosm.frame import units as frame_units + +from ._asec_current_money_codec import current_money_content_sha256 +from .asec_current_money import ( + MICROUNIT_VERSION, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _require, + _sha, + _validate_money, +) +from .asec_current_money_source import ( + AuthenticatedCurrentMoneySource, + _detach_frame_axes, + _frame_signature, + _same_table, + _source_verification_identity, +) +from .asec_student_controls import ( + ALIASES as STUDENT_ALIASES, +) +from .asec_student_controls import ( + CONTROLS as STUDENT_CONTROLS, +) +from .asec_student_controls import ( + StudentControlsAttachedAsec, +) + +MICROUNIT_CURRENT_MONEY_COLUMNS = tuple( + "WSAL_VAL SEMP_VAL FRSE_VAL INT_VAL DIV_VAL RNT_VAL CAP_VAL UC_VAL OI_VAL ANN_VAL PNSN_VAL SS_VAL PTOTVAL".split() +) +_STRUCTURAL_COLUMNS = ( + "PH_SEQ", + "A_LINENO", + "A_AGE", + "A_MARITL", + "A_SPOUSE", + "PEPAR1", + "PEPAR2", + "A_EXPRRP", +) +_CONTROL_COLUMNS = ( + "A_ENRLW", + "A_FTPT", + "A_HSCOL", + "PEDISDRS", + "PEDISEAR", + "PEDISEYE", + "PEDISOUT", + "PEDISPHY", + "PEDISREM", +) +MICROUNIT_CURRENT_INPUT_COLUMNS = ( + _STRUCTURAL_COLUMNS + _CONTROL_COLUMNS + MICROUNIT_CURRENT_MONEY_COLUMNS +) +TAX_PERSON_OUTPUTS = ( + "person_tax_unit_id", + "tax_unit_role_input", + "is_related_to_head_or_spouse", +) +_RESULT_TOKEN = object() + + +class MicrounitControlRefusalError(MoneyRefusalError): + """Closed control-field diagnostics without changing the money-source schema.""" + + def __init__(self, reason, column): + super().__init__(reason) + self.field = ( + column if column in _STRUCTURAL_COLUMNS + _CONTROL_COLUMNS else "contract" + ) + self.args = (f"{self.field}: {reason}",) + + +def _require_control(condition, reason, column): + if not condition: + raise MicrounitControlRefusalError(reason, column) + + +# Exact 0.1.0 import/runtime closure: __init__ imports core/diagnostics/registry; +# tax construction calls rule_helpers, whose cached threshold reads this YAML. +# No PolicyEngine package is needed for those rules. +_MICROUNIT_EXECUTION_PINS = { + "__init__.py": "6164ebb289659a027d2bdd419781bd1025483bb2baf5576487f8e3f4ac594e01", + "core.py": "95ec837e23e9a61a20bb48bcf20f87ffbfa536aaf0536706d020c6f79dc47701", + "diagnostics.py": "c1a683b98ceac7a44438653eb992240e5d9ed130119f75f69f6bed3a02e6c7c6", + "registry.py": "3ae5442a7bc637f8f786913abae97f63997baba5cedac948a80ce3b00f8cf673", + "rule_helpers.py": "7d32f1b3ab855a21ec0ca2e6d30431c4dcb030ffb250a28ef1978c50ed334243", + "tax_unit_construction.py": "79007065113f9600601433dc2633f4392c675e28b87a7383ab8a1f7c6279d47e", + "data/dependent_gross_income_limit.yaml": "3e20dd8f44d5ff919497fe38c2f68aa97a9f333539fbea7d0ca4f2c29e588da0", +} + + +def _runtime_file_bytes(path): + _require(path.stat().st_size <= 1024 * 1024, "MICROUNIT_EXECUTION_SIZE") + value = path.read_bytes() + _require(len(value) <= 1024 * 1024, "MICROUNIT_EXECUTION_SIZE") + return value + + +def _execution(): + try: + distribution = metadata.distribution("microunit") + _require(distribution.version == MICROUNIT_VERSION, "MICROUNIT_VERSION") + root = Path(distribution.locate_file("microunit")).resolve() + inventory = {str(p) for p in distribution.files or ()} + for name, expected in _MICROUNIT_EXECUTION_PINS.items(): + _require( + "microunit/" + name in inventory + and _sha(_runtime_file_bytes(root / name)) == expected, + "MICROUNIT_EXECUTION_PIN", + ) + import microunit + + _require( + Path(microunit.__file__).resolve() == root / "__init__.py", + "MICROUNIT_IMPORT_ORIGIN", + ) + _require( + inspect.getsourcefile(microunit.construct_tax_units) + == str(root / "tax_unit_construction.py"), + "MICROUNIT_IMPORT_ORIGIN", + ) + for name in _MICROUNIT_EXECUTION_PINS: + if name.endswith(".py"): + module_name = ( + "microunit" if name == "__init__.py" else "microunit." + name[:-3] + ) + module = sys.modules.get(module_name) + _require( + module is not None + and Path(module.__file__).resolve() == root / name, + "MICROUNIT_IMPORT_ORIGIN", + ) + package = resources.files(__package__) + files = { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_current_money_units.py", + "asec_current_money_source.py", + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_resources.py", + "asec_student_controls.py", + ) + } + files["frame.units"] = _sha(Path(frame_units.__file__).read_bytes()) + identity = { + "version": distribution.version, + "microunit": _MICROUNIT_EXECUTION_PINS, + "adapter_modules": files, + "dependencies": { + n: metadata.version(n) for n in ("numpy", "pandas", "pyyaml") + }, + "year": 2024, + "mode": "policyengine", + } + return microunit.construct_tax_units, _json(identity) + except (metadata.PackageNotFoundError, OSError, ImportError): + raise MoneyRefusalError("MICROUNIT_EXECUTION_UNAVAILABLE") from None + + +def current_money_microunit_view(source, ready): + """Project only reviewed controls and corrected amounts, in actual person order.""" + _require( + type(source) in (AuthenticatedCurrentMoneySource, StudentControlsAttachedAsec) + and type(ready) is ReadyCurrentMoney, + "AUTHENTICATED_MONEY_REQUIRED", + ) + expected = source.ready() + _require( + ready.bindings == expected.bindings and ready.fields == expected.fields, + "MONEY_CONTENT_MISMATCH", + ) + _require(ready.bindings.spec == source.spec, "SPEC_MISMATCH") + _validate_money(ready, source.spec, nominal=False) + _require( + all(field.validity.all() for field in ready.fields), "MISSING_REQUIRED_AMOUNT" + ) + person = source.frame.person + _require( + set(_STRUCTURAL_COLUMNS + _CONTROL_COLUMNS) <= set(person), + "MICROUNIT_INPUT_ROSTER", + ) + view = ( + person.loc[:, _STRUCTURAL_COLUMNS + _CONTROL_COLUMNS] + .copy() + .reset_index(drop=True) + ) + if type(source) is StudentControlsAttachedAsec: + # Only authenticated, code-owned aliases replace the transient logical + # controls. Original source observations, including NaNs, stay untouched. + source.validate() + for name, alias in zip(STUDENT_CONTROLS, STUDENT_ALIASES, strict=True): + view[name] = person[alias].to_numpy(copy=True) + for column in view: + values = view[column] + _require_control( + pd.api.types.is_numeric_dtype(values.dtype) + and not pd.api.types.is_bool_dtype(values.dtype), + "MICROUNIT_CONTROL_DTYPE", + column, + ) + _require_control(not values.isna().any(), "MICROUNIT_CONTROL_MISSING", column) + array = values.to_numpy(dtype=np.float64) + _require_control( + np.isfinite(array).all() and (array == np.floor(array)).all(), + "MICROUNIT_CONTROL_VALUES", + column, + ) + _require( + np.array_equal(view.PH_SEQ.to_numpy(), person.person_household_id.to_numpy()), + "MICROUNIT_HOUSEHOLD_BINDING", + ) + _require( + not view[["PH_SEQ", "A_LINENO"]].duplicated().any() + and (view.A_LINENO > 0).all(), + "MICROUNIT_PERSON_LINES", + ) + for field in MICROUNIT_CURRENT_MONEY_COLUMNS: + view[field] = ready.field(field).amounts.copy() + _require( + tuple(view.columns) == MICROUNIT_CURRENT_INPUT_COLUMNS, "MICROUNIT_INPUT_ROSTER" + ) + return view + + +def _partitions(frame): + grouped = {} + for person, group in zip( + frame.person.person_id, frame.person.person_tax_unit_id, strict=True + ): + grouped.setdefault(int(group), []).append(int(person)) + return tuple(sorted(tuple(sorted(members)) for members in grouped.values())) + + +@dataclass(frozen=True) +class CurrentMoneyTaxUnitResult: + frame: Frame + money: ReadyCurrentMoney + _receipt: bytes + _token: InitVar[object] = None + _student_source: StudentControlsAttachedAsec | None = None + + def __post_init__(self, _token): + _require(_token is _RESULT_TOKEN, "TAX_RESULT_CONSTRUCTOR_UNAVAILABLE") + + @property + def receipt(self): + return _parse(self._receipt) + + def validate(self): + """Check the permitted tax-only reattachment before consuming a live result.""" + receipt = self.receipt + if self._student_source is not None: + _require( + type(self._student_source) is StudentControlsAttachedAsec, + "STUDENT_TAX_PARENT", + ) + self._student_source.validate() + _require( + self.money.bindings == self._student_source.money.bindings + and self.money.fields == self._student_source.money.fields + and receipt["student_controls"] == self._student_source.receipt, + "STUDENT_TAX_PARENT", + ) + else: + _require(receipt["student_controls"] is None, "STUDENT_TAX_PARENT") + _validate_money(self.money, self.money.bindings.spec, nominal=False) + source_evidence = _parse(self.money.bindings.spec.source.identity) + _require( + _source_verification_identity(source_evidence) + == source_evidence["verification_sha256"], + "SOURCE_IMPLEMENTATION_CHANGED", + ) + try: + frame_signature = _frame_signature(self.frame) + except (ValueError, TypeError, KeyError, AssertionError): + raise MoneyRefusalError("TAX_RESULT_CHANGED") from None + _require( + _sha(self.money.header) == receipt["money_header_sha256"] + and current_money_content_sha256(self.money) + == receipt["money_content_sha256"] + and frame_signature == receipt["output_frame_sha256"], + "TAX_RESULT_CHANGED", + ) + _require( + _execution()[1] == _json(receipt["implementation"]), + "MICROUNIT_EXECUTION_CHANGED", + ) + + +def reconstruct_current_money_tax_units(source, ready): + """Run the pinned real engine, graft tax outputs, and verify preservation.""" + try: + return _reconstruct(source, ready) + except MoneyRefusalError: + raise + except (ValueError, TypeError, KeyError, AssertionError): + raise MoneyRefusalError("TAX_RECONSTRUCTION_REFUSAL") from None + + +def _reconstruct(source, ready): + view = current_money_microunit_view(source, ready) + construct, implementation = _execution() + original = source.frame + assignments, tax_unit = frame_units._construct_tax_units( + construct, view, year=2024, mode="policyengine" + ) + tax_unit = canonicalize_table_string_dtypes( + tax_unit, + boundary="ASEC current-money tax reconstruction", + table_name="tax_unit", + ) + _require( + assignments.index.equals(view.index) and len(assignments) == len(view), + "MICROUNIT_OUTPUT_ROWS", + ) + person = original.person.copy(deep=True) + for target, output in zip( + TAX_PERSON_OUTPUTS, + ("TAX_ID", "tax_unit_role_input", "is_related_to_head_or_spouse"), + strict=True, + ): + dtype = person[target].dtype if target in person else assignments[output].dtype + person[target] = pd.array(assignments[output].to_numpy(copy=True), dtype=dtype) + tables = {entity: original.table(entity) for entity in original.entities} + tables["person"] = person + tables["tax_unit"] = tax_unit + result = Frame( + tables, + original.schema, + { + entity: Weights( + original.weights_for(entity).values, original.weights_for(entity).kind + ) + for entity in original.weighted_entities + }, + original.strata, + mass_log=original.mass_log, + metadata=original.metadata, + ) + _detach_frame_axes(result) + source.validate() + _require(_execution()[1] == implementation, "MICROUNIT_EXECUTION_CHANGED") + for entity in original.entities: + if entity not in ("person", "tax_unit"): + _same_table(result.table(entity), original.table(entity)) + untouched = [name for name in original.person if name not in TAX_PERSON_OUTPUTS] + _same_table(result.person[untouched], original.person[untouched]) + pd.testing.assert_series_equal(result.strata, original.strata, check_exact=True) + _require( + result.metadata == original.metadata and result.mass_log == original.mass_log, + "TAX_CONTEXT_PRESERVATION", + ) + for entity in original.weighted_entities: + _require( + result.weights_for(entity).kind == original.weights_for(entity).kind + and result.weights_for(entity).values.tobytes() + == original.weights_for(entity).values.tobytes(), + "TAX_WEIGHT_PRESERVATION", + ) + receipt = { + "schema_version": 1, + "artifact_kind": "microcosm.asec_current_money_tax_units", + "release_eligible": False, + "all_current_money_consumers_wired": False, + "year": 2024, + "mode": "policyengine", + "money_header_sha256": _sha(ready.header), + "money_content_sha256": current_money_content_sha256(ready), + "source_frame_sha256": _parse(source.source.identity)["frame_sha256"], + "student_controls": source.receipt + if type(source) is StudentControlsAttachedAsec + else None, + "output_frame_sha256": _frame_signature(result), + "implementation": _parse(implementation), + "person_outputs": TAX_PERSON_OUTPUTS, + "tax_table_outputs": tuple(tax_unit.columns), + "old_tax_units": original.n("tax_unit"), + "new_tax_units": result.n("tax_unit"), + "old_partition_sha256": _sha(_json(_partitions(original))), + "new_partition_sha256": _sha(_json(_partitions(result))), + "scope_sha256": _parse(ready.header)["scope_sha256"], + } + return CurrentMoneyTaxUnitResult( + result, + ready, + _json(receipt), + _token=_RESULT_TOKEN, + _student_source=source if type(source) is StudentControlsAttachedAsec else None, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py new file mode 100644 index 000000000..0a227ebb3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py @@ -0,0 +1,1519 @@ +"""Explicit ASEC demographic source contract: sex, its allocation flag, and the +household reference-person code, restored from the pinned original CSV cohorts. + +Scope fence. This module declares and implements the contract and its reader. +It is deliberately **not** wired into +:mod:`.asec_prepared_source`: the root adjudication of 2026-09-06 authorised +preparing this source code and tiny fixtures independently and authorised no +additional full-source read and no new binding. Nothing here is executed +against a genuine Census member by this lane. + +What the contract refuses to assume, stated once so a later reader cannot +mistake an omission for an oversight: + +* A declared value range is not measured proof that every delivered token is + inside it. ``A_SEX`` is validated against its own printed codes, and a token + that is not a printed code leaves that person's sex **unbound**. Nothing here + maps "not 2" onto male. +* ``AXSEX`` prints two codes inside a wider declared range. A token in the + range but outside the printed codes is an unresolved allocation provenance, + so it leaves that person's sex unbound rather than being smoothed into + "reported". +* ``A_EXPRRP`` codes 1 and 2 are the printed, self-labelled household + reference person. They are the only admitted head evidence here. +* ``P_SEQ`` labels no code in its own dictionary entry. It is carried and + diagnosed against the reference code, and it is never a fallback: a + household whose reference code is absent, duplicated or indeterminate stays + unbound even when exactly one of its members has ``P_SEQ == 1``. +* ``A_FAMREL`` is a **family** relationship and is not read here. ``A_LINENO`` + is a Basic-CPS roster line number read only as a join crosscheck. Neither + substitutes for the household reference code. + +Every printed fact below was read this session from the three pinned public +dictionaries named in :data:`ASEC_DICTIONARY_AUTHORITY`, at the PDF pages +recorded there, and the four entries are textually identical across the three +years. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import struct +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import asec_current_money_source as legacy +from . import asec_person_income_source as restoration +from .asec_student_controls import _snapshot +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +MAGIC = b"MCASDEMO\x01" +FILENAME = "asec_demographic_source.bin" +ARTIFACT_KIND = "microcosm.asec_demographic_source.v1" +_HEADER_MAX = 262_144 +_MAX_PERSONS = 600_000 +_TOKEN = object() +_COMPOSE_TOKEN = object() + + +class DemographicSourceRefusalError(ValueError): + """Value-free refusal; never carries a row, an identifier or an amount.""" + + def __init__(self, reason: str, field: str = "contract"): + self.reason = reason + self.field = field + super().__init__(f"{field}: {reason}") + + +def _require(condition: bool, reason: str, field: str = "contract") -> None: + if not condition: + raise DemographicSourceRefusalError(reason, field) + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +_ASEC_DICTIONARY_URL_TEMPLATE = ( + "https://www2.census.gov/programs-surveys/cps/datasets/" + "{survey_year}/march/asec{survey_year}_ddl_pub_full.pdf" +) +#: Survey year -> the pinned public-use dictionary that prints these entries. +#: The three digests are the same documents the reviewed household +#: reported-income contract already pins; no new pin is introduced here. +ASEC_DICTIONARY_AUTHORITY: Mapping[int, Mapping[str, object]] = MappingProxyType( + { + 2023: MappingProxyType( + { + "url": _ASEC_DICTIONARY_URL_TEMPLATE.format(survey_year=2023), + "sha256": ( + "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117" + ), + } + ), + 2024: MappingProxyType( + { + "url": _ASEC_DICTIONARY_URL_TEMPLATE.format(survey_year=2024), + "sha256": ( + "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840" + ), + } + ), + 2025: MappingProxyType( + { + "url": _ASEC_DICTIONARY_URL_TEMPLATE.format(survey_year=2025), + "sha256": ( + "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f" + ), + } + ), + } +) + + +@dataclass(frozen=True, eq=False) +class AsecDemographicField: + """One printed dictionary entry, restated exactly, with its own role. + + ``named_codes`` holds only codes the dictionary actually prints. Codes + inside ``printed_range`` that it does not print are exposed as + :attr:`unnamed_in_range_codes` and are never treated as a named value. + """ + + name: str + concept: str + printed_length: int + printed_range: tuple[int, int] + printed_position: int + named_codes: Mapping[int, str] + universe_as_printed: str + role: str + quoted_locator: str + pdf_page_1based_by_survey_year: Mapping[int, int] + + @property + def declared_domain(self) -> tuple[int, ...]: + low, high = self.printed_range + return tuple(range(low, high + 1)) + + @property + def unnamed_in_range_codes(self) -> tuple[int, ...]: + return tuple( + code for code in self.declared_domain if code not in self.named_codes + ) + + @property + def token_pattern(self) -> str: + return rf"[0-9]{{1,{self.printed_length}}}" + + def document(self) -> dict: + return { + "name": self.name, + "concept": self.concept, + "printed_length": self.printed_length, + "printed_range": list(self.printed_range), + "printed_position": self.printed_position, + "named_codes": { + str(code): self.named_codes[code] for code in sorted(self.named_codes) + }, + "unnamed_in_range_codes": list(self.unnamed_in_range_codes), + "universe_as_printed": self.universe_as_printed, + "role": self.role, + "quoted_locator": self.quoted_locator, + "token_pattern": self.token_pattern, + "authority": [ + { + "survey_year": year, + "url": ASEC_DICTIONARY_AUTHORITY[year]["url"], + "sha256": ASEC_DICTIONARY_AUTHORITY[year]["sha256"], + "pdf_page_1based": page, + } + for year, page in sorted(self.pdf_page_1based_by_survey_year.items()) + ], + } + + +A_SEX = AsecDemographicField( + name="A_SEX", + concept="Sex", + printed_length=1, + printed_range=(1, 2), + printed_position=92, + named_codes=MappingProxyType({1: "Male", 2: "Female"}), + universe_as_printed="All Persons", + role="restored_observation", + quoted_locator="A_SEX / Sex / 1 (1:2) 92 / Values: 1 = Male 2 = Female / " + "Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 22, 2024: 23, 2025: 24}), +) +AXSEX = AsecDemographicField( + name="AXSEX", + concept="Allocation flag for A_SEX", + printed_length=1, + printed_range=(0, 4), + printed_position=161, + named_codes=MappingProxyType({0: "No change", 4: "Allocated"}), + universe_as_printed="All Persons", + role="restored_observation", + quoted_locator="AXSEX / Allocation flag for A_SEX / 1 (0:4) 161 / " + "Values: 0 = No change 4 = Allocated / Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 27, 2024: 28, 2025: 29}), +) +A_EXPRRP = AsecDemographicField( + name="A_EXPRRP", + concept="Expanded relationship code", + printed_length=2, + printed_range=(1, 14), + printed_position=82, + named_codes=MappingProxyType( + { + 1: "Reference person with relatives", + 2: "Reference person without relatives", + 3: "Husband", + 4: "Wife", + 5: "Own child", + 7: "Grandchild", + 8: "Parent", + 9: "Brother/sister", + 10: "Other relative", + 11: "Foster child", + 12: "Nonrelative with relatives", + 13: "Partner/roommate", + 14: "Nonrelative without relatives", + } + ), + universe_as_printed="All Persons", + role="restored_observation", + quoted_locator="A_EXPRRP / Expanded relationship code / 2 (1:14) 82 / " + "Values: 1 = Reference person with relatives 2 = Reference person " + "without relatives 3 = Husband 4 = Wife 5 = Own child 7 = Grandchild " + "8 = Parent 9 = Brother/sister 10 = Other relative 11 = Foster child " + "12 = Nonrelative with relatives 13 = Partner/roommate 14 = Nonrelative " + "without relatives / Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 21, 2024: 22, 2025: 23}), +) +P_SEQ = AsecDemographicField( + name="P_SEQ", + concept="Sequence number of person in hhld", + printed_length=2, + printed_range=(0, 16), + printed_position=10, + named_codes=MappingProxyType({}), + universe_as_printed="All Persons", + role="diagnostic_crosscheck", + quoted_locator="P_SEQ / Sequence number of person in hhld / 2 (00:16) 10 / " + "Values: 0-16 / Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 20, 2024: 21, 2025: 22}), +) + +#: The four printed entries this contract covers, in output order. +ASEC_DEMOGRAPHIC_FIELDS: tuple[AsecDemographicField, ...] = ( + A_SEX, + AXSEX, + A_EXPRRP, + P_SEQ, +) +OBSERVATIONS: tuple[str, ...] = tuple(field.name for field in ASEC_DEMOGRAPHIC_FIELDS) +ALIASES: tuple[str, ...] = tuple(f"asec_{name}" for name in OBSERVATIONS) +#: The only codes admitted as household reference-person evidence. Both are +#: printed, self-labelled reference-person codes; the split is with/without +#: relatives and is irrelevant to headship itself. +HOUSEHOLD_REFERENCE_CODES: tuple[int, ...] = (1, 2) + +#: Signals deliberately not adopted as semantic proof of headship, each with +#: the reason it is refused. Declared so the refusal survives a later reader. +NON_ADOPTED_HEAD_SIGNALS: tuple[Mapping[str, str], ...] = ( + MappingProxyType( + { + "signal": "P_SEQ == 1", + "kind": "positional_sequence_within_household", + "refusal": ( + "P_SEQ's own dictionary entry labels no code, including 1, as " + "reference person or household head; the ordering statement " + "that would support it is expressly limited to the ASCII file " + "while this restoration reads the CSV member" + ), + "carried_as": "diagnostic_crosscheck", + } + ), + MappingProxyType( + { + "signal": "A_FAMREL == 1", + "kind": "family_relationship", + "refusal": ( + "A_FAMREL is a family relationship scoped to primary-family " + "membership; a subfamily reference person is not its code 1, " + "so it is not a household role and is not read here" + ), + "carried_as": "not_read", + } + ), + MappingProxyType( + { + "signal": "A_LINENO == 1", + "kind": "basic_cps_roster_line_number", + "refusal": ( + "A_LINENO is a separately assigned Basic-CPS roster line " + "number with no source guarantee of row-for-row agreement " + "with any ASEC sequence; it is read only as a join crosscheck" + ), + "carried_as": "join_crosscheck", + } + ), +) + +#: Person-level derived production fields, with their declared code meanings. +SEX_BINDING_STATES: Mapping[int, str] = MappingProxyType( + {0: "unbound", 1: "bound_male", 2: "bound_female"} +) +SEX_UNBOUND_REASONS: Mapping[int, str] = MappingProxyType( + { + 0: "bound", + 1: "sex_code_outside_printed_codes", + 2: "allocation_flag_outside_printed_codes", + } +) +SEX_ALLOCATION_STATES: Mapping[int, str] = MappingProxyType( + {0: "undetermined", 1: "no_change", 2: "census_allocated"} +) +HOUSEHOLD_REFERENCE_STATES: Mapping[int, str] = MappingProxyType( + { + 0: "unbound", + 1: "bound_household_reference_person", + 2: "bound_not_household_reference_person", + } +) +HOUSEHOLD_REFERENCE_UNBOUND_REASONS: Mapping[int, str] = MappingProxyType( + { + 0: "bound", + 1: "relationship_code_outside_printed_codes", + 2: "household_reference_absent", + 3: "household_reference_duplicated", + 4: "household_relationship_code_indeterminate", + } +) +DERIVED: tuple[str, ...] = ( + "asec_sex_binding_state", + "asec_sex_unbound_reason", + "asec_sex_allocation_state", + "asec_household_reference_state", + "asec_household_reference_unbound_reason", +) + +COORDINATES: tuple[str, ...] = ( + "person_id", + "income_year", + "source_household_id", + "A_LINENO", + "A_AGE", +) +#: Every column the artifact carries, in body order. +COLUMNS: tuple[str, ...] = COORDINATES + ALIASES + DERIVED +#: The exact readset: the only columns read from any Census member here. +ASEC_DEMOGRAPHIC_SOURCE_COLUMNS: tuple[str, ...] = ( + "PERIDNUM", + "PH_SEQ", + "A_LINENO", + "A_AGE", + *OBSERVATIONS, +) +_COORDINATE_COLUMNS = ("PH_SEQ", "A_LINENO", "A_AGE") +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +#: Income year -> survey year. The observation is the interview household one +#: year after the income year; this module harmonizes no period. +COHORTS: Mapping[int, int] = MappingProxyType( + {year: year + 1 for year, *_rest in _MEMBER_PINS} +) +OBSERVED_PERIOD_KIND = "interview_household_one_year_after_income_year" + + +def _code_document(codes: Mapping[int, str]) -> dict: + """Codes are documented with string keys so a JSON replay compares equal.""" + return {str(code): codes[code] for code in sorted(codes)} + + +def contract_document() -> dict: + """The full declared contract, as it is recorded in every receipt.""" + return { + "schema_version": 1, + "artifact_kind": ARTIFACT_KIND, + "fields": [field.document() for field in ASEC_DEMOGRAPHIC_FIELDS], + "readset": list(ASEC_DEMOGRAPHIC_SOURCE_COLUMNS), + "missing_token_policy": "refuse_source_member_without_filling_or_coercion", + "incumbent_relationship_policy": ( + "A_EXPRRP presence does not attest source observation; compare and " + "diagnose discrepancies without replacing or trusting the incumbent recode" + ), + "production_fields": { + "restored_observations": [ + { + "column": alias, + "source_column": name, + "role": field.role, + "dtype": "int64", + } + for alias, name, field in zip( + ALIASES, OBSERVATIONS, ASEC_DEMOGRAPHIC_FIELDS, strict=True + ) + ], + "derived": [ + { + "column": "asec_sex_binding_state", + "dtype": "int64", + "codes": _code_document(SEX_BINDING_STATES), + }, + { + "column": "asec_sex_unbound_reason", + "dtype": "int64", + "codes": _code_document(SEX_UNBOUND_REASONS), + }, + { + "column": "asec_sex_allocation_state", + "dtype": "int64", + "codes": _code_document(SEX_ALLOCATION_STATES), + }, + { + "column": "asec_household_reference_state", + "dtype": "int64", + "codes": _code_document(HOUSEHOLD_REFERENCE_STATES), + }, + { + "column": "asec_household_reference_unbound_reason", + "dtype": "int64", + "codes": _code_document(HOUSEHOLD_REFERENCE_UNBOUND_REASONS), + }, + ], + }, + "household_reference_codes": list(HOUSEHOLD_REFERENCE_CODES), + "non_adopted_head_signals": [dict(item) for item in NON_ADOPTED_HEAD_SIGNALS], + "household_and_family_roles_separate": True, + "acs_diagnostic_reference_contract": { + "source_column": "RELSHIPP", + "declared_range": [20, 38], + "housing_unit_reference_code": 20, + "group_quarters_codes": { + "37": "institutionalized", + "38": "noninstitutionalized", + }, + "authority": { + "url": "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf", + "sha256": "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff", + "pdf_page_1based": 43, + }, + "production_acs_values_changed": False, + }, + "family_relationship_field_read": False, + "line_number_used_for_headship": False, + "sex_binding_requires": [ + "A_SEX in printed codes", + "AXSEX in printed codes", + ], + "household_reference_binding_requires": [ + "every A_EXPRRP in the household inside printed codes", + "exactly one household member carrying code 1 or 2", + ], + "household_membership_verification": { + "grouping_column": HOUSEHOLD_GROUPING_COLUMN, + "published_household_key": list(HOUSEHOLD_MEMBERSHIP_KEY), + "equivalence": "bidirectional_partition_bijection", + "verified_before_classification": True, + "merged_published_households": "refused", + "split_published_households": "refused", + "native_household_id_reuse_across_cohorts": "admitted_and_counted", + }, + "other_than_two_is_male": False, + "cohorts": { + str(year): { + "income_year": year, + "survey_year": survey, + "observed_period_kind": OBSERVED_PERIOD_KIND, + "aged_from_a_date": False, + } + for year, survey in sorted(COHORTS.items()) + }, + "release_eligible": False, + "wired_into_prepared_source": False, + } + + +def _implementation() -> dict: + package = resources.files(__package__) + return { + "schema": 1, + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_demographic_source.py", + "asec_person_income_source.py", + "asec_student_controls.py", + ) + }, + "legacy_verification": legacy._verification_identity(), + "dependencies": {n: metadata.version(n) for n in ("numpy", "pandas")}, + "member_pins": _MEMBER_PINS, + "contract": contract_document(), + } + + +def read_demographic_member(path, *, rows: int, size: int) -> pd.DataFrame: + """Parse one member's readset as exact tokens; never fill, never coerce. + + Lexical admission comes from the dictionary's own printed field length: + a token must be digits only and no longer than the printed width, so a + two-character ``A_SEX`` or a signed token is refused here rather than + reaching the domain classification as a plausible code. + """ + path = Path(path) + _require(path.stat().st_size == size, "MEMBER_SIZE") + with path.open("r", encoding="utf-8", newline="") as handle: + header = next(csv.reader(handle), []) + _require( + len(header) == len(set(header)) + and set(ASEC_DEMOGRAPHIC_SOURCE_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + table = pd.read_csv( + path, + usecols=list(ASEC_DEMOGRAPHIC_SOURCE_COLUMNS), + dtype="string", + na_filter=False, + ) + _require(len(table) == rows, "MEMBER_ROWS") + _require( + bool(table.PERIDNUM.str.fullmatch(r"[0-9]{22}").all()), "MEMBER_PERSON_KEY" + ) + for name in _COORDINATE_COLUMNS: + _require( + bool(table[name].str.fullmatch(r"[0-9]+").all()), + "MEMBER_INTEGER", + name, + ) + table[name] = table[name].astype("int64") + for field in ASEC_DEMOGRAPHIC_FIELDS: + _require( + bool(table[field.name].str.fullmatch(field.token_pattern).all()), + "MEMBER_TOKEN_WIDTH", + field.name, + ) + table[field.name] = table[field.name].astype("int64") + _require( + not table.PERIDNUM.duplicated().any() + and not table.duplicated(["PH_SEQ", "A_LINENO"]).any() + and bool((table.PH_SEQ > 0).all()) + and bool((table.A_LINENO > 0).all()), + "MEMBER_COORDINATES", + ) + return table + + +def _counts(codes: np.ndarray, meanings: Mapping[int, str]) -> dict: + return {meanings[code]: int((codes == code).sum()) for code in sorted(meanings)} + + +#: The published household key. ``A_EXPRRP`` names a person's role inside the +#: household the Census published, and that household is identified by its +#: native ``PH_SEQ`` **within one income year**: an unrelated household in +#: another year legitimately carries the same native id. +HOUSEHOLD_MEMBERSHIP_KEY: tuple[str, ...] = ("income_year", "source_household_id") +#: The parent roster column headship is grouped by. It is a remapped id, not +#: the published one, so its partition is proven equal to the key above rather +#: than assumed to be. +HOUSEHOLD_GROUPING_COLUMN = "person_household_id" +#: Counters below that any nonzero value would have refused before an artifact +#: exists. Named so a later reader cannot read their zero as a measurement. +MEMBERSHIP_REFUSAL_COUNTERS: tuple[str, ...] = ( + "grouping_households_spanning_multiple_cohorts", + "grouping_households_spanning_multiple_published_households", + "published_households_spanning_multiple_grouping_households", +) +ZERO_COUNTER_SEMANTICS = ( + "derived from the delivered values, but any nonzero value refuses the run " + "before an artifact exists, so a zero in a successful receipt attests that " + "refusal and is not an independent measurement of the source" +) + + +def _dense_codes(*keys: np.ndarray) -> tuple[np.ndarray, int]: + """Number the distinct key tuples 0..n-1 without combining them numerically. + + Key columns are compared component-wise after a lexicographic sort, so no + two coordinates are ever multiplied or concatenated into one integer and + no width assumption can silently collide two distinct households. + """ + order = np.lexsort(tuple(reversed(keys))) + changed = np.zeros(len(order), dtype=bool) + changed[0] = True + for values in keys: + sorted_values = values[order] + changed[1:] |= sorted_values[1:] != sorted_values[:-1] + codes = np.empty(len(order), dtype=np.int64) + codes[order] = np.cumsum(changed) - 1 + return codes, int(changed.sum()) + + +def _representatives(codes: np.ndarray, count: int) -> np.ndarray: + """One arbitrary but deterministic row position per distinct code.""" + positions = np.zeros(count, dtype=np.int64) + positions[codes] = np.arange(len(codes), dtype=np.int64) + return positions + + +def verify_household_membership( + *, + income_year: np.ndarray, + source_household_id: np.ndarray, + household_id: np.ndarray, +) -> dict: + """Prove the grouping column partitions persons exactly as the publisher did. + + Household reference status is a statement about the household the Census + published: the cardinality of ``A_EXPRRP`` 1/2 is evidence only inside + ``(income_year, source_household_id)``. Classification groups instead by + the parent roster's remapped :data:`HOUSEHOLD_GROUPING_COLUMN`, and a + per-row equality between that roster's ``source_household_id`` and the + member's ``PH_SEQ`` says nothing about how either column *groups*. So the + two partitions are proven identical in both directions here, before any + household verdict is formed. + + Refused in both directions: a grouping household carrying persons from + more than one published household (a merge, whether inside one cohort or + across cohorts), and a published household split across more than one + grouping household. Admitted and counted: the same native household id + reused by unrelated households in different income years, which is + ordinary and is what makes the income year part of the key. + + Returns the aggregate verification recorded in the artifact receipt. It + carries counts, column names and cohort years only; no person, household + or native identifier appears in it or in any refusal raised here. + """ + rows = len(income_year) + _require(0 < rows <= _MAX_PERSONS, "MEMBERSHIP_ROWS") + arrays = { + "income_year": income_year, + "source_household_id": source_household_id, + HOUSEHOLD_GROUPING_COLUMN: household_id, + } + for name, values in arrays.items(): + _require( + isinstance(values, np.ndarray) + and values.dtype == np.dtype("int64") + and values.shape == (rows,), + "MEMBERSHIP_INPUT", + name, + ) + _require( + bool(np.isin(income_year, list(COHORTS)).all()), + "MEMBERSHIP_COHORT", + "income_year", + ) + for name in ("source_household_id", HOUSEHOLD_GROUPING_COLUMN): + _require(bool((arrays[name] > 0).all()), "MEMBERSHIP_COORDINATE", name) + + published, published_count = _dense_codes(income_year, source_household_id) + grouping, grouping_count = _dense_codes(household_id) + edges, edge_count = _dense_codes(grouping, published) + # The relation between the two partitions is exactly its distinct + # (grouping household, published household) pairs; take one row per pair. + edge_rows = _representatives(edges, edge_count) + edge_grouping = grouping[edge_rows] + edge_published = published[edge_rows] + edge_year = income_year[edge_rows] + + _, grouping_degree = np.unique(edge_grouping, return_counts=True) + _, published_degree = np.unique(edge_published, return_counts=True) + cohort_edges, cohort_edge_count = _dense_codes(edge_grouping, edge_year) + _, cohort_degree = np.unique( + edge_grouping[_representatives(cohort_edges, cohort_edge_count)], + return_counts=True, + ) + native_rows = _representatives(published, published_count) + _, native_degree = np.unique(source_household_id[native_rows], return_counts=True) + + collisions = int((cohort_degree > 1).sum()) + merged = int((grouping_degree > 1).sum()) + split = int((published_degree > 1).sum()) + _require(collisions == 0, "MEMBERSHIP_COHORT_COLLISION", HOUSEHOLD_GROUPING_COLUMN) + _require( + merged == 0, "MEMBERSHIP_MERGED_NATIVE_HOUSEHOLDS", HOUSEHOLD_GROUPING_COLUMN + ) + _require( + split == 0, "MEMBERSHIP_SPLIT_NATIVE_HOUSEHOLDS", HOUSEHOLD_GROUPING_COLUMN + ) + # A relation whose edge count equals both vertex counts, with every vertex + # incident to at least one edge by construction, is a bijection. Asserted + # independently of the three counters above so a gap in any one of them + # still cannot let an unequal partition through. + _require( + edge_count == grouping_count == published_count, + "MEMBERSHIP_NOT_BIJECTIVE", + HOUSEHOLD_GROUPING_COLUMN, + ) + + return { + "schema_version": 1, + "grouping_column": HOUSEHOLD_GROUPING_COLUMN, + "published_household_key": list(HOUSEHOLD_MEMBERSHIP_KEY), + "equivalence": "bidirectional_partition_bijection", + "verified_before_classification": True, + "rows": rows, + "grouping_households": grouping_count, + "published_households": published_count, + "membership_edges": edge_count, + # Recomputable from the three counts recorded immediately above it. + "bijective": edge_count == grouping_count == published_count, + "grouping_households_spanning_multiple_cohorts": collisions, + "grouping_households_spanning_multiple_published_households": merged, + "published_households_spanning_multiple_grouping_households": split, + "native_household_ids_reused_across_cohorts": int((native_degree > 1).sum()), + "native_household_id_reuse_across_cohorts_is_admitted": True, + "by_cohort": { + str(year): { + "income_year": int(year), + "survey_year": int(COHORTS[int(year)]), + "rows": int((income_year == year).sum()), + "grouping_households": int( + np.unique(household_id[income_year == year]).size + ), + "published_households": int( + np.unique(published[income_year == year]).size + ), + "native_household_ids": int( + np.unique(source_household_id[income_year == year]).size + ), + } + for year in sorted({int(value) for value in np.unique(income_year)}) + }, + "counters_zero_in_every_successful_receipt": list(MEMBERSHIP_REFUSAL_COUNTERS), + "zero_counter_semantics": ZERO_COUNTER_SEMANTICS, + } + + +@dataclass(frozen=True) +class DemographicClassification: + """Per-person states plus value-free counts. Holds no source identifier.""" + + states: Mapping[str, np.ndarray] + diagnostics: dict + + +def classify_asec_demographic_observations( + *, + income_year: np.ndarray, + household_id: np.ndarray, + a_sex: np.ndarray, + axsex: np.ndarray, + a_exprrp: np.ndarray, + p_seq: np.ndarray, +) -> DemographicClassification: + """Bind only what the printed codes and the household cardinality support. + + Sex binds when the sex code is printed and its allocation flag is printed; + the allocation state is carried alongside, never folded into the value. + Household headship binds when every relationship code in the household is + printed and exactly one member carries a reference-person code. Every other + person is reported unbound with the reason, and ``P_SEQ`` is diagnosed + against the result without ever supplying it. + + ``household_id`` is the grouping column, and the reference-person + cardinality this reads is only evidence about the household the Census + published. The caller must therefore have proven that grouping equal to + ``(income_year, source_household_id)`` with + :func:`verify_household_membership` first; the cohort-uniformity check + below is a weaker self-check and is not that proof. + """ + arrays = { + "income_year": income_year, + "household_id": household_id, + "A_SEX": a_sex, + "AXSEX": axsex, + "A_EXPRRP": a_exprrp, + "P_SEQ": p_seq, + } + rows = len(income_year) + _require(0 < rows <= _MAX_PERSONS, "CLASSIFY_ROWS") + for name, values in arrays.items(): + _require( + isinstance(values, np.ndarray) + and values.dtype == np.dtype("int64") + and values.shape == (rows,), + "CLASSIFY_INPUT", + name, + ) + _require( + bool(np.isin(income_year, list(COHORTS)).all()) + and bool((household_id > 0).all()), + "CLASSIFY_COORDINATES", + ) + + sex_state = np.zeros(rows, dtype=np.int64) + sex_reason = np.zeros(rows, dtype=np.int64) + allocation = np.zeros(rows, dtype=np.int64) + named_sex = np.isin(a_sex, list(A_SEX.named_codes)) + named_flag = np.isin(axsex, list(AXSEX.named_codes)) + allocation[named_flag & (axsex == 0)] = 1 + allocation[named_flag & (axsex == 4)] = 2 + admissible = named_sex & named_flag + sex_state[admissible & (a_sex == 1)] = 1 + sex_state[admissible & (a_sex == 2)] = 2 + sex_reason[~named_sex] = 1 + sex_reason[named_sex & ~named_flag] = 2 + + named_relationship = np.isin(a_exprrp, list(A_EXPRRP.named_codes)) + reference = np.isin(a_exprrp, list(HOUSEHOLD_REFERENCE_CODES)) + head_state = np.zeros(rows, dtype=np.int64) + head_reason = np.zeros(rows, dtype=np.int64) + order = np.argsort(household_id, kind="stable") + grouped = household_id[order] + starts = np.flatnonzero(np.concatenate(([True], grouped[1:] != grouped[:-1]))) + bounds = np.append(starts, len(grouped)) + households = len(starts) + household_verdicts = np.zeros(households, dtype=np.int64) + for index in range(households): + positions = order[bounds[index] : bounds[index + 1]] + _require( + len(np.unique(income_year[positions])) == 1, "CLASSIFY_HOUSEHOLD_COHORT" + ) + unnamed = ~named_relationship[positions] + if unnamed.any(): + head_reason[positions] = 4 + head_reason[positions[unnamed]] = 1 + household_verdicts[index] = 4 + continue + references = positions[reference[positions]] + if len(references) == 0: + head_reason[positions] = 2 + household_verdicts[index] = 2 + continue + if len(references) > 1: + head_reason[positions] = 3 + household_verdicts[index] = 3 + continue + head_state[positions] = 2 + head_state[references[0]] = 1 + household_verdicts[index] = 1 + + # P_SEQ is diagnosed, never consulted. Nothing above reads it. + p_seq_one = p_seq == 1 + in_declared_range = np.isin(p_seq, list(P_SEQ.declared_domain)) + household_years = income_year[order[bounds[:-1]]] + household_p_seq_ones = np.add.reduceat(p_seq_one[order].astype(np.int64), starts) + + diagnostics: dict = { + "rows": rows, + "households": households, + "by_cohort": {}, + "totals": {}, + "p_seq_crosscheck": {}, + } + cohorts = sorted({int(year) for year in np.unique(income_year)}) + for scope, mask, household_mask in [ + ("totals", np.ones(rows, dtype=bool), np.ones(households, dtype=bool)), + *[ + ( + str(year), + income_year == year, + household_years == year, + ) + for year in cohorts + ], + ]: + block = { + "income_year": None if scope == "totals" else int(scope), + "survey_year": None + if scope == "totals" + else int(COHORTS.get(int(scope), int(scope) + 1)), + "observed_period_kind": OBSERVED_PERIOD_KIND, + "rows": int(mask.sum()), + "households": int(household_mask.sum()), + "field_domains": { + field.name: { + "rows_in_declared_range": int( + np.isin(arrays[field.name][mask], field.declared_domain).sum() + ), + "rows_outside_declared_range": int( + ( + ~np.isin(arrays[field.name][mask], field.declared_domain) + ).sum() + ), + "rows_in_named_codes": int( + np.isin(arrays[field.name][mask], list(field.named_codes)).sum() + ), + "rows_in_unnamed_range_codes": int( + np.isin( + arrays[field.name][mask], field.unnamed_in_range_codes + ).sum() + ), + "role": field.role, + } + for field in ASEC_DEMOGRAPHIC_FIELDS + }, + "sex_binding_state": _counts(sex_state[mask], SEX_BINDING_STATES), + "sex_unbound_reason": _counts(sex_reason[mask], SEX_UNBOUND_REASONS), + "sex_allocation_state": _counts(allocation[mask], SEX_ALLOCATION_STATES), + "household_reference_state": _counts( + head_state[mask], HOUSEHOLD_REFERENCE_STATES + ), + "household_reference_unbound_reason": _counts( + head_reason[mask], HOUSEHOLD_REFERENCE_UNBOUND_REASONS + ), + "household_reference_verdict": _counts( + household_verdicts[household_mask], + MappingProxyType( + { + 1: "exactly_one_reference", + 2: "no_reference", + 3: "multiple_references", + 4: "indeterminate_relationship_code", + } + ), + ), + "p_seq_crosscheck": { + "declared_role": P_SEQ.concept, + "named_codes": {}, + "is_semantic_proof_of_headship": False, + "used_as_fallback": False, + "rows_with_p_seq_one": int(p_seq_one[mask].sum()), + "rows_outside_declared_range": int((~in_declared_range[mask]).sum()), + "households_with_one_p_seq_one": int( + (household_p_seq_ones[household_mask] == 1).sum() + ), + "households_with_no_p_seq_one": int( + (household_p_seq_ones[household_mask] == 0).sum() + ), + "households_with_multiple_p_seq_one": int( + (household_p_seq_ones[household_mask] > 1).sum() + ), + "agreement_rows": int((p_seq_one & reference)[mask].sum()), + "p_seq_one_without_reference_code": int( + (p_seq_one & ~reference)[mask].sum() + ), + "reference_code_without_p_seq_one": int( + (reference & ~p_seq_one)[mask].sum() + ), + "unbound_households_with_one_p_seq_one": int( + ((household_verdicts != 1) & (household_p_seq_ones == 1))[ + household_mask + ].sum() + ), + }, + } + if scope == "totals": + diagnostics["totals"] = block + diagnostics["p_seq_crosscheck"] = block["p_seq_crosscheck"] + else: + diagnostics["by_cohort"][scope] = block + diagnostics["family_relationship_field_read"] = False + diagnostics["line_number_used_for_headship"] = False + diagnostics["household_and_family_roles_separate"] = True + return DemographicClassification( + MappingProxyType( + { + "asec_sex_binding_state": sex_state, + "asec_sex_unbound_reason": sex_reason, + "asec_sex_allocation_state": allocation, + "asec_household_reference_state": head_state, + "asec_household_reference_unbound_reason": head_reason, + } + ), + diagnostics, + ) + + +#: The ACS side is *not* rewritten by this contract. These are the pinned 2024 +#: 1-year PUMS codes the existing reader already branches on, restated so the +#: shared classifier below can read the ACS arm without inventing a head. +ACS_OBSERVED_REFERENCE_CODE = 20 +ACS_GROUP_QUARTERS_CODES: tuple[int, int] = (37, 38) +#: What ``acs_pums`` already writes for a group-quarters-only household. It is +#: a printed A_EXPRRP nonrelative code, deliberately not a reference code, so a +#: synthetic group-quarters representative can never be read as an observed +#: housing-unit reference person. +ACS_SYNTHETIC_GROUP_QUARTERS_RELATIONSHIP_CODE = 14 +ACS_REFERENCE_STATES: Mapping[int, str] = MappingProxyType( + { + 0: "unbound", + 1: "observed_housing_unit_reference_person", + 2: "observed_not_reference_person", + 3: "group_quarters_no_observed_reference_person", + } +) + + +def acs_household_reference_states( + *, household_id: np.ndarray, relshipp: np.ndarray +) -> tuple[np.ndarray, dict]: + """Read the ACS arm's own observed reference person; never manufacture one. + + ``RELSHIPP == 20`` is the self-labelled reference person of a housing unit. + Codes 37 and 38 are the institutional and noninstitutional group-quarters + populations; they are neither a reference person nor a relationship to one, + so their households report *no* observed reference person and stay + distinguishable from a housing unit throughout. + """ + rows = len(household_id) + _require(0 < rows <= _MAX_PERSONS, "ACS_ROWS") + for name, values in (("household_id", household_id), ("RELSHIPP", relshipp)): + _require( + isinstance(values, np.ndarray) + and values.dtype == np.dtype("int64") + and values.shape == (rows,), + "ACS_INPUT", + name, + ) + group_quarters = np.isin(relshipp, list(ACS_GROUP_QUARTERS_CODES)) + valid_relationship = np.isin(relshipp, np.arange(20, 39)) + reference = relshipp == ACS_OBSERVED_REFERENCE_CODE + states = np.where(reference, 1, 2).astype(np.int64) + order = np.argsort(household_id, kind="stable") + grouped = household_id[order] + starts = np.flatnonzero(np.concatenate(([True], grouped[1:] != grouped[:-1]))) + bounds = np.append(starts, len(grouped)) + housing_units = 0 + gq_households = 0 + unbound_housing_units = 0 + bad_reference_cardinality = 0 + mixed_households = 0 + invalid_households = 0 + for index in range(len(starts)): + positions = order[bounds[index] : bounds[index + 1]] + if group_quarters[positions].all(): + states[positions] = 3 + gq_households += 1 + continue + housing_units += 1 + bad_reference_cardinality += int(int(reference[positions].sum()) != 1) + mixed_households += int(group_quarters[positions].any()) + invalid_households += int(not valid_relationship[positions].all()) + if ( + group_quarters[positions].any() + or not valid_relationship[positions].all() + or int(reference[positions].sum()) != 1 + ): + states[positions] = 0 + unbound_housing_units += 1 + diagnostics = { + "rows": rows, + "households": len(starts), + "housing_unit_households": housing_units, + "group_quarters_only_households": gq_households, + "unbound_housing_unit_households": unbound_housing_units, + "housing_unit_households_without_exactly_one_reference": bad_reference_cardinality, + "mixed_housing_unit_group_quarters_households": mixed_households, + "households_with_invalid_relationship_codes": invalid_households, + "state_counts": _counts(states, ACS_REFERENCE_STATES), + "group_quarters_rows": int(group_quarters.sum()), + "institutional_group_quarters_rows": int((relshipp == 37).sum()), + "noninstitutional_group_quarters_rows": int((relshipp == 38).sum()), + "manufactured_group_quarters_reference_person": False, + "synthetic_group_quarters_relationship_code": ( + ACS_SYNTHETIC_GROUP_QUARTERS_RELATIONSHIP_CODE + ), + } + return states, diagnostics + + +@dataclass(frozen=True) +class AuthenticatedAsecDemographicSource: + """Immutable owned numeric buffers issued only by closed reconstruction.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "DEMOGRAPHIC_CONSTRUCTOR_UNAVAILABLE") + self.validate() + + @property + def receipt(self) -> dict: + return json.loads(self._header.decode("ascii")) + + @property + def content_sha256(self) -> str: + return _sha(self._header + self._body) + + def validate(self) -> None: + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "DEMOGRAPHIC_HEADER_SIZE", + ) + header = self.receipt + rows = header["rows"] + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "DEMOGRAPHIC_ROWS") + _require( + type(self._body) is bytes + and len(self._body) == rows * len(COLUMNS) * 8 + and _sha(self._body) == header["body_sha256"] + and header["columns"] == list(COLUMNS) + and _json(header["contract"]) == _json(contract_document()) + and _json(header["implementation"]) == _json(_implementation()), + "DEMOGRAPHIC_CONTENT_CHANGED", + ) + + def array(self, name: str) -> np.ndarray: + self.validate() + _require(name in COLUMNS, "DEMOGRAPHIC_COLUMN") + rows = self.receipt["rows"] + offset = COLUMNS.index(name) * rows * 8 + return np.frombuffer(self._body, dtype=" bytes: + value.validate() + payload = ( + MAGIC + struct.pack(" None: + _require( + type(source) is legacy.AuthenticatedCurrentMoneySource, + "DEMOGRAPHIC_AUTHENTICATED_PARENT", + ) + source.validate() + + +def _reconstruct(source, member_paths: Mapping[int, str | Path]): + restoration._paths(member_paths) + _parent(source) + before = _implementation() + person = source.frame.person + rows = len(person) + _require(0 < rows <= _MAX_PERSONS, "DEMOGRAPHIC_ROWS") + _require(not set(ALIASES) & set(person), "DEMOGRAPHIC_ALREADY_ATTACHED") + _require( + set(COORDINATES[2:]) <= set(person) and "person_household_id" in person, + "DEMOGRAPHIC_PARENT_ROSTER", + ) + years = np.asarray(source.scope.person_years, dtype=np.int64) + keys = np.asarray(source.scope.person_native_keys) + output: dict[str, np.ndarray] = { + "person_id": np.asarray(source.scope.person_ids, dtype=np.int64), + "income_year": years, + } + for name in COORDINATES[2:]: + _require( + person[name].dtype == np.dtype("int64"), + "DEMOGRAPHIC_COORDINATE_DTYPE", + name, + ) + output[name] = person[name].to_numpy(copy=True) + # The grouping column is a coordinate of the headship verdict, so it is + # typed here rather than coerced by the conversion that consumes it. + _require( + person[HOUSEHOLD_GROUPING_COLUMN].dtype == np.dtype("int64"), + "DEMOGRAPHIC_COORDINATE_DTYPE", + HOUSEHOLD_GROUPING_COLUMN, + ) + household_id = person[HOUSEHOLD_GROUPING_COLUMN].to_numpy(copy=True) + for alias in ALIASES: + output[alias] = np.empty(rows, dtype=np.int64) + joins = [] + with tempfile.TemporaryDirectory( + prefix="microcosm-demographic-members-" + ) as directory: + for year, member, archive_pin, pin, member_rows, size in _MEMBER_PINS: + staged = Path(directory) / member + _require( + _snapshot(member_paths[year], staged, size=size) == pin, + "DEMOGRAPHIC_SOURCE_BYTES", + ) + raw = read_demographic_member(staged, rows=member_rows, size=size) + staged.unlink() + positions = np.flatnonzero(years == year) + _require(len(positions) == member_rows, "DEMOGRAPHIC_COHORT_ROWS") + recipient = pd.Index(keys[positions]) + _require(not recipient.has_duplicates, "DEMOGRAPHIC_PERSON_KEY") + indices = pd.Index(raw.PERIDNUM).get_indexer(recipient) + _require( + bool((indices >= 0).all()) and len(np.unique(indices)) == member_rows, + "DEMOGRAPHIC_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + coordinate_conflicts = 0 + for name, raw_name in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + conflicts = int( + (output[name][positions] != joined[raw_name].to_numpy()).sum() + ) + coordinate_conflicts += conflicts + _require(conflicts == 0, "DEMOGRAPHIC_NATIVE_KEY_OR_AGE", name) + compared = {} + incumbent_conflicts = 0 + relationship_crosscheck = { + "incumbent_present": "A_EXPRRP" in person, + "compared_rows": 0, + "missing_rows": 0, + "mismatch_rows": 0, + "presence_certifies_source_observation": False, + "used_as_semantic_proof": False, + } + for alias, name in zip(ALIASES, OBSERVATIONS, strict=True): + observed = joined[name].to_numpy(dtype=np.int64) + output[alias][positions] = observed + if name not in person: + compared[name] = None + continue + incumbent = person[name].iloc[positions] + known = ~incumbent.isna().to_numpy() + equal = ( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known] + == observed[known] + ) + if name == "A_EXPRRP": + # Older prepared cohorts carry an A_LINENO-derived recode. + # The restored alias owns the CSV observation; an unverified + # incumbent may diagnose a discrepancy but cannot veto it. + relationship_crosscheck.update( + compared_rows=int(known.sum()), + missing_rows=int((~known).sum()), + mismatch_rows=int((~equal).sum()), + ) + compared[name] = int(known.sum()) + continue + conflicts = int((~equal).sum()) + incumbent_conflicts += conflicts + _require(conflicts == 0, "DEMOGRAPHIC_INCUMBENT_CONFLICT", name) + compared[name] = int(known.sum()) + joins.append( + { + "income_year": year, + "survey_year": COHORTS[year], + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": member_rows, + "joined_rows": len(positions), + "unreferenced_source_rows": int( + member_rows - len(np.unique(indices)) + ), + "incumbent_compared_rows": compared, + "incumbent_relationship_crosscheck": relationship_crosscheck, + "accepted_source_missing_tokens": dict.fromkeys(OBSERVATIONS, 0), + "incumbent_absent_columns": sorted( + name for name, value in compared.items() if value is None + ), + "incumbent_conflicts": incumbent_conflicts, + "native_key_or_age_conflicts": coordinate_conflicts, + "counters_zero_in_every_successful_receipt": [ + "accepted_source_missing_tokens", + "incumbent_conflicts", + "native_key_or_age_conflicts", + "unreferenced_source_rows", + ], + "zero_counter_semantics": ZERO_COUNTER_SEMANTICS, + } + ) + membership = verify_household_membership( + income_year=years, + source_household_id=output["source_household_id"], + household_id=household_id, + ) + classification = classify_asec_demographic_observations( + income_year=years, + household_id=household_id, + a_sex=output["asec_A_SEX"], + axsex=output["asec_AXSEX"], + a_exprrp=output["asec_A_EXPRRP"], + p_seq=output["asec_P_SEQ"], + ) + output.update({name: values for name, values in classification.states.items()}) + _parent(source) + _require( + _json(_implementation()) == _json(before), + "DEMOGRAPHIC_IMPLEMENTATION_CHANGED", + ) + buffers = [output[name].astype(" AuthenticatedAsecDemographicSource: + """Reconstruct from pinned members; a candidate supplies bytes, not content.""" + try: + expected = _reconstruct(source, member_paths) + if candidate_path is not None: + payload = _encode(expected) + with tempfile.TemporaryDirectory( + prefix="microcosm-demographic-candidate-" + ) as directory: + digest = _snapshot( + candidate_path, Path(directory) / FILENAME, size=len(payload) + ) + _require(digest == _sha(payload), "DEMOGRAPHIC_CANONICAL_BYTES") + _parent(source) + expected.validate() + return expected + except DemographicSourceRefusalError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + OverflowError, + UnicodeError, + csv.Error, + re.error, + ): + raise DemographicSourceRefusalError("DEMOGRAPHIC_SOURCE_CONTRACT") from None + + +def _attachment_receipt(frame, source, observations) -> dict: + return { + "schema_version": 1, + "artifact_kind": "microcosm.asec_demographic_source_attachment.v1", + "source_identity_sha256": _sha(source.source.identity), + "observations_content_sha256": observations.content_sha256, + "columns": list(ALIASES + DERIVED), + "output_frame_sha256": legacy._frame_signature(frame), + "release_eligible": False, + } + + +def _check_attachment_parents(source, observations) -> None: + _parent(source) + _require( + type(observations) is AuthenticatedAsecDemographicSource, + "DEMOGRAPHIC_AUTHENTICATED_OBSERVATIONS", + ) + observations.validate() + _require( + observations.receipt["source_identity"].encode() == source.source.identity, + "DEMOGRAPHIC_ATTACHMENT_PARENT", + ) + + +@dataclass(frozen=True) +class DemographicSourceAttachedAsec: + """Owned additive Frame beside an unchanged authenticated money parent.""" + + frame: Frame + parent: legacy.AuthenticatedCurrentMoneySource + observations: AuthenticatedAsecDemographicSource + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require( + _token is _COMPOSE_TOKEN, + "DEMOGRAPHIC_ATTACHMENT_CONSTRUCTOR_UNAVAILABLE", + ) + + @property + def receipt(self) -> dict: + return json.loads(self._receipt.decode("ascii")) + + def validate(self) -> None: + try: + _check_attachment_parents(self.parent, self.observations) + _require(type(self.frame) is Frame, "DEMOGRAPHIC_ATTACHMENT_FRAME") + for name in ALIASES + DERIVED: + values = self.frame.person[name] + _require( + values.dtype == np.dtype("int64") + and np.array_equal( + values.to_numpy(), self.observations.array(name) + ), + "DEMOGRAPHIC_ATTACHMENT_OUTPUT", + name, + ) + recovered = restoration._owned_frame(self.frame) + recovered.person.drop(columns=list(ALIASES + DERIVED), inplace=True) + _require( + legacy._frame_signature(recovered) + == self.observations.receipt["source_frame_sha256"] + and self.receipt + == _attachment_receipt(self.frame, self.parent, self.observations), + "DEMOGRAPHIC_ATTACHMENT_CHANGED", + ) + except DemographicSourceRefusalError: + raise + except (ValueError, TypeError, KeyError, AttributeError) as error: + raise DemographicSourceRefusalError( + "DEMOGRAPHIC_ATTACHMENT_CHANGED" + ) from error + + +def attach_asec_demographic_source( + source, observations +) -> DemographicSourceAttachedAsec: + """Append the restored observations and their states; change nothing else.""" + _check_attachment_parents(source, observations) + _require( + not set(ALIASES + DERIVED) & set(source.frame.person), + "DEMOGRAPHIC_ALREADY_ATTACHED", + ) + frame = restoration._owned_frame(source.frame) + for name in ALIASES + DERIVED: + frame.person[name] = observations.array(name).copy() + result = DemographicSourceAttachedAsec( + frame, + source, + observations, + _json(_attachment_receipt(frame, source, observations)), + _token=_COMPOSE_TOKEN, + ) + result.validate() + return result + + +def write_asec_demographic_source(source, *, member_paths, output_dir) -> dict: + """Produce a new immutable local bundle; never replace an existing parent.""" + destination = Path(output_dir) + if destination.exists(): + raise FileExistsError(destination) + value = load_authenticated_asec_demographic_source( + source, member_paths=member_paths + ) + payload = _encode(value) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".asec-demographic-", dir=destination.parent + ) as directory: + staging = Path(directory) + (staging / FILENAME).write_bytes(payload) + receipt = { + **value.receipt, + "content_sha256": value.content_sha256, + "output_file": FILENAME, + "output_sha256": _sha(payload), + } + (staging / "receipt.json").write_bytes(_json(receipt) + b"\n") + destination.mkdir() + for path in staging.iterdir(): + os.link(path, destination / path.name) + return receipt + + +__all__ = [ + "ACS_GROUP_QUARTERS_CODES", + "ACS_OBSERVED_REFERENCE_CODE", + "ACS_REFERENCE_STATES", + "ACS_SYNTHETIC_GROUP_QUARTERS_RELATIONSHIP_CODE", + "ALIASES", + "ARTIFACT_KIND", + "ASEC_DEMOGRAPHIC_FIELDS", + "ASEC_DEMOGRAPHIC_SOURCE_COLUMNS", + "ASEC_DICTIONARY_AUTHORITY", + "COHORTS", + "COLUMNS", + "DERIVED", + "HOUSEHOLD_GROUPING_COLUMN", + "HOUSEHOLD_MEMBERSHIP_KEY", + "HOUSEHOLD_REFERENCE_CODES", + "HOUSEHOLD_REFERENCE_STATES", + "HOUSEHOLD_REFERENCE_UNBOUND_REASONS", + "MEMBERSHIP_REFUSAL_COUNTERS", + "NON_ADOPTED_HEAD_SIGNALS", + "OBSERVATIONS", + "OBSERVED_PERIOD_KIND", + "SEX_ALLOCATION_STATES", + "SEX_BINDING_STATES", + "SEX_UNBOUND_REASONS", + "ZERO_COUNTER_SEMANTICS", + "AsecDemographicField", + "AuthenticatedAsecDemographicSource", + "DemographicClassification", + "DemographicSourceAttachedAsec", + "DemographicSourceRefusalError", + "acs_household_reference_states", + "attach_asec_demographic_source", + "classify_asec_demographic_observations", + "contract_document", + "load_authenticated_asec_demographic_source", + "read_demographic_member", + "verify_household_membership", + "write_asec_demographic_source", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py new file mode 100644 index 000000000..5648d95a4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py @@ -0,0 +1,473 @@ +"""Closure-admitted PolicyEngine-US evaluation of the corrected money leaves. + +An output is admitted only when its complete static input closure is produced by +this graph. The defaults allowlist is empty by decision, so a closure leaf this +slice does not map blocks its root outright rather than being zero-filled or +split into an unreviewed surface. Each admitted root carries a pinned closure +identity; engine or closure drift refuses before any simulation runs. + +Outputs are formula-owned. They are retained in an evaluation artifact and are +never written back as population cells. +""" + +from __future__ import annotations + +import struct +import sys +from dataclasses import dataclass +from hashlib import sha256 +from importlib import resources +from importlib.metadata import distribution +from importlib.util import find_spec +from pathlib import Path + +import numpy as np + +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph import ArtifactType +from microcosm.graph.canonical import canonical_json + +from .operator_boundary import PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + +US_ASEC_ENGINE_EVALUATION_TYPE = ArtifactType("microcosm.us.asec_engine_evaluation", 1) +ENGINE_PACKAGE = "policyengine-us" +ENGINE_DEFAULTS_RESOURCE = "asec_current_money_engine_defaults_v1.json" +ENGINE_DEFAULTS_SHA256 = ( + "090dd200ecdb837127dba1d1f22903d152cef8a434085e4ead4d379350ee5b82" +) +ENGINE_DEFAULTS_ARTIFACT_KIND = "microcosm.us.asec_current_money_engine_defaults.v1" +EVALUATION_MAGIC = b"MCASECEV\x01" +EVALUATION_ARTIFACT_KIND = "microcosm.us.asec_engine_evaluation.v1" +EVALUATION_PERIOD = 2024 +EVALUATION_HEADER_MAX_BYTES = 64 * 1024 +EVALUATION_MAX_ROWS = 1_000_000 +_PRIOR_YEAR_MARKERS = ("last_year", "previous_year", "prior_year") + + +class EngineAdmissionError(ValueError): + """A candidate output is not admitted, or a pinned identity drifted.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise EngineAdmissionError(reason) + + +@dataclass(frozen=True) +class EngineOutputContract: + """Pinned identity of one admitted root's static PolicyEngine-US closure.""" + + root: str + entity: str + engine_version: str + input_leaves: tuple[str, ...] + formula_node_count: int + edge_count: int + sha256: str + + +ADMITTED_ENGINE_OUTPUT_CONTRACTS: tuple[EngineOutputContract, ...] = ( + EngineOutputContract( + root="capital_gains", + entity="person", + engine_version="1.819.0", + input_leaves=( + "long_term_capital_gains_before_response", + "short_term_capital_gains", + ), + formula_node_count=4, + edge_count=6, + sha256="1768cb833b183b3898567910b463e645a2744c2138688c50b5fc1cc17c3319df", + ), + EngineOutputContract( + root="dividend_income", + entity="person", + engine_version="1.819.0", + input_leaves=("non_qualified_dividend_income", "qualified_dividend_income"), + formula_node_count=2, + edge_count=3, + sha256="a27f07536f4909361de7444831f7852a3f976e12d40d6afe4059c6a5fbdfbbe3", + ), + EngineOutputContract( + root="employment_income", + entity="person", + engine_version="1.819.0", + input_leaves=("employment_income_before_lsr",), + formula_node_count=3, + edge_count=4, + sha256="bdcd8da23a79ff228a610a4d0b483f5fd2068d968f31e96c07cea8a86cc3366b", + ), + EngineOutputContract( + root="social_security", + entity="person", + engine_version="1.819.0", + input_leaves=( + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + ), + formula_node_count=1, + edge_count=4, + sha256="3f5a5304ac1e8c8d785614069867590176325b212a0a080457f80594a6fd53c0", + ), +) +BLOCKED_ENGINE_OUTPUTS: tuple[str, ...] = ( + "interest_income", + "pension_income", + "self_employment_income", +) +ADMITTED_ROOTS: tuple[str, ...] = tuple( + contract.root for contract in ADMITTED_ENGINE_OUTPUT_CONTRACTS +) + + +def load_engine_defaults() -> dict: + """Read and pin-verify the defaults resource; the allowlist must stay empty.""" + payload = ( + resources.files(__package__).joinpath(ENGINE_DEFAULTS_RESOURCE).read_bytes() + ) + _require( + sha256(payload).hexdigest() == ENGINE_DEFAULTS_SHA256, + "ENGINE_DEFAULTS_FINGERPRINT", + ) + import json + + document = json.loads(payload) + _require( + document["schema_version"] == 1 + and document["artifact_kind"] == ENGINE_DEFAULTS_ARTIFACT_KIND, + "ENGINE_DEFAULTS_SCHEMA", + ) + _require(document["allowlist"] == [], "ENGINE_DEFAULTS_ALLOWLIST") + _require( + tuple(sorted(document["admitted_roots"])) == ADMITTED_ROOTS, + "ENGINE_DEFAULTS_ROOTS", + ) + _require( + tuple(sorted(document["blocked_roots"])) == BLOCKED_ENGINE_OUTPUTS, + "ENGINE_DEFAULTS_BLOCKED", + ) + _require(document["engine_package"] == ENGINE_PACKAGE, "ENGINE_DEFAULTS_PACKAGE") + _require(document["release_eligible"] is False, "ENGINE_DEFAULTS_RELEASE_CLAIM") + return document + + +def _runtime_package_identity(package: str, module: str) -> dict: + """Hash installed model code and parameter files, never population data. + + Include relative names so additions, removals and relocation have explicit + semantics. Recompute on every invocation: cache admission must see an edit + made after a previous evaluation in the same process. + """ + installed = distribution(package) + root = Path(installed.locate_file(module)) + files = sorted( + path + for path in root.rglob("*") + if path.is_file() + and ( + path.suffix == ".py" + or ( + "parameters" in path.relative_to(root).parts + and path.suffix in (".yaml", ".yml", ".json") + ) + ) + ) + records = [ + [path.relative_to(root).as_posix(), sha256(path.read_bytes()).hexdigest()] + for path in files + ] + return { + "version": installed.version, + "files": len(records), + "source_and_parameters_sha256": sha256(canonical_json(records)).hexdigest(), + } + + +def _verify_runtime_origin(package: str, module: str) -> None: + """The adapter must import the same distribution whose bytes are pinned.""" + root = Path(distribution(package).locate_file(module)).resolve() + spec = find_spec(module) + _require( + spec is not None + and spec.origin is not None + and Path(spec.origin).resolve() == root / "__init__.py" + and spec.submodule_search_locations is not None + and [Path(path).resolve() for path in spec.submodule_search_locations] + == [root], + "ENGINE_RUNTIME_ORIGIN", + ) + for name, loaded in tuple(sys.modules.items()): + if name != module and not name.startswith(module + "."): + continue + filename = getattr(loaded, "__file__", None) + if filename is not None: + _require( + Path(filename).resolve().is_relative_to(root), "ENGINE_RUNTIME_ORIGIN" + ) + for path in getattr(loaded, "__path__", ()): + _require(Path(path).resolve().is_relative_to(root), "ENGINE_RUNTIME_ORIGIN") + + +def engine_runtime_identity() -> dict: + """Verify the pinned baseline runtime before execution or a cache lookup.""" + document = load_engine_defaults() + for package, module in ( + ("policyengine-us", "policyengine_us"), + ("policyengine-core", "policyengine_core"), + ): + _verify_runtime_origin(package, module) + observed = { + package: _runtime_package_identity(package, module) + for package, module in ( + ("policyengine-us", "policyengine_us"), + ("policyengine-core", "policyengine_core"), + ) + } + _require(observed == document["baseline_runtime"], "ENGINE_RUNTIME_DRIFT") + return {"policy": "installed_baseline_no_reforms", "packages": observed} + + +def _prior_year_leaves() -> frozenset[str]: + family = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["prior_year_income"] + return frozenset(name for columns in family.values() for name in columns) + + +def _closure(index, root: str): + try: + return index.variable_dependency_closure(root) + except ValueError as error: + raise EngineAdmissionError(f"UNKNOWN_ENGINE_ROOT:{root}") from error + + +def _check_contract(closure, contract: EngineOutputContract) -> None: + observed = { + "engine_version": closure.engine_version, + "root": closure.root, + "input_leaves": tuple(closure.input_leaves), + "formula_node_count": len(closure.formula_nodes), + "edge_count": len(closure.edges), + "sha256": closure.sha256, + } + expected = { + "engine_version": contract.engine_version, + "root": contract.root, + "input_leaves": contract.input_leaves, + "formula_node_count": contract.formula_node_count, + "edge_count": contract.edge_count, + "sha256": contract.sha256, + } + if observed != expected: + raise EngineAdmissionError( + "PolicyEngine-US closure drifted for " + f"{contract.root!r}; expected={expected}, observed={observed}." + ) + + +def admit_engine_outputs( + index, + *, + produced_leaves, + defaults: dict | None = None, +) -> tuple[EngineOutputContract, ...]: + """Admit only roots whose complete closure this graph produces.""" + document = load_engine_defaults() if defaults is None else defaults + _require(document["allowlist"] == [], "ENGINE_DEFAULTS_ALLOWLIST") + produced = frozenset(produced_leaves) + prior_year = _prior_year_leaves() + for contract in ADMITTED_ENGINE_OUTPUT_CONTRACTS: + closure = _closure(index, contract.root) + _check_contract(closure, contract) + metadata = index.variable_metadata(contract.root) + _require(metadata.entity == contract.entity, "ADMITTED_ROOT_ENTITY") + missing = sorted(set(closure.input_leaves) - produced) + if missing: + raise EngineAdmissionError( + f"Root {contract.root!r} has unproduced closure leaves {missing}; " + "the engine-default allowlist is empty by decision." + ) + reaching = sorted(set(closure.input_leaves) & prior_year) + _require(not reaching, f"PRIOR_YEAR_LEAF_IN_CLOSURE:{contract.root}") + _require( + not any( + marker in leaf + for leaf in closure.input_leaves + for marker in _PRIOR_YEAR_MARKERS + ), + f"PRIOR_YEAR_LEAF_IN_CLOSURE:{contract.root}", + ) + for root in BLOCKED_ENGINE_OUTPUTS: + closure = _closure(index, root) + blocked = sorted(set(closure.input_leaves) - produced) + _require(bool(blocked), f"BLOCKED_ROOT_NOW_COMPLETE:{root}") + return ADMITTED_ENGINE_OUTPUT_CONTRACTS + + +@dataclass(frozen=True) +class EngineEvaluation: + """Engine outputs retained as evidence; never population cells.""" + + header: dict + values: dict[str, np.ndarray] + + @property + def roots(self) -> tuple[str, ...]: + return tuple(sorted(self.values)) + + +def _aggregates(values: np.ndarray) -> dict[str, object]: + return { + "rows": int(values.size), + "nonzero_rows": int(np.count_nonzero(values)), + "unweighted_total": float(values.sum()), + "minimum": float(values.min()), + "maximum": float(values.max()), + } + + +def materialize_engine_outputs( + frame: Frame, + engine, + contracts: tuple[EngineOutputContract, ...], + *, + period: int = EVALUATION_PERIOD, + context_bindings: dict | None = None, +) -> EngineEvaluation: + """Run the real adapter for the admitted roots and retain the arrays only.""" + _require(type(frame) is Frame and frame.schema == US_SCHEMA, "ENGINE_FRAME_SCHEMA") + _require(getattr(engine, "country", None) == "us", "ENGINE_COUNTRY") + _require(period == EVALUATION_PERIOD, "ENGINE_PERIOD") + _require(bool(contracts), "ENGINE_NO_ADMITTED_OUTPUTS") + _require(contracts == ADMITTED_ENGINE_OUTPUT_CONTRACTS, "ENGINE_CONTRACT_ROSTER") + runtime = engine_runtime_identity() + rows = frame.n("person") + _require(0 < rows <= EVALUATION_MAX_ROWS, "ENGINE_ROW_BOUND") + roots = tuple(sorted(contract.root for contract in contracts)) + _require(len(set(roots)) == len(roots), "ENGINE_DUPLICATE_ROOT") + results = engine.materialize(frame, list(roots), period) + _require(set(results) == set(roots), "ENGINE_RESULT_ROSTER") + values: dict[str, np.ndarray] = {} + for root in roots: + array = np.asarray(results[root], dtype="float64") + _require(array.shape == (rows,), "ENGINE_RESULT_SHAPE") + _require(bool(np.isfinite(array).all()), "ENGINE_RESULT_NONFINITE") + values[root] = np.ascontiguousarray(array, dtype=" tuple[bytes, ...]: + _require(type(evaluation) is EngineEvaluation, "TYPED_EVALUATION_REQUIRED") + header = canonical_json(evaluation.header) + _require(0 < len(header) <= EVALUATION_HEADER_MAX_BYTES, "HEADER_SIZE") + roots = tuple(evaluation.header["roots"]) + _require(roots == tuple(sorted(evaluation.values)), "ENGINE_RESULT_ROSTER") + rows = evaluation.header["person_rows"] + parts = [EVALUATION_MAGIC, struct.pack(" bytes: + """Encode the evaluation header and arrays with a transport checksum.""" + parts = _evaluation_parts(evaluation) + checksum = sha256() + for part in parts: + checksum.update(part) + return b"".join((*parts, checksum.digest())) + + +def decode_engine_evaluation(payload: bytes) -> EngineEvaluation: + """Decode a stored evaluation, verifying framing, checksum and shapes.""" + _require( + type(payload) is bytes and len(payload) > len(EVALUATION_MAGIC) + 4 + 32, + "PAYLOAD_SIZE", + ) + _require(payload.startswith(EVALUATION_MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" None: + if not condition: + raise HouseholdCoverageSourceError(reason) + + +def _producer() -> dict: + _require(_csv_builtin.csv_reader_bound(csv), "SOURCE_CSV_READER_CHANGED") + return { + "protocol": _SCHEMA, + "owner_sha256": shared._sha( + resources.files(__package__) + .joinpath("asec_household_coverage_fields.py") + .read_bytes() + ), + "bounded_reader_dependency": shared._implementation(), + "csv_builtin_sha256": shared._sha( + resources.files(__package__).joinpath("source_csv_builtin.py").read_bytes() + ), + "columns": COLUMNS, + "fields": _FIELDS, + "interpretation_notes": _INTERPRETATION_NOTES, + } + + +def _code_state(name: str, token: str, interview_code: int | None) -> dict: + contract = _FIELDS[name] + code, label = None, None + if token == "": + status = "missing" + elif re.fullmatch(rf"[0-9]{{1,{contract['printed_length']}}}", token) is None: + status = "malformed" + else: + code = int(token) + label = contract["labels"].get(str(code)) + status = "valid" if label is not None else "unlabelled" + universe = ( + "in" + if contract["universe_as_printed"] == "All Households" or interview_code == 1 + else "outside" + if interview_code in (2, 3) + else "unresolved" + ) + return { + "code": code, + "label": label, + "code_status": status, + "universe_status": universe, + } + + +def _project(row: Sequence[str]) -> dict: + _require( + all(len(token) <= shared._MAX_FIELD_CHARS for token in row), "MEMBER_FIELD_SIZE" + ) + native = row[0] + _require( + re.fullmatch(r"[0-9]+", native) is not None and 1 <= int(native) <= 99999, + "MEMBER_NATIVE_KEY", + ) + literals = dict(zip(COLUMNS, row, strict=True)) + interview = _code_state("H_HHTYPE", literals["H_HHTYPE"], None) + interview_code = interview["code"] if interview["code_status"] == "valid" else None + states = { + name: _code_state(name, literals[name], interview_code) for name in COLUMNS[1:] + } + hr = ( + states["HRHTYPE"]["code"] + if states["HRHTYPE"]["code_status"] == "valid" + else None + ) + count = ( + states["H_NUMPER"]["code"] + if states["H_NUMPER"]["code_status"] == "valid" + else None + ) + diagnostics = [] + if interview_code == 1: + if hr == 0: + diagnostics.append("interview_with_noninterview_household_type") + if count == 0: + diagnostics.append("interview_with_noninterview_person_count") + elif interview_code in (2, 3): + if hr is not None and hr > 0: + diagnostics.append("noninterview_with_interview_household_type") + if count is not None and count > 0: + diagnostics.append("noninterview_with_positive_person_count") + if states["H_LIVQRT"]["code"] == 11: + diagnostics.append("student_quarters_dictionary_methodology_unresolved") + return { + **literals, + "native_household_id": int(native), + "field_states": states, + "reported_person_count": count + if interview_code == 1 and count is not None and count > 0 + else None, + "diagnostics": diagnostics, + "coverage_interpretation": "unresolved", + } + + +def _read_capture(capture: Path, pin) -> list[dict]: + """Compose existing bounded byte/record primitives with this field projection.""" + csv_reader = _csv_builtin.capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + result, seen = [], set() + member = SourceMember(pin.canonical_member_id, pin.member_sha256) + with open(capture, "rb", buffering=0, opener=shared._regular_opener) as raw: + before = os.fstat(raw.fileno()) + _require(before.st_size == pin.size_bytes, "CAPTURE_SIZE") + digest = shared._DigestReader(raw, pin.size_bytes) + with io.TextIOWrapper( + io.BufferedReader(digest), encoding="utf-8-sig", newline="" + ) as text: + lines = shared._RecordLines(text) + reader = csv_reader(lines, strict=True) + header = next(reader, []) + _require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + positions = [header.index(column) for column in COLUMNS] + while True: + lines.characters = 0 + try: + row = next(reader) + except StopIteration: + break + _require(len(row) == len(header), "MEMBER_ROW_WIDTH") + _require(len(result) < pin.rows, "MEMBER_ROWS") + record = _project([row[i] for i in positions]) + native = record["native_household_id"] + _require(native not in seen, "MEMBER_DUPLICATE_KEY") + seen.add(native) + record.update( + income_year=pin.income_year, + survey_year=pin.survey_year, + member_id=pin.canonical_member_id, + member_sha256=pin.member_sha256, + member_row_1based=len(result) + 1, + origin_key=origin_key( + AsecHouseholdOrigin(member, pin.income_year, native) + ), + ) + result.append(record) + _require(len(result) == pin.rows, "MEMBER_ROWS") + _require( + digest.count == pin.size_bytes + and digest.digest.hexdigest() == pin.member_sha256, + "CAPTURE_CHANGED", + ) + after = os.fstat(raw.fileno()) + _require( + all( + getattr(before, key) == getattr(after, key) + for key in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ), + "CAPTURE_CHANGED", + ) + return result + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsecHouseholdCoverageFields: + """Reader-issued immutable original observations, without Frame authority.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token: object) -> None: + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + try: + digest = shared._issue_capsule(self, self.payload) + except shared.HouseholdWeightSourceError as error: + raise HouseholdCoverageSourceError(str(error)) from None + object.__setattr__(self, "_issued_sha256", digest) + + @property + def document(self) -> dict: + """Return a defensive view after checking issuance and current producer.""" + return verify_asec_household_coverage_fields(self) + + def households_for( + self, native_roster: Sequence[tuple[int, int]] + ) -> tuple[dict, ...]: + """Look up exact cohort/native keys; preserve unresolved observations.""" + document = self.document + _require( + isinstance(native_roster, (list, tuple)) + and 0 < len(native_roster) <= shared._MAX_ROWS, + "LOOKUP_ROSTER", + ) + keys = [] + for key in native_roster: + _require( + isinstance(key, (list, tuple)) + and len(key) == 2 + and all(type(value) is int for value in key), + "LOOKUP_ROSTER", + ) + keys.append(tuple(key)) + _require(len(set(keys)) == len(keys), "LOOKUP_ROSTER") + records = { + (row["income_year"], row["native_household_id"]): row + for row in document["records"] + } + _require(all(key in records for key in keys), "LOOKUP_UNKNOWN") + return tuple(records[key] for key in keys) + + +def verify_asec_household_coverage_fields( + source: AuthenticatedAsecHouseholdCoverageFields, +) -> dict: + """Validate a reader-issued capsule; JSON is not an admission credential.""" + _require(type(source) is AuthenticatedAsecHouseholdCoverageFields, "SOURCE_TYPE") + try: + payload = shared._checked_capsule_payload(source) + document = json.loads(payload) + _require( + shared._encode(document["producer"]) == shared._encode(_producer()), + "PRODUCER_CHANGED", + ) + except shared.HouseholdWeightSourceError as error: + raise HouseholdCoverageSourceError(str(error)) from None + return document + + +def load_authenticated_asec_household_coverage_fields( + member_paths: Mapping[int, str | Path], *, candidate: bytes | None = None +) -> AuthenticatedAsecHouseholdCoverageFields: + """Reconstruct whole requested original cohorts, never caller-provided rows.""" + try: + pins = shared._registry() + paths = shared._member_path_snapshot(member_paths, pins) + _require( + candidate is None + or ( + type(candidate) is bytes and len(candidate) <= shared._MAX_PAYLOAD_BYTES + ), + "CANDIDATE_SIZE", + ) + producer_document = _producer() + _require( + shared._encode(producer_document["bounded_reader_dependency"]["registry"]) + == shared._encode([asdict(pin) for pin in pins]), + "REGISTRY_TRANSITION", + ) + producer = shared._encode(producer_document) + selected = [pin for pin in pins if pin.income_year in paths] + _require(bool(selected), "MEMBER_PATHS") + _require(sum(pin.rows for pin in selected) <= shared._MAX_ROWS, "REQUEST_ROWS") + records = [] + with tempfile.TemporaryDirectory( + prefix="asec-household-coverage-fields-" + ) as tmp: + for pin in selected: + capture = Path(tmp) / f"{pin.income_year}.csv" + try: + digest = shared._capture_owner._snapshot( + paths[pin.income_year], capture, size=pin.size_bytes + ) + except (OSError, ValueError, TypeError): + raise HouseholdCoverageSourceError("MEMBER_CAPTURE") from None + _require(digest == pin.member_sha256, "MEMBER_SHA256") + try: + records.extend(_read_capture(capture, pin)) + except ( + HouseholdCoverageSourceError, + shared.HouseholdWeightSourceError, + ): + raise + except (OSError, UnicodeError, csv.Error, ValueError, OverflowError): + raise HouseholdCoverageSourceError("MEMBER_ENCODING") from None + _require(shared._encode(_producer()) == producer, "PRODUCER_CHANGED") + payload = shared._encode( + { + "schema": _SCHEMA, + "producer": json.loads(producer), + "members": [asdict(pin) for pin in selected], + "fields": _FIELDS, + "dictionary_authorities": [ + { + "survey_year": year, + "sha256": digest, + "url": "https://www2.census.gov/programs-surveys/cps/datasets/" + f"{year}/march/asec{year}_ddl_pub_full.pdf", + } + for year, _page, digest in shared._DICTIONARIES + ], + "interpretation_notes": _INTERPRETATION_NOTES, + "native_roster_sha256": shared._sha( + shared._encode( + [ + [row["income_year"], row["native_household_id"]] + for row in records + ] + ) + ), + "projection_sha256": shared._sha(shared._encode(records)), + "records": records, + "source_authenticated": True, + "population_binding_authenticated": False, + "release_eligible": False, + "coverage_classification_performed": False, + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + return AuthenticatedAsecHouseholdCoverageFields(payload, _token=_TOKEN) + except shared.HouseholdWeightSourceError as error: + raise HouseholdCoverageSourceError(str(error)) from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py new file mode 100644 index 000000000..c0ba8eb56 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py @@ -0,0 +1,392 @@ +"""Attach authenticated raw ASEC household observations by exact original keys. + +The caller authenticates the v4 checkpoint bytes and loads its actual Frame and +metadata with ``load_asec_raw_stage_checkpoint_v4`` before calling this helper. +Structural checks here do not authenticate arbitrary caller-supplied Frame +values. The augmented source has a new attachment receipt; it is not the input +checkpoint's v4 content identity. No graph registration or price conversion runs. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO + +import h5py +import numpy as np +import pandas as pd + +from microcosm.build.outer_stage_runtime import FrameIdentity, frame_identity +from microcosm.build.us_runtime import asec_checkpoint +from microcosm.build.us_runtime.operator_boundary import ( + assert_operator_free_source_frame, +) +from microcosm.frame import Frame + +ASEC_HOUSEHOLD_OBSERVATION_COLUMNS = ( + "HTOTVAL", + "H_LIVQRT", + "HRHTYPE", + "H_HHTYPE", + "H_SEQ", + "H_TENURE", + "GESTFIPS", +) +_OUTPUT_COLUMNS = tuple( + f"asec_{column}" for column in ASEC_HOUSEHOLD_OBSERVATION_COLUMNS +) +_SHA256 = re.compile(r"[0-9a-f]{64}") +_CONTRACT_LABEL = Path("authenticated-v4-frame") + + +def _require_sha(value: object, label: str) -> str: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise ValueError(f"{label} must be a lowercase SHA-256 digest.") + return value + + +@dataclass(frozen=True) +class AsecHouseholdObservationSource: + """Explicit raw household file binding to the checkpoint's income year.""" + + year: int + path: Path + sha256: str + + def __post_init__(self) -> None: + if type(self.year) is not int: + raise ValueError("ASEC household source income year must be an integer.") + _require_sha(self.sha256, "ASEC household source pin") + object.__setattr__(self, "path", Path(self.path)) + + +@dataclass(frozen=True) +class AsecHouseholdObservationResult: + """The augmented full source Frame and a reproducible, value-free receipt.""" + + frame: Frame + _receipt_json: str + + @property + def receipt(self) -> dict[str, object]: + """Return an independent JSON-safe copy of the immutable receipt.""" + return json.loads(self._receipt_json) + + +def _stat_identity(stat: os.stat_result) -> tuple[int, ...]: + return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns) + + +def _hash_file(handle: BinaryIO) -> str: + handle.seek(0) + return hashlib.file_digest(handle, "sha256").hexdigest() + + +def _hard_child(group: h5py.Group, name: str) -> h5py.Group | h5py.Dataset: + if not isinstance(group.get(name, getlink=True), h5py.HardLink): + raise ValueError(f"ASEC household HDF requires local hard-linked {name!r}.") + child = group[name] + if isinstance(child, h5py.Dataset) and (child.is_virtual or child.external): + raise ValueError("ASEC household HDF cannot read virtual or external storage.") + return child + + +def _dataset(group: h5py.Group, name: str) -> h5py.Dataset: + child = _hard_child(group, name) + if not isinstance(child, h5py.Dataset): + raise ValueError(f"ASEC household HDF {name!r} must be a dataset.") + return child + + +def _column_names(group: h5py.Group, name: str) -> tuple[str, ...]: + dataset = _dataset(group, name) + if dataset.ndim != 1 or dataset.dtype.kind != "S": + raise ValueError("ASEC household column metadata must be fixed-width strings.") + try: + values = tuple(value.decode("utf-8") for value in dataset[()]) + except UnicodeDecodeError as error: + raise ValueError("ASEC household column metadata must be UTF-8.") from error + if len(values) != len(set(values)) or any(not name for name in values): + raise ValueError("ASEC household column metadata is empty or duplicated.") + return values + + +def _read_numeric_household( + handle: BinaryIO, + *, + columns: Sequence[str] = ASEC_HOUSEHOLD_OBSERVATION_COLUMNS, + require_same_block: bool = False, +) -> pd.DataFrame: + """Read selected primitive observations without decoding any object block.""" + if ( + isinstance(columns, str) + or not columns + or any(not isinstance(name, str) or not name for name in columns) + or len(set(columns)) != len(columns) + ): + raise ValueError("ASEC household numeric projection requires unique names.") + requested = tuple(columns) + with h5py.File(handle, "r") as h5: + group = _hard_child(h5, "household") + if not isinstance(group, h5py.Group): + raise ValueError("ASEC household HDF lacks its household group.") + if group.attrs.get("pandas_type") != b"frame" or group.attrs.get("ndim") != 2: + raise ValueError( + "ASEC household HDF requires the pandas fixed frame format." + ) + nblocks = group.attrs.get("nblocks") + if ( + isinstance(nblocks, (bool, np.bool_)) + or not isinstance(nblocks, (int, np.integer)) + or nblocks < 1 + ): + raise ValueError("ASEC household HDF has an invalid block count.") + columns = _column_names(group, "axis0") + if set(requested) - set(columns): + raise ValueError("ASEC household HDF lacks required observed columns.") + axis1 = _dataset(group, "axis1") + if axis1.ndim != 1: + raise ValueError("ASEC household HDF row axis must be one-dimensional.") + rows = axis1.shape[0] + if not rows: + raise ValueError("ASEC household HDF has no household rows.") + all_items = [] + output = {} + for index in range(int(nblocks)): + names = _column_names(group, f"block{index}_items") + all_items.extend(names) + wanted = [i for i, name in enumerate(names) if name in requested] + if not wanted: + # Do not even open an unrelated object/VLARRAY value dataset. + continue + if require_same_block and output: + raise ValueError( + "ASEC household selected observations require one shared block." + ) + values = _dataset(group, f"block{index}_values") + if values.dtype != np.dtype(" pd.DataFrame: + with source.path.open("rb", buffering=0) as handle: + before = _stat_identity(os.fstat(handle.fileno())) + if ( + _stat_identity(source.path.stat()) != before + or _hash_file(handle) != source.sha256 + ): + raise ValueError(f"ASEC household source {source.year} SHA-256 mismatch.") + handle.seek(0) + table = _read_numeric_household(handle) + if ( + _hash_file(handle) != source.sha256 + or _stat_identity(os.fstat(handle.fileno())) != before + or _stat_identity(source.path.stat()) != before + ): + raise ValueError( + f"ASEC household source {source.year} changed during reading." + ) + if table.H_SEQ.duplicated().any(): + raise ValueError(f"ASEC household source {source.year} repeats H_SEQ.") + return table + + +def _int64_column(table: pd.DataFrame, name: str) -> np.ndarray: + if name not in table or table[name].dtype != np.dtype("int64"): + raise ValueError( + f"ASEC household join requires complete ordinary int64 {name}." + ) + return table[name].to_numpy(copy=False) + + +def _validate_input(frame: Frame, metadata: Mapping[str, object]) -> str: + if not isinstance(frame, Frame) or not isinstance(metadata, Mapping): + raise TypeError( + "ASEC household attachment requires an actual v4 Frame and metadata." + ) + asec_checkpoint._validate_raw_stage_binding( + metadata, + path=_CONTRACT_LABEL, + policy=asec_checkpoint._RAW_STAGE_V4, + ) + asec_checkpoint._validate_asec_frame( + frame, path=_CONTRACT_LABEL, artifact_label="ASEC v4 source" + ) + assert_operator_free_source_frame(frame, label="ASEC household observation input") + asec_checkpoint._validate_raw_stage_source_columns( + frame, + path=_CONTRACT_LABEL, + policy=asec_checkpoint._RAW_STAGE_V4, + ) + asec_checkpoint._validate_coverage_v4_binding(frame, metadata, path=_CONTRACT_LABEL) + actual = frame_identity(frame) + for key in ("identity", "source_construction_identity"): + if FrameIdentity.from_payload(metadata[key], label=key) != actual: + raise ValueError( + "ASEC household input structural identity differs from v4 metadata." + ) + if set(_OUTPUT_COLUMNS) & set(frame.table("household").columns): + raise ValueError( + "ASEC household observation attachment refuses existing output columns." + ) + return actual.sha256 + + +def _household_keys(frame: Frame) -> pd.DataFrame: + person, household = frame.person, frame.table("household") + person_keys = pd.DataFrame( + { + "household_id": _int64_column(person, "person_household_id"), + "source_year": _int64_column(person, "source_year"), + "H_SEQ": _int64_column(person, "source_household_id"), + } + ) + keys = person_keys.drop_duplicates() + if keys.household_id.duplicated().any(): + raise ValueError("ASEC persons disagree on the source key within a household.") + if keys.duplicated(["source_year", "H_SEQ"]).any(): + raise ValueError( + "ASEC raw source key is claimed by multiple normalized households." + ) + ids = _int64_column(household, "household_id") + if set(keys.household_id) != set(ids): + raise ValueError( + "ASEC household source keys do not cover actual household IDs." + ) + return keys.set_index("household_id").loc[ids].reset_index() + + +def with_asec_household_observations( + frame: Frame, + *, + checkpoint_metadata: Mapping[str, object], + checkpoint_sha256: str, + sources: Sequence[AsecHouseholdObservationSource], +) -> AsecHouseholdObservationResult: + """Augment the actual authenticated v4 source without changing its input. + + ``checkpoint_sha256`` records the caller's authenticated input-byte pin; this + in-memory helper cannot verify that file. Explicit household sources must + match all per-year pins in the loaded checkpoint receipt. Every raw source + is authenticated locally before and after numeric-only reading. Source codes + remain unclassified; nominal signed amounts remain exact int64 observations. + """ + checkpoint_sha256 = _require_sha(checkpoint_sha256, "Input checkpoint pin") + input_identity = _validate_input(frame, checkpoint_metadata) + declared = { + item["year"]: item["sha256"] + for item in checkpoint_metadata["source_receipt"]["sources"] + } + if any( + not isinstance(source, AsecHouseholdObservationSource) for source in sources + ): + raise TypeError("ASEC household sources require explicit typed file bindings.") + actual = {source.year: source.sha256 for source in sources} + if len(actual) != len(sources) or actual != declared: + raise ValueError("ASEC household source years/pins differ from the v4 receipt.") + keys = _household_keys(frame) + household = frame.table("household").copy() + _int64_column(household, "state_fips") + _int64_column(household, "H_TENURE") + outputs = { + name: np.empty(len(household), dtype=np.int64) for name in _OUTPUT_COLUMNS + } + source_receipts = [] + for source in sorted(sources, key=lambda value: value.year): + raw = _load_source(source) + positions = np.flatnonzero(keys.source_year.to_numpy() == source.year) + source_positions = pd.Index(raw.H_SEQ).get_indexer(keys.H_SEQ.iloc[positions]) + if (source_positions < 0).any(): + raise ValueError( + f"ASEC household source {source.year} lacks a referenced H_SEQ." + ) + joined = raw.iloc[source_positions] + for raw_name, existing_name in ( + ("H_TENURE", "H_TENURE"), + ("GESTFIPS", "state_fips"), + ): + if not np.array_equal( + joined[raw_name].to_numpy(), + household[existing_name].iloc[positions].to_numpy(), + ): + raise ValueError( + f"ASEC household joined {raw_name} disagrees with carried {existing_name}." + ) + for column in ASEC_HOUSEHOLD_OBSERVATION_COLUMNS: + outputs[f"asec_{column}"][positions] = joined[column].to_numpy(copy=False) + source_receipts.append( + { + "income_year": source.year, + "sha256": source.sha256, + "source_rows": len(raw), + "joined_rows": len(positions), + "unreferenced_source_rows": len(raw) - len(positions), + } + ) + for name, values in outputs.items(): + household[name] = values + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = household + augmented = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + assert_operator_free_source_frame( + augmented, label="ASEC household observation output" + ) + if frame_identity(augmented).sha256 != input_identity: + raise AssertionError("ASEC household attachment changed structural identity.") + receipt = { + "schema_version": 1, + "artifact_kind": "microcosm.asec_household_observation_attachment", + "input_checkpoint_sha256": checkpoint_sha256, + "input_structural_identity_sha256": input_identity, + "reader": "pandas_fixed_hdf_numeric_int64_v1", + "join_keys": ["income_year", "H_SEQ"], + "sources": source_receipts, + "household_identity": { + "rows": len(household), + "id_dtype": "int64", + "ordered_ids_sha256": hashlib.sha256( + household.household_id.to_numpy(dtype=" None: + if not condition: + raise HousingStatusRefusalError(reason) + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _parse(value: bytes) -> dict: + _require(type(value) is bytes and 0 < len(value) <= HEADER_MAX_BYTES, "HEADER_SIZE") + try: + parsed = json.loads(value) + _require(type(parsed) is dict and _json(parsed) == value, "HEADER_CANONICAL") + return parsed + except (ValueError, TypeError, UnicodeError, RecursionError): + raise HousingStatusRefusalError("HEADER_FORMAT") from None + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _codebook() -> dict: + return { + "status": { + "unknown": 0, + "receipt": 1, + "nonreceipt": 2, + "not_in_universe": 3, + "conflict": 4, + }, + "route": { + "unresolved": 0, + "public_housing": 1, + "lower_rent": 2, + "both_negative": 3, + "not_in_universe": 4, + }, + "group_quarters": {"not_gq": 0, "gq": 1, "unavailable": 2}, + "conflict_bits": { + "excluded_scope": 1, + "parent_route": 2, + "public_allocation": 4, + "lower_allocation": 8, + "context": 16, + }, + "unknown_reason_bits": {"parent_answer": 1, "lower_answer": 2, "scope": 4}, + "quality": { + "not_applicable": 0, + "no_allocation_code_origin_unresolved": 1, + "allocated": 2, + "conflict": 3, + }, + "validity": "1 iff substantive response; receipt_valid additionally requires consistent known receipt/nonreceipt", + "zero_origin": {"not_zero": 0, "frozen_zero_origin_unresolved": 1}, + } + + +def _definition_identity() -> str: + return _sha( + _json( + { + "module": _sha( + resources.files(__package__) + .joinpath("asec_housing_status.py") + .read_bytes() + ), + "codebook": _codebook(), + "dictionaries": DICTIONARIES, + "zero_origin_evidence": ZERO_ORIGIN_EVIDENCE, + "dependencies": { + name: metadata.version(name) for name in ("numpy", "pandas") + }, + } + ) + ) + + +def _derive(raw: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + """Vectorized dictionary-domain checks and explicit interpretation axes.""" + n = len(raw["household_id"]) + _require(0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + all(v.shape == (n,) and v.dtype == np.dtype("int64") for v in raw.values()), + "RAW_INT64", + ) + for name, maximum in ( + ("HPUBLIC", 2), + ("HLORENT", 2), + ("I_HPUBLI", 1), + ("I_HLOREN", 1), + ): + _require(bool(((raw[name] >= 0) & (raw[name] <= maximum)).all()), "CODE_DOMAIN") + for name, maximum in (("H_TENURE", 3), ("HRHTYPE", 10), ("H_HHTYPE", 3)): + # Zero context remains unresolved under the frozen fill provenance. + _require( + bool(((raw[name] >= 0) & (raw[name] <= maximum)).all()), "CONTEXT_DOMAIN" + ) + _require(bool(np.isin(raw["income_year"], [2022, 2023, 2024]).all()), "INCOME_YEAR") + _require(bool((raw["survey_year"] == raw["income_year"] + 1).all()), "SURVEY_YEAR") + _require(bool((raw["H_SEQ"] > 0).all()), "NATIVE_KEY") + _require(len(np.unique(raw["household_id"])) == n, "DUPLICATE_HOUSEHOLD_ID") + keys = pd.MultiIndex.from_arrays([raw["income_year"], raw["H_SEQ"]]) + _require(keys.is_unique, "DUPLICATE_NATIVE_KEY") + public, lower = raw["HPUBLIC"], raw["HLORENT"] + interview = raw["H_HHTYPE"] == 1 + noninterview = raw["H_HHTYPE"] >= 2 + known = interview & np.isin(raw["H_TENURE"], [2, 3]) + excluded = noninterview | (interview & (raw["H_TENURE"] == 1)) + scope_unknown = ~(known | excluded) + context_conflict = (interview & (raw["HRHTYPE"] == 0)) | ( + noninterview & ((raw["HRHTYPE"] > 0) | (raw["H_TENURE"] > 0)) + ) + parent_conflict = (lower > 0) & (public != 2) + public_alloc_conflict = (raw["I_HPUBLI"] == 1) & ((public == 0) | excluded) + lower_alloc_conflict = (raw["I_HLOREN"] == 1) & ( + (lower == 0) | (public != 2) | excluded + ) + conflicts = ( + ((excluded & ((public > 0) | (lower > 0))).astype("uint8")) + | (parent_conflict.astype("uint8") * 2) + | (public_alloc_conflict.astype("uint8") * 4) + | (lower_alloc_conflict.astype("uint8") * 8) + | (context_conflict.astype("uint8") * 16) + ) + status = np.zeros(n, dtype="uint8") + route = np.zeros(n, dtype="uint8") + for mask, state, path in ( + (excluded, 3, 4), + (known & (public == 1) & (lower == 0), 1, 1), + (known & (public == 2) & (lower == 1), 1, 2), + (known & (public == 2) & (lower == 2), 2, 3), + ): + status[mask], route[mask] = state, path + status[conflicts > 0] = 4 + route[conflicts > 0] = 0 + unknown = (known & (public == 0)).astype("uint8") + unknown |= (known & (public == 2) & (lower == 0)).astype("uint8") * 2 + unknown |= scope_unknown.astype("uint8") * 4 + quality = [] + for answer, flag, invalid in ( + (public, raw["I_HPUBLI"], public_alloc_conflict), + (lower, raw["I_HLOREN"], lower_alloc_conflict), + ): + q = np.where(answer > 0, np.where(flag == 1, 2, 1), 0).astype("uint8") + q[invalid] = 3 + quality.append(q) + return { + "status": status, + "route": route, + "receipt_valid": np.isin(status, [1, 2]).astype("uint8"), + "group_quarters": np.where( + raw["HRHTYPE"] == 0, 2, np.isin(raw["HRHTYPE"], [9, 10]) + ).astype("uint8"), + "conflicts": conflicts, + "unknown_reasons": unknown, + "public_quality": quality[0], + "lower_quality": quality[1], + "HPUBLIC_valid": (public > 0).astype("uint8"), + "HLORENT_valid": (lower > 0).astype("uint8"), + **{ + name + "_zero_origin": (raw[name] == 0).astype("uint8") + for name in OBSERVED_COLUMNS + }, + } + + +@dataclass(frozen=True) +class _HousingStatus: + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _STATUS_TOKEN, "STATUS_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + """Independent provenance copy; modifying it cannot change the artifact.""" + return _parse(self.header) + + def array(self, name: str) -> np.ndarray: + """Return an immutable byte-backed view in declared household order.""" + if name not in COLUMNS: + raise KeyError(name) + return np.frombuffer( + self.buffers[COLUMNS.index(name)], + dtype=" None: + """Recheck shape, actual classifier closure, and all derived evidence.""" + _require( + type(self) in (SyntheticHousingStatus, AuthenticatedHousingStatus), + "STATUS_TYPE", + ) + data = self.header_data + _require( + data["definition_sha256"] == _definition_identity(), "DEFINITION_CHANGED" + ) + n = data["household_rows"] + _require(type(n) is int and 0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + type(self.buffers) is tuple and len(self.buffers) == len(COLUMNS), + "BUFFER_ROSTER", + ) + for name, buf in zip(COLUMNS, self.buffers, strict=True): + _require( + type(buf) is bytes + and len(buf) == n * (8 if name in RAW_COLUMNS else 1), + "BUFFER_SIZE", + ) + _require( + data["source_authentication"] + == ( + "checkpoint_bytes_verified" + if type(self) is AuthenticatedHousingStatus + else "synthetic_unverified" + ), + "STATUS_AUTHENTICATION", + ) + raw = {name: self.array(name) for name in RAW_COLUMNS} + derived = _derive(raw) + _require( + all( + derived[name].tobytes() == self.buffers[COLUMNS.index(name)] + for name in DERIVED_COLUMNS + ), + "DERIVED_EVIDENCE", + ) + if type(self) is AuthenticatedHousingStatus: + from .asec_housing_status_source import _implementation + + _require( + data["implementation"] == _implementation(), "IMPLEMENTATION_CHANGED" + ) + + @property + def content_sha256(self) -> str: + """Complete header and raw/derived buffer identity, without a large join.""" + self.validate() + digest = hashlib.sha256(b"microcosm/asec-housing-status-content/1\0") + for part in _parts(self): + digest.update(part) + return digest.hexdigest() + + +class SyntheticHousingStatus(_HousingStatus): + """Invented observations; never accepted by the authenticated attachment.""" + + +class AuthenticatedHousingStatus(_HousingStatus): + """Only the fixed-cohort source loader may issue this status evidence.""" + + +def _issue( + raw: pd.DataFrame, *, source: dict | None = None, implementation: dict | None = None +) -> _HousingStatus: + _require( + type(raw) is pd.DataFrame + and raw.columns.is_unique + and set(raw.columns) == set(RAW_COLUMNS), + "RAW_COLUMNS", + ) + _require( + all(raw[name].dtype == np.dtype("int64") for name in RAW_COLUMNS), "RAW_INT64" + ) + # Copy into immutable bytes before either validating values or deriving output. + buffers = tuple( + raw[name].to_numpy(dtype=" SyntheticHousingStatus: + """Classify explicitly invented rows; this API never mints source authority.""" + return _issue(observations) + + +def _parts(status): + yield MAGIC + yield struct.pack(" bytes: + """Serialize primitive buffers with a bounded header and transport checksum.""" + _require( + type(status) in (SyntheticHousingStatus, AuthenticatedHousingStatus), + "STATUS_TYPE", + ) + status.validate() + parts = tuple(_parts(status)) + digest = hashlib.sha256() + for part in parts: + digest.update(part) + return b"".join((*parts, digest.digest())) + + +def decode_housing_status( + payload: bytes, *, expected: _HousingStatus +) -> _HousingStatus: + """Require independently issued COMPLETE expected content before replay authority.""" + _require( + type(expected) in (SyntheticHousingStatus, AuthenticatedHousingStatus), + "EXPECTED_STATUS", + ) + expected.validate() + _require( + type(payload) is bytes + and len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + view = memoryview(payload) + _require(payload.startswith(MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" str: + """Write an artifact and return its complete-content identity.""" + Path(path).write_bytes(encode_housing_status(status)) + return status.content_sha256 + + +def read_housing_status( + path: str | Path, *, expected: _HousingStatus +) -> _HousingStatus: + """Bound disk reads and require independently source-issued expected content.""" + with Path(path).open("rb") as handle: + payload = handle.read(PAYLOAD_MAX_BYTES + 1) + return decode_housing_status(payload, expected=expected) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py new file mode 100644 index 000000000..eb0152e31 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py @@ -0,0 +1,338 @@ +"""Fixed-cohort numeric source authentication and tax-result composition. + +This module does not calculate housing benefits. Public source authority requires +an existing verified ASEC source and privately staged exact cohort bytes. +""" + +from __future__ import annotations + +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, Weights + +from . import asec_current_money_source as money_source +from . import asec_household_observations as household_reader +from . import asec_housing_status as status_module +from .asec_current_money import MoneyRefusalError +from .asec_current_money_units import CurrentMoneyTaxUnitResult +from .asec_housing_status import ( + CONTEXT_COLUMNS, + DERIVED_COLUMNS, + OBSERVED_COLUMNS, + RAW_COLUMNS, + AuthenticatedHousingStatus, + HousingStatusRefusalError, + _json, + _parse, + _require, + _sha, +) +from .operator_boundary import assert_operator_free_source_frame + +_READ_COLUMNS = ("H_SEQ", "H_YEAR") + CONTEXT_COLUMNS + OBSERVED_COLUMNS +ATTACHED_COLUMNS = tuple("asec_housing_observed_" + name for name in DERIVED_COLUMNS) +_COMPOSE_TOKEN = object() + + +def _implementation() -> dict: + """Actual classifier, reader, source, Frame, codec and composition closure.""" + package = resources.files(__package__) + return { + "source_verification_sha256": money_source._verification_identity(), + "definition_sha256": status_module._definition_identity(), + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_housing_status_source.py", + "asec_housing_status.py", + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_units.py", + ) + }, + } + + +def _read_cohort(path: Path) -> pd.DataFrame: + with path.open("rb") as handle: + return household_reader._read_numeric_household( + handle, columns=_READ_COLUMNS, require_same_block=True + ) + + +def load_authenticated_housing_status( + source: money_source.AuthenticatedCurrentMoneySource, + *, + cohort_paths: Mapping[int, str | Path], +) -> AuthenticatedHousingStatus: + """Join declared full cohort snapshots to the verified source's native keys. + + No caller pin override. This does not certify release or annual participation. + The source's sealed scope supplies ordered IDs/year/native keys; the freshly + authenticated cohort context must also equal the carried attachment context. + """ + try: + return _load(source, cohort_paths) + except HousingStatusRefusalError: + raise + except MoneyRefusalError as error: + raise HousingStatusRefusalError(error.reason) from None + except (OSError, ValueError, KeyError, TypeError, OverflowError): + raise HousingStatusRefusalError("SOURCE_CONTRACT_REFUSAL") from None + + +def _load(source, cohort_paths: Mapping[int, str | Path]): + _require( + type(source) is money_source.AuthenticatedCurrentMoneySource, + "AUTHENTICATED_SOURCE", + ) + source.validate() + before = _implementation() + evidence = _parse(source.source.identity) + pins = tuple((year, pin) for year, pin in evidence["cohorts"]) + _require( + isinstance(cohort_paths, Mapping) + and set(cohort_paths) == {year for year, _ in pins} + and all(type(year) is int for year in cohort_paths), + "COHORT_PATHS", + ) + # Scope tuples were sealed from the actual Frame by the source loader. + # They cannot borrow mutable pandas index/column backing arrays. + scope = source.scope + output = { + "household_id": np.asarray(scope.household_ids, dtype=np.int64), + "income_year": np.asarray(scope.household_years, dtype=np.int64), + "survey_year": np.empty(len(scope.household_ids), dtype=np.int64), + "H_SEQ": np.asarray( + [int(key) for key in scope.household_native_keys], dtype=np.int64 + ), + **{ + name: np.empty(len(scope.household_ids), dtype=np.int64) + for name in CONTEXT_COLUMNS + OBSERVED_COLUMNS + }, + } + carried = ( + source.frame.table("household") + .loc[:, ["household_id"] + ["asec_" + n for n in CONTEXT_COLUMNS]] + .copy(deep=True) + ) + carried.index = carried.index.copy(deep=True) + carried.columns = carried.columns.copy(deep=True) + _require( + np.array_equal(carried.household_id.to_numpy(), output["household_id"]), + "HOUSEHOLD_ORDER", + ) + _require( + all(carried[name].dtype == np.dtype("int64") for name in carried), + "CARRIED_INT64", + ) + joins = [] + with tempfile.TemporaryDirectory(prefix="microcosm-housing-status-") as directory: + for year, pin in pins: + staged = Path(directory) / f"cohort-{year}.h5" + money_source._stage_verified(cohort_paths[year], pin, staged) + raw = _read_cohort(staged) + staged.unlink() + _require( + not raw.H_SEQ.duplicated().any() and bool((raw.H_SEQ > 0).all()), + "NATIVE_KEY", + ) + _require(bool((raw.H_YEAR == year + 1).all()), "SURVEY_YEAR") + positions = np.flatnonzero(output["income_year"] == year) + source_positions = pd.Index(raw.H_SEQ).get_indexer( + output["H_SEQ"][positions] + ) + _require(bool((source_positions >= 0).all()), "SOURCE_KEY_COVERAGE") + joined = raw.iloc[source_positions] + for name in CONTEXT_COLUMNS: + _require( + np.array_equal( + joined[name].to_numpy(), + carried["asec_" + name].iloc[positions].to_numpy(), + ), + "CARRIED_CONTEXT", + ) + output["survey_year"][positions] = joined.H_YEAR.to_numpy() + for name in CONTEXT_COLUMNS + OBSERVED_COLUMNS: + output[name][positions] = joined[name].to_numpy() + joins.append( + { + "income_year": year, + "sha256": pin, + "source_rows": len(raw), + "joined_rows": len(positions), + "unreferenced_source_rows": len(raw) - len(positions), + } + ) + source.validate() + _require(_implementation() == before, "IMPLEMENTATION_CHANGED") + return status_module._issue( + pd.DataFrame(output, columns=RAW_COLUMNS), + source={"identity": source.source.identity.decode(), "joins": joins}, + implementation=before, + ) + + +def _owned_frame(frame: Frame) -> Frame: + # Frame copies native buffers; explicitly detach axes and weight vectors too. + # This is one complete result copy, not a partial/fabricated US population. + strata = frame.strata.copy(deep=True) + strata.index = money_source._owned_index(strata.index) + result = Frame( + { + **{entity: frame.table(entity) for entity in frame.entities}, + **{name: frame.link(name) for name in frame.links}, + }, + frame.schema, + { + entity: Weights( + frame.weights_for(entity).values, frame.weights_for(entity).kind + ) + for entity in frame.weighted_entities + }, + strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + money_source._detach_frame_axes(result) + return result + + +def _check_parent_status(parent, status): + _require(type(parent) is CurrentMoneyTaxUnitResult, "TAX_RESULT_REQUIRED") + _require(type(status) is AuthenticatedHousingStatus, "AUTHENTICATED_STATUS") + parent.validate() + status.validate() + _require( + status.header_data["source"]["identity"].encode() + == parent.money.bindings.spec.source.identity, + "STATUS_SOURCE_BINDING", + ) + household = parent.frame.table("household") + _require( + np.array_equal(household.household_id.to_numpy(), status.array("household_id")), + "STATUS_HOUSEHOLD_BINDING", + ) + for name in CONTEXT_COLUMNS: + _require( + np.array_equal(household["asec_" + name].to_numpy(), status.array(name)), + "STATUS_CONTEXT_BINDING", + ) + + +def validate_attached_frame( + frame: Frame, + *, + parent: CurrentMoneyTaxUnitResult, + status: AuthenticatedHousingStatus, +) -> None: + """Verify a live or checkpoint-reloaded composition against both trusted parents.""" + try: + _check_parent_status(parent, status) + _require(type(frame) is Frame, "COMPOSED_FRAME") + frame.revalidate() + assert_operator_free_source_frame(frame, label="ASEC housing status attachment") + household = frame.table("household") + _require(set(ATTACHED_COLUMNS) <= set(household), "ATTACHMENT_COLUMNS") + for name, attached in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + _require( + household[attached].dtype == np.dtype("uint8") + and household[attached].to_numpy().tobytes() + == status.array(name).tobytes(), + "ATTACHMENT_OUTPUT", + ) + # Drop ONLY our columns, then compare the existing full Frame signature. + recovered = _owned_frame(frame) + recovered.table("household").drop(columns=list(ATTACHED_COLUMNS), inplace=True) + _require( + money_source._frame_signature(recovered) + == parent.receipt["output_frame_sha256"], + "PARENT_FRAME_RECOVERY", + ) + parent.validate() + except HousingStatusRefusalError: + raise + except MoneyRefusalError as error: + raise HousingStatusRefusalError(error.reason) from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingStatusRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None + + +@dataclass(frozen=True) +class HousingStatusAttachedAsec: + """Separate composed result retaining the identical sealed tax/money parents.""" + + frame: Frame + tax_result: CurrentMoneyTaxUnitResult + status: AuthenticatedHousingStatus + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _COMPOSE_TOKEN, "ATTACHMENT_CONSTRUCTOR_UNAVAILABLE") + + @property + def money(self): + return self.tax_result.money + + @property + def receipt(self): + return _parse(self._receipt) + + def validate(self) -> None: + validate_attached_frame(self.frame, parent=self.tax_result, status=self.status) + _require( + self.receipt["housing_content_sha256"] == self.status.content_sha256 + and self.receipt["tax_receipt_sha256"] + == _sha(_json(self.tax_result.receipt)) + and self.receipt["output_frame_sha256"] + == money_source._frame_signature(self.frame), + "COMPOSED_RESULT_CHANGED", + ) + + +def attach_housing_status( + tax_result: CurrentMoneyTaxUnitResult, status: AuthenticatedHousingStatus +) -> HousingStatusAttachedAsec: + """Compose source-status evidence after actual tax reconstruction, without mutation.""" + try: + _check_parent_status(tax_result, status) + _require( + not set(ATTACHED_COLUMNS) & set(tax_result.frame.table("household")), + "EXISTING_ATTACHMENT_COLUMNS", + ) + result = _owned_frame(tax_result.frame) + _require( + money_source._frame_signature(result) + == tax_result.receipt["output_frame_sha256"], + "PARENT_CAPTURE_CHANGED", + ) + for name, output in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + result.table("household")[output] = status.array(name).copy() + validate_attached_frame(result, parent=tax_result, status=status) + receipt = { + "schema_version": 1, + "artifact_kind": "microcosm.asec_housing_status_attachment.v1", + "release_eligible": False, + "tax_receipt_sha256": _sha(_json(tax_result.receipt)), + "housing_content_sha256": status.content_sha256, + "parent_frame_sha256": tax_result.receipt["output_frame_sha256"], + "output_frame_sha256": money_source._frame_signature(result), + "outputs": ATTACHED_COLUMNS, + } + return HousingStatusAttachedAsec( + result, tax_result, status, _json(receipt), _token=_COMPOSE_TOKEN + ) + except HousingStatusRefusalError: + raise + except MoneyRefusalError as error: + raise HousingStatusRefusalError(error.reason) from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingStatusRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py new file mode 100644 index 000000000..e67327f8d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py @@ -0,0 +1,423 @@ +"""Source-specific occupied-HU evidence; never a population or release verdict.""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import struct +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from .asec_housing_status import DICTIONARIES, ZERO_ORIGIN_EVIDENCE, ZERO_POLICY + +RAW_COLUMNS = ( + "household_id", + "income_year", + "H_SEQ", + "H_HHTYPE", + "H_LIVQRT", + "HRHTYPE", + "H_TENURE", +) +CONTEXT_COLUMNS = ("H_HHTYPE", "H_LIVQRT", "HRHTYPE", "H_TENURE") +DERIVED_COLUMNS = ( + "interview_scope", + "physical_unit", + "household_kind", + "tenure_subtype", + "occupied_hu", + "hu_tenure_class", + "unresolved_reasons", +) +COLUMNS = RAW_COLUMNS + DERIVED_COLUMNS +MAGIC = b"MCAHUNIV\x01" +HEADER_MAX_BYTES = 65536 +MAX_HOUSEHOLDS = 400_000 +PAYLOAD_MAX_BYTES = ( + len(MAGIC) + + 4 + + HEADER_MAX_BYTES + + MAX_HOUSEHOLDS * (8 * len(RAW_COLUMNS) + len(DERIVED_COLUMNS)) + + 32 +) +_UNIVERSE_TOKEN = object() + + +class HousingUniverseRefusalError(ValueError): + """Sanitized reason for unavailable or inconsistent source evidence.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise HousingUniverseRefusalError(reason) + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _parse(value: bytes) -> dict: + _require(type(value) is bytes and 0 < len(value) <= HEADER_MAX_BYTES, "HEADER_SIZE") + try: + result = json.loads(value) + _require(type(result) is dict and _json(result) == value, "HEADER_CANONICAL") + return result + except (ValueError, TypeError, UnicodeError, RecursionError): + raise HousingUniverseRefusalError("HEADER_FORMAT") from None + + +def _codebook() -> dict: + return { + "interview_scope": { + "unavailable": 0, + "interview": 1, + "type_a_noninterview": 2, + "type_bc_noninterview": 3, + }, + "physical_unit": {"unavailable": 0, "housing_unit": 1, "other_unit": 2}, + "household_kind": { + "source_zero_or_not_established": 0, + "primary_household": 1, + "group_quarters": 2, + }, + "tenure_subtype": { + "niu_or_zero_unresolved": 0, + "owned_or_being_bought": 1, + "cash_renter": 2, + "no_cash_rent": 3, + }, + "occupied_hu": { + "not_established": 0, + "occupied_housing_unit": 1, + "outside_housing_unit": 2, + }, + "hu_tenure_class": { + "outside_or_unestablished_hu": 0, + "owner": 1, + "renter": 2, + "occupied_hu_unresolved_tenure": 3, + }, + "unresolved_reason_bits": { + "outside_interview_scope": 1, + "interview_zero": 2, + "quarters_zero": 4, + "household_kind_zero": 8, + "hu_quarters_gq_kind": 16, + "context_conflict": 32, + "hu_tenure_zero": 64, + "other_quarters_primary_kind": 128, + }, + } + + +def _definition_identity() -> str: + return _sha( + _json( + { + "module": _sha( + resources.files(__package__) + .joinpath("asec_housing_universe.py") + .read_bytes() + ), + "codebook": _codebook(), + "dictionaries": DICTIONARIES, + "zero_origin_evidence": ZERO_ORIGIN_EVIDENCE, + "dependencies": { + name: metadata.version(name) for name in ("numpy", "pandas") + }, + } + ) + ) + + +def _derive(raw: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + n = len(raw["household_id"]) + _require(0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + set(raw) == set(RAW_COLUMNS) + and all(v.shape == (n,) and v.dtype == np.dtype("int64") for v in raw.values()), + "RAW_INT64", + ) + for name, maximum in ( + ("H_HHTYPE", 3), + ("H_LIVQRT", 12), + ("HRHTYPE", 10), + ("H_TENURE", 3), + ): + _require( + bool(((raw[name] >= 0) & (raw[name] <= maximum)).all()), "CONTEXT_DOMAIN" + ) + _require(bool(np.isin(raw["income_year"], (2022, 2023, 2024)).all()), "INCOME_YEAR") + _require(bool((raw["H_SEQ"] > 0).all()), "NATIVE_KEY") + _require(len(np.unique(raw["household_id"])) == n, "DUPLICATE_HOUSEHOLD_ID") + _require( + pd.MultiIndex.from_arrays([raw["income_year"], raw["H_SEQ"]]).is_unique, + "DUPLICATE_NATIVE_KEY", + ) + interview = raw["H_HHTYPE"] == 1 + noninterview = raw["H_HHTYPE"] >= 2 + quarters, kind, tenure = raw["H_LIVQRT"], raw["HRHTYPE"], raw["H_TENURE"] + hu_quarters = (quarters >= 1) & (quarters <= 7) + other_quarters = quarters >= 8 + primary = (kind >= 1) & (kind <= 8) + gq = kind >= 9 + context_conflict = (interview & (kind == 0)) | ( + noninterview & ((kind > 0) | (tenure > 0)) + ) + hu = interview & hu_quarters & primary & ~context_conflict + outside = interview & other_quarters & gq & ~context_conflict + reasons = noninterview.astype("uint8") + for mask, bit in ( + (raw["H_HHTYPE"] == 0, 2), + (interview & (quarters == 0), 4), + (interview & (kind == 0), 8), + (interview & hu_quarters & gq, 16), + (context_conflict, 32), + (hu & (tenure == 0), 64), + (interview & other_quarters & primary, 128), + ): + reasons |= mask.astype("uint8") * bit + return { + "interview_scope": raw["H_HHTYPE"].astype("uint8"), + "physical_unit": np.where( + hu_quarters, 1, np.where(other_quarters, 2, 0) + ).astype("uint8"), + "household_kind": np.where(primary, 1, np.where(gq, 2, 0)).astype("uint8"), + "tenure_subtype": tenure.astype("uint8"), + "occupied_hu": np.where(hu, 1, np.where(outside, 2, 0)).astype("uint8"), + "hu_tenure_class": np.where( + hu, np.where(tenure == 1, 1, np.where(tenure >= 2, 2, 3)), 0 + ).astype("uint8"), + "unresolved_reasons": reasons, + } + + +@dataclass(frozen=True) +class _HousingUniverse: + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _UNIVERSE_TOKEN, "UNIVERSE_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + return _parse(self.header) + + def array(self, name: str) -> np.ndarray: + if name not in COLUMNS: + raise KeyError(name) + return np.frombuffer( + self.buffers[COLUMNS.index(name)], + dtype=" None: + """Recompute every derived code from immutable raw evidence.""" + _require( + type(self) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "UNIVERSE_TYPE", + ) + data = self.header_data + try: + _require( + data["definition_sha256"] == _definition_identity(), + "DEFINITION_CHANGED", + ) + n = data["household_rows"] + _require(type(n) is int and 0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + type(self.buffers) is tuple and len(self.buffers) == len(COLUMNS), + "BUFFER_ROSTER", + ) + for name, buf in zip(COLUMNS, self.buffers, strict=True): + _require( + type(buf) is bytes + and len(buf) == n * (8 if name in RAW_COLUMNS else 1), + "BUFFER_SIZE", + ) + authenticated = type(self) is AuthenticatedHousingUniverse + _require( + data["source_authentication"] + == ( + "checkpoint_bytes_verified" + if authenticated + else "synthetic_unverified" + ), + "UNIVERSE_AUTHENTICATION", + ) + derived = _derive({name: self.array(name) for name in RAW_COLUMNS}) + _require( + all( + derived[name].tobytes() == self.buffers[COLUMNS.index(name)] + for name in DERIVED_COLUMNS + ), + "DERIVED_EVIDENCE", + ) + if authenticated: + from .asec_housing_universe_source import _implementation + + _require( + data["implementation"] == _implementation(), + "IMPLEMENTATION_CHANGED", + ) + except (KeyError, TypeError, OverflowError): + raise HousingUniverseRefusalError("UNIVERSE_CONTRACT") from None + + @property + def content_sha256(self) -> str: + self.validate() + digest = hashlib.sha256(b"microcosm/asec-housing-universe-content/1\0") + for part in _parts(self): + digest.update(part) + return digest.hexdigest() + + +class SyntheticHousingUniverse(_HousingUniverse): + """Invented observations, with no source authority.""" + + +class AuthenticatedHousingUniverse(_HousingUniverse): + """Issued only from an authenticated current-money source.""" + + +def _issue(raw: pd.DataFrame, *, source=None, implementation=None) -> _HousingUniverse: + _require( + type(raw) is pd.DataFrame + and raw.columns.is_unique + and set(raw.columns) == set(RAW_COLUMNS), + "RAW_COLUMNS", + ) + _require( + all(raw[name].dtype == np.dtype("int64") for name in RAW_COLUMNS), "RAW_INT64" + ) + _require(0 < len(raw) <= MAX_HOUSEHOLDS, "ROW_COUNT") + buffers = tuple( + raw[name].to_numpy(dtype=" SyntheticHousingUniverse: + """Classify invented rows without granting source authority.""" + return _issue(observations) + + +def _parts(value): + yield MAGIC + yield struct.pack(" bytes: + """Canonical primitive bytes with a bounded header and transport checksum.""" + _require( + type(value) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "UNIVERSE_TYPE", + ) + value.validate() + payload = b"".join(_parts(value)) + return payload + hashlib.sha256(payload).digest() + + +def decode_housing_universe( + payload: bytes, *, expected: _HousingUniverse +) -> _HousingUniverse: + """Only independently issued complete expected evidence confers replay authority.""" + _require( + type(expected) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "EXPECTED_UNIVERSE", + ) + expected.validate() + _require( + type(payload) is bytes + and len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + # Compare before parsing any candidate header or buffer; no alternate encoding. + _require(payload == encode_housing_universe(expected), "EXPECTED_CONTENT") + return type(expected)(expected.header, expected.buffers, _token=_UNIVERSE_TOKEN) + + +def write_housing_universe(value: _HousingUniverse, path: str | Path) -> str: + """Create an immutable artifact; never overwrite an existing path.""" + payload = encode_housing_universe(value) + try: + with Path(path).open("xb") as handle: + handle.write(payload) + except OSError: + raise HousingUniverseRefusalError("ARTIFACT_WRITE") from None + return value.content_sha256 + + +def read_housing_universe( + path: str | Path, *, expected: _HousingUniverse +) -> _HousingUniverse: + """Bound regular-file reads; no FIFO, device or candidate header parsing.""" + _require( + type(expected) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "EXPECTED_UNIVERSE", + ) + size = len(encode_housing_universe(expected)) + descriptor = None + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + info = os.fstat(descriptor) + _require(stat.S_ISREG(info.st_mode) and info.st_size == size, "ARTIFACT_FILE") + with os.fdopen(descriptor, "rb") as handle: + descriptor = None + payload = handle.read(size + 1) + return decode_housing_universe(payload, expected=expected) + except (OSError, TypeError): + raise HousingUniverseRefusalError("ARTIFACT_READ") from None + finally: + if descriptor is not None: + os.close(descriptor) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py new file mode 100644 index 000000000..23deb4385 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py @@ -0,0 +1,336 @@ +"""Authenticate carried ASEC HU evidence and preserve the chosen parent chain.""" + +from __future__ import annotations + +from dataclasses import InitVar, dataclass +from importlib import resources + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import asec_current_money_source as money_source +from . import asec_housing_status_source as housing_source +from . import asec_housing_universe as universe_module +from .asec_current_money import MoneyRefusalError +from .asec_current_money_units import CurrentMoneyTaxUnitResult +from .asec_housing_status import HousingStatusRefusalError +from .asec_housing_status_source import HousingStatusAttachedAsec, _owned_frame +from .asec_housing_universe import ( + CONTEXT_COLUMNS, + DERIVED_COLUMNS, + RAW_COLUMNS, + AuthenticatedHousingUniverse, + HousingUniverseRefusalError, + _json, + _parse, + _require, + _sha, +) +from .operator_boundary import assert_operator_free_source_frame + +ATTACHED_COLUMNS = tuple("asec_housing_universe_" + name for name in DERIVED_COLUMNS) +_COMPOSE_TOKEN = object() + + +def _implementation() -> dict: + """Bind new interpretation/composition without editing any old verifier roster.""" + package = resources.files(__package__) + return { + "source_verification_sha256": money_source._verification_identity(), + "definition_sha256": universe_module._definition_identity(), + "housing_implementation": housing_source._implementation(), + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ("asec_housing_universe_source.py", "asec_housing_universe.py") + }, + } + + +def load_authenticated_housing_universe( + source: money_source.AuthenticatedCurrentMoneySource, +) -> AuthenticatedHousingUniverse: + """Read only the already-authenticated household attachment, with no new joins.""" + try: + _require( + type(source) is money_source.AuthenticatedCurrentMoneySource, + "AUTHENTICATED_SOURCE", + ) + source.validate() + implementation = _implementation() + scope = source.scope + table = source.frame.table("household") + columns = ("household_id",) + tuple( + "asec_" + name for name in ("H_SEQ",) + CONTEXT_COLUMNS + ) + _require( + table.columns.is_unique and set(columns) <= set(table), "CARRIED_COLUMNS" + ) + _require( + all(table[name].dtype == np.dtype("int64") for name in columns), + "CARRIED_INT64", + ) + raw = pd.DataFrame( + { + "household_id": np.asarray(scope.household_ids, dtype="int64"), + "income_year": np.asarray(scope.household_years, dtype="int64"), + "H_SEQ": np.asarray( + [int(key) for key in scope.household_native_keys], dtype="int64" + ), + **{ + name: table["asec_" + name].to_numpy(copy=True) + for name in CONTEXT_COLUMNS + }, + }, + columns=RAW_COLUMNS, + ) + _require( + np.array_equal(table.household_id.to_numpy(), raw.household_id.to_numpy()), + "HOUSEHOLD_ORDER", + ) + _require( + np.array_equal(table.asec_H_SEQ.to_numpy(), raw.H_SEQ.to_numpy()), + "NATIVE_KEY_BINDING", + ) + result = universe_module._issue( + raw, + source={ + "identity": source.source.identity.decode(), + "carried_columns": columns, + "income_year_origin": "authenticated_money_scope", + }, + implementation=implementation, + ) + source.validate() + _require(_implementation() == implementation, "IMPLEMENTATION_CHANGED") + return result + except HousingUniverseRefusalError: + raise + except MoneyRefusalError as error: + raise HousingUniverseRefusalError(error.reason) from None + except (ValueError, TypeError, KeyError, OverflowError, AssertionError, OSError): + raise HousingUniverseRefusalError("SOURCE_CONTRACT_REFUSAL") from None + + +def _tax_parent(parent) -> CurrentMoneyTaxUnitResult: + _require( + type(parent) in (CurrentMoneyTaxUnitResult, HousingStatusAttachedAsec), + "TAX_OR_HOUSING_PARENT_REQUIRED", + ) + return parent if type(parent) is CurrentMoneyTaxUnitResult else parent.tax_result + + +def _check_parent(parent, universe): + tax = _tax_parent(parent) + _require(type(universe) is AuthenticatedHousingUniverse, "AUTHENTICATED_UNIVERSE") + parent.validate() + universe.validate() + _require( + universe.header_data["source"]["identity"].encode() + == tax.money.bindings.spec.source.identity, + "UNIVERSE_SOURCE_BINDING", + ) + household = parent.frame.table("household") + _require( + household.household_id.dtype == np.dtype("int64") + and np.array_equal( + household.household_id.to_numpy(), universe.array("household_id") + ), + "UNIVERSE_HOUSEHOLD_BINDING", + ) + _require( + _parse(tax.money.header)["household_year_sha256"] + == _sha(universe.array("income_year").tobytes()), + "UNIVERSE_YEAR_BINDING", + ) + for name in ("H_SEQ",) + CONTEXT_COLUMNS: + column = household["asec_" + name] + _require( + column.dtype == np.dtype("int64") + and column.to_numpy().tobytes() == universe.array(name).tobytes(), + "UNIVERSE_CONTEXT_BINDING", + ) + + +def validate_attached_frame( + frame: Frame, + *, + parent: CurrentMoneyTaxUnitResult | HousingStatusAttachedAsec, + universe: AuthenticatedHousingUniverse, +) -> None: + """Check a whole owned parent extension; a selected child cannot inherit this authority.""" + try: + _check_parent(parent, universe) + _require(type(frame) is Frame, "COMPOSED_FRAME") + frame.revalidate() + assert_operator_free_source_frame( + frame, label="ASEC housing universe attachment" + ) + household = frame.table("household") + _require(set(ATTACHED_COLUMNS) <= set(household), "ATTACHMENT_COLUMNS") + for name, attached in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + column = household[attached] + _require( + column.dtype == np.dtype("uint8") + and column.to_numpy().tobytes() == universe.array(name).tobytes(), + "ATTACHMENT_OUTPUT", + ) + recovered = _owned_frame(frame) + recovered.table("household").drop(columns=list(ATTACHED_COLUMNS), inplace=True) + _require( + money_source._frame_signature(recovered) + == parent.receipt["output_frame_sha256"], + "PARENT_FRAME_RECOVERY", + ) + parent.validate() + universe.validate() + except HousingUniverseRefusalError: + raise + except MoneyRefusalError as error: + raise HousingUniverseRefusalError(error.reason) from None + except HousingStatusRefusalError: + raise HousingUniverseRefusalError("HOUSING_PARENT_REFUSAL") from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingUniverseRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None + + +@dataclass(frozen=True) +class HousingUniverseAttachedAsec: + """Owned HU extension retaining its exact tax or housing-status parent.""" + + frame: Frame + parent: CurrentMoneyTaxUnitResult | HousingStatusAttachedAsec + universe: AuthenticatedHousingUniverse + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _COMPOSE_TOKEN, "ATTACHMENT_CONSTRUCTOR_UNAVAILABLE") + + @property + def tax_result(self): + return _tax_parent(self.parent) + + @property + def money(self): + return self.tax_result.money + + @property + def receipt(self): + return _parse(self._receipt) + + def validate(self) -> None: + validate_attached_frame(self.frame, parent=self.parent, universe=self.universe) + _require( + self.receipt == _receipt(self.parent, self.universe, self.frame), + "COMPOSED_RESULT_CHANGED", + ) + + +def _receipt(parent, universe, frame) -> dict: + return { + "schema_version": 1, + "artifact_kind": "microcosm.asec_housing_universe_attachment.v1", + "release_eligible": False, + "parent_kind": type(parent).__name__, + "parent_receipt_sha256": _sha(_json(parent.receipt)), + "tax_receipt_sha256": _sha(_json(_tax_parent(parent).receipt)), + "money_header_sha256": _sha(parent.money.header), + "universe_content_sha256": universe.content_sha256, + "parent_frame_sha256": parent.receipt["output_frame_sha256"], + "output_frame_sha256": money_source._frame_signature(frame), + "outputs": list(ATTACHED_COLUMNS), + } + + +def attach_housing_universe( + parent: CurrentMoneyTaxUnitResult | HousingStatusAttachedAsec, + universe: AuthenticatedHousingUniverse, +) -> HousingUniverseAttachedAsec: + """Preserve all chosen parent columns, including student and subsidy evidence.""" + try: + _check_parent(parent, universe) + _require( + not set(ATTACHED_COLUMNS) & set(parent.frame.table("household")), + "EXISTING_ATTACHMENT_COLUMNS", + ) + frame = _owned_frame(parent.frame) + _require( + money_source._frame_signature(frame) + == parent.receipt["output_frame_sha256"], + "PARENT_CAPTURE_CHANGED", + ) + for name, alias in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + frame.table("household")[alias] = universe.array(name).copy() + validate_attached_frame(frame, parent=parent, universe=universe) + receipt = _receipt(parent, universe, frame) + result = HousingUniverseAttachedAsec( + frame, parent, universe, _json(receipt), _token=_COMPOSE_TOKEN + ) + result.validate() + return result + except HousingUniverseRefusalError: + raise + except MoneyRefusalError as error: + raise HousingUniverseRefusalError(error.reason) from None + except HousingStatusRefusalError: + raise HousingUniverseRefusalError("HOUSING_PARENT_REFUSAL") from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingUniverseRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None + + +def verify_housing_universe_rows( + household_table: pd.DataFrame, + universe: AuthenticatedHousingUniverse, + *, + lineage_ids: np.ndarray, +) -> None: + """Verify source-derived cells of subsets/clones, without issuing Frame authority. + + ``lineage_ids`` are the caller's original source household IDs, repeated for + clones. Their custody/assembly provenance is a separate obligation. This + checks carried H_SEQ/context and derived values, not a selected Frame receipt. + """ + try: + _require( + type(universe) is AuthenticatedHousingUniverse, "AUTHENTICATED_UNIVERSE" + ) + universe.validate() + _require( + type(household_table) is pd.DataFrame and household_table.columns.is_unique, + "HOUSEHOLD_TABLE", + ) + _require( + type(lineage_ids) is np.ndarray + and lineage_ids.dtype == np.dtype("int64") + and lineage_ids.shape == (len(household_table),), + "LINEAGE_INT64", + ) + positions = pd.Index(universe.array("household_id")).get_indexer(lineage_ids) + _require(bool((positions >= 0).all()), "LINEAGE_COVERAGE") + if "household_source_id" in household_table: + carried_ids = household_table.household_source_id + _require( + carried_ids.dtype == np.dtype("int64") + and np.array_equal(carried_ids.to_numpy(), lineage_ids), + "LINEAGE_BINDING", + ) + for name, column, dtype in ( + *((n, "asec_" + n, "int64") for n in ("H_SEQ",) + CONTEXT_COLUMNS), + *( + (n, c, "uint8") + for n, c in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True) + ), + ): + values = household_table[column] + _require( + values.dtype == np.dtype(dtype) + and np.array_equal(values.to_numpy(), universe.array(name)[positions]), + "ROW_EVIDENCE", + ) + universe.validate() + except HousingUniverseRefusalError: + raise + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingUniverseRefusalError("ROW_CONTRACT_REFUSAL") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py new file mode 100644 index 000000000..0c0283d17 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py @@ -0,0 +1,574 @@ +"""Closed PAW source observations, independent of the sealed 33-field money body. + +Only exact source reconstruction issues this authority. Candidate replay compares +canonical expected bytes before parsing any candidate. No Frame columns are added. +""" + +from __future__ import annotations + +import csv +import hashlib +import os +import stat +import struct +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as legacy +from . import asec_person_income_source as restoration +from ._asec_current_money_codec import current_money_content_sha256 +from .asec_current_money import ( + RESOURCE_PINS, + RESTORED_SOURCE_KIND, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _require, + _sha, + _validate_money, +) +from .asec_student_controls import _snapshot +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +MAGIC = b"MCAINCOB\x01" +FILENAME = "asec_income_observations.bin" +ARTIFACT_KIND = "microcosm.us.asec_income_observations.v1" +_HEADER_MAX = 65536 +_MAX_PERSONS = 600_000 +INCOME_PAYLOAD_MAX_BYTES = len(MAGIC) + 4 + _HEADER_MAX + 8 * 8 * _MAX_PERSONS + 32 +COORDINATES = ("person_id", "income_year", "source_household_id", "A_LINENO", "A_AGE") +COLUMNS = COORDINATES + ("PAW_YN", "PAW_VAL", "PAW_VAL_2024_price") +COLUMN_DTYPES = { + name: "little_endian_float64" if name == COLUMNS[-1] else "little_endian_int64" + for name in COLUMNS +} +_READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "PAW_VAL", "PAW_YN") +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +RESOURCE = "asec_income_observations_v1.json" +RESOURCE_SHA256 = "6e37855ddd6642d02231efc664644029ed6131885eb743a00197261bc5a6a51f" +ZERO_CLASSES = ( + "observed_amount", + "niu_below_receipt_age", + "reported_receipt_zero_amount", + "reported_no_receipt", + "niu_in_age_universe", +) +_TOKEN = object() +_CSV_DIALECT = "excel" +_CSV_ENCODING = "utf-8" +_PRICE_TARGET = "174.4" +_PRICE_YEARS = (2022, 2023, 2024) + + +def parser_profile() -> dict: + """Read the effective CSV parser limits and dialect, without changing them. + + ``csv.field_size_limit`` is process state that silently changes which member + files parse. Binding it into the source identity makes any change to it a + different artifact identity instead of an invisible reparse. + """ + # _read_member uses csv.reader's built-in default, which is independent of + # the mutable named-dialect registry. Inspect that same reader's dialect. + dialect = csv.reader(()).dialect + return { + "header_engine": "stdlib_csv_reader", + "table_engine": "pandas_read_csv", + "dialect": _CSV_DIALECT, + "encoding": _CSV_ENCODING, + "newline": "", + "delimiter": dialect.delimiter, + "quotechar": dialect.quotechar, + "doublequote": bool(dialect.doublequote), + "skipinitialspace": bool(dialect.skipinitialspace), + "strict": bool(dialect.strict), + "quoting": int(dialect.quoting), + "csv.field_size_limit": csv.field_size_limit(), + } + + +def price_factors(contract) -> tuple[tuple[int, str, str], ...]: + """The single restatement-factor derivation, shared with graph transport.""" + return tuple( + ( + year, + f"{_PRICE_TARGET}/{contract['price']['cells'][str(year)]}", + struct.pack( + " 0).all()) + and bool((table.A_LINENO > 0).all()), + "INCOME_MEMBER_COORDINATES", + ) + _validate_observations( + table.A_AGE.to_numpy(), table.PAW_YN.to_numpy(), table.PAW_VAL.to_numpy() + ) + return table + + +def _validate_observations( + age: np.ndarray, receipt_code: np.ndarray, nominal: np.ndarray +): + _require( + all( + type(a) is np.ndarray + and a.dtype == np.dtype("int64") + and a.ndim == 1 + and a.shape == nominal.shape + for a in (age, receipt_code, nominal) + ), + "INCOME_OBSERVATION_ARRAYS", + ) + _require( + bool(((nominal >= 0) & (nominal <= 99999)).all()), + "INCOME_CODE_DOMAIN", + "PAW_VAL", + ) + _require( + bool(np.isin(receipt_code, [0, 1, 2]).all()), "INCOME_CODE_DOMAIN", "PAW_YN" + ) + _require( + bool((receipt_code[nominal != 0] == 1).all()) + and bool((nominal[receipt_code != 1] == 0).all()) + and bool((receipt_code[age < 15] == 0).all()) + and bool((nominal[age < 15] == 0).all()), + "INCOME_SOURCE_UNIVERSE", + ) + + +def _restate_amounts(nominal, years, factors): + """The single PAW restatement path, also used to verify graph transport.""" + amounts = nominal.astype("= 15) & (receipt_code == 1)] = 2 + result[zero & (age >= 15) & (receipt_code == 2)] = 3 + result[zero & (age >= 15) & (receipt_code == 0)] = 4 + return result + + +@dataclass(frozen=True) +class AuthenticatedIncomeObservations: + """Immutable source authority; only the complete reconstructed join issues it.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "INCOME_CONSTRUCTOR_UNAVAILABLE") + self.validate() + + @property + def receipt(self): + return _parse(self._header) + + @property + def content_sha256(self): + return _sha(self._header + self._body) + + def validate(self): + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "INCOME_HEADER_SIZE", + ) + header = self.receipt + _require( + set(header) == _HEADER_KEYS and _json(header) == self._header, + "INCOME_HEADER_SCHEMA", + ) + rows = header["rows"] + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "INCOME_ROWS") + _require( + header["schema_version"] == 1 + and header["artifact_kind"] == ARTIFACT_KIND + and type(self._body) is bytes + and len(self._body) == rows * len(COLUMNS) * 8 + and _sha(self._body) == header["body_sha256"] + and header["columns"] == list(COLUMNS) + and header["column_dtypes"] == COLUMN_DTYPES + and _json(header["implementation"]) == _json(_implementation()) + and header["column_sha256"] + == { + name: _sha(self._body[i * rows * 8 : (i + 1) * rows * 8]) + for i, name in enumerate(COLUMNS) + }, + "INCOME_CONTENT_CHANGED", + ) + + def array(self, name): + self.validate() + _require(name in COLUMNS, "INCOME_COLUMN") + n = self.receipt["rows"] + return np.frombuffer( + self._body, + dtype="= 0).all()) and len(np.unique(indices)) == rows, + "INCOME_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + for name, raw_name in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + _require( + np.array_equal( + output[name][positions], joined[raw_name].to_numpy() + ), + "INCOME_NATIVE_KEY_OR_AGE", + ) + incumbent = person.PAW_VAL.iloc[positions] + known = ~incumbent.isna().to_numpy() + observed = joined.PAW_VAL.to_numpy(dtype=np.int64) + _require( + np.array_equal( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known], + observed[known], + ), + "INCOME_INCUMBENT_CONFLICT", + "PAW_VAL", + ) + for name in ("PAW_YN", "PAW_VAL"): + output[name][positions] = joined[name].to_numpy(dtype=np.int64) + classes = zero_origin_classes( + output["A_AGE"][positions], output["PAW_YN"][positions], observed + ) + counts = { + name: int((classes == i).sum()) for i, name in enumerate(ZERO_CLASSES) + } + joins.append( + { + "income_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": rows, + "joined_rows": len(positions), + "unreferenced_source_rows": 0, + "incumbent_compared_rows": {"PAW_VAL": int(known.sum())}, + "incumbent_conflicts": 0, + "native_key_or_age_conflicts": 0, + "zero_origin_counts": counts, + "paw_yn_zero_at_age_15_plus": counts["niu_in_age_universe"], + } + ) + output["PAW_VAL_2024_price"] = _restate_amounts( + output["PAW_VAL"], years, ready.bindings.spec.factors + ) + _parent(source, ready) + _require(_json(_implementation()) == _json(before), "INCOME_IMPLEMENTATION_CHANGED") + buffers = [ + output[name] + .astype(" None: + if not condition: + raise HouseholdWeightSourceError(reason) + + +def _sha(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _issue_capsule(source: object, payload: bytes) -> str: + """Seal this identity outside writable capsule state without retaining it.""" + _require( + type(payload) is bytes and len(payload) <= _MAX_PAYLOAD_BYTES, "SOURCE_BYTES" + ) + digest = _sha(payload) + identity = id(source) + + def discard(reference): + entry = _CAPSULE_ISSUANCE.get(identity) + if entry is not None and entry[0] is reference: + del _CAPSULE_ISSUANCE[identity] + + # WeakKeyDictionary would merge distinct, value-equal frozen dataclasses. + _CAPSULE_ISSUANCE[identity] = (weakref.ref(source, discard), digest) + return digest + + +def _checked_capsule_payload(source: object) -> bytes: + """Return exactly the checked bytes; copies and deserialization are unissued.""" + entry = _CAPSULE_ISSUANCE.get(id(source)) + _require(entry is not None and entry[0]() is source, "SOURCE_MUTATION") + payload = getattr(source, "payload", None) + _require( + type(payload) is bytes + and len(payload) <= _MAX_PAYLOAD_BYTES + and _sha(payload) == entry[1], + "SOURCE_MUTATION", + ) + return payload + + +def _member_path_snapshot( + member_paths: Mapping[int, str | Path], + pins: tuple[_origin.AsecNativeMemberPin, ...], +) -> dict[int, Path]: + """Bound the closed cohort snapshot before allocation and each value lookup.""" + _require(isinstance(member_paths, Mapping), "MEMBER_PATHS") + try: + count = len(member_paths) + _require(0 < count <= len(pins), "MEMBER_PATHS") + allowed = {pin.income_year for pin in pins} + paths = {} + for year in member_paths: + _require( + len(paths) < count + and type(year) is int + and year in allowed + and year not in paths, + "MEMBER_PATHS", + ) + path = member_paths[year] + _require(isinstance(path, (str, Path)), "MEMBER_PATHS") + paths[year] = Path(path) + _require(len(paths) == count == len(member_paths), "MEMBER_PATHS") + except (TypeError, ValueError, RuntimeError, KeyError, OverflowError): + raise HouseholdWeightSourceError("MEMBER_PATHS") from None + return paths + + +def _encode(value: object) -> bytes: + """Canonical JSON with a bound during encoding, not after a huge allocation.""" + encoder = json.JSONEncoder( + ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ) + result = bytearray() + for fragment in encoder.iterencode(value): + encoded = fragment.encode("ascii") + _require(len(result) + len(encoded) <= _MAX_PAYLOAD_BYTES, "ENCODING_SIZE") + result.extend(encoded) + return bytes(result) + + +def _registry() -> tuple[_origin.AsecNativeMemberPin, ...]: + pins = tuple(_origin._ASEC_MEMBER_PINS) + _require(0 < len(pins) <= 3, "MEMBER_REGISTRY") + try: + for pin in pins: + _origin.AsecNativeMemberPin(**asdict(pin)) + _require( + pin.income_year in (2022, 2023, 2024) + and pin.size_bytes <= _MAX_MEMBER_BYTES + and pin.rows <= _MAX_ROWS, + "MEMBER_REGISTRY", + ) + _require( + len({p.income_year for p in pins}) == len(pins) + and len({p.canonical_member_id for p in pins}) == len(pins), + "MEMBER_REGISTRY", + ) + except (TypeError, ValueError, AttributeError): + raise HouseholdWeightSourceError("MEMBER_REGISTRY") from None + return tuple(sorted(pins, key=lambda pin: pin.income_year)) + + +def _implementation() -> dict: + """Bind the live owner, registry, private capture, codec and field policy.""" + _require(_csv_builtin.csv_reader_bound(csv), "SOURCE_CSV_READER_CHANGED") + parser_binary = getattr(_csv, "__file__", None) + if parser_binary is None: + parser_binary = ( + Path(sysconfig.get_config_var("LIBDIR")) + / sysconfig.get_config_var("LDLIBRARY") + if sysconfig.get_config_var("Py_ENABLE_SHARED") + else Path(sys.executable) + ) + return { + "schema": _SCHEMA, + "modules": { + name: _sha(resources.files(__package__).joinpath(name).read_bytes()) + for name in ( + "asec_original_household_weights.py", + "source_csv_builtin.py", + "native_household_origin.py", + "asec_student_controls.py", + "education_assistance_source.py", + ) + }, + "origin_codec_sha256": _sha( + resources.files("microcosm.build.cd_benchmark") + .joinpath("origin.py") + .read_bytes() + ), + "python": sys.version, + "csv_python_sha256": _sha(Path(csv.__file__).read_bytes()), + "csv_native_origin": _csv.__spec__.origin, + "csv_native_provider": Path(parser_binary).name, + "csv_native_sha256": _sha(Path(parser_binary).read_bytes()), + "registry": [asdict(pin) for pin in _registry()], + "fields": _FIELD_CONTRACT, + "dictionaries": _DICTIONARIES, + "limits": [ + _MAX_MEMBER_BYTES, + _MAX_ROWS, + _MAX_RECORD_CHARS, + _MAX_FIELD_CHARS, + _MAX_PAYLOAD_BYTES, + ], + } + + +class _DigestReader(io.RawIOBase): + """Bound and hash the actual captured bytes consumed by UTF-8 decoding.""" + + def __init__(self, raw: io.FileIO, size: int): + self.raw = raw + self.size = size + self.count = 0 + self.digest = hashlib.sha256() + + def readable(self) -> bool: + return True + + def readinto(self, buffer) -> int: + limit = min(len(buffer), self.size - self.count + 1) + count = self.raw.readinto(memoryview(buffer)[:limit]) + self.count += count + _require(self.count <= self.size, "CAPTURE_SIZE") + self.digest.update(memoryview(buffer)[:count]) + return count + + +class _RecordLines: + """Limit the complete logical CSV record, including embedded newlines.""" + + def __init__(self, text: io.TextIOWrapper): + self.text = text + self.characters = 0 + + def __iter__(self): + return self + + def __next__(self) -> str: + line = self.text.readline(_MAX_RECORD_CHARS - self.characters + 1) + if not line: + raise StopIteration + self.characters += len(line) + _require(self.characters <= _MAX_RECORD_CHARS, "MEMBER_RECORD_SIZE") + return line + + +def _record_fields(tokens: Sequence[str]) -> dict: + native_token, household_type, weight_token = tokens + _require(all(len(t) <= _MAX_FIELD_CHARS for t in tokens), "MEMBER_FIELD_SIZE") + _require( + re.fullmatch(r"[0-9]+", native_token) is not None + and 1 <= int(native_token) <= 99999, + "MEMBER_NATIVE_KEY", + ) + if household_type == "1": + universe = "in" + elif re.fullmatch(r"(?:0|[2-9])", household_type): + # Establish only that the printed weight predicate is false, not the + # meaning/validity of any other household classification. + universe = "outside" + else: + universe = "unresolved" + units = None + if weight_token == "": + state = "missing" + elif re.fullmatch(r"[0-9]+", weight_token) is None: + state = "malformed" + else: + units = int(weight_token) + state = "integer" if units <= 999999999 else "out_of_range" + return { + "H_SEQ": native_token, + "native_household_id": int(native_token), + "H_HHTYPE": household_type, + "HSUP_WGT": weight_token, + "universe_status": universe, + "weight_token_status": state, + "weight_integer_units": units, + "weight_denominator": 100 if units is not None else None, + "weight_status": ( + "valid_in_universe" + if universe == "in" and state == "integer" + else "unresolved_weight" + if universe == "in" + else "outside_universe" + if universe == "outside" + else "unresolved_universe" + ), + } + + +def _regular_opener(path, flags): + fd = os.open(path, flags | os.O_NONBLOCK) + try: + _require(stat.S_ISREG(os.fstat(fd).st_mode), "CAPTURE_KIND") + except BaseException: + os.close(fd) + raise + return fd + + +def _read_capture(capture: Path, pin: _origin.AsecNativeMemberPin) -> list[dict]: + csv_reader = _csv_builtin.capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + member = SourceMember(pin.canonical_member_id, pin.member_sha256) + result, seen = [], set() + with open(capture, "rb", buffering=0, opener=_regular_opener) as raw: + before = os.fstat(raw.fileno()) + _require(before.st_size == pin.size_bytes, "CAPTURE_SIZE") + digest_reader = _DigestReader(raw, pin.size_bytes) + with io.TextIOWrapper( + io.BufferedReader(digest_reader), encoding="utf-8-sig", newline="" + ) as text: + lines = _RecordLines(text) + reader = csv_reader(lines, strict=True) + header = next(reader, []) + _require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + positions = [header.index(column) for column in _COLUMNS] + while True: + lines.characters = 0 + try: + row = next(reader) + except StopIteration: + break + _require(len(row) == len(header), "MEMBER_ROW_WIDTH") + _require(len(result) < pin.rows, "MEMBER_ROWS") + record = _record_fields([row[i] for i in positions]) + native = record["native_household_id"] + _require(native not in seen, "MEMBER_DUPLICATE_KEY") + seen.add(native) + record.update( + income_year=pin.income_year, + survey_year=pin.survey_year, + member_id=pin.canonical_member_id, + member_sha256=pin.member_sha256, + member_row_1based=len(result) + 1, + origin_key=origin_key( + AsecHouseholdOrigin(member, pin.income_year, native) + ), + ) + result.append(record) + _require(len(result) == pin.rows, "MEMBER_ROWS") + _require( + digest_reader.count == pin.size_bytes + and digest_reader.digest.hexdigest() == pin.member_sha256, + "CAPTURE_CHANGED", + ) + after = os.fstat(raw.fileno()) + _require( + all( + getattr(before, attribute) == getattr(after, attribute) + for attribute in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ), + "CAPTURE_CHANGED", + ) + return result + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsecHouseholdWeights: + """Reader-issued immutable source evidence, not a population-weight vector.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token: object) -> None: + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + object.__setattr__(self, "_issued_sha256", _issue_capsule(self, self.payload)) + + @property + def document(self) -> dict: + """Fresh defensive decoding, checked against the live source producer.""" + return verify_asec_household_weights_source(self) + + def original_weight_fractions( + self, native_roster: Sequence[tuple[int, int]] + ) -> tuple[Fraction, ...]: + """Exact weights for known in-universe keys; no Frame binding or stamping.""" + doc = self.document + _require( + isinstance(native_roster, (list, tuple)) + and 0 < len(native_roster) <= _MAX_ROWS, + "ANCHOR_ROSTER", + ) + keys = [] + for key in native_roster: + _require( + isinstance(key, (list, tuple)) + and len(key) == 2 + and all(type(value) is int for value in key), + "ANCHOR_ROSTER", + ) + keys.append(tuple(key)) + _require(len(set(keys)) == len(keys), "ANCHOR_ROSTER") + records = { + (row["income_year"], row["native_household_id"]): row + for row in doc["records"] + } + values = [] + for key in keys: + _require(key in records, "ANCHOR_UNKNOWN") + row = records[key] + _require(row["weight_status"] == "valid_in_universe", "ANCHOR_UNRESOLVED") + values.append(Fraction(row["weight_integer_units"], 100)) + return tuple(values) + + +def verify_asec_household_weights_source( + source: AuthenticatedAsecHouseholdWeights, +) -> dict: + """Validate reader-issued evidence; arbitrary JSON/digests cannot issue it.""" + _require(type(source) is AuthenticatedAsecHouseholdWeights, "SOURCE_TYPE") + payload = _checked_capsule_payload(source) + doc = json.loads(payload) + _require(_encode(doc["producer"]) == _encode(_implementation()), "PRODUCER_CHANGED") + return doc + + +def load_authenticated_asec_household_weights( + member_paths: Mapping[int, str | Path], *, candidate: bytes | None = None +) -> AuthenticatedAsecHouseholdWeights: + """Reconstruct complete requested cohorts from closed original member pins. + + Mapping keys are income/source years, not survey years. Any nonempty subset + of the closed cohorts is explicit; every requested original member is read + completely. Optional candidate bytes must equal the independent canonical + reconstruction. There is no public pin override or candidate-only admission. + """ + pins = _registry() + paths = _member_path_snapshot(member_paths, pins) + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _MAX_PAYLOAD_BYTES), + "CANDIDATE_SIZE", + ) + producer = _implementation() + _require(producer["registry"] == [asdict(pin) for pin in pins], "PRODUCER_CHANGED") + implementation = _encode(producer) + selected = [pin for pin in pins if pin.income_year in paths] + records = [] + with tempfile.TemporaryDirectory(prefix="asec-original-household-weights-") as tmp: + for pin in selected: + capture = Path(tmp) / f"{pin.income_year}.csv" + try: + digest = _capture_owner._snapshot( + paths[pin.income_year], capture, size=pin.size_bytes + ) + except (OSError, ValueError, TypeError): + raise HouseholdWeightSourceError("MEMBER_CAPTURE") from None + _require(digest == pin.member_sha256, "MEMBER_SHA256") + try: + records.extend(_read_capture(capture, pin)) + except HouseholdWeightSourceError: + raise + except (OSError, UnicodeError, csv.Error, ValueError, OverflowError): + raise HouseholdWeightSourceError("MEMBER_ENCODING") from None + _require(_encode(_implementation()) == implementation, "PRODUCER_CHANGED") + payload = _encode( + { + "schema": _SCHEMA, + "producer": json.loads(implementation), + "members": [asdict(pin) for pin in selected], + "fields": _FIELD_CONTRACT, + "dictionary_authorities": [ + { + "survey_year": year, + "pdf_page_1based": page, + "printed_page": "6A-1", + "sha256": digest, + "url": "https://www2.census.gov/programs-surveys/cps/datasets/" + f"{year}/march/asec{year}_ddl_pub_full.pdf", + } + for year, page, digest in _DICTIONARIES + ], + "native_roster_sha256": _sha( + _encode([[r["income_year"], r["native_household_id"]] for r in records]) + ), + "projection_sha256": _sha(_encode(records)), + "records": records, + "source_authenticated": True, + "population_binding_authenticated": False, + "release_eligible": False, + "period_and_coverage_policy": "not_selected_by_this_source_owner", + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + return AuthenticatedAsecHouseholdWeights(payload, _token=_TOKEN) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py new file mode 100644 index 000000000..eef9b20ae --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py @@ -0,0 +1,299 @@ +"""Bounded, non-authoritative original-CSV ASEC PRPERTYP projection. + +Only invented cohorts have been exercised. This reader neither authenticates +source bytes nor attaches observations to a Frame. Genuine use requires a +separately reviewed authentication/binding wrapper and source-read approval. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import re +import stat +from collections.abc import Mapping +from pathlib import Path + +import pandas as pd + +from . import source_csv_builtin as _csv_builtin + +PROTOCOL = "microcosm.asec_person_coverage_source.v1" +# Existing ASEC adapters use income/source year as the cohort key. This is an +# identity convention only, not a decision about the observation's period. +SOURCE_YEARS = (2022, 2023, 2024) +KEYS = ("source_year", "PERIDNUM") +CROSSCHECKS = ("source_household_id", "A_LINENO", "A_AGE") +ROSTER_COLUMNS = KEYS + CROSSCHECKS +READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "PRPERTYP") +MAX_PERSONS = 600_000 +MAX_MEMBER_BYTES = 1_000_000_000 +_INT64_MAX = 2**63 - 1 + + +class CoverageSourceRefusalError(ValueError): + """Static refusal codes only; no path, key, row or source token.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise CoverageSourceRefusalError(reason) + + +def coverage_field_contract() -> dict: + """Fresh metadata from the public dictionaries reviewed on 2026-09-07. + + The review's receipt.json and field-pages.json establish these definitions; + their document hashes are metadata, not original CSV authentication pins. + """ + documents = ( + (2023, 26, "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117"), + (2024, 27, "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840"), + (2025, 28, "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f"), + ) + return { + "protocol": PROTOCOL, + "survey": "CPS ASEC", + "grain": "person", + "field": "PRPERTYP", + "concept_as_printed": "Type of person record recode", + "universe_as_printed": "All Persons", + "printed_length": 1, + "printed_position": 155, + "printed_range": [-4, 3], + "codes": { + "1": "Child household member", + "2": "Adult civilian household member", + "3": "Adult Armed Forces household member", + }, + "unlabelled_in_range_codes": [-4, -3, -2, -1, 0], + "blank_meaning": "unresolved", + "authority": [ + { + "survey_year": year, + "source_year": year - 1, + "url": "https://www2.census.gov/programs-surveys/cps/datasets/" + f"{year}/march/asec{year}_ddl_pub_full.pdf", + "sha256": digest, + "pdf_page_1based": page, + "printed_page": "6C-7", + } + for year, page, digest in documents + ], + "join_keys": list(KEYS), + "crosschecks": dict(zip(CROSSCHECKS, READ_COLUMNS[1:4], strict=True)), + "roster_scope": "all_person_rows_in_each_of_the_three_source_cohorts", + "period_harmonized": False, + "cross_survey_coverage_equivalence_established": False, + } + + +def _state(token: str) -> str: + if token in ("1", "2", "3"): + return "observed_code" + if token == "": + return "blank_unresolved" + if token in ("-4", "-3", "-2", "-1", "0"): + return "unlabelled_in_range" + if re.fullmatch(r"-?(?:0|[1-9][0-9]*)", token) and token != "-0": + return "out_of_range" + return "malformed_token" + + +def _roster(person_roster: pd.DataFrame) -> pd.DataFrame: + _require( + isinstance(person_roster, pd.DataFrame) + and person_roster.columns.is_unique + and set(ROSTER_COLUMNS) <= set(person_roster.columns), + "ROSTER_COLUMNS", + ) + _require(0 < len(person_roster) <= MAX_PERSONS, "ROSTER_ROWS") + result = person_roster.loc[:, list(ROSTER_COLUMNS)].copy().reset_index(drop=True) + _require(not result.isna().any().any(), "ROSTER_MISSING") + _require( + all(str(result[name].dtype) == "int64" for name in (KEYS[0], *CROSSCHECKS)), + "ROSTER_INTEGER_DTYPE", + ) + _require(set(result.source_year) == set(SOURCE_YEARS), "ROSTER_COHORTS") + _require( + result.PERIDNUM.map( + lambda value: ( + isinstance(value, str) and re.fullmatch(r"[0-9]{22}", value) is not None + ) + ).all(), + "ROSTER_PERSON_KEY", + ) + _require( + (result.source_household_id > 0).all() + and (result.A_LINENO > 0).all() + and (result.A_AGE >= 0).all(), + "ROSTER_COORDINATES", + ) + _require(not result.duplicated(list(KEYS)).any(), "ROSTER_DUPLICATE_KEY") + _require( + not result.duplicated([KEYS[0], *CROSSCHECKS[:2]]).any(), + "ROSTER_DUPLICATE_COORDINATES", + ) + return result + + +def _coordinate(token: str) -> int: + _require(re.fullmatch(r"[0-9]{1,19}", token) is not None, "MEMBER_INTEGER") + number = int(token) + _require(number <= _INT64_MAX, "MEMBER_INTEGER") + return number + + +def _digest(table: pd.DataFrame) -> str: + # Stream canonical records into the hash; never log native values or retain + # a second full JSON copy. Row order is part of the projection identity. + digest = hashlib.sha256() + digest.update(json.dumps(list(table.columns), separators=(",", ":")).encode()) + for row in table.itertuples(index=False, name=None): + digest.update(b"\n") + digest.update(json.dumps(row, separators=(",", ":"), allow_nan=False).encode()) + return digest.hexdigest() + + +def _regular_member_opener(path, flags): + # Check the opened descriptor, not a race-prone path snapshot. Nonblocking + # open lets us refuse a FIFO even when no writer has opened its other end. + descriptor = os.open(path, flags | os.O_NONBLOCK) + try: + snapshot = os.fstat(descriptor) + _require(stat.S_ISREG(snapshot.st_mode), "MEMBER_FILE_TYPE") + _require(snapshot.st_size <= MAX_MEMBER_BYTES, "MEMBER_BYTES") + except BaseException: + os.close(descriptor) + raise + return descriptor + + +class _BoundedMember(io.RawIOBase): + """Bound raw reads beneath buffering/decoding; the caller owns the raw file.""" + + def __init__(self, raw): + super().__init__() + self._raw = raw + self._remaining = MAX_MEMBER_BYTES + + def readable(self): + return True + + def readinto(self, buffer): + # One excess byte distinguishes exact-limit EOF from a growing member. + # Never pass that probe byte to the decoder or issue an unbounded read. + count = self._raw.readinto(memoryview(buffer)[: self._remaining + 1]) + self._remaining -= count + _require(self._remaining >= 0, "MEMBER_BYTES") + return count + + +def read_asec_person_coverage_source( + member_paths: Mapping[int, str | Path], *, person_roster: pd.DataFrame +) -> tuple[pd.DataFrame, dict]: + """Read exact whole cohorts, returning literals in requested native order. + + Paths are keyed by source/income years 2022, 2023, 2024 (survey years + 2023, 2024, 2025), following the existing ASEC source adapters. The caller + supplies PERIDNUM and int64 source_year/source_household_id/A_LINENO/A_AGE. + Every CSV row must match exactly one requested person in its own cohort; + PH_SEQ, A_LINENO and A_AGE crosscheck the requested native coordinates. + Age is only a crosscheck: it never supplies or overrides PRPERTYP meaning. + + All PRPERTYP cells remain strings, including blanks and malformed tokens. + The state column describes lexical/printed-code evidence, not coverage or + eligibility. Receipt digests bind this local projection, not source custody. + """ + csv_reader = _csv_builtin.capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + _require( + isinstance(member_paths, Mapping) + and set(member_paths) == set(SOURCE_YEARS) + and all(type(year) is int for year in member_paths) + and all(isinstance(path, (str, Path)) for path in member_paths.values()), + "MEMBER_PATHS", + ) + expected = _roster(person_roster) + lookup = { + row[:2]: (position, row[2:]) + for position, row in enumerate(expected.itertuples(index=False, name=None)) + } + tokens = [""] * len(expected) + seen = set() + cohort_counts = [] + for year in SOURCE_YEARS: + coordinates = set() + rows = 0 + try: + with ( + open( + member_paths[year], + "rb", + buffering=0, + opener=_regular_member_opener, + ) as raw, + io.BufferedReader(_BoundedMember(raw)) as bounded, + io.TextIOWrapper(bounded, encoding="utf-8", newline="") as handle, + ): + reader = csv_reader(handle, strict=True) + header = next(reader, []) + _require( + len(header) == len(set(header)) + and set(READ_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + indices = [header.index(name) for name in READ_COLUMNS] + for cells in reader: + rows += 1 + _require(len(seen) < MAX_PERSONS, "MEMBER_ROWS") + _require(len(cells) == len(header), "MEMBER_ROW_WIDTH") + key, household, line, age, token = (cells[i] for i in indices) + _require( + re.fullmatch(r"[0-9]{22}", key) is not None, "MEMBER_PERSON_KEY" + ) + native_key = (year, key) + _require(native_key not in seen, "MEMBER_DUPLICATE_KEY") + _require(native_key in lookup, "MEMBER_EXTRA_KEY") + actual = tuple( + _coordinate(value) for value in (household, line, age) + ) + _require(actual[0] > 0 and actual[1] > 0, "MEMBER_COORDINATES") + _require( + actual[:2] not in coordinates, "MEMBER_DUPLICATE_COORDINATES" + ) + position, wanted = lookup[native_key] + _require(actual == wanted, "MEMBER_CROSSCHECK") + coordinates.add(actual[:2]) + seen.add(native_key) + tokens[position] = token + except CoverageSourceRefusalError: + raise + except (OSError, UnicodeError, csv.Error, ValueError, OverflowError): + raise CoverageSourceRefusalError("MEMBER_READ") from None + cohort_counts.append( + {"source_year": year, "survey_year": year + 1, "rows": rows} + ) + _require(len(seen) == len(expected), "MEMBER_MISSING_KEY") + result = expected.copy() + result["PRPERTYP"] = pd.array(tokens, dtype="string") + result["PRPERTYP_state"] = pd.array( + [_state(token) for token in tokens], dtype="string" + ) + receipt = { + "protocol": PROTOCOL, + "source_authenticated": False, + "population_binding_authenticated": False, + "coverage_status": "literal_source_fields_only", + "release_eligible": False, + "contract": coverage_field_contract(), + "read_columns": list(READ_COLUMNS), + "cohorts": cohort_counts, + "person_rows": len(result), + "native_roster_sha256": _digest(expected), + "projection_sha256": _digest(result), + } + return result, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py new file mode 100644 index 000000000..245a8caef --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py @@ -0,0 +1,488 @@ +"""Restore an observed person total from closed, authenticated Census members. + +P (v4) and H (household attachment) remain immutable. T appends one nominal +observation. A candidate T digest is a locator identity, never source authority: +verification reconstructs all of T from authenticated parents and CSV members. +""" + +from __future__ import annotations + +import csv +import hashlib +import os +import tempfile +from collections.abc import Mapping +from dataclasses import dataclass +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build import frame_checkpoint as checkpoint +from microcosm.frame import Frame, Weights + +from . import asec_current_money as money_schema +from . import asec_current_money_source as legacy +from .asec_current_money import MoneyRefusalError, _json, _require, _sha +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +CHECKPOINT_FILENAME = "asec_person_income_source.checkpoint.h5" +ARTIFACT_KIND = "microcosm.asec_person_income_observation_attachment" +OBSERVED_COLUMN = "asec_PTOTVAL" +_READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "PTOTVAL") +# Closed code-owned source pins. No public caller pin authority exists. +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +_ATTACHMENT_MAX_BYTES = 2_000_000_000 +_ENCODING_CONTRACT = "independently_reconstructed_canonical_checkpoint_bytes_v1" + + +def _implementation() -> str: + return _sha( + _json( + { + "schema": 1, + "legacy_verification": legacy._verification_identity(), + "schema_module": _sha( + resources.files(__package__) + .joinpath("asec_current_money.py") + .read_bytes() + ), + "source_kind": money_schema.RESTORED_SOURCE_KIND, + "field_columns": dict(money_schema.RESTORED_FIELD_COLUMNS), + "field_zero_policy": dict(money_schema.RESTORED_FIELD_ZERO_POLICY), + "reader_producer": _sha( + resources.files(__package__) + .joinpath("asec_person_income_source.py") + .read_bytes() + ), + "member_pins": _MEMBER_PINS, + "columns": _READ_COLUMNS, + "output": OBSERVED_COLUMN, + "origin": "authenticated_census_csv_encoded_zero_v1", + "encoding_contract": _ENCODING_CONTRACT, + } + ) + ) + + +def _paths(member_paths): + _require( + isinstance(member_paths, Mapping) + and set(member_paths) == {2022, 2023, 2024} + and all(type(y) is int for y in member_paths) + and all(isinstance(p, (str, Path)) for p in member_paths.values()), + "MEMBER_PATHS", + ) + + +def _read_member(path: Path, *, rows: int, size: int) -> pd.DataFrame: + _require(path.stat().st_size == size, "MEMBER_SIZE") + with path.open("r", encoding="utf-8", newline="") as handle: + header = next(csv.reader(handle), []) + _require( + len(header) == len(set(header)) and set(_READ_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + # Preserve source tokens until validation. In particular, never fillna and + # never let permissive numeric coercion turn absent observations into zeros. + table = pd.read_csv( + path, usecols=list(_READ_COLUMNS), dtype="string", na_filter=False + ) + _require(len(table) == rows, "MEMBER_ROWS") + _require( + bool(table.PERIDNUM.str.fullmatch(r"[0-9]{22}").all()), "MEMBER_PERSON_KEY" + ) + for column in _READ_COLUMNS[1:]: + _require( + bool(table[column].str.fullmatch(r"-?[0-9]+").all()), + "MEMBER_INTEGER" if column == "PTOTVAL" else f"MEMBER_INTEGER_{column}", + column, + ) + table[column] = table[column].astype("int64") + _require( + bool(table.PTOTVAL.between(-99999, 99999999).all()), + "MEMBER_AMOUNT_DOMAIN", + "PTOTVAL", + ) + _require( + not table.PERIDNUM.duplicated().any() + and not table.duplicated(["PH_SEQ", "A_LINENO"]).any(), + "MEMBER_DUPLICATE_KEY", + ) + return table + + +def _owned_frame(parent: Frame) -> Frame: + strata = parent.strata.copy(deep=True) + strata.index = legacy._owned_index(strata.index) + result = Frame( + { + **{e: parent.table(e) for e in parent.entities}, + **{n: parent.link(n) for n in parent.links}, + }, + parent.schema, + { + e: Weights(parent.weights_for(e).values, parent.weights_for(e).kind) + for e in parent.weighted_entities + }, + strata, + mass_log=parent.mass_log, + metadata=parent.metadata, + ) + legacy._detach_frame_axes(result) + return result + + +def _reconstruct(source, member_paths: Mapping[int, str | Path]): + _paths(member_paths) + source.validate() + before = _implementation() + parent = source.frame + person = parent.person + _require(OBSERVED_COLUMN not in person, "RESTORATION_ALREADY_ATTACHED") + _require( + all( + c in person for c in ("source_household_id", "A_LINENO", "A_AGE", "PTOTVAL") + ), + "RESTORATION_SOURCE_COLUMNS", + ) + for c in ("source_household_id", "A_LINENO", "A_AGE"): + _require( + person[c].dtype == np.dtype("int64") and not person[c].isna().any(), + "RESTORATION_COORDINATE_DTYPE", + ) + years = np.asarray(source.scope.person_years, dtype=np.int64) + keys = np.asarray(source.scope.person_native_keys) + output = np.empty(len(person), dtype=np.int64) + joins = [] + with tempfile.TemporaryDirectory( + prefix="microcosm-person-income-members-" + ) as directory: + for year, member, archive_pin, pin, rows, size in _MEMBER_PINS: + staged = Path(directory) / member + legacy._stage_verified(member_paths[year], pin, staged) + raw = _read_member(staged, rows=rows, size=size) + staged.unlink() + positions = np.flatnonzero(years == year) + _require(len(positions) == rows, "RESTORATION_COHORT_ROWS") + recipient_keys = pd.Index(keys[positions]) + _require(not recipient_keys.has_duplicates, "RESTORATION_PERSON_KEY") + indices = pd.Index(raw.PERIDNUM).get_indexer(recipient_keys) + _require( + bool((indices >= 0).all()) and len(np.unique(indices)) == rows, + "RESTORATION_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + for source_col, raw_col in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + _require( + np.array_equal( + person[source_col].iloc[positions].to_numpy(), + joined[raw_col].to_numpy(), + ), + "RESTORATION_NATIVE_KEY_OR_AGE", + ) + incumbent = person.PTOTVAL.iloc[positions] + observed = joined.PTOTVAL.to_numpy(dtype=np.int64) + known = ~incumbent.isna().to_numpy() + _require( + np.array_equal( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known], + observed[known], + ), + "RESTORATION_INCUMBENT_CONFLICT", + "PTOTVAL", + ) + # The full current cohort has an incumbent and must be checked, not + # silently treated as another missing-column restoration. + if year == 2024: + _require( + bool(known.all()), "RESTORATION_INCUMBENT_INCOMPLETE", "PTOTVAL" + ) + output[positions] = observed + joins.append( + { + "income_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": rows, + "joined_rows": len(positions), + "unreferenced_source_rows": 0, + "incumbent_compared_rows": int(known.sum()), + "incumbent_conflicts": 0, + "native_key_or_age_conflicts": 0, + } + ) + result = _owned_frame(parent) + result.person[OBSERVED_COLUMN] = output + source.validate() + _require(_implementation() == before, "RESTORATION_IMPLEMENTATION_CHANGED") + evidence = source.source.evidence + from .asec_current_money import _parse + + parent_evidence = _parse(evidence) + receipt = { + "schema_version": 1, + "artifact_kind": ARTIFACT_KIND, + "original_v4_sha256": parent_evidence["parent_sha256"], + "household_attachment_sha256": parent_evidence["attachment_sha256"], + "parent_frame_sha256": legacy._frame_signature(parent), + "output_frame_sha256": legacy._frame_signature(result), + "reader": "authenticated_census_csv_exact_integer_no_fill_v1", + "join_keys": ["source_year", "PERIDNUM"], + "crosscheck_columns": ["source_household_id", "A_LINENO", "A_AGE"], + "sources": joins, + "output_column": OBSERVED_COLUMN, + "logical_field": "PTOTVAL", + "nominal_basis": "income_year_us_dollars", + "dtype": "int64", + "observation_sha256": _sha(output.astype(" _VerifiedRestoration: + """Authenticate and reconstruct T fully; this function does not issue money.""" + try: + return _verify( + parent_path, + household_attachment_path, + person_income_attachment_path, + member_paths, + ) + except MoneyRefusalError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + OverflowError, + UnicodeError, + csv.Error, + StopIteration, + ): + raise MoneyRefusalError("RESTORATION_SOURCE_CONTRACT") from None + + +def restore_asec_person_income_source( + parent_path: str | Path, + household_attachment_path: str | Path, + *, + member_paths: Mapping[int, str | Path], + output_dir: str | Path, +) -> dict[str, object]: + """Write a new local T and receipt from closed sources; never replace parents.""" + destination = Path(output_dir) + if destination.exists(): + raise FileExistsError(destination) + try: + _paths(member_paths) + source = legacy.load_authenticated_current_money_source( + parent_path, household_attachment_path + ) + frame, receipt = _reconstruct(source, member_paths) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".asec-person-income-", dir=destination.parent + ) as directory: + staging = Path(directory) + output = staging / CHECKPOINT_FILENAME + checkpoint.write_frame_checkpoint( + output, frame, metadata=_document(receipt) + ) + verified = _verify( + parent_path, household_attachment_path, output, member_paths + ) + report = { + **receipt, + "output_sha256": verified.attachment_sha256, + "output_file": CHECKPOINT_FILENAME, + } + (staging / "restoration.receipt.json").write_bytes(_json(report) + b"\n") + # Exclusive final directory creation avoids replacing even a raced + # empty directory. No historical artifact is opened for writing. + destination.mkdir() + for path in staging.iterdir(): + os.link(path, destination / path.name) + return report + except FileExistsError: + raise + except MoneyRefusalError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + OverflowError, + UnicodeError, + csv.Error, + StopIteration, + ): + raise MoneyRefusalError("RESTORATION_SOURCE_CONTRACT") from None + + +def load_authenticated_restored_current_money_source( + parent_path: str | Path, + household_attachment_path: str | Path, + person_income_attachment_path: str | Path, + *, + member_paths: Mapping[int, str | Path], +) -> legacy.AuthenticatedCurrentMoneySource: + """Issue restored authority only after closed-source complete reconstruction.""" + from .asec_current_money import ( + _SOURCE_TOKEN, + RESTORED_FIELD_COLUMNS, + RESTORED_FIELD_ZERO_POLICY, + RESTORED_SOURCE_KIND, + AuthenticatedAsecSource, + _parse, + compile_asec_current_money_spec, + ) + from .asec_current_money_resources import load_current_money_resources + + checked = verify_asec_person_income_source( + parent_path, + household_attachment_path, + person_income_attachment_path, + member_paths=member_paths, + ) + frame = checked.frame + scope = legacy._scope(frame) + views = legacy._views(frame, scope, restored=True) + scope_sha, input_sha = legacy._input_binding(views) + evidence = _parse(checked.parent_source.source.identity) + evidence.update( + schema_version=2, + source_kind=RESTORED_SOURCE_KIND, + person_income_attachment_sha256=checked.attachment_sha256, + field_source_columns=dict(RESTORED_FIELD_COLUMNS), + field_zero_origin_policy=dict(RESTORED_FIELD_ZERO_POLICY), + scope_sha256=scope_sha, + input_sha256=input_sha, + frame_sha256=legacy._frame_signature(frame), + verification_sha256=checked.receipt["implementation_sha256"], + ) + checked.parent_source.validate() + _require( + _implementation() == evidence["verification_sha256"], + "RESTORATION_IMPLEMENTATION_CHANGED", + ) + authority = AuthenticatedAsecSource(_json(evidence), _token=_SOURCE_TOKEN) + spec = compile_asec_current_money_spec(load_current_money_resources(), authority) + result = legacy.AuthenticatedCurrentMoneySource( + frame, scope, authority, spec, _token=legacy._LOAD_TOKEN + ) + result.validate() + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py new file mode 100644 index 000000000..03c83d4d7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py @@ -0,0 +1,684 @@ +"""Authenticate the complete 2024 ASEC catalogue before population construction. + +The closed parent still authenticates three cohorts. Only original 2024 records +are exposed here. No descendant Frame, unit assignment, domain classification, +selection, or weight allocation is performed. Household literals are retained; +A_AGE and A_LINENO are canonical strings of authenticated numeric originals. +""" + +from __future__ import annotations + +import hashlib +import json +import operator +import sys +import weakref +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from itertools import chain +from pathlib import Path +from typing import NamedTuple + +from . import asec_2024_native_population as native +from . import survey_population_domains as domains + +ARTIFACT_KIND = "microcosm.us.asec-source-catalogue.v1" +_MAX_ENVELOPE_BYTES = 1024**2 +_MAX_RECORD_BYTES = 128 * 1024**2 +_ISSUED: dict[int, tuple[weakref.ReferenceType, bytes, object]] = {} + + +class AsecSourceCatalogueError(ValueError): + """Static refusal codes, without source paths or source values.""" + + +def _require(condition, code): + if not condition: + raise AsecSourceCatalogueError(code) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _encode(value): + # Only compact identities and individual records use this encoder. The + # complete catalogue is hashed incrementally, never encoded as another body. + result = bytearray() + encoder = json.JSONEncoder( + ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ) + for part in encoder.iterencode(value): + part = part.encode("ascii") + _require(len(result) + len(part) <= _MAX_ENVELOPE_BYTES, "ENVELOPE_SIZE") + result.extend(part) + return bytes(result) + + +def _implementation(): + modules = (sys.modules[__name__], domains) + runtime = {m.__name__: native.coverage_owner._runtime_code(m) for m in modules} + code = {m.__name__: _sha(Path(m.__file__).read_bytes()) for m in modules} + _require( + runtime == _RUNTIME_AUTHORITY and code == _BYTE_AUTHORITY, + "PRODUCER_CODE_CHANGED", + ) + return { + "kind": ARTIFACT_KIND, + "runtime": runtime, + "code": code, + "source_owner": native._implementation(), + "limits": [_MAX_ENVELOPE_BYTES, _MAX_RECORD_BYTES], + "periods": [2024, 2025], + } + + +def _source_authority(): + """Snapshot exact code-owned source pin values without file/resource I/O. + + Canonical bytes detach the seal from mutable registry objects. The original + household registry helper validates its records but reads no source files. + """ + return _encode( + { + "parent": native.parent_owner._SOURCE_PINS, + "restoration": native.restoration._MEMBER_PINS, + "coverage": native.coverage_owner._MEMBER_PINS, + "household": [asdict(pin) for pin in native.anchor_owner._registry()], + } + ) + + +@dataclass(frozen=True, slots=True) +class UnrepresentedAsecHousehold: + """Original noninterview household, absent from the authenticated roster. + + This is a custody ledger, not a scientific coverage classification. No empty + person list is supplied as a claim about the household's actual membership. + """ + + key: domains.HouseholdKey + h_hhtype: str + hrhtype: str + h_livqrt: str + h_numper: str + hsup_wgt: str + reason: str = "original_noninterview_not_in_authenticated_person_roster" + + +def _key_record(key): + _require(type(key) is domains.HouseholdKey, "RECORD_TYPE") + _require(key.source is domains.Source.ASEC, "RECORD_SOURCE") + return [key.source.value, key.source_year, key.survey_year, key.native_id] + + +def _record(row): + _require( + type(row) in (domains.AsecHousehold, UnrepresentedAsecHousehold), + "RECORD_TYPE", + ) + result = { + "key": _key_record(row.key), + "H_HHTYPE": row.h_hhtype, + "HRHTYPE": row.hrhtype, + "H_LIVQRT": row.h_livqrt, + "H_NUMPER": row.h_numper, + "HSUP_WGT": row.hsup_wgt, + } + if type(row) is UnrepresentedAsecHousehold: + result["unrepresented_reason"] = row.reason + else: + _require(type(row.persons) is tuple, "RECORD_TYPE") + persons = [] + for person in row.persons: + _require(type(person) is domains.AsecPerson, "RECORD_TYPE") + persons.append( + [ + person.peridnum, + person.a_lineno, + person.age, + person.prpertyp, + person.prpertyp_state, + _key_record(person.household_key), + ] + ) + result["persons"] = persons + return result + + +class _RecordsMemo(NamedTuple): + """The issued record roots, their projected leaves, and the identity streamed.""" + + households: tuple + ledger: tuple + household_leaves: tuple + ledger_leaves: tuple + identity: tuple + + +def _records_identity(households, ledger, *, memo=False): + """Stream every canonical record; optionally bind a leaf snapshot memo. + + With ``memo=True`` the result is ``(identity, memo)``. The memo's leaves are + projected from the exact rows encoded in this same pass, so it can only + stand for this identity. It is ``None`` when any record carries a leaf + that is not an exact ``str``/``int`` or the ASEC source member. + """ + _require(type(households) is tuple and type(ledger) is tuple, "RECORD_TYPE") + digest, count, size, persons = hashlib.sha256(), 0, 0, 0 + household_leaves, ledger_leaves, eligible = [], [], memo + for kind, rows in (("household", households), ("unrepresented", ledger)): + for row in rows: + data = _encode([kind, _record(row)]) + b"\n" + size += len(data) + _require(size <= _MAX_RECORD_BYTES, "RECORD_SIZE") + digest.update(data) + count += 1 + if kind == "household": + persons += len(row.persons) + if eligible: + projected = _eligible_leaves(row, kind) + if projected is None: + eligible = False + elif kind == "household": + household_leaves.append(projected) + else: + ledger_leaves.append(projected) + identity = (digest.hexdigest(), size, count, persons) + if not memo: + return identity + if not eligible: + return identity, None + return identity, _RecordsMemo( + households, ledger, tuple(household_leaves), tuple(ledger_leaves), identity + ) + + +_KEY_LEAVES = operator.attrgetter("source", "source_year", "survey_year", "native_id") +_HOUSEHOLD_LITERALS = operator.attrgetter( + "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt" +) +_HOUSEHOLD_LEAVES = operator.attrgetter( + "key", "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt", "persons" +) +_PERSON_LITERALS = operator.attrgetter( + "peridnum", "a_lineno", "age", "prpertyp", "prpertyp_state" +) +_PERSON_LEAVES = operator.attrgetter( + "peridnum", "a_lineno", "age", "prpertyp", "prpertyp_state", "household_key" +) +_LEDGER_LITERALS = operator.attrgetter( + "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt", "reason" +) +_LEDGER_LEAVES = operator.attrgetter( + "key", "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt", "reason" +) + + +def _key_leaves(key): + # The encoder writes ``source.value``; an enum member is a mutable object, + # so its value object is a leaf in its own right. Classes are leaves too: + # ``__class__`` can be reassigned in place, and the encoder refuses + # anything but the exact record types on every full pass. + source, *rest = _KEY_LEAVES(key) + return (type(key), source, source.value, *rest) + + +def _household_leaves(row): + key, *literals, persons = _HOUSEHOLD_LEAVES(row) + return ( + type(row), + key, + *_key_leaves(key), + *literals, + type(persons), + persons, + *chain.from_iterable( + (type(person), *_PERSON_LEAVES(person), *_key_leaves(person.household_key)) + for person in persons + ), + ) + + +def _ledger_leaves(row): + key, *literals = _LEDGER_LEAVES(row) + return (type(row), key, *_key_leaves(key), *literals) + + +def _eligible_key(key): + return ( + type(key) is domains.HouseholdKey + and key.source is domains.Source.ASEC + and type(key.source.value) is str + and type(key.source_year) is int + and type(key.survey_year) is int + and type(key.native_id) is str + ) + + +def _eligible_leaves(row, kind): + """Project one record's leaves when every one is an exact immutable value.""" + if not _eligible_key(row.key): + return None + if kind == "household": + if type(row) is not domains.AsecHousehold or type(row.persons) is not tuple: + return None + literals = list(_HOUSEHOLD_LITERALS(row)) + for person in row.persons: + if type(person) is not domains.AsecPerson or not _eligible_key( + person.household_key + ): + return None + literals.extend(_PERSON_LITERALS(person)) + leaves = _household_leaves(row) + else: + if type(row) is not UnrepresentedAsecHousehold: + return None + literals = _LEDGER_LITERALS(row) + leaves = _ledger_leaves(row) + if any(type(item) is not str for item in literals): + return None + return leaves + + +def _leaves_unchanged(rows, snapshot, project): + if len(rows) != len(snapshot): + return False + for row, expected in zip(rows, snapshot, strict=True): + try: + current = project(row) + except (AttributeError, TypeError): + return False + if len(current) != len(expected) or not all( + map(operator.is_, current, expected) + ): + return False + return True + + +def _memoized_records_identity( + households, ledger, memo=None, *, expected_identity=None +): + """Reuse the issued records identity only over the same unchanged leaves. + + A hit needs the exact issued household and ledger tuples, a memo identity + equal to the owner's retained one, and every projected leaf (literals, + keys, person rosters and their members' fields) to be the very object seen + while the identity streamed. Frozen dataclasses stay writable through + ``object.__setattr__``, so the leaves carry the proof, not the records. + Anything else falls back to the full canonical encoding and its refusals. + """ + if ( + type(households) is tuple + and type(ledger) is tuple + and type(memo) is _RecordsMemo + and households is memo.households + and ledger is memo.ledger + and type(memo.household_leaves) is tuple + and type(memo.ledger_leaves) is tuple + and type(memo.identity) is tuple + and memo.identity == expected_identity + and _leaves_unchanged(households, memo.household_leaves, _household_leaves) + and _leaves_unchanged(ledger, memo.ledger_leaves, _ledger_leaves) + ): + return memo.identity + return _records_identity(households, ledger) + + +def _bound_records_identity(rows, roster): + """Stream complete receiving/native keys, source positions and exact anchors.""" + digest, count, size = hashlib.sha256(), 0, 0 + for kind, values in (("household", rows), ("person", roster)): + for row in values: + data = _encode([kind, row]) + b"\n" + size += len(data) + _require(size <= _MAX_RECORD_BYTES, "RECORD_SIZE") + digest.update(data) + count += 1 + return {"sha256": digest.hexdigest(), "canonical_bytes": size, "records": count} + + +def _rows(rows, roster, anchors, fields): + people = {} + for person in roster: + people.setdefault(int(person["source_household_id"]), []).append(person) + households = [] + for row in sorted(rows, key=lambda r: r["source"]["member_row_1based"]): + original = row["household_fields"] + key = domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, original["H_SEQ"]) + members = tuple( + domains.AsecPerson( + p["PERIDNUM"], + str(int(p["A_LINENO"])), + str(int(p["A_AGE"])), + p["PRPERTYP"], + p["PRPERTYP_state"], + key, + ) + for p in people[row["native_key"][1]] + ) + households.append( + domains.AsecHousehold( + key, + original["H_HHTYPE"], + original["HRHTYPE"], + original["H_LIVQRT"], + original["H_NUMPER"], + row["HSUP_WGT"], + members, + ) + ) + # _roster(None) already authenticates every original member row and refuses + # interview rows absent from the represented roster or noninterviews in it. + weights = {r["H_SEQ"]: r for r in anchors.document["records"]} + ledger = tuple( + UnrepresentedAsecHousehold( + domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, r["H_SEQ"]), + r["H_HHTYPE"], + r["HRHTYPE"], + r["H_LIVQRT"], + r["H_NUMPER"], + weights[r["H_SEQ"]]["HSUP_WGT"], + ) + for r in fields.document["records"] + if r["H_HHTYPE"] != "1" + ) + return tuple(households), ledger + + +class _State(NamedTuple): + parent: object + parent_frame: object + coverage: object + anchors: object + fields: object + source_files: tuple + producer: bytes + source_authority: bytes + parent_identity: str + attached_evidence: tuple + households: tuple + ledger: tuple + records_identity: tuple + records_memo: _RecordsMemo | None = None + + +def _current_records_identity(state): + return _memoized_records_identity( + state.households, + state.ledger, + state.records_memo, + expected_identity=state.records_identity, + ) + + +def _validate_state(state): + _require(_source_authority() == state.source_authority, "SOURCE_AUTHORITY_CHANGED") + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require( + state.parent.frame is state.parent_frame + and native._frame_identity(state.parent.frame) == state.parent_identity, + "PARENT_CHANGED", + ) + _require( + _current_records_identity(state) == state.records_identity, "RECORDS_CHANGED" + ) + state.parent.validate() + native.coverage_owner.verify_asec_coverage_parent(state.coverage, state.parent) + native.anchor_owner.verify_asec_household_weights_source(state.anchors) + native.field_owner.verify_asec_household_coverage_fields(state.fields) + for path, expected, maximum, size in state.source_files: + native._file_identity(path, expected, maximum, size) + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require( + state.parent.frame is state.parent_frame + and native._frame_identity(state.parent.frame) == state.parent_identity, + "PARENT_CHANGED", + ) + _require( + _current_records_identity(state) == state.records_identity, "RECORDS_CHANGED" + ) + # Pure final seals after source/resource I/O; no recursively repeated reads. + current = native._attached_evidence( + state.parent, state.coverage, state.anchors, state.fields + ) + _require( + current[0] is state.attached_evidence[0] + and current[1:] == state.attached_evidence[1:], + "ATTACHED_EVIDENCE_CHANGED", + ) + # Producer helpers read registry values before later helper source-code I/O. + # A current borrow must refuse a late registry change, without waiting for + # the next producer pass. This final comparison is pure and bounded. + _require(_source_authority() == state.source_authority, "SOURCE_AUTHORITY_CHANGED") + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsecSourceCatalogue: + """Process-issued immutable records with checked borrows and compact evidence.""" + + payload: bytes + + def _checked(self): + try: + entry = _ISSUED.get(id(self)) + _require( + type(self) is AuthenticatedAsecSourceCatalogue + and entry is not None + and entry[0]() is self + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + _validate_state(entry[2]) + _require( + _ISSUED.get(id(self)) is entry + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + except AsecSourceCatalogueError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + OverflowError, + ): + raise AsecSourceCatalogueError("CATALOGUE_BINDING_REFUSAL") from None + + @property + def households(self) -> tuple[domains.AsecHousehold, ...]: + return self._checked()[2].households + + @property + def exclusion_ledger(self) -> tuple[UnrepresentedAsecHousehold, ...]: + return self._checked()[2].ledger + + @property + def receipt(self) -> dict: + return json.loads(self._checked()[1]) + + def to_bytes(self) -> bytes: + return self._checked()[1] + + def validate(self) -> None: + self._checked() + + +def verify_asec_source_catalogue(value: object) -> AuthenticatedAsecSourceCatalogue: + _require(type(value) is AuthenticatedAsecSourceCatalogue, "UNISSUED_OR_CHANGED") + value.validate() + return value + + +def issue_asec_source_catalogue( + parent_path: str | Path, + household_attachment_path: str | Path, + person_income_attachment_path: str | Path, + *, + person_member_paths: Mapping[int, str | Path], + household_member_path: str | Path, + candidate: bytes | None = None, +) -> AuthenticatedAsecSourceCatalogue: + """Reconstruct the complete 2024 source catalogue from authenticated paths. + + Candidate bytes are compared only after actual reconstruction. No caller + Frame, record list, weights or receipt can grant source authority. + """ + try: + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _MAX_ENVELOPE_BYTES), + "CANDIDATE_SIZE_OR_TYPE", + ) + parent_path, household_path, restored_path, persons, member, _, candidate = ( + native._inputs( + parent_path, + household_attachment_path, + person_income_attachment_path, + person_member_paths, + household_member_path, + None, + candidate, + ) + ) + source_authority = _source_authority() + producer = _encode(_implementation()) + parent = native.restoration.load_authenticated_restored_current_money_source( + parent_path, household_path, restored_path, member_paths=persons + ) + parent_identity = native._frame_identity(parent.frame) + coverage = native.coverage_owner.authenticate_asec_coverage( + parent, member_paths=persons + ) + anchors = native.anchor_owner.load_authenticated_asec_household_weights( + {2024: member} + ) + fields = native.field_owner.load_authenticated_asec_household_coverage_fields( + {2024: member} + ) + _, _, rows, roster = native._roster(parent, coverage, anchors, fields, None) + households, ledger = _rows(rows, roster, anchors, fields) + records_identity, records_memo = _records_identity( + households, ledger, memo=True + ) + identity = json.loads(parent.source.identity) + files = [ + ( + parent_path, + identity["parent_sha256"], + native._MAX_CHECKPOINT_BYTES, + None, + ), + ( + household_path, + identity["attachment_sha256"], + native._MAX_CHECKPOINT_BYTES, + None, + ), + ( + restored_path, + identity["person_income_attachment_sha256"], + native._MAX_CHECKPOINT_BYTES, + None, + ), + ] + sources = coverage.receipt["sources"] + for source in sources: + files.append( + ( + persons[source["source_year"]], + source["member_sha256"], + source["member_bytes"], + source["member_bytes"], + ) + ) + pin = anchors.document["members"][0] + files.append( + (member, pin["member_sha256"], pin["size_bytes"], pin["size_bytes"]) + ) + attached = native._attached_evidence(parent, coverage, anchors, fields) + state = _State( + parent, + parent.frame, + coverage, + anchors, + fields, + tuple(files), + producer, + source_authority, + parent_identity, + attached, + households, + ledger, + records_identity, + records_memo, + ) + payload = _encode( + { + "kind": ARTIFACT_KIND, + "source_year": 2024, + "survey_year": 2025, + "counts": { + "households": len(households), + "persons": records_identity[3], + "unrepresented_households": len(ledger), + }, + "records": { + "sha256": records_identity[0], + "canonical_bytes": records_identity[1], + "count": records_identity[2], + }, + "native_binding": _bound_records_identity(rows, roster), + "producer_sha256": _sha(producer), + "source_authority_sha256": _sha(source_authority), + "parent_custody": { + "cohorts": [2022, 2023, 2024], + "catalogue_cohorts": [2024], + "identity": identity, + "frame_sha256": parent_identity, + }, + "source_members": sources, + "household_member": pin, + "attached_capsules_sha256": [_sha(value) for value in attached[1:]], + "age_representation": "canonical_integer_string_of_authenticated_original_A_AGE_not_retained_CSV_lexeme", + "line_representation": "canonical_integer_string_of_authenticated_original_A_LINENO_not_retained_CSV_lexeme", + "household_key_representation": "original_H_SEQ_CSV_lexeme", + "classification_performed": False, + "weights_applied": False, + "selection_performed": False, + "release_eligible": False, + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + _validate_state(state) + result = AuthenticatedAsecSourceCatalogue(payload) + key = id(result) + + def discard(reference): + entry = _ISSUED.get(key) + if entry is not None and entry[0] is reference: + del _ISSUED[key] + + _ISSUED[key] = (weakref.ref(result, discard), payload, state) + result.validate() + return result + except AsecSourceCatalogueError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError, OverflowError): + raise AsecSourceCatalogueError("CATALOGUE_SOURCE_REFUSAL") from None + + +_RUNTIME_AUTHORITY = { + m.__name__: native.coverage_owner._runtime_code(m) + for m in (sys.modules[__name__], domains) +} + +_BYTE_AUTHORITY = { + m.__name__: _sha(Path(m.__file__).read_bytes()) + for m in (sys.modules[__name__], domains) +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py new file mode 100644 index 000000000..0aa7ae532 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py @@ -0,0 +1,567 @@ +"""Full-source ASEC preparation for the current-money graph slice. + +The graph source is one directory holding exactly the reviewed restoration +inputs: the raw-stage parent P, the household-observation attachment H, the +restored person-income attachment T with its receipt, the three housing cohort +HDFs, and the three official Census PERSON CSV members. No ZIP archive is +accepted and no file is discovered by pattern: the roster below is the contract. + +The executor keys a directory source by relative name plus file bytes, which is +a cache identity, not an authentication. Authentication stays where it already +is — the pinned digests inside the reviewed loaders and the canonical byte +replay of T — and this module simply refuses to run unless those loaders issue +authority. + +The authenticated ``ReadyCurrentMoney`` exists only inside this call. What +leaves is the canonical encoded body, a receipt of identities, and an +operator-free frame whose dtypes have been promoted to graph tokens. +""" + +from __future__ import annotations + +import os +import stat +from dataclasses import dataclass +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.population import dtype_for_token, token_for_dtype + +from . import asec_current_money_source as money_source +from . import asec_housing_status_source as housing_source +from . import asec_housing_universe as housing_universe +from . import asec_housing_universe_source as housing_universe_source +from . import asec_income_observations as income_source +from . import asec_person_income_source as person_income +from . import asec_student_controls as student_source +from ._asec_current_money_codec import ( + current_money_content_sha256, + encode_current_money, +) +from .asec_checkpoint import ASEC_RAW_STAGE_CHECKPOINT_FILENAME +from .asec_current_money import ( + ENCODED_ZERO_POLICY, + FIELDS, + RESTORED_SOURCE_KIND, + ZERO_POLICY, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _sha, +) +from .asec_current_money import ( + _digest as _digest_shape, +) +from .asec_current_money_units import reconstruct_current_money_tax_units +from .asec_housing_status_source import ( + attach_housing_status, + load_authenticated_housing_status, +) +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES +from .operator_boundary import assert_operator_free_source_frame + +PREPARED_SOURCE_KIND = "us_asec_prepared_current_money_v3" +PREPARED_RECEIPT_SCHEMA = "microcosm.us.asec_prepared_receipt.v3" +PREPARATION_BOUNDARY = "US graph ASEC prepared source" +SOURCE_YEARS: tuple[int, ...] = (2022, 2023, 2024) +PARENT_FILENAME = ASEC_RAW_STAGE_CHECKPOINT_FILENAME +HOUSEHOLD_ATTACHMENT_FILENAME = "asec_household_observations.checkpoint.h5" +PERSON_INCOME_FILENAME = person_income.CHECKPOINT_FILENAME +RESTORATION_RECEIPT_FILENAME = "restoration.receipt.json" +COHORT_FILENAMES: tuple[tuple[int, str], ...] = tuple( + (year, f"asec_household_cohort_{year}.h5") for year in SOURCE_YEARS +) +MEMBER_FILENAMES: tuple[tuple[int, str], ...] = tuple( + (year, ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year].member) for year in SOURCE_YEARS +) +PREPARED_SOURCE_FILES: tuple[str, ...] = ( + PARENT_FILENAME, + HOUSEHOLD_ATTACHMENT_FILENAME, + PERSON_INCOME_FILENAME, + RESTORATION_RECEIPT_FILENAME, + *(name for _year, name in COHORT_FILENAMES), + *(name for _year, name in MEMBER_FILENAMES), +) +RESTORATION_RECEIPT_MAX_BYTES = 256 * 1024 +_INTEGER_WIDTH_TARGETS = frozenset({"int8", "int16", "uint8", "uint16", "uint32"}) +_NULLABLE_INTEGER_TARGETS = frozenset( + {"Int8", "Int16", "Int32", "UInt8", "UInt16", "UInt32", "UInt64"} +) + + +class PreparedSourceRefusalError(ValueError): + """Sanitized preparation refusal; never carries rows, values or paths.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedSourceRefusalError(reason) + + +def _roster(source_dir) -> dict[str, Path]: + directory = Path(source_dir) + _require(directory.is_dir(), "SOURCE_DIRECTORY") + present = sorted(entry.name for entry in directory.iterdir()) + _require(present == sorted(PREPARED_SOURCE_FILES), "SOURCE_FILE_ROSTER") + paths = {} + for name in PREPARED_SOURCE_FILES: + path = directory / name + _require(path.is_file() and not path.is_symlink(), "SOURCE_REGULAR_FILE") + paths[name] = path + return paths + + +RESTORATION_READER = "authenticated_census_csv_exact_integer_no_fill_v1" +RESTORATION_JOIN_KEYS = ["source_year", "PERIDNUM"] +RESTORATION_CROSSCHECK_COLUMNS = ["source_household_id", "A_LINENO", "A_AGE"] +RESTORATION_LOGICAL_FIELD = "PTOTVAL" +RESTORATION_NOMINAL_BASIS = "income_year_us_dollars" +RESTORATION_ZERO_CLAIM = "encoded_source_value_not_respondent_answer" +_RESTORATION_RECEIPT_KEYS = frozenset( + { + "artifact_kind", + "crosscheck_columns", + "dtype", + "encoding_contract", + "household_attachment_sha256", + "implementation_sha256", + "join_keys", + "logical_field", + "nominal_basis", + "observation_sha256", + "original_v4_sha256", + "output_column", + "output_file", + "output_frame_sha256", + "output_sha256", + "parent_frame_sha256", + "reader", + "schema_version", + "sources", + "zero_claim", + "zero_origin_policy", + } +) +_RESTORATION_SOURCE_KEYS = frozenset( + { + "archive_sha256", + "income_year", + "incumbent_compared_rows", + "incumbent_conflicts", + "joined_rows", + "member", + "member_sha256", + "native_key_or_age_conflicts", + "source_rows", + "survey_year", + "unreferenced_source_rows", + } +) + + +def _restoration_sources(rows, source) -> None: + _require( + type(rows) is list and len(rows) == len(SOURCE_YEARS), "RESTORATION_SOURCES" + ) + for row, pins in zip(rows, person_income._MEMBER_PINS, strict=True): + year, member, archive_pin, member_pin, source_rows, _size = pins + positions = np.flatnonzero(np.asarray(source.scope.person_years) == year) + known_incumbents = int( + source.frame.person.PTOTVAL.iloc[positions].notna().sum() + ) + _require( + type(row) is dict and set(row) == _RESTORATION_SOURCE_KEYS, + "RESTORATION_SOURCES", + ) + _require( + row["income_year"] == year + and row["survey_year"] == year + 1 + and row["member"] == member + and row["archive_sha256"] == archive_pin + and row["member_sha256"] == member_pin + and row["source_rows"] == source_rows, + "RESTORATION_SOURCE_PINS", + ) + _require( + all( + type(row[name]) is int and row[name] >= 0 + for name in ( + "joined_rows", + "unreferenced_source_rows", + "incumbent_compared_rows", + ) + ) + and row["joined_rows"] == len(positions) == source_rows + and row["unreferenced_source_rows"] == 0 + and row["incumbent_compared_rows"] == known_incumbents + and row["incumbent_conflicts"] == 0 + and row["native_key_or_age_conflicts"] == 0, + "RESTORATION_SOURCE_COVERAGE", + ) + + +def _restoration_receipt(path: Path, evidence: dict, source) -> dict: + """Bind the shipped restoration receipt to the authority the loader issued.""" + # Refuse special files even if the roster check raced with a replacement, + # and bound allocation before parsing an untrusted sidecar. + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + try: + info = os.fstat(descriptor) + _require(stat.S_ISREG(info.st_mode), "RESTORATION_RECEIPT") + _require( + 0 < info.st_size <= RESTORATION_RECEIPT_MAX_BYTES, "RESTORATION_RECEIPT" + ) + with os.fdopen(descriptor, "rb", closefd=False) as stream: + payload = stream.read(RESTORATION_RECEIPT_MAX_BYTES + 1) + finally: + os.close(descriptor) + _require(0 < len(payload) <= RESTORATION_RECEIPT_MAX_BYTES, "RESTORATION_RECEIPT") + _require(payload.endswith(b"\n"), "RESTORATION_RECEIPT") + try: + receipt = _parse(payload[:-1], RESTORATION_RECEIPT_MAX_BYTES) + except MoneyRefusalError as error: + raise PreparedSourceRefusalError("RESTORATION_RECEIPT") from error + _require(_json(receipt) == payload[:-1], "RESTORATION_RECEIPT_CANONICAL") + _require(set(receipt) == _RESTORATION_RECEIPT_KEYS, "RESTORATION_RECEIPT_SCHEMA") + _require( + receipt["schema_version"] == 1 + and receipt["artifact_kind"] == person_income.ARTIFACT_KIND + and receipt["output_file"] == PERSON_INCOME_FILENAME + and receipt["output_column"] == person_income.OBSERVED_COLUMN + and receipt["encoding_contract"] == person_income._ENCODING_CONTRACT + and receipt["reader"] == RESTORATION_READER + and receipt["join_keys"] == RESTORATION_JOIN_KEYS + and receipt["crosscheck_columns"] == RESTORATION_CROSSCHECK_COLUMNS + and receipt["logical_field"] == RESTORATION_LOGICAL_FIELD + and receipt["nominal_basis"] == RESTORATION_NOMINAL_BASIS + and receipt["dtype"] == "int64" + and receipt["zero_origin_policy"] == ENCODED_ZERO_POLICY + and receipt["zero_claim"] == RESTORATION_ZERO_CLAIM, + "RESTORATION_RECEIPT_SCHEMA", + ) + _require( + all( + _digest_shape(receipt[name]) + for name in ( + "observation_sha256", + "output_frame_sha256", + "parent_frame_sha256", + ) + ), + "RESTORATION_RECEIPT_DIGEST", + ) + _restoration_sources(receipt["sources"], source) + # The reconstruction the loader just performed is the authority; the shipped + # receipt must agree with the evidence it issued, digest for digest. + _require( + receipt["output_sha256"] == evidence["person_income_attachment_sha256"] + and receipt["original_v4_sha256"] == evidence["parent_sha256"] + and receipt["household_attachment_sha256"] == evidence["attachment_sha256"] + and receipt["implementation_sha256"] == evidence["verification_sha256"], + "RESTORATION_RECEIPT_BINDING", + ) + # T1's pinned producer only appends the observed person-income column to an + # owned copy of the authenticated parent. Recover that exact parent view + # without reading another source or changing the original T1 authority. + # The sidecar must describe those reconstructed facts, not merely contain + # syntactically valid digest strings beside an authentic checkpoint. + source.validate() + observed = source.frame.person[person_income.OBSERVED_COLUMN] + _require(observed.dtype == np.dtype("int64"), "RESTORATION_OBSERVATION_DTYPE") + _require( + receipt["observation_sha256"] + == _sha(observed.to_numpy(dtype=" str: + try: + return token_for_dtype(dtype) + except Exception as error: # PopulationError is a ValueError subclass + del error + name = getattr(dtype, "name", None) + if isinstance(dtype, pd.api.extensions.ExtensionDtype): + _require(name in _NULLABLE_INTEGER_TARGETS, "UNPROMOTABLE_DTYPE") + return "Int64" + normalized = np.dtype(dtype) + _require( + normalized.name in _INTEGER_WIDTH_TARGETS or normalized.name == "float16", + "UNPROMOTABLE_DTYPE", + ) + return "float64" if normalized.name == "float16" else "int64" + + +def _promote(frame: Frame) -> tuple[dict[str, str], ...]: + """Promote every column to its graph dtype token, value- and null-preserving.""" + canonicalize_frame_string_dtypes( + frame, boundary=PREPARATION_BOUNDARY, in_place=True + ) + transitions: list[dict[str, str]] = [] + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in list(table.columns): + _require(isinstance(column, str), "COLUMN_NAME") + original = table[column] + token = _target_token(original.dtype) + target = dtype_for_token(token) + if original.dtype == target: + continue + promoted = original.astype(target) + _require(original.isna().equals(promoted.isna()), "DTYPE_PROMOTION_NULLS") + observed = original.notna() + _require( + np.array_equal( + original[observed].to_numpy(dtype=object), + promoted[observed].to_numpy(dtype=object), + ), + "DTYPE_PROMOTION_VALUES", + ) + table[column] = promoted + transitions.append( + { + "entity": entity, + "column": column, + "from": str(original.dtype), + "to": token, + } + ) + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table.columns: + token_for_dtype(table[column].dtype) + return tuple(transitions) + + +def _entity_rows(frame: Frame) -> dict[str, int]: + return {entity: frame.n(entity) for entity in US_SCHEMA.entities} + + +def prepared_source_implementation_identity() -> str: + """Bind this preparation's own code plus the reviewed loaders it calls.""" + package = resources.files(__package__) + modules = { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_prepared_source.py", + "asec_income_observations.py", + "asec_current_money_selection.py", + "cps_carried_current.py", + "graph_housing_universe.py", + ) + } + return _sha( + _json( + { + "schema": 3, + "source_kind": PREPARED_SOURCE_KIND, + "modules": modules, + "money_source_verification": money_source._verification_identity(), + "restoration_verification": person_income._implementation(), + "student_controls_verification": student_source._implementation(), + "income_observations_verification": income_source._implementation(), + "housing_verification": housing_source._implementation(), + "housing_universe_verification": housing_universe_source._implementation(), + "file_roster": list(PREPARED_SOURCE_FILES), + "boundary": PREPARATION_BOUNDARY, + } + ) + ) + + +@dataclass(frozen=True) +class PreparedAsecPopulation: + """An operator-free prepared frame with its money body and identity receipt.""" + + frame: Frame + money_payload: bytes + receipt: dict + field_entities: tuple[tuple[str, str], ...] + housing_universe_payload: bytes + income_observations_payload: bytes + + @property + def receipt_payload(self) -> bytes: + return _json(self.receipt) + + @property + def receipt_sha256(self) -> str: + return _sha(self.receipt_payload) + + +def prepare_asec_current_money_population(source_dir) -> PreparedAsecPopulation: + """Load, reconstruct, attach and promote the full authenticated population.""" + paths = _roster(source_dir) + identity = prepared_source_implementation_identity() + source = person_income.load_authenticated_restored_current_money_source( + paths[PARENT_FILENAME], + paths[HOUSEHOLD_ATTACHMENT_FILENAME], + paths[PERSON_INCOME_FILENAME], + member_paths={year: paths[name] for year, name in MEMBER_FILENAMES}, + ) + evidence = _parse(source.source.identity) + _require(evidence["source_kind"] == RESTORED_SOURCE_KIND, "RESTORED_SOURCE_KIND") + _require(evidence["zero_origin_policy"] == ZERO_POLICY, "ZERO_ORIGIN_POLICY") + _require(evidence["field_roster"] == list(FIELDS), "FIELD_ROSTER") + restoration = _restoration_receipt( + paths[RESTORATION_RECEIPT_FILENAME], evidence, source + ) + ready = source.ready() + _require(type(ready) is ReadyCurrentMoney, "AUTHENTICATED_READINESS_REQUIRED") + income = income_source.load_authenticated_income_observations( + source, + ready, + member_paths={year: paths[name] for year, name in MEMBER_FILENAMES}, + ) + # S is separate source authority. Its public loader authenticates the same + # official members against the unchanged T1 and money parents; attachment + # adds two aliases while retaining original missing controls untouched. + controls = student_source.load_authenticated_student_controls( + source, + ready, + member_paths={year: paths[name] for year, name in MEMBER_FILENAMES}, + ) + with_students = student_source.attach_student_controls(source, ready, controls) + tax = reconstruct_current_money_tax_units(with_students, ready) + # Housing still authenticates original T1, never the transient tax/S view. + status = load_authenticated_housing_status( + source, cohort_paths={year: paths[name] for year, name in COHORT_FILENAMES} + ) + attached = attach_housing_status(tax, status) + attached.validate() + # HU authenticates original T1 and retains the complete S/tax/housing parent. + # Its strict uint8 attachment validation runs before graph width promotion. + universe = housing_universe_source.load_authenticated_housing_universe(source) + with_universe = housing_universe_source.attach_housing_universe(attached, universe) + with_universe.validate() + frame = housing_source._owned_frame(with_universe.frame) + pre_promotion = money_source._frame_signature(frame) + _require( + pre_promotion == with_universe.receipt["output_frame_sha256"], "PARENT_CAPTURE" + ) + transitions = _promote(frame) + post_promotion = money_source._frame_signature(frame) + # The graph carries metadata as a typed context artifact, and the store's + # frame codec does not persist Frame.metadata at all. A non-empty prepared + # metadata mapping would be silently dropped, so refuse instead. + _require(dict(frame.metadata) == {}, "PREPARED_FRAME_METADATA") + _require(frame.mass_log == (), "PREPARED_FRAME_MASS_LOG") + _require(frame.weighted_entities == ("household",), "PREPARED_FRAME_WEIGHTS") + frame.revalidate() + assert_operator_free_source_frame(frame, label=PREPARATION_BOUNDARY) + _require( + prepared_source_implementation_identity() == identity, + "PREPARATION_IMPLEMENTATION_CHANGED", + ) + money_payload = encode_current_money(ready) + universe_payload = housing_universe.encode_housing_universe(universe) + income_payload = income_source.encode_income_observations(income) + field_entities = tuple( + (domain.name, domain.entity) for domain in ready.bindings.spec.fields + ) + receipt = { + "schema": PREPARED_RECEIPT_SCHEMA, + "source_kind": PREPARED_SOURCE_KIND, + "restored_source_kind": evidence["source_kind"], + "release_eligible": False, + "all_current_money_consumers_wired": False, + "source_evidence_sha256": _sha(source.source.identity), + "t1_sha256": evidence["person_income_attachment_sha256"], + "income_observations": { + "parser_profile": income_source.parser_profile(), + "header_sha256": _sha(income._header), + "content_sha256": income.content_sha256, + "payload_sha256": _sha(income_payload), + "rows": income.receipt["rows"], + "columns": list(income_source.COLUMNS), + "kind": income_source.ARTIFACT_KIND, + "scope_sha256": income.receipt["scope_sha256"], + }, + "student_controls_receipt_sha256": _sha(_json(controls.receipt)), + "student_controls_content_sha256": controls.content_sha256, + "student_controls_attachment_receipt_sha256": _sha( + _json(with_students.receipt) + ), + "student_controls_reference_period": controls.receipt["reference_period"], + "annual_five_month_student_status_validated": False, + "money_header_sha256": _sha(ready.header), + "money_content_sha256": current_money_content_sha256(ready), + "money_spec_sha256": ready.bindings.spec.sha256, + "field_entities": [list(item) for item in field_entities], + "scope_sha256": _parse(ready.header)["scope_sha256"], + "tax_receipt_sha256": _sha(_json(tax.receipt)), + "tax_old_partition_sha256": tax.receipt["old_partition_sha256"], + "tax_new_partition_sha256": tax.receipt["new_partition_sha256"], + "housing_content_sha256": status.content_sha256, + "housing_attachment_receipt_sha256": _sha(_json(attached.receipt)), + "pre_housing_universe_frame_sha256": attached.receipt["output_frame_sha256"], + "housing_universe": { + "header_sha256": _sha(universe.header), + "content_sha256": universe.content_sha256, + "payload_sha256": _sha(universe_payload), + "definition_sha256": universe.header_data["definition_sha256"], + "attachment_receipt_sha256": _sha(_json(with_universe.receipt)), + "parent_kind": "HousingStatusAttachedAsec", + "source_period_kind": "interview_household_universe", + "native_dtype": "uint8", + "graph_dtype": "int64", + "aliases": list(housing_universe_source.ATTACHED_COLUMNS), + }, + "restoration_receipt_sha256": _sha(_json(restoration)), + "pre_promotion_frame_sha256": pre_promotion, + "post_promotion_frame_sha256": post_promotion, + "dtype_transitions": [dict(item) for item in transitions], + "entity_rows": _entity_rows(frame), + "implementation_sha256": identity, + "file_roster": list(PREPARED_SOURCE_FILES), + } + return PreparedAsecPopulation( + frame, money_payload, receipt, field_entities, universe_payload, income_payload + ) + + +def load_graph_asec_prepared(path, *, store=None) -> Frame: + """Registered source codec: the prepared operator-free population only. + + The executor verifies that this loader is installed and content-keys the + directory; it never calls it. The declared computation that invokes the + preparation is the CREATE kernel, which needs the money body and receipt + this signature cannot return, so the two paths share one implementation + rather than preparing the population twice. + """ + del store + return prepare_asec_current_money_population(path).frame + + +__all__ = [ + "COHORT_FILENAMES", + "HOUSEHOLD_ATTACHMENT_FILENAME", + "MEMBER_FILENAMES", + "PARENT_FILENAME", + "PERSON_INCOME_FILENAME", + "PREPARATION_BOUNDARY", + "PREPARED_RECEIPT_SCHEMA", + "PREPARED_SOURCE_FILES", + "PREPARED_SOURCE_KIND", + "PreparedAsecPopulation", + "PreparedSourceRefusalError", + "RESTORATION_RECEIPT_FILENAME", + "load_graph_asec_prepared", + "prepare_asec_current_money_population", + "prepared_source_implementation_identity", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_raw_stage_v4.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_raw_stage_v4.py new file mode 100644 index 000000000..5e60f0a0e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_raw_stage_v4.py @@ -0,0 +1,194 @@ +"""Explicit local v3-to-v4 exact-source restoration, never schema relabeling.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +import platform +import re +import tempfile +from collections.abc import Mapping +from importlib.metadata import version +from pathlib import Path + +from microcosm.build.frame_checkpoint import write_frame_checkpoint +from microcosm.build.us_runtime import asec_checkpoint, reported_coverage_source +from microcosm.frame import Frame + + +def _file_sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def restore_asec_raw_stage_v4( + checkpoint_path: str | Path, + *, + expected_sha256: str, + coverage_paths: Mapping[int, str | Path], + output_dir: str | Path, +) -> dict[str, object]: + """Authenticate and restore all seven recodes into a new local bundle. + + The input is strictly v3 and operator-untouched. Every pooled income year + must have an explicit local official member or ZIP; this API never fetches. + Existing nonmissing recodes must agree with that source. The output directory + must not exist, so neither historical artifacts nor receipts are overwritten. + + Whole-file hashes authenticate immutable local inputs relative to caller pins; + they do not establish trust in an unknown producer. Structural Frame identity + alone does not bind person values. Source code and serialization versions are + recorded, but no country model runs and no model accuracy is certified here. + """ + source_path = Path(checkpoint_path).expanduser().resolve() + destination = Path(output_dir).expanduser().resolve() + if destination.exists(): + raise FileExistsError(destination) + if not isinstance(expected_sha256, str) or not re.fullmatch( + r"[0-9a-f]{64}", expected_sha256 + ): + raise ValueError("expected_sha256 must be a lowercase SHA-256 digest") + if _file_sha256(source_path) != expected_sha256: + raise ValueError("ASEC v3 input SHA-256 mismatch") + frame, metadata = asec_checkpoint.load_asec_raw_stage_checkpoint(source_path) + if _file_sha256(source_path) != expected_sha256: + raise ValueError("ASEC v3 input SHA-256 changed during loading") + years = tuple(sorted({int(year) for year in frame.table("person")["source_year"]})) + receipt_years = {source["year"] for source in metadata["source_receipt"]["sources"]} + if set(years) != receipt_years: + raise ValueError("ASEC v3 frame/source receipt year coverage differs") + if set(coverage_paths) != set(years) or any( + type(year) is not int or not isinstance(path, (str, Path)) + for year, path in coverage_paths.items() + ): + raise ValueError( + "explicit local coverage paths must cover exactly the pooled years" + ) + sidecar = reported_coverage_source.load_asec_reported_coverage_sources( + coverage_paths, income_years=years + ) + person = reported_coverage_source.fill_asec_reported_coverage_source( + frame.table("person"), sidecar + ) + restored = Frame( + { + entity: person if entity == "person" else frame.table(entity) + for entity in frame.entities + }, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + binding = copy.deepcopy(metadata) + binding["schema_version"] = asec_checkpoint.ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION + pins = reported_coverage_source.ASEC_EDUCATION_ASSISTANCE_ARCHIVES + source_pins = [ + { + "income_year": year, + "locator": pins[year].zip_url, + "member": pins[year].member, + "sha256": pins[year].zip_sha256, + "member_sha256": pins[year].member_sha256, + } + for year in years + ] + audit = sidecar.attrs["source_audit"] + for column in reported_coverage_source.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + binding["raw_source_mappings"][column] = { + "column": column, + "entity": "person", + "operation": "exact_source_join", + "join_keys": ["source_year", "PERIDNUM"], + "source_pins": source_pins, + "audit": { + str(year): { + "rows": audit[year]["rows"], + **audit[year]["columns"][column], + } + for year in years + }, + } + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".asec-v4-", dir=destination.parent + ) as temp: + temporary = Path(temp) + output_path = temporary / asec_checkpoint.ASEC_RAW_STAGE_CHECKPOINT_FILENAME + write_frame_checkpoint(output_path, restored, metadata=binding) + # Validate the actual serialization, not only the in-memory construction. + checked, checked_binding = asec_checkpoint.load_asec_raw_stage_checkpoint_v4( + output_path + ) + del checked, checked_binding + implementations = { + Path(module.__file__).name: _file_sha256(Path(module.__file__)) + for module in (asec_checkpoint, reported_coverage_source) + } + implementations[Path(__file__).name] = _file_sha256(Path(__file__)) + receipt = { + "schema": "microcosm.asec_raw_stage_restoration.v1", + "input_schema_version": 3, + "output_schema_version": 4, + "input_sha256": expected_sha256, + "output_sha256": _file_sha256(output_path), + "output_file": output_path.name, + "operation": "authenticated_exact_source_join", + "source_pins": source_pins, + "source_years": list(years), + "person_rows": len(person), + "implementation_sha256": implementations, + "runtime_versions": { + "python": platform.python_version(), + **{ + name: version(name) + for name in ("microcosm-build", "pandas", "numpy", "h5py") + }, + }, + "model_execution": False, + "release_certification": False, + "pipeline_lineage_sha256": metadata["pipeline_sha256"], + } + (temporary / "restoration.receipt.json").write_text( + json.dumps(receipt, sort_keys=True, indent=2, allow_nan=False) + "\n" + ) + if destination.exists(): + raise FileExistsError(destination) + os.rename(temporary, destination) + return receipt + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--expected-sha256", required=True) + parser.add_argument( + "--coverage", action="append", required=True, metavar="YEAR=LOCAL_PATH" + ) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args(argv) + paths = {} + for item in args.coverage: + year_text, separator, path = item.partition("=") + if not separator or not year_text.isdecimal() or not path: + parser.error("--coverage must be YEAR=LOCAL_PATH") + year = int(year_text) + if year in paths: + parser.error("--coverage repeats a year") + paths[year] = Path(path) + receipt = restore_asec_raw_stage_v4( + args.input, + expected_sha256=args.expected_sha256, + coverage_paths=paths, + output_dir=args.output_dir, + ) + print(json.dumps(receipt, sort_keys=True, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py new file mode 100644 index 000000000..ddf127896 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py @@ -0,0 +1,511 @@ +"""Closed Census interview-week controls, separate from the T1 money source. + +These observations do not establish annual or five-month tax-student status. +Candidate bytes never supply parsed content or authority: replay reconstructs S +from the authenticated T1 parent and the code-owned official CSV members. +""" + +from __future__ import annotations + +import csv +import hashlib +import os +import stat +import struct +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import asec_current_money_source as legacy +from . import asec_person_income_source as restoration +from ._asec_current_money_codec import current_money_content_sha256 +from .asec_current_money import ( + RESTORED_SOURCE_KIND, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _require, + _sha, + _validate_money, +) +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +CONTROLS = ("A_ENRLW", "A_FTPT") +ALIASES = ("asec_A_ENRLW", "asec_A_FTPT") +COORDINATES = ("person_id", "income_year", "source_household_id", "A_LINENO", "A_AGE") +COLUMNS = COORDINATES + CONTROLS +_READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE") + CONTROLS +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +MAGIC = b"MCASTUD\x01" +FILENAME = "asec_student_controls.bin" +_HEADER_MAX = 65536 +_MAX_PERSONS = 600_000 +_TOKEN = object() +_COMPOSE_TOKEN = object() + + +def _implementation(): + package = resources.files(__package__) + return { + "schema": 1, + "t1_verification_sha256": restoration._implementation(), + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ("asec_student_controls.py", "_asec_current_money_codec.py") + }, + "dependencies": {n: metadata.version(n) for n in ("numpy", "pandas")}, + "member_pins": _MEMBER_PINS, + "aliases": dict(zip(CONTROLS, ALIASES, strict=True)), + } + + +def _parent(source, ready, *, reconstruct_money=False): + _require( + type(source) is legacy.AuthenticatedCurrentMoneySource + and type(ready) is ReadyCurrentMoney, + "STUDENT_AUTHENTICATED_PARENT", + ) + source.validate() + _require( + _parse(source.source.identity)["source_kind"] == RESTORED_SOURCE_KIND, + "STUDENT_RESTORED_PARENT_REQUIRED", + ) + _require(ready.bindings.spec == source.spec, "STUDENT_MONEY_PARENT") + _validate_money(ready, source.spec, nominal=False) + if reconstruct_money: + expected = source.ready() + _require( + ready.bindings == expected.bindings and ready.fields == expected.fields, + "STUDENT_MONEY_PARENT", + ) + + +def _snapshot(path, destination, *, size): + """Exact-size private capture; no candidate parser and no unbounded copy.""" + digest = hashlib.sha256() + # Opening a FIFO in ordinary blocking mode would wait before fstat could + # reject it. Use one nonblocking descriptor, then admit only regular files. + descriptor = os.open(Path(path), os.O_RDONLY | os.O_NONBLOCK) + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "STUDENT_SOURCE_FILE_KIND") + _require(before.st_size == size, "STUDENT_SOURCE_SIZE") + count = 0 + with ( + os.fdopen(descriptor, "rb", closefd=False) as src, + destination.open("xb") as dst, + ): + while chunk := src.read(min(1024 * 1024, size - count + 1)): + count += len(chunk) + _require(count <= size, "STUDENT_SOURCE_SIZE") + digest.update(chunk) + dst.write(chunk) + after = os.fstat(descriptor) + attrs = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + _require( + count == size + and all(getattr(before, a) == getattr(after, a) for a in attrs), + "STUDENT_SOURCE_CHANGED", + ) + finally: + os.close(descriptor) + return digest.hexdigest() + + +def _read_member(path, *, rows): + with path.open("r", encoding="utf-8", newline="") as handle: + header = next(csv.reader(handle), []) + _require( + len(header) == len(set(header)) and set(_READ_COLUMNS) <= set(header), + "STUDENT_MEMBER_HEADER", + ) + table = pd.read_csv( + path, usecols=list(_READ_COLUMNS), dtype="string", na_filter=False + ) + _require(len(table) == rows, "STUDENT_MEMBER_ROWS") + _require( + bool(table.PERIDNUM.str.fullmatch(r"[0-9]{22}").all()), + "STUDENT_MEMBER_PERSON_KEY", + ) + for name in _READ_COLUMNS[1:]: + _require( + bool(table[name].str.fullmatch(r"[0-9]+").all()), + "STUDENT_MEMBER_INTEGER", + name, + ) + table[name] = table[name].astype("int64") + _require( + not table.PERIDNUM.duplicated().any() + and not table.duplicated(["PH_SEQ", "A_LINENO"]).any() + and bool((table.PH_SEQ > 0).all()) + and bool((table.A_LINENO > 0).all()), + "STUDENT_MEMBER_COORDINATES", + ) + for name in CONTROLS: + _require(bool(table[name].isin([0, 1, 2]).all()), "STUDENT_CODE_DOMAIN", name) + _require( + bool((table.A_ENRLW[~table.A_AGE.between(16, 54)] == 0).all()) + and bool((table.A_FTPT[table.A_ENRLW == 1] != 0).all()) + and bool((table.A_FTPT[table.A_ENRLW != 1] == 0).all()), + "STUDENT_SOURCE_UNIVERSE", + ) + return table + + +@dataclass(frozen=True) +class AuthenticatedStudentControls: + """Immutable owned numeric buffers; only closed source reconstruction issues S.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "STUDENT_CONSTRUCTOR_UNAVAILABLE") + self.validate() + + @property + def receipt(self): + return _parse(self._header) + + @property + def content_sha256(self): + return _sha(self._header + self._body) + + def validate(self): + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "STUDENT_HEADER_SIZE", + ) + header = self.receipt + rows = header["rows"] + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "STUDENT_ROWS") + _require( + type(self._body) is bytes + and len(self._body) == rows * len(COLUMNS) * 8 + and _sha(self._body) == header["body_sha256"] + and header["columns"] == list(COLUMNS) + and header["aliases"] == dict(zip(CONTROLS, ALIASES, strict=True)) + and _json(header["implementation"]) == _json(_implementation()), + "STUDENT_CONTENT_CHANGED", + ) + + def array(self, name): + self.validate() + _require(name in COLUMNS, "STUDENT_COLUMN") + n = self.receipt["rows"] + offset = COLUMNS.index(name) * n * 8 + # A bytes-backed buffer cannot be made writeable by the caller. + return np.frombuffer(self._body, dtype="= 0).all()) and len(np.unique(indices)) == rows, + "STUDENT_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + for name, raw_name in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + _require( + np.array_equal( + output[name][positions], joined[raw_name].to_numpy() + ), + "STUDENT_NATIVE_KEY_OR_AGE", + ) + compared = {} + for name in CONTROLS: + incumbent = person[name].iloc[positions] + known = ~incumbent.isna().to_numpy() + observed = joined[name].to_numpy(dtype=np.int64) + _require( + np.array_equal( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known], + observed[known], + ), + "STUDENT_INCUMBENT_CONFLICT", + name, + ) + if year == 2024: + _require(bool(known.all()), "STUDENT_INCUMBENT_INCOMPLETE", name) + output[name][positions] = observed + compared[name] = int(known.sum()) + joins.append( + { + "income_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": rows, + "joined_rows": len(positions), + "unreferenced_source_rows": 0, + "incumbent_compared_rows": compared, + "incumbent_conflicts": 0, + "native_key_or_age_conflicts": 0, + } + ) + _parent(source, ready) + _require( + _json(_implementation()) == _json(before), "STUDENT_IMPLEMENTATION_CHANGED" + ) + buffers = [output[name].astype(" tuple[bytes, bytes]: + """Normalize exact original API responses and mapping bytes, with a receipt. + + Each ordered item is ``(state, block_response, single_state_total_response)``. + There is exactly one item per selected state, drawn from the 50 states plus + DC. Expected API requests appear in the descriptive receipt with repeated + ``in`` parameters preserved and no credential parameter. The caller owns + acquisition and publisher qualification; the response cannot prove its URL. + + Response byte limits precede JSON parsing; the row limit is enforced after + JSON allocation. CD ZIP metadata and inflation follow the existing bounded + source reader. All rows, including zero-population blocks, are counted; + every positive block is retained exactly once and reconciled to its separate + state response. No county/tract aggregation or synthetic PL file is used. + """ + _require( + type(state_fips) is tuple + and bool(state_fips) + and all( + type(state) is str and state in sources._GEO_MEMBERS for state in state_fips + ) + and state_fips == tuple(sorted(set(state_fips))), + "STATE_ROSTER", + ) + identities = normalized._sources(source_ids) + bounds = { + "max_source_bytes": max_source_bytes, + "max_member_bytes": max_member_bytes, + "max_total_geography_bytes": max_total_geography_bytes, + "max_zip_members": max_zip_members, + "max_line_bytes": max_line_bytes, + "max_response_rows": max_response_rows, + } + _require( + all(type(value) is int and 0 < value <= 2**63 - 1 for value in bounds.values()), + "BOUNDS", + ) + expanded = [0] + puma_payload, puma_provenance = _raw_source( + tract_to_puma, bounds=bounds, expanded=expanded + ) + cd_payload, cd_provenance = sources._selected_zip_member( + cd_archive, sources.CD_MEMBER, bounds=bounds, expanded=expanded + ) + cd_statistics, unassigned = {"source_records": 0, "delegate_records": 0}, set() + districts = parse_national_cd_bef( + sources._cd_lines( + cd_payload, + max_line_bytes=max_line_bytes, + statistics=cd_statistics, + unassigned=unassigned, + ) + ) + _require(not unassigned.intersection(districts), "CD_CONFLICTING_UNASSIGNED") + _require( + cd_statistics["source_records"] == len(districts) + len(unassigned), + "CD_RECONCILIATION", + ) + cd_provenance.update( + source_id=identities["district"], + **cd_statistics, + assigned_blocks=len(districts), + unassigned_blocks=len(unassigned), + delegate_normalization="98_to_00", + district_relation="official_tabulation", + ) + del cd_payload, unassigned + pumas = parse_tract_to_puma_relationship( + sources._lines(puma_payload, encoding="utf-8", max_line_bytes=max_line_bytes), + allowed_state_fips=frozenset(state_fips), + ) + puma_provenance.update( + source_id=identities["puma"], selected_state_tract_mappings=len(pumas) + ) + del puma_payload + + iterator, exhausted = iter(population_responses), object() + population, population_provenance, state_totals = {}, [], {} + for state in state_fips: + item = next(iterator, exhausted) + _require( + type(item) is tuple and len(item) == 3 and item[0] == state, + "POPULATION_SOURCE_ROSTER", + ) + block_payload, block_provenance = _raw_source( + item[1], bounds=bounds, expanded=expanded + ) + total_payload, total_provenance = _raw_source( + item[2], bounds=bounds, expanded=expanded + ) + blocks, statistics = _state_population( + block_payload, total_payload, state=state, max_rows=max_response_rows + ) + _require(not population.keys() & blocks.keys(), "OVERLAPPING_STATES") + population.update(blocks) + state_totals[state] = statistics["state_population"] + population_provenance.append( + { + "source_id": identities["population"], + "state_fips": state, + "blocks": {**block_provenance, "request": _request(state, blocks=True)}, + "state_total": { + **total_provenance, + "request": _request(state, blocks=False), + }, + **statistics, + } + ) + del item, block_payload, total_payload, blocks + _require(next(iterator, exhausted) is exhausted, "POPULATION_SOURCE_ROSTER") + payload = normalized.assemble_atomic_block_support( + block_population=population, + cd_by_block=districts, + puma_by_tract=pumas, + source_ids=identities, + ) + _require(len(payload) <= RAW_BYTES_MAX_BYTES, "SUPPORT_BYTES") + support = decode_atomic_support(payload) + output_totals = dict.fromkeys(state_fips, 0) + for state, value in zip( + support.arrays["state"], support.arrays["population"], strict=True + ): + output_totals[str(state)] += int(value) + ordered_blocks = sorted(population) + _require( + support.arrays["area"].tolist() == [f"{block:015d}" for block in ordered_blocks] + and support.arrays["population"].tolist() + == [population[block] for block in ordered_blocks] + and support.arrays["puma"].tolist() + == [f"{pumas[block // 10000]:07d}" for block in ordered_blocks] + and support.arrays["district"].tolist() + == [f"{districts[block]:04d}" for block in ordered_blocks] + and output_totals == state_totals + and len(support.arrays["area"]) == len(population), + "OUTPUT_RECONCILIATION", + ) + receipt = canonical_json( + { + "protocol": PROTOCOL, + "state_fips": list(state_fips), + "source_ids": identities, + "limits": bounds, + "sources": { + "population": population_provenance, + "district": cd_provenance, + "puma": puma_provenance, + }, + "reconciliation": { + "state_population": state_totals, + "output_state_population": output_totals, + "population_total": sum(state_totals.values()), + "populated_blocks": len(population), + "selected_geography_bytes": expanded[0], + }, + "support_sha256": sources._sha(payload), + "exact_input_hashes_verified": True, + "publisher_provenance_established": False, + "request_origin_verified": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + return payload, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py new file mode 100644 index 000000000..aea5b8366 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py @@ -0,0 +1,411 @@ +"""Pinned Census source bytes to normalized atomic-block support, without I/O. + +The caller establishes publisher identity and obtains the bytes. This adapter +checks those bytes against explicit pins, checks full ZIP member rosters, and +decompresses only the selected geography members. Unselected PL segment contents +are neither opened nor CRC-checked. No source or release admission is issued. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from io import BytesIO +from pathlib import PurePosixPath +from zipfile import ZIP_DEFLATED, ZIP_STORED, BadZipFile, ZipFile +from zlib import error as zlib_error + +from microcosm.build.atomic_geography import decode_atomic_support +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES + +from . import atomic_block_support as normalized +from .block_ladder_sources import ( + US_STATES, + parse_national_cd_bef, + parse_pl_geo_blocks, +) +from .puma_ladder_sources import parse_tract_to_puma_relationship + +PROTOCOL = "microcosm.us.atomic-block-sources.v1" +CD_MEMBER = "NationalCD119.txt" +_GEO_MEMBERS = {state: f"{usps.lower()}geo2020.pl" for state, usps, _ in US_STATES} +_CHUNK_BYTES = 64 * 1024 + + +@dataclass(frozen=True, slots=True) +class AtomicBlockSourceBytes: + """Literal bytes and expected digest/roster; construction conveys no authority.""" + + payload: bytes + sha256: str + zip_members: tuple[str, ...] = () + + +def _require(condition, reason): + if not condition: + raise ValueError("ATOMIC_BLOCK_SOURCES_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _checked_source(record, *, archive, max_source_bytes): + _require(type(record) is AtomicBlockSourceBytes, "SOURCE_TYPE") + payload, digest, members = record.payload, record.sha256, record.zip_members + _require( + type(payload) is bytes and 0 < len(payload) <= max_source_bytes, + "SOURCE_BYTES", + ) + _require( + type(digest) is str + and re.fullmatch(r"[0-9a-f]{64}", digest) is not None + and _sha(payload) == digest, + "SOURCE_DIGEST", + ) + _require(type(members) is tuple, "MEMBER_ROSTER") + if archive: + _require( + bool(members) + and all( + type(name) is str + and 0 < len(name) <= 255 + and "\0" not in name + and "\\" not in name + and not name.endswith("/") + and not PurePosixPath(name).is_absolute() + and all(part not in {"", ".", ".."} for part in name.split("/")) + for name in members + ) + and len(set(members)) == len(members), + "MEMBER_ROSTER", + ) + else: + _require(members == (), "RAW_MEMBER_ROSTER") + return payload, digest, members + + +def _selected_zip_member(record, member, *, bounds, expanded): + source_payload, source_digest, expected_members = _checked_source( + record, archive=True, max_source_bytes=bounds["max_source_bytes"] + ) + provenance = {"sha256": source_digest, "size_bytes": len(source_payload)} + _require( + member in expected_members + and len(expected_members) <= bounds["max_zip_members"], + "SELECTED_MEMBER", + ) + try: + with ZipFile(BytesIO(source_payload)) as archive: + members = archive.infolist() + names = tuple(info.filename for info in members) + _require( + len(names) == len(expected_members) + and len(set(names)) == len(names) + and all(info.orig_filename == info.filename for info in members) + and set(names) == set(expected_members), + "ZIP_MEMBER_ROSTER", + ) + info = archive.getinfo(member) + _require( + not info.is_dir() + and not info.flag_bits & 1 + and info.compress_type in {ZIP_STORED, ZIP_DEFLATED}, + "ZIP_MEMBER_FORMAT", + ) + _require( + 0 < info.file_size <= bounds["max_member_bytes"], "ZIP_MEMBER_SIZE" + ) + _require( + expanded[0] + info.file_size <= bounds["max_total_geography_bytes"], + "TOTAL_GEOGRAPHY_BYTES", + ) + # ZipExtFile validates the selected member's CRC when exhausted. + # No other member is opened, even to check its CRC. + output = BytesIO() + with archive.open(info) as stream: + while chunk := stream.read(_CHUNK_BYTES): + _require( + output.tell() + len(chunk) <= info.file_size, + "ZIP_MEMBER_SIZE", + ) + output.write(chunk) + payload = output.getvalue() + _require(len(payload) == info.file_size, "ZIP_MEMBER_SIZE") + expanded[0] += len(payload) + provenance.update( + { + "zip_members": sorted(names), + "selected_member": member, + "selected_member_sha256": _sha(payload), + "selected_member_size_bytes": len(payload), + "crc_checked_members": [member], + "unselected_member_contents_read": False, + } + ) + except (BadZipFile, EOFError, RuntimeError, NotImplementedError, zlib_error): + raise ValueError("ATOMIC_BLOCK_SOURCES_ZIP_INVALID") from None + return payload, provenance + + +def _lines(payload, *, encoding, max_line_bytes): + stream = BytesIO(payload) + while line := stream.readline(max_line_bytes + 1): + _require(len(line) <= max_line_bytes and b"\0" not in line, "TEXT_LINE") + yield line.decode(encoding) + + +def _pl_lines(payload, *, state, max_line_bytes, statistics): + """Guard parser skips: count every block, including zero-population rows.""" + seen = set() + for line in _lines(payload, encoding="latin-1", max_line_bytes=max_line_bytes): + if not line.strip(): + continue + fields = line.rstrip("\r\n").split("|") + _require(len(fields) == 97, "PL_ROW_WIDTH") + statistics["geography_rows"] += 1 + level = fields[2] + if level in {"040", "750"}: + raw = fields[90].strip() + _require(re.fullmatch(r"[0-9]+", raw) is not None, "PL_POPULATION") + population = int(raw) + if level == "040": + statistics["state_rows"] += 1 + _require(statistics["state_rows"] == 1, "PL_STATE_ROWS") + statistics["state_population"] = population + else: + geoid = fields[9].strip() + _require( + re.fullmatch(r"[0-9]{15}", geoid) is not None + and geoid.startswith(state), + "PL_BLOCK_STATE", + ) + _require(geoid not in seen, "PL_DUPLICATE_BLOCK") + seen.add(geoid) + statistics["block_rows"] += 1 + statistics["zero_population_blocks"] += int(population == 0) + yield line + + +def _cd_lines(payload, *, max_line_bytes, statistics, unassigned): + first = True + for line in _lines(payload, encoding="latin-1", max_line_bytes=max_line_bytes): + if not line.strip(): + continue + if first: + first = False + else: + parts = line.strip().split(",") + _require(len(parts) == 2, "CD_ROW_WIDTH") + statistics["source_records"] += 1 + geoid, district = (part.strip() for part in parts) + if district == "ZZ": + _require(re.fullmatch(r"[0-9]{15}", geoid) is not None, "CD_BLOCK") + block = int(geoid) + _require(block not in unassigned, "CD_DUPLICATE_BLOCK") + unassigned.add(block) + statistics["delegate_records"] += int(district == "98") + yield line + + +def assemble_atomic_block_sources( + *, + pl_archives: Iterable[tuple[str, AtomicBlockSourceBytes]], + cd_archive: AtomicBlockSourceBytes, + tract_to_puma: AtomicBlockSourceBytes, + state_fips: tuple[str, ...], + source_ids: Mapping[str, str], + max_source_bytes: int = 512 * 1024**2, + max_member_bytes: int = 2 * 1024**3, + max_total_geography_bytes: int = 32 * 1024**3, + max_zip_members: int = 16, + max_line_bytes: int = 64 * 1024, +) -> tuple[bytes, bytes]: + """Return deterministic normalized support and a canonical source receipt. + + ``state_fips`` is a nonempty sorted tuple drawn from the 50 states plus DC. + ``pl_archives`` yields exactly that ordered roster, one archive at a time; + only the current PL archive/member is retained while accumulating block maps. + Required members are ``{usps_lower}geo2020.pl`` and ``NationalCD119.txt``. + The tract relationship is raw UTF-8 text. ZIP target text uses the maintained + source readers' Latin-1 convention. All positive-POP100 blocks are retained; + zero-population exclusions are counted in each state's receipt. + + The source byte cap and hash check precede ZIP metadata parsing. Member + roster/count and expanded-byte limits precede selected-member inflation and + text parsing, with a cumulative bound on selected geography plus the raw + relationship. The member-count limit does not bound ZipFile's initial + metadata allocation. No unselected segment is decompressed or parsed. + Normalized output must fit the receiving raw-bytes-v1 codec's 64 MiB cap. + + Matching caller-supplied hashes proves byte integrity, not that the caller's + pins identify an official publisher, a current source, or an admitted dataset. + """ + _require( + type(state_fips) is tuple + and bool(state_fips) + and all(type(state) is str and state in _GEO_MEMBERS for state in state_fips) + and state_fips == tuple(sorted(set(state_fips))), + "STATE_ROSTER", + ) + identities = normalized._sources(source_ids) + bounds = { + "max_source_bytes": max_source_bytes, + "max_member_bytes": max_member_bytes, + "max_total_geography_bytes": max_total_geography_bytes, + "max_zip_members": max_zip_members, + "max_line_bytes": max_line_bytes, + } + _require( + all(type(value) is int and 0 < value <= 2**63 - 1 for value in bounds.values()), + "BOUNDS", + ) + puma_payload, puma_digest, _members = _checked_source( + tract_to_puma, archive=False, max_source_bytes=max_source_bytes + ) + puma_provenance = {"sha256": puma_digest, "size_bytes": len(puma_payload)} + _require( + len(puma_payload) <= min(max_member_bytes, max_total_geography_bytes), + "TOTAL_GEOGRAPHY_BYTES", + ) + expanded = [len(puma_payload)] + cd_payload, cd_provenance = _selected_zip_member( + cd_archive, CD_MEMBER, bounds=bounds, expanded=expanded + ) + cd_statistics = {"source_records": 0, "delegate_records": 0} + unassigned = set() + districts = parse_national_cd_bef( + _cd_lines( + cd_payload, + max_line_bytes=max_line_bytes, + statistics=cd_statistics, + unassigned=unassigned, + ) + ) + _require(not unassigned.intersection(districts), "CD_CONFLICTING_UNASSIGNED") + _require( + cd_statistics["source_records"] == len(districts) + len(unassigned), + "CD_RECONCILIATION", + ) + cd_provenance.update( + source_id=identities["district"], + **cd_statistics, + assigned_blocks=len(districts), + unassigned_blocks=len(unassigned), + delegate_normalization="98_to_00", + district_relation="official_tabulation", + ) + del cd_payload, unassigned + pumas = parse_tract_to_puma_relationship( + _lines(puma_payload, encoding="utf-8", max_line_bytes=max_line_bytes), + allowed_state_fips=frozenset(state_fips), + ) + puma_provenance.update( + source_id=identities["puma"], selected_state_tract_mappings=len(pumas) + ) + + iterator, exhausted = iter(pl_archives), object() + population, population_provenance, state_totals = {}, [], {} + for state in state_fips: + item = next(iterator, exhausted) + _require( + type(item) is tuple and len(item) == 2 and item[0] == state, + "PL_SOURCE_ROSTER", + ) + record = item[1] + payload, provenance = _selected_zip_member( + record, _GEO_MEMBERS[state], bounds=bounds, expanded=expanded + ) + statistics = { + "geography_rows": 0, + "state_rows": 0, + "state_population": 0, + "block_rows": 0, + "zero_population_blocks": 0, + } + state_population = parse_pl_geo_blocks( + _pl_lines( + payload, + state=state, + max_line_bytes=max_line_bytes, + statistics=statistics, + ), + state_fips=state, + ) + _require( + statistics["state_rows"] == 1 + and statistics["state_population"] == sum(state_population.values()) + and statistics["block_rows"] + == len(state_population) + statistics["zero_population_blocks"], + "PL_RECONCILIATION", + ) + _require( + not population.keys() & state_population.keys(), "PL_OVERLAPPING_STATES" + ) + population.update(state_population) + state_totals[state] = statistics["state_population"] + provenance.update( + source_id=identities["population"], + state_fips=state, + populated_blocks=len(state_population), + **statistics, + ) + population_provenance.append(provenance) + del item, record, payload, state_population + _require(next(iterator, exhausted) is exhausted, "PL_SOURCE_ROSTER") + payload = normalized.assemble_atomic_block_support( + block_population=population, + cd_by_block=districts, + puma_by_tract=pumas, + source_ids=identities, + ) + _require(len(payload) <= RAW_BYTES_MAX_BYTES, "SUPPORT_BYTES") + support = decode_atomic_support(payload) + output_totals = dict.fromkeys(state_fips, 0) + for state, value in zip( + support.arrays["state"], support.arrays["population"], strict=True + ): + output_totals[str(state)] += int(value) + ordered_blocks = sorted(population) + _require( + support.arrays["area"].tolist() == [f"{block:015d}" for block in ordered_blocks] + and support.arrays["population"].tolist() + == [population[block] for block in ordered_blocks] + and support.arrays["puma"].tolist() + == [f"{pumas[block // 10000]:07d}" for block in ordered_blocks] + and support.arrays["district"].tolist() + == [f"{districts[block]:04d}" for block in ordered_blocks] + and output_totals == state_totals + and len(support.arrays["area"]) == len(population), + "OUTPUT_RECONCILIATION", + ) + receipt = canonical_json( + { + "protocol": PROTOCOL, + "state_fips": list(state_fips), + "source_ids": identities, + "limits": bounds, + "sources": { + "population": population_provenance, + "district": cd_provenance, + "puma": puma_provenance, + }, + "reconciliation": { + "state_population": state_totals, + "output_state_population": output_totals, + "population_total": sum(state_totals.values()), + "populated_blocks": len(population), + "selected_geography_bytes": expanded[0], + }, + "support_sha256": _sha(payload), + "exact_input_hashes_verified": True, + "unselected_zip_member_contents_read": False, + "publisher_provenance_established": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + return payload, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py new file mode 100644 index 000000000..20fa3686c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py @@ -0,0 +1,276 @@ +"""Normalize US block mappings for shared geography operators; no new kernel. + +Publisher parsers supply block POP100, the CD block-equivalency mapping and the +2020 tract-to-PUMA mapping. Source authentication remains upstream. This module +retains block rows and supplies a country declaration; assignment and geographic +derivation are executed by the shared country-neutral operators. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from numbers import Integral + +import numpy as np + +from microcosm.build.atomic_geography import ( + encode_atomic_support, + validate_assignment_spec, +) + +SYSTEM = "us_census_block_2020" +SOURCE = "us_atomic_block_support" +_STATES = frozenset( + { + 1, + 2, + 4, + 5, + 6, + 8, + 9, + 10, + 11, + 12, + 13, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 53, + 54, + 55, + 56, + } +) + + +def _require(condition, message): + if not condition: + raise ValueError("US atomic support: " + message) + + +def _integer(value, *, minimum=0, maximum=2**53): + return ( + isinstance(value, Integral) + and not isinstance(value, (bool, np.bool_)) + and minimum <= value <= maximum + ) + + +def _sources(source_ids): + _require( + isinstance(source_ids, Mapping) + and set(source_ids) == {"population", "district", "puma"} + and all( + isinstance(v, str) and v and v.strip() == v for v in source_ids.values() + ), + "three explicit nonempty source identities required", + ) + return dict(source_ids) + + +def assemble_atomic_block_support( + *, + block_population: Mapping, + cd_by_block: Mapping, + puma_by_tract: Mapping, + source_ids: Mapping, +) -> bytes: + """Keep each supplied block, with string codes and source-labeled mappings. + + Population is the declared 2020-person sampling proxy. This does not imply + household counts or support for subsequent construction in unpopulated + 2020 blocks. Existing PL parsers supply populated blocks only. Extra mapping + entries may cover unselected areas; every supplied support row must map. + """ + sources = _sources(source_ids) + _require( + all( + isinstance(m, Mapping) + for m in (block_population, cd_by_block, puma_by_tract) + ) + and bool(block_population), + "nonempty block support and mappings required", + ) + for block, population in block_population.items(): + _require( + _integer(block, minimum=10**13, maximum=10**15 - 1) + and int(block) // 10**13 in _STATES, + "invalid block GEOID", + ) + _require(_integer(population), "invalid population weight") + blocks = sorted(int(block) for block in block_population) + pumas, districts = [], [] + for block in blocks: + state, tract = block // 10**13, block // 10000 + puma, district = puma_by_tract.get(tract), cd_by_block.get(block) + _require( + _integer(puma, minimum=1, maximum=9999999) + and int(puma) // 100000 == state + and int(puma) % 100000 > 0, + "missing or inconsistent tract-to-PUMA mapping", + ) + _require( + _integer(district, minimum=100, maximum=5699) + and int(district) // 100 == state + and int(district) % 100 <= 53, + "missing or inconsistent block-to-district mapping", + ) + pumas.append(f"{int(puma):07d}") + districts.append(f"{int(district):04d}") + arrays = { + "area": np.asarray([f"{v:015d}" for v in blocks]), + "state": np.asarray([f"{v // 10**13:02d}" for v in blocks]), + "county": np.asarray([f"{v // 10**10:05d}" for v in blocks]), + "tract": np.asarray([f"{v // 10000:011d}" for v in blocks]), + "puma": np.asarray(pumas), + "district": np.asarray(districts), + "population": np.asarray( + [int(block_population[v]) for v in blocks], dtype=np.int64 + ), + } + columns = { + name: { + "kind": "code", + "source": sources["population"], + "vintage": "2020", + "relation": "exact", + } + for name in ("area", "state", "county", "tract") + } + columns.update( + { + "puma": { + "kind": "code", + "source": sources["puma"], + "vintage": "2020", + "relation": "exact", + }, + "district": { + "kind": "code", + "source": sources["district"], + "vintage": "119th_congress", + "relation": "official_tabulation", + }, + "population": { + "kind": "weight", + "source": sources["population"], + "basis": "2020_census_persons", + }, + } + ) + return encode_atomic_support( + { + "version": 1, + "system": SYSTEM, + "level": "block", + "code_system": "census_geoid", + "vintage": "2020", + "columns": columns, + }, + arrays, + ) + + +def assignment_definition( + *, + identity: Sequence[str], + state_column: str, + puma_column: str | None, + source_ids: Mapping, + seed: int, +) -> dict: + """Declare one block draw constrained by normalized observed state/PUMA. + + Input codes must already be strings from the survey's source projection. + Callers declare stable source plus completed initial-clone identity columns. + Assignment follows those clones; subsequent views retain the block and all + functionally derived geographies. + """ + sources = _sources(source_ids) + _require(type(seed) is int and 0 <= seed < 2**63, "invalid seed") + _require(not isinstance(identity, (str, bytes)), "identity must be columns") + constraints = [{"input": state_column, "support": "state", "required": True}] + if puma_column is not None: + constraints.append({"input": puma_column, "support": "puma", "required": False}) + layers = [ + { + "input": name, + "output": output, + "vintage": vintage, + "relation": relation, + "source": sources[source], + } + for name, output, vintage, relation, source in ( + ("state", "assigned_state_fips", "2020", "exact", "population"), + ("county", "county_fips", "2020", "exact", "population"), + ("tract", "census_tract_geoid", "2020", "exact", "population"), + ("puma", "assigned_puma_geoid", "2020", "exact", "puma"), + ( + "district", + "congressional_district_geoid", + "119th_congress", + "official_tabulation", + "district", + ), + ) + ] + return validate_assignment_spec( + { + "version": 1, + "identity": list(identity), + "stream": ["sha256-u53-v1", "us-atomic-geography", 0, seed], + "outputs": { + "area": "census_block_geoid", + "system": "geography_system", + "basis": "geography_assignment_basis", + }, + "systems": [ + { + "id": SYSTEM, + "level": "block", + "code_system": "census_geoid", + "vintage": "2020", + "source": SOURCE, + "selector": {}, + "constraints": constraints, + "observed_area": None, + "stages": [{"level": "area", "weight": "population"}], + "layers": layers, + } + ], + } + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference.py new file mode 100644 index 000000000..41a120d73 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference.py @@ -0,0 +1,652 @@ +"""Public CD references and uncertainty, without target or candidate activation. + +This inventory retains published observations. Crosswalk shares describe alignment +exposure only; this module neither allocates fiscal cells nor rescales state totals. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import io +import math +import re +import sys +import zipfile +from collections import Counter, defaultdict +from pathlib import Path +from xml.etree import ElementTree as ET + +from .cd_reference_sources import ( + TABLES, + capture_request, + census_requests, + read_census_key, + strict_json, + write_json, +) + +US_STATE_FIPS = frozenset( + "01 02 04 05 06 08 09 10 11 12 13 15 16 17 18 19 20 21 22 23 24 25 26 " + "27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 44 45 46 47 48 49 50 51 53 54 55 56".split() +) +AT_LARGE_117 = frozenset("02 10 11 30 38 46 50 56".split()) +SENTINELS = { + "-666666666": "insufficient_sample", + "-999999999": "insufficient_geographic_sample", + "-888888888": "not_applicable_or_available", + "-222222222": "moe_insufficient_sample", + "-333333333": "moe_open_interval", + "-555555555": "controlled_estimate", +} +ANNOTATIONS = { + "-": "insufficient_sample", + "N": "insufficient_geographic_sample", + "(X)": "not_applicable_or_available", + "**": "moe_insufficient_sample", + "***": "moe_open_interval", + "*****": "controlled_estimate", +} +AUTHORITIES = { + "acs_annotations": "https://www.census.gov/data/developers/data-sets/acs-1year/notes-on-acs-estimate-and-annotation-values.html", + "acs_annotation_precedence": "https://www.census.gov/data/developers/data-sets/acs-1year/notes-on-acs-api-variable-types.html", + "acs_geography": "https://www.census.gov/programs-surveys/acs/geography-acs/geography-boundaries-by-year/2024.html", + "acs_confidence": "https://www.census.gov/programs-surveys/acs/methodology/sample-size-and-data-quality/sample-size-definitions.html", + "irs_guide": "https://www.irs.gov/pub/irs-soi/22incddocguide.docx", +} + + +def classify_acs_value(raw, annotation) -> dict: + """Annotation fields take precedence; retain sentinels and conflicts verbatim.""" + sentinel = SENTINELS.get(str(raw)) + annotation_class = None + if annotation not in (None, ""): + if not isinstance(annotation, str): + raise ValueError("ACS annotation must be text or null") + annotation_class = ANNOTATIONS.get(annotation) + if annotation_class is None: + annotation_class = ( + "open_interval" + if re.fullmatch(r"[\d,.]+[+-]", annotation) + else "unknown_annotation" + ) + classification = annotation_class or sentinel + numeric = None + if classification is None: + if raw in (None, ""): + classification = "missing" + else: + try: + numeric = float(raw) + if not math.isfinite(numeric): + raise ValueError + except (ValueError, TypeError): + raise ValueError("unrecognized ACS numeric value") from None + classification = "numeric" + return { + "raw": raw, + "annotation": annotation, + "class": classification, + "numeric_value": numeric, + "annotation_conflict": bool( + sentinel and annotation_class and sentinel != annotation_class + ), + } + + +def _district_id(state: str, district: str, congress: int) -> str: + if not re.fullmatch(r"\d{2}", state) or not re.fullmatch(r"\d{2}", district): + raise ValueError("invalid district identity") + return f"500{congress - 100:02d}00US{state}{'00' if district == '98' else district}" + + +def acs_table_inventory(table: str, metadata: dict, data_rows: list) -> dict: + if table not in TABLES: + raise ValueError("undeclared ACS table") + variables = metadata["variables"] + stems = sorted( + key[:-1] + for key in variables + if key.startswith(table + "_") and key.endswith("E") + ) + if not stems or not data_rows: + raise ValueError("ACS table has no estimate variables or data") + header = data_rows[0] + if len(header) != len(set(header)): + raise ValueError("duplicate ACS header") + required = {"NAME", "state", "congressional district"} + required.update( + stem + suffix for stem in stems for suffix in ("E", "M", "EA", "MA") + ) + missing = sorted(required - set(header)) + if missing: + raise ValueError("missing ACS columns: " + ", ".join(missing)) + for stem in stems: + if any(stem + suffix not in variables for suffix in ("E", "M", "EA", "MA")): + raise ValueError("missing ACS variable metadata") + cells, geographies, excluded = [], {}, [] + seen = set() + counts = {"estimate": Counter(), "moe": Counter()} + for values in data_rows[1:]: + if len(values) != len(header): + raise ValueError("ACS row width mismatch") + row = dict(zip(header, values, strict=True)) + state, district = row["state"], row["congressional district"] + identity = _district_id(state, district, 119) + if identity in seen: + raise ValueError("duplicate geography in ACS table") + seen.add(identity) + if state not in US_STATE_FIPS: + excluded.append( + { + "state_fips": state, + "district": district, + "name": row["NAME"], + "reason": "outside_us_50_states_dc", + } + ) + continue + geographies[identity] = { + "geography_id": identity, + "state_fips": state, + "published_district": district, + "name": row["NAME"], + "published_geo_id": row.get("GEO_ID"), + } + for stem in stems: + estimate = classify_acs_value(row[stem + "E"], row[stem + "EA"]) + moe = classify_acs_value(row[stem + "M"], row[stem + "MA"]) + if moe["numeric_value"] is not None and moe["numeric_value"] < 0: + raise ValueError("negative numeric ACS MOE") + counts["estimate"][estimate["class"]] += 1 + counts["moe"][moe["class"]] += 1 + cells.append( + { + "geography_id": identity, + "variable": stem, + "estimate": estimate, + "moe": moe, + } + ) + return { + "table": table, + "year": 2024, + "congress": 119, + "confidence_level": 0.9, + "uncertainty": { + "kind": "published_margin_of_error", + "variance_conversion": "not_performed", + "controlled_estimates": "raw sentinel retained; no invented variance", + }, + "grouping": "subject_age_and_sex_published_groups" + if table == "S0101" + else "detailed_sex_by_age_published_groups" + if table == "B01001" + else "published_detailed_table_cells", + "reservation": { + "status": "reserved_same_source_holdout" + if table in ("B19001", "B25003") + else "inventory_only", + "covers_derived_versions": True, + "targets_allowed": False, + "tuning_allowed": False, + "source_independent": False, + "reason": "ACS tabulations and ACS PUMS share a source; no candidate scoring or tuning here", + }, + "lineage": "published", + "geography_ids": sorted(geographies), + "geographies": [geographies[key] for key in sorted(geographies)], + "excluded_geographies": excluded, + "variables": { + stem: { + suffix: variables[stem + suffix] for suffix in ("E", "M", "EA", "MA") + } + for stem in stems + }, + "classification_counts": {key: dict(value) for key, value in counts.items()}, + "cells": sorted( + cells, key=lambda cell: (cell["geography_id"], cell["variable"]) + ), + } + + +def irs_csv_inventory(payload: bytes) -> dict: + reader = csv.DictReader(io.StringIO(payload.decode("utf-8-sig"))) + header = reader.fieldnames or [] + identity_fields = {"STATEFIPS", "STATE", "CONG_DISTRICT", "agi_stub"} + if len(header) != len(set(header)) or not identity_fields.issubset(header): + raise ValueError("IRS CSV identity columns missing or duplicated") + metrics = [field for field in header if field not in identity_fields] + counts = {field: Counter() for field in metrics} + rows, seen = [], set() + stubs = Counter() + for row in reader: + if None in row or None in row.values(): + raise ValueError("IRS row width mismatch") + state, district, stub = row["STATEFIPS"], row["CONG_DISTRICT"], row["agi_stub"] + identity = state, district, stub + if identity in seen: + raise ValueError("duplicate IRS source row") + seen.add(identity) + _district_id(state, district, 117) + if state not in US_STATE_FIPS | {"00"} or not re.fullmatch(r"\d+", stub): + raise ValueError("IRS source row outside declared scope") + if state == "00" and district != "00": + raise ValueError("national IRS row has district code") + level = ( + "national" + if state == "00" + else "state" + if district == "00" + else "congressional_district" + ) + rows.append( + { + "state_fips": state, + "state": row["STATE"], + "district": district, + "agi_stub": stub, + "published_geography_level": level, + "published_geography_id": _district_id(state, district, 117) + if level == "congressional_district" + else None, + "at_large_proxy_geography_id": _district_id(state, "00", 117) + if state in AT_LARGE_117 and district == "00" + else None, + } + ) + stubs[stub] += 1 + for field in metrics: + raw = row[field] + if raw in ("", "**"): + classification = ( + "missing" if raw == "" else "suppression_or_combination_marker" + ) + else: + try: + value = float(raw) + if not math.isfinite(value): + raise ValueError + except ValueError: + raise ValueError("unrecognized IRS metric cell") from None + classification = ( + "numeric_zero_disclosure_unknown" + if value == 0 + else "numeric_disclosure_unknown" + ) + counts[field][classification] += 1 + return { + "tax_year": 2022, + "processing_year": 2023, + "congress": 117, + "lineage": "published", + "rows": rows, + "row_count": len(rows), + "columns": header, + "metric_classification_counts": { + field: dict(count) for field, count in counts.items() + }, + "agi_stub_inventory": { + "counts": dict(sorted(stubs.items())), + "guide_documented_codes": list(range(11)), + "labels_status": "needs_method_resolution", + "note": "Retain published codes. Guide section G describes 0–10; fetched CSV contains 0–9. Do not infer income-bin labels.", + }, + "uncertainty": { + "variance_status": "not_published_needs_method_resolution", + "confidence_level": None, + "note": "Guide section C describes a population; footnote 1 identifies an SOI sample input. No variance is inferred.", + }, + "disclosure": { + "numeric_zero_proves_unsuppressed": False, + "csv_cell_suppression_status": "not_identifiable_from_numeric_cells", + "published_rules": [ + "fewer than 20 returns: combine adjacent AGI cells", + "district item totals under 20 excluded", + "dominant single-return items suppressed using unpublished threshold", + ], + "xlsx_markers": "kept separately by sheet and cell; not joined to CSV AGI slices", + }, + "units": { + "monetary_amounts": "thousands_of_dollars", + "rounding_increment": None, + "rounding_status": "not_established_by_unit_label", + }, + "coverage_caveats": [ + "tax filers only", + "tax address may differ from residence", + "ZIP-based district assignment", + "US national total is not a congressional district", + "at-large state totals are explicit proxies, not directly published district observations", + ], + } + + +def crosswalk_alignment(rows: list[dict]) -> dict: + """Summarize incoming target composition, not fiscal allocation error.""" + incoming, outgoing = defaultdict(list), defaultdict(list) + seen = set() + for row in rows: + source, target = row["source_geography_id"], row["target_geography_id"] + if not re.fullmatch(r"5001700US\d{4}", source) or not re.fullmatch( + r"5001900US\d{4}", target + ): + raise ValueError("crosswalk geography vintage mismatch") + if source[9:11] not in US_STATE_FIPS or source[9:11] != target[9:11]: + raise ValueError("crosswalk outside US50+DC or crosses state") + if (source, target) in seen: + raise ValueError("duplicate crosswalk pair") + seen.add((source, target)) + population, weight = float(row["pair_population"]), float(row["weight"]) + if ( + not math.isfinite(population) + or population <= 0 + or not math.isfinite(weight) + or weight <= 0 + ): + raise ValueError( + "crosswalk population and weight must be positive and finite" + ) + incoming[target].append((source, population)) + outgoing[source].append((population, weight)) + if not seen: + raise ValueError("empty crosswalk") + for pairs in outgoing.values(): + total = math.fsum(population for population, _ in pairs) + if not math.isfinite(total) or any( + not math.isclose(weight, population / total, rel_tol=1e-9, abs_tol=1e-12) + for population, weight in pairs + ): + raise ValueError("crosswalk outgoing weights disagree with pair population") + targets = [] + for target, pairs in sorted(incoming.items()): + total = math.fsum(population for _, population in pairs) + contributions = [ + { + "source_geography_id": source, + "pair_population": population, + "incoming_share": population / total, + } + for source, population in sorted(pairs) + ] + maximum = max(item["incoming_share"] for item in contributions) + targets.append( + { + "target_geography_id": target, + "pair_population": total, + "contributions": contributions, + "max_incoming_share": maximum, + "non_dominant_incoming_share": 1 - maximum, + "known_allocation_error": None, + } + ) + return { + "source_congress": 117, + "target_congress": 119, + "population_vintage": "2020_tabulation_blocks", + "basis": "pair_population_normalized_per_target", + "interpretation": "alignment exposure, not known allocation error", + "targets": targets, + } + + +def xlsx_disclosure_inventory(payload: bytes) -> dict: + ns = {"x": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + shared = ET.fromstring(archive.read("xl/sharedStrings.xml")) + strings = ["".join(item.itertext()) for item in shared.findall("x:si", ns)] + cells = [] + for name in sorted(archive.namelist()): + if not re.fullmatch(r"xl/worksheets/sheet\d+\.xml", name): + continue + root = ET.fromstring(archive.read(name)) + for cell in root.findall(".//x:c", ns): + if cell.attrib.get("t") == "s": + value = cell.findtext("x:v", namespaces=ns) + text = strings[int(value)] if value is not None else "" + else: + text = ( + "".join(cell.find("x:is", ns).itertext()) + if cell.find("x:is", ns) is not None + else cell.findtext("x:v", default="", namespaces=ns) + ) + if "**" in text: + cells.append( + { + "sheet_part": name, + "cell": cell.attrib["r"], + "text": text, + "class": "suppression_or_combination_marker" + if text.strip() == "**" + else "disclosure_note", + } + ) + return { + "markers": cells, + "csv_cell_mapping": "not_performed", + "note": "Published workbook cells and disclosure notes remain separate from CSV AGI rows.", + } + + +def _pin_local(output: Path, payload: bytes, suffix: str, **metadata) -> dict: + digest = hashlib.sha256(payload).hexdigest() + relative = f"raw/{digest}{suffix}" + path = output / relative + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and path.read_bytes() != payload: + raise ValueError("local source digest collision") + if not path.exists(): + path.write_bytes(payload) + return {**metadata, "sha256": digest, "size_bytes": len(payload), "path": relative} + + +def build_bundle( + output: Path, + irs_authority: Path, + crosswalk: Path, + crosswalk_provenance: Path, + *, + offline: bool = False, + api_key: str | None = None, + fetch=None, +) -> dict: + """Create or replay an immutable inventory. No candidate values are accepted.""" + output = Path(output) + captures, tables = [], {} + for request in census_requests(): + descriptor = capture_request( + output, + request, + offline=offline, + api_key=api_key, + **({"fetch": fetch} if fetch else {}), + ) + captures.append(descriptor) + tables.setdefault(request["table"], {})[request["kind"]] = strict_json( + (output / descriptor["path"]).read_bytes() + ) + acs = { + table: acs_table_inventory( + table, tables[table]["metadata"], tables[table]["data"] + ) + for table in TABLES + } + expected = acs["S0101"]["geography_ids"] + if any(item["geography_ids"] != expected for item in acs.values()): + raise ValueError("ACS table geography universes differ") + index_payload = (irs_authority / "source-index.json").read_bytes() + index = strict_json(index_payload) + by_name = {item["filename"]: item for item in index} + if len(by_name) != len(index): + raise ValueError("duplicate IRS authority source identity") + local, payloads = [], {} + for name in ("22incd.csv", "22incdall.xlsx"): + descriptor = by_name[name] + if descriptor["url"] != "https://www.irs.gov/pub/irs-soi/" + name: + raise ValueError("IRS authority URL mismatch") + payload = (irs_authority / name).read_bytes() + if ( + hashlib.sha256(payload).hexdigest() != descriptor["sha256"] + or len(payload) != descriptor["size_bytes"] + ): + raise ValueError("IRS authority source digest mismatch") + payloads[name] = payload + local.append( + _pin_local( + output, payload, Path(name).suffix, url=descriptor["url"], filename=name + ) + ) + guide = (irs_authority / "22incddocguide.docx").read_bytes() + if ( + hashlib.sha256(guide).hexdigest() + != (irs_authority / "22incddocguide.sha256").read_text().split()[0] + ): + raise ValueError("IRS guide digest mismatch") + local.append( + _pin_local( + output, + guide, + ".docx", + url=AUTHORITIES["irs_guide"], + filename="22incddocguide.docx", + ) + ) + local.append(_pin_local(output, index_payload, ".json", role="irs_source_index")) + with zipfile.ZipFile(io.BytesIO(guide)) as archive: + guide_xml = archive.read("word/document.xml") + guide_text = "\n".join(ET.fromstring(guide_xml).itertext()).encode() + local.append( + _pin_local( + output, + guide_text, + ".txt", + role="irs_guide_extracted_text", + derived_from_sha256=hashlib.sha256(guide).hexdigest(), + derivation="word/document.xml text nodes in document order", + ) + ) + crosswalk_bytes, provenance_bytes = ( + crosswalk.read_bytes(), + crosswalk_provenance.read_bytes(), + ) + provenance = strict_json(provenance_bytes) + if any( + provenance.get(key) != value + for key, value in { + "source_geography_vintage": "117th_congress", + "target_geography_vintage": "119th_congress", + "block_vintage": "2020_tabulation_blocks", + }.items() + ): + raise ValueError("crosswalk provenance vintage mismatch") + if hashlib.sha256(crosswalk_bytes).hexdigest() != provenance["crosswalk_sha256"]: + raise ValueError("crosswalk provenance digest mismatch") + local.append(_pin_local(output, crosswalk_bytes, ".csv", role="crosswalk")) + local.append( + _pin_local(output, provenance_bytes, ".json", role="crosswalk_provenance") + ) + alignment = crosswalk_alignment( + list(csv.DictReader(io.StringIO(crosswalk_bytes.decode()))) + ) + irs = irs_csv_inventory(payloads["22incd.csv"]) + irs["workbook_disclosure"] = xlsx_disclosure_inventory(payloads["22incdall.xlsx"]) + published = sorted( + { + row["published_geography_id"] + for row in irs["rows"] + if row["published_geography_id"] + } + ) + proxies = sorted( + { + row["at_large_proxy_geography_id"] + for row in irs["rows"] + if row["at_large_proxy_geography_id"] + } + ) + crosswalk_sources = { + item["source_geography_id"] + for target in alignment["targets"] + for item in target["contributions"] + } + targets = {item["target_geography_id"] for item in alignment["targets"]} + bundle = { + "schema_version": 1, + "kind": "us_cd_reference_inventory", + "purpose": "reference_inventory_only", + "candidate_evaluation": "not_performed", + "target_activation": False, + "authorities": AUTHORITIES, + "scope": { + "country": "us", + "states": "50_states_and_dc", + "puerto_rico": "excluded", + "acs_year": 2024, + "target_congress": 119, + "irs_tax_year": 2022, + "irs_congress": 117, + }, + "sources": {"census": captures, "local": local}, + "acs": acs, + "irs": irs, + "alignment": alignment, + "reconciliation": { + "irs_published_district_ids": published, + "irs_at_large_state_proxy_ids": proxies, + "irs_missing_crosswalk_source_ids": sorted( + crosswalk_sources - set(published) - set(proxies) + ), + "irs_unmapped_source_ids": sorted( + (set(published) | set(proxies)) - crosswalk_sources + ), + "acs_missing_crosswalk_target_ids": sorted(targets - set(expected)), + "acs_unmapped_target_ids": sorted(set(expected) - targets), + }, + "lineage_products": { + "published": "included", + "crosswalked_values": "not_generated", + "state_rescaled_values": "not_generated", + }, + } + write_json(output / "reference-inventory.json", bundle) + return bundle + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--irs-authority-dir", type=Path, required=True) + parser.add_argument("--crosswalk", type=Path, required=True) + parser.add_argument("--crosswalk-provenance", type=Path, required=True) + parser.add_argument("--offline", action="store_true") + parser.add_argument("--census-key-service") + parser.add_argument("--census-key-account") + args = parser.parse_args(argv) + if bool(args.census_key_service) != bool(args.census_key_account): + parser.error("credential service and account must be supplied together") + try: + key = ( + read_census_key(args.census_key_service, args.census_key_account) + if args.census_key_service and not args.offline + else None + ) + bundle = build_bundle( + args.output_dir, + args.irs_authority_dir, + args.crosswalk, + args.crosswalk_provenance, + offline=args.offline, + api_key=key, + ) + except (ValueError, OSError, KeyError, zipfile.BadZipFile) as error: + print(f"Reference bundle failed: {error}", file=sys.stderr) + return 1 + print( + f"Reference inventory saved: {args.output_dir / 'reference-inventory.json'}; {len(bundle['acs']['S0101']['geography_ids'])} ACS districts; no candidate evaluation" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference_sources.py new file mode 100644 index 000000000..9a1e9b137 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference_sources.py @@ -0,0 +1,188 @@ +"""Pinned public ACS acquisition; credentials never enter stored provenance.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import HTTPRedirectHandler, Request, build_opener + +TABLES = ("S0101", "B01001", "B19001", "B25003") +MAX_RESPONSE_BYTES = 32 * 1024 * 1024 + + +def census_requests() -> list[dict]: + """The complete, fixed public reference request surface.""" + requests = [] + for table in TABLES: + dataset = "acs/acs1/subject" if table == "S0101" else "acs/acs1" + base = f"https://api.census.gov/data/2024/{dataset}" + for kind, url in ( + ("metadata", f"{base}/groups/{table}.json"), + ( + "data", + base + + "?" + + urlencode( + { + "get": f"group({table})", + "for": "congressional district:*", + "in": "state:*", + } + ), + ), + ): + requests.append( + { + "table": table, + "kind": kind, + "year": 2024, + "dataset": dataset, + "url": url, + } + ) + return requests + + +def strict_json(payload: bytes): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + def nonfinite(_): + raise ValueError("nonfinite JSON number") + + try: + return json.loads(payload, object_pairs_hook=pairs, parse_constant=nonfinite) + except (UnicodeError, json.JSONDecodeError): + raise ValueError("source is not valid UTF-8 JSON") from None + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *_args, **_kwargs): + return None + + +def fetch_census(url: str, api_key: str | None) -> bytes: + """Fetch a declared request, suppressing URL-bearing transport errors.""" + if url not in {item["url"] for item in census_requests()}: + raise ValueError("request is not declared") + request_url = url + if api_key: + request_url += ("&" if "?" in url else "?") + urlencode({"key": api_key}) + try: + with build_opener(_NoRedirect()).open( + Request(request_url, headers={"User-Agent": "Microcosm-CD-reference/1"}), + timeout=90, + ) as response: + payload = response.read(MAX_RESPONSE_BYTES + 1) + except HTTPError as error: + raise ValueError(f"Census request failed (HTTP {error.code})") from None + except (URLError, OSError, ValueError): + raise ValueError("Census request failed (transport error)") from None + if len(payload) > MAX_RESPONSE_BYTES: + raise ValueError("Census response exceeds size limit") + return payload + + +def read_census_key(service: str, account: str) -> str: + """Read only the explicitly requested agent-secret into process memory.""" + try: + result = subprocess.run( + ["agent-secret", "get", service, account], + capture_output=True, + check=False, + timeout=30, + ) + except (OSError, subprocess.SubprocessError): + raise ValueError("Census credential helper failed") from None + if result.returncode or not result.stdout.strip(): + raise ValueError("Census credential unavailable") + try: + return result.stdout.decode("utf-8").strip() + except UnicodeError: + raise ValueError("Census credential is not text") from None + + +def write_json(path: Path, value) -> None: + payload = ( + json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + b"\n" + ) + if path.exists(): + if path.read_bytes() != payload: + raise ValueError(f"refusing to replace immutable artifact: {path.name}") + return + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as stream: + stream.write(payload) + + +def capture_request( + root: Path, + request: dict, + *, + api_key: str | None = None, + fetch=fetch_census, + offline: bool = False, +) -> dict: + """Fetch once, pin exact response bytes, then verify on every replay.""" + if request not in census_requests(): + raise ValueError("request metadata is not declared") + root = Path(root) + request_key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor_path = root / "requests" / f"{request_key}.json" + if descriptor_path.exists(): + descriptor = strict_json(descriptor_path.read_bytes()) + digest = descriptor.get("sha256", "") + expected_path = f"raw/{digest}.json" + if ( + len(digest) != 64 + or any(char not in "0123456789abcdef" for char in digest) + or descriptor.get("path") != expected_path + or any(descriptor.get(key) != value for key, value in request.items()) + ): + raise ValueError("cached request descriptor mismatch") + payload = (root / expected_path).read_bytes() + if ( + hashlib.sha256(payload).hexdigest() != digest + or len(payload) != descriptor["size_bytes"] + ): + raise ValueError("cached source digest mismatch") + strict_json(payload) + return descriptor + if offline: + raise ValueError("offline source is missing") + payload = fetch(request["url"], api_key) + if api_key and api_key.encode() in payload: + raise ValueError("response contains credential; refusing to persist") + strict_json(payload) + digest = hashlib.sha256(payload).hexdigest() + relative_path = f"raw/{digest}.json" + raw_path = root / relative_path + raw_path.parent.mkdir(parents=True, exist_ok=True) + if raw_path.exists(): + if raw_path.read_bytes() != payload: + raise ValueError("raw source digest collision") + else: + with raw_path.open("xb") as stream: + stream.write(payload) + descriptor = { + **request, + "sha256": digest, + "path": relative_path, + "size_bytes": len(payload), + "retrieved_at": datetime.now(UTC).isoformat(), + } + write_json(descriptor_path, descriptor) + return descriptor diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py b/packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py new file mode 100644 index 000000000..f7671534f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py @@ -0,0 +1,234 @@ +"""Pure export comparison against a supplied common parent; no source issuer. + +The host must authenticate the actual enriched Population and calibration +ancestry, including targets, ordered weights and geographic scope. A supplied +reference, specification or returned digest cannot establish that ancestry. +The host must also seal those inputs across writer/readback I/O and finish its +owner checks before accepting an export. This module performs no I/O or solve. +""" + +from __future__ import annotations + +import hashlib + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, MassChange, WeightKind, Weights +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import storage_equal + +PROTOCOL = "microcosm.us.supplied-parent-export-comparison.v1" +MAX_SPECIFICATION_BYTES = 1024 * 1024 + + +class RetainedFrameExportError(ValueError): + """An export differs from its supplied parent or declared weight/scope inputs.""" + + +def _require(condition, code): + if not condition: + raise RetainedFrameExportError(code) + + +def _vector(value, dtype, code): + _require( + type(value) is np.ndarray and value.ndim == 1 and value.dtype == dtype, + code, + ) + return value + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _array_binding(value): + _require(value.dtype.kind in ("i", "u", "f"), "NUMERICAL_BINDING_DTYPE") + return { + "dtype": value.dtype.str, + "rows": len(value), + "sha256": _sha(np.ascontiguousarray(value).tobytes()), + } + + +def _same_series(expected, actual, *, readback): + """Allow only masked-cell codec canonicalization on logical readback.""" + if expected.dtype != actual.dtype or len(expected) != len(actual): + return False + expected_null = expected.isna().to_numpy(dtype=np.bool_) + actual_null = actual.isna().to_numpy(dtype=np.bool_) + if not np.array_equal(expected_null, actual_null): + return False + if readback and expected.dtype == object: + # Checkpoint supports object-string strata. Raw object pointers cannot + # identify values across a serialization boundary. Arbitrary object + # equality is outside this small numerical/string contract. + left = expected.to_numpy()[~expected_null].tolist() + right = actual.to_numpy()[~actual_null].tolist() + return all(type(v) is str for v in (*left, *right)) and left == right + return storage_equal(expected, actual, ~expected_null if readback else None) + + +def verify_retained_frame_export( + parent: Frame, + candidate: Frame, + *, + parent_reference: str, + ordered_household_ids: np.ndarray, + calibrated_weights: np.ndarray, + calibration_specification: bytes, + scope_household_ids: np.ndarray | None = None, + prune_zero_weight: bool = True, + comparison: str = "storage", + expected_binding: bytes | None = None, +) -> bytes: + """Compare complete retained inputs and bind supplied numerical export inputs. + + IDs retain the parent's exact signed/unsigned integer dtype; weights are + exact float64 arrays in the *whole parent* + household order. Scope IDs must be a unique subset in that same order; + None explicitly denotes all households. Pruning removes exactly zero + weights, with no tolerance or default origin-quality requirement. Household + selection carries all its persons and precisely their referenced groups. + + Call before writing, then on the serializer's logical readback with + comparison="frame-checkpoint-readback" and the prior expected_binding. + That closed mode allows only existing + codec normalization under null masks; known values, dtype and masks remain + exact. Extra or missing columns and record reordering refuse. Pandas row + indices may be reset: entity IDs, not incidental indices, define alignment. + + No Population ownership, design anchor, mass ledger, target interpretation, + geographic predicate meaning or source authenticity is established here. + The host must bind parent_reference to actual graph ancestry and verify the + specification/scope against that ancestry; opaque bytes are not authority. + Empty support, scopes with no positive weight, and experimental separate + link tables refuse explicitly, matching the Frame weight contract. + """ + _require(isinstance(parent, Frame) and isinstance(candidate, Frame), "FRAME_TYPE") + _require( + type(parent_reference) is str and bool(parent_reference), "PARENT_REFERENCE" + ) + _require( + type(calibration_specification) is bytes + and 0 < len(calibration_specification) <= MAX_SPECIFICATION_BYTES, + "CALIBRATION_SPECIFICATION", + ) + _require(type(prune_zero_weight) is bool, "OPTIONS") + _require( + type(comparison) is str + and comparison in ("storage", "frame-checkpoint-readback"), + "COMPARISON_MODE", + ) + readback = comparison == "frame-checkpoint-readback" + _require( + expected_binding is None or type(expected_binding) is bytes, "BINDING_TYPE" + ) + _require(parent.schema == candidate.schema, "SCHEMA") + _require( + not parent.schema.links and parent.links == candidate.links == (), + "SEPARATE_LINK_TABLES_UNSUPPORTED", + ) + _require("household" in parent.schema.group_entities, "HOUSEHOLD_SCHEMA") + _require( + "household" in parent.weighted_entities + and parent.weighted_entities == candidate.weighted_entities, + "WEIGHT_TOPOLOGY", + ) + _require(parent.metadata == candidate.metadata, "FRAME_METADATA") + household_id = parent.schema.entity_id_column("household") + parent_ids = parent.table("household")[household_id].to_numpy() + _require(parent_ids.dtype.kind in ("i", "u"), "INTEGER_HOUSEHOLD_IDS") + ids = _vector(ordered_household_ids, parent_ids.dtype, "HOUSEHOLD_ID_VECTOR") + weights = _vector(calibrated_weights, np.dtype("float64"), "WEIGHT_VECTOR") + _require( + parent_ids.dtype == ids.dtype + and np.array_equal(parent_ids, ids) + and len(np.unique(ids)) == len(ids), + "COMPLETE_ORDERED_HOUSEHOLD_IDS", + ) + _require( + weights.shape == ids.shape + and np.isfinite(weights).all() + and (weights >= 0).all(), + "FINITE_ALIGNED_WEIGHTS", + ) + if scope_household_ids is None: + scope = ids + scope_mask = np.ones(len(ids), dtype=np.bool_) + else: + scope = _vector(scope_household_ids, ids.dtype, "SCOPE_ID_VECTOR") + scope_mask = np.isin(ids, scope) + _require(np.array_equal(ids[scope_mask], scope), "ORDERED_SCOPE_SUBSET") + keep = scope_mask & (weights > 0) if prune_zero_weight else scope_mask + _require(keep.any() and (weights[keep] > 0).any(), "EMPTY_EXPORT_SUPPORT") + membership = parent.schema.membership_column("household") + comparison_parent = parent.with_weights( + "household", + Weights(weights, WeightKind.CALIBRATED), + mass=MassChange( + factor=None, + reason="Temporary supplied-weight view for retained export comparison", + ), + ) + selected = comparison_parent.select( + np.isin(parent.person[membership].to_numpy(), ids[keep]) + ) + _require(selected.entities == candidate.entities, "ENTITY_ROSTER") + entity_ids = {} + for entity in selected.entities: + expected, actual = selected.table(entity), candidate.table(entity) + _require( + not actual.columns.has_duplicates + and tuple(expected.columns) == tuple(actual.columns), + "COLUMN_ROSTER:" + entity, + ) + id_column = parent.schema.entity_id_column(entity) + _require( + _same_series(expected[id_column], actual[id_column], readback=False), + "RETAINED_ENTITY_IDS:" + entity, + ) + for name in expected.columns: + _require( + _same_series(expected[name], actual[name], readback=readback), + "RETAINED_INPUT:" + entity + "." + str(name), + ) + entity_ids[entity] = _array_binding(expected[id_column].to_numpy()) + _require( + selected.strata.name == candidate.strata.name + and _same_series(selected.strata, candidate.strata, readback=readback), + "RETAINED_STRATA", + ) + for entity in selected.weighted_entities: + actual = candidate.weights_for(entity) + if entity == "household": + expected_values, expected_kind = weights[keep], WeightKind.CALIBRATED + else: + expected_weight = selected.weights_for(entity) + expected_values, expected_kind = ( + expected_weight.values, + expected_weight.kind, + ) + _require( + actual.kind is expected_kind + and storage_equal(pd.Series(expected_values), pd.Series(actual.values)), + "RETAINED_WEIGHTS:" + entity, + ) + binding = canonical_json( + { + "protocol": PROTOCOL, + "scope": "comparison_against_supplied_parent_only", + "parent_reference": parent_reference, + "actual_graph_calibration_ancestry_verified": False, + "release_eligible": False, + "calibration_specification_sha256": _sha(calibration_specification), + "complete_ordered_household_ids": _array_binding(ids), + "complete_calibrated_weights": _array_binding(weights), + "scope_household_ids": _array_binding(scope), + "prune_zero_weight": prune_zero_weight, + "retained_entity_ids": entity_ids, + } + ) + _require(expected_binding is None or binding == expected_binding, "EXPORT_BINDING") + return binding diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py index 3182ef056..e1b5f7210 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py @@ -7,6 +7,7 @@ import json from collections.abc import Iterable, Mapping from importlib.resources import files +from io import BytesIO from pathlib import Path from typing import Any @@ -96,6 +97,21 @@ def default_congressional_district_vintage_crosswalk_path() -> Path: ) +def decode_congressional_district_vintage_crosswalk(payload: bytes) -> pd.DataFrame: + """Decode immutable CSV artifact bytes under the existing crosswalk contract. + + The caller separately declares whether national roster validation is + required. Generic crosswalks retain the same behavior as the file loader. + """ + if not isinstance(payload, bytes): + raise TypeError("CD vintage crosswalk payload must be immutable bytes.") + return _prepare_crosswalk( + pd.read_csv(BytesIO(payload)), + source_prefix=SOURCE_CONGRESSIONAL_DISTRICT_PREFIX, + target_prefix=CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, + ) + + def load_default_congressional_district_vintage_crosswalk() -> pd.DataFrame: """Load the packaged Census-built default 117th->119th CD crosswalk. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py index 82c172e56..12ff2315d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py @@ -9,6 +9,7 @@ from __future__ import annotations from collections.abc import Mapping +from dataclasses import dataclass import numpy as np import pandas as pd @@ -21,6 +22,9 @@ PAW_TYPE_TANF_CODES, fill_asec_public_assistance_type_source, ) +from microcosm.build.us_runtime.reported_coverage_source import ( + fill_asec_reported_coverage_source, +) from microcosm.frame import US_SCHEMA, Frame __all__ = [ @@ -35,6 +39,8 @@ "US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS", "WIC_CARRIER_ADJUDICATION_URL", "derive_us_cps_carried_inputs", + "derive_us_cps_carried_tables", + "CpsCarriedTables", "reported_snap_receipt_by_spm_unit", "reported_tanf_enrollment_by_spm_unit", "reported_wic_receipt_carrier", @@ -132,17 +138,75 @@ } ) +# Actual person reads for the no-sidecar measured mapping. Optional restoration +# remains a separate explicit argument on the legacy/table entrypoints. +CPS_CARRIED_RAW_PERSON_COLUMNS = frozenset( + { + "A_AGE", + "A_SEX", + "WICYN", + "WSAL_VAL", + "SEMP_VAL", + "INT_VAL", + "DIV_VAL", + "CAP_VAL", + "SS_VAL", + "RESNSS1", + "RESNSS2", + "PNSN_VAL", + "ANN_VAL", + "OI_VAL", + "OI_OFF", + "RNT_VAL", + "FRSE_VAL", + "UC_VAL", + "PHIP_VAL", + "PMED_VAL", + "POTC_VAL", + "NOW_MRK", + "NOW_NONM", + "NOW_MCAID", + "NOW_GRP", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "PAW_VAL", + "PAW_TYP", + "SPM_SNAPSUB", + "SPM_CHILDCAREXPNS", + *(f"DST_SC{suffix}" for suffix in ("1", "2", "1_YNG", "2_YNG")), + *(f"DST_VAL{suffix}" for suffix in ("1", "2", "1_YNG", "2_YNG")), + } +) +CPS_CARRIED_BOOLEAN_INPUTS = frozenset( + { + "is_female", + "receives_wic", + "receives_tanf", + "receives_snap", + *US_REPORTED_COVERAGE_PERSON_INPUTS, + } +) + def derive_us_cps_carried_inputs( frame: Frame, *, public_assistance_type_source: pd.DataFrame | None = None, + reported_coverage_source: pd.DataFrame | None = None, ) -> Frame: """Carry raw CPS ASEC values onto PE input leaves. - Existing leaf input columns are preserved, making the transform idempotent. - The transform refuses to run on a non-US frame and never creates - formula-owned aggregate variables. + Existing leaves keep the original presence rules; the raw other-income + pair intentionally recomputes its three split leaves. The transform + refuses a non-US frame and never creates formula-owned aggregates. + + Optional ``reported_coverage_source`` restores seven measured at-interview + recodes before deriving coverage leaves. It requires an exact source join; + conflicting observations fail. Native ``NOW_GRP`` and ``NOW_MRK`` remain + unchanged. These observed inputs do not certify a fitted model or release. ``PAW_VAL``, ``SPM_SNAPSUB``, and ``WICYN`` are annual reported facts, while the engine's ``receives_tanf``, ``receives_snap``, and @@ -164,7 +228,49 @@ def derive_us_cps_carried_inputs( if frame.schema != US_SCHEMA: raise ValueError("CPS-carried derivations require the US schema.") tables = {entity: frame.table(entity).copy() for entity in frame.entities} - person = tables["person"] + produced = derive_us_cps_carried_tables( + tables["person"], + tables["spm_unit"], + public_assistance_type_source=public_assistance_type_source, + reported_coverage_source=reported_coverage_source, + ) + tables["person"] = produced.person + tables["spm_unit"] = produced.spm_unit + + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +@dataclass(frozen=True) +class CpsCarriedTables: + """The two actual entity tables touched by the CPS-carried operator.""" + + person: pd.DataFrame + spm_unit: pd.DataFrame + + +def derive_us_cps_carried_tables( + person: pd.DataFrame, + spm_unit: pd.DataFrame, + *, + public_assistance_type_source: pd.DataFrame | None = None, + reported_coverage_source: pd.DataFrame | None = None, +) -> CpsCarriedTables: + """Run the original measured mapping on isolated person and SPM views. + + Group outputs align by actual SPM IDs. Monetary amounts retain their + existing nominal source-year basis; this operation does no price alignment. + """ + person = person.copy(deep=True) + spm_unit = spm_unit.copy(deep=True) + if reported_coverage_source is not None: + person = fill_asec_reported_coverage_source(person, reported_coverage_source) _fill_missing(person, "age", _source(person, "A_AGE")) _fill_bool_missing(person, "is_female", _integer_source(person, "A_SEX") == 2) @@ -222,7 +328,6 @@ def derive_us_cps_carried_inputs( _fill_missing(person, "taxable_ira_distributions", _ira_distributions(person)) person = derive_us_alimony_from_asec(person) - tables["person"] = person direct_sources: Mapping[str, str] = { "rental_income": "RNT_VAL", @@ -241,10 +346,10 @@ def derive_us_cps_carried_inputs( _fill_health_coverage_inputs(person) _fill_spm_unit_reported_enrollment_inputs( person, - tables["spm_unit"], + spm_unit, public_assistance_type_source=public_assistance_type_source, ) - _fill_spm_unit_childcare_inputs(person, tables["spm_unit"]) + _fill_spm_unit_childcare_inputs(person, spm_unit) formula_owned = sorted(CPS_CARRIED_FORMULA_OWNED_COLUMNS.intersection(person)) if formula_owned: @@ -253,14 +358,7 @@ def derive_us_cps_carried_inputs( f"columns: {formula_owned}." ) - return Frame( - tables, - frame.schema, - {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, - frame.strata, - mass_log=frame.mass_log, - metadata=frame.metadata, - ) + return CpsCarriedTables(person, spm_unit) def reported_tanf_enrollment_by_spm_unit( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py new file mode 100644 index 000000000..7270593ea --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py @@ -0,0 +1,377 @@ +"""Corrected CPS monetary leaves derived from a typed selected-money subset. + +Amounts come only from the selected current-money artifact; routing codes come +only from declared raw code columns. A raw dollar column is never an alternate +input here: the derivation refuses one outright, so a corrected leaf cannot +silently fall back to the nominal source value. + +The split fractions, the RESNSS reason routing with its age-62 fallback, the +OI_OFF 20/12 partition, the DST code-4 IRA rule and the SPM childcare grain are +the legacy modelling choices, reused unchanged and imported from +``cps_carried``/``alimony`` rather than redeclared. Only the amounts they +consume change: nominal source dollars become the 2024-basis restated amounts +the money recipe already produced. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from .alimony import US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS, derive_us_alimony_from_asec +from .asec_current_money import FIELDS +from .asec_current_money_selection import SelectedCurrentMoney +from .cps_carried import ( + LONG_TERM_CAPITAL_GAIN_FRACTION, + QUALIFIED_DIVIDEND_FRACTION, + TAXABLE_INTEREST_FRACTION, + TAXABLE_PENSION_FRACTION, +) + +CPS_CARRIED_CURRENT_ROUTING_COLUMNS: tuple[str, ...] = ( + "A_AGE", + "DST_SC1", + "DST_SC1_YNG", + "DST_SC2", + "DST_SC2_YNG", + "OI_OFF", + "RESNSS1", + "RESNSS2", +) +CPS_CARRIED_CURRENT_MONEY_FIELDS: tuple[str, ...] = ( + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PHIP_VAL", + "PMED_VAL", + "PNSN_VAL", + "POTC_VAL", + "RNT_VAL", + "SEMP_VAL", + "SPM_CHILDCAREXPNS", + "SS_VAL", + "UC_VAL", + "WSAL_VAL", +) +CPS_CARRIED_CURRENT_PERSON_LEAVES: tuple[str, ...] = ( + "age", + "alimony_income", + "employment_income_before_lsr", + "farm_operations_income", + "health_insurance_premiums_without_medicare_part_b", + "long_term_capital_gains_before_response", + "miscellaneous_income", + "non_qualified_dividend_income", + "other_medical_expenses", + "over_the_counter_health_expenses", + "qualified_dividend_income", + "rental_income", + "self_employment_income_before_lsr", + "short_term_capital_gains", + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + "strike_benefits", + "tax_exempt_private_pension_income", + "taxable_interest_income", + "taxable_ira_distributions", + "taxable_private_pension_income", + "unemployment_compensation", +) +CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES: tuple[str, ...] = ( + "spm_unit_pre_subsidy_childcare_expenses", +) +CPS_CARRIED_CURRENT_CONTRACT_SCHEMA = "microcosm.us.cps-carried-current-leaves.v1" +_IRA_DISTRIBUTION_CODE = 4 +_SOCIAL_SECURITY_RETIREMENT_AGE_FALLBACK = 62 + + +class CpsCarriedCurrentRefusalError(ValueError): + """Sanitized refusal; never carries row identifiers or amounts.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise CpsCarriedCurrentRefusalError(reason) + + +def _routing(routing, rows: int) -> dict[str, np.ndarray]: + _require(type(routing) is pd.DataFrame, "ROUTING_TABLE") + _require(len(routing) == rows, "ROUTING_ROW_ALIGNMENT") + columns = tuple(routing.columns) + _require(len(set(columns)) == len(columns), "ROUTING_ROSTER") + # A source dollar column can never reach this derivation, even by accident. + _require( + not any(name in FIELDS or name.endswith("_VAL") for name in columns), + "RAW_MONEY_INPUT_REFUSED", + ) + _require( + tuple(sorted(columns)) == CPS_CARRIED_CURRENT_ROUTING_COLUMNS, "ROUTING_ROSTER" + ) + values = {} + for name in CPS_CARRIED_CURRENT_ROUTING_COLUMNS: + series = routing[name] + _require(pd.api.types.is_integer_dtype(series.dtype), "ROUTING_DTYPE") + _require(not bool(series.isna().any()), "ROUTING_MISSING") + values[name] = series.to_numpy(dtype="int64", copy=True) + return values + + +def _amounts(selected: SelectedCurrentMoney, rows: int) -> dict[str, np.ndarray]: + values = {} + for name in CPS_CARRIED_CURRENT_MONEY_FIELDS: + _require(selected.entity_of(name) == "person", "MONEY_FIELD_ENTITY") + amounts = selected.amounts(name) + _require(len(amounts) == rows, "MONEY_ROW_ALIGNMENT") + _require(bool(np.isfinite(amounts).all()), "MONEY_NONFINITE") + values[name] = amounts + return values + + +def _social_security(amount: np.ndarray, routing) -> dict[str, np.ndarray]: + """Reuse the legacy RESNSS reason routing and its age-62 fallback exactly.""" + reason_1 = routing["RESNSS1"] + reason_2 = routing["RESNSS2"] + age = routing["A_AGE"] + is_retirement = (reason_1 == 1) | (reason_2 == 1) + is_disability = (reason_1 == 2) | (reason_2 == 2) + is_survivor = np.isin(reason_1, [3, 5]) | np.isin(reason_2, [3, 5]) + is_dependent = np.isin(reason_1, [4, 6, 7]) | np.isin(reason_2, [4, 6, 7]) + unclassified = ( + (amount > 0) & ~is_retirement & ~is_disability & ~is_survivor & ~is_dependent + ) + threshold = _SOCIAL_SECURITY_RETIREMENT_AGE_FALLBACK + return { + "social_security_retirement": np.where( + is_retirement | (unclassified & (age >= threshold)), amount, 0.0 + ), + "social_security_disability": np.where( + (is_disability & ~is_retirement) | (unclassified & (age < threshold)), + amount, + 0.0, + ), + "social_security_survivors": np.where( + is_survivor & ~is_retirement & ~is_disability, amount, 0.0 + ), + "social_security_dependents": np.where( + is_dependent & ~is_retirement & ~is_disability & ~is_survivor, amount, 0.0 + ), + } + + +def _ira_distributions(amounts, routing, rows: int) -> np.ndarray: + values = np.zeros(rows, dtype=np.float64) + for suffix in ("1", "2", "1_YNG", "2_YNG"): + code = routing[f"DST_SC{suffix}"] + values += np.where( + code == _IRA_DISTRIBUTION_CODE, amounts[f"DST_VAL{suffix}"], 0.0 + ) + return values + + +def _other_income(amount: np.ndarray, routing) -> dict[str, np.ndarray]: + """Delegate the OI_OFF 20/12 partition to the registered legacy helper.""" + table = pd.DataFrame({"OI_VAL": amount, "OI_OFF": routing["OI_OFF"]}) + split = derive_us_alimony_from_asec(table) + return { + name: split[name].to_numpy(dtype="float64", copy=True) + for name in US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS + } + + +def _childcare(amount: np.ndarray, membership: np.ndarray, ids: np.ndarray): + """Carry the SPM-unit childcare value; members must already agree exactly.""" + _require( + type(membership) is np.ndarray + and membership.dtype == np.dtype("int64") + and membership.shape == amount.shape, + "SPM_MEMBERSHIP", + ) + _require( + type(ids) is np.ndarray and ids.dtype == np.dtype("int64") and ids.ndim == 1, + "SPM_IDS", + ) + _require(len(ids) == len(set(ids.tolist())), "SPM_IDS") + _require(np.array_equal(np.sort(ids), ids), "SPM_IDS") + _require(np.array_equal(np.unique(membership), ids), "SPM_MEMBERSHIP_COVERAGE") + positions = np.searchsorted(ids, membership) + values = np.zeros(len(ids), dtype=np.float64) + seen = np.zeros(len(ids), dtype=bool) + for position, value in zip(positions.tolist(), amount.tolist(), strict=True): + if seen[position]: + # The source repeats one SPM value on every member; a disagreement + # is a source contract failure, never something to reduce away. + _require(values[position] == value, "INCONSISTENT_SPM_AMOUNT") + continue + values[position] = value + seen[position] = True + return values + + +@dataclass(frozen=True) +class CpsCarriedCurrentLeaves: + """Corrected person and SPM-unit leaves, aligned to the selected rows.""" + + person: Mapping[str, np.ndarray] + spm_unit: Mapping[str, np.ndarray] + + +CPS_CURRENT_PREDICTOR_MONEY_FIELDS = ( + "WSAL_VAL", + "SEMP_VAL", + "INT_VAL", + "DIV_VAL", + "CAP_VAL", +) +CPS_CURRENT_PREDICTOR_PERSON_LEAVES = ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", +) + + +def derive_cps_current_predictor_leaves(amounts): + """Pure five-field split, sharing the maintained current-money judgments. + + Callers qualify the actual money owner or modeled target artifacts. This + numerical function grants no source authority and never fills an unknown. + Inputs are aligned physical float64 arrays, already in the declared dollar + basis. INT/DIV/CAP fitted for ACS remain modeled, including their splits. + """ + _require(isinstance(amounts, Mapping), "PREDICTOR_MONEY_MAPPING") + _require( + set(amounts) == set(CPS_CURRENT_PREDICTOR_MONEY_FIELDS), + "PREDICTOR_MONEY_ROSTER", + ) + lengths = set() + for value in amounts.values(): + _require( + type(value) is np.ndarray + and value.ndim == 1 + and value.dtype == np.dtype("float64"), + "PREDICTOR_MONEY_TYPE", + ) + _require(bool(np.isfinite(value).all()), "PREDICTOR_MONEY_UNKNOWN") + lengths.add(len(value)) + _require(len(lengths) == 1 and next(iter(lengths)) > 0, "PREDICTOR_MONEY_AXIS") + dividends, gains = amounts["DIV_VAL"], amounts["CAP_VAL"] + return { + "employment_income_before_lsr": amounts["WSAL_VAL"].copy(), + "self_employment_income_before_lsr": amounts["SEMP_VAL"].copy(), + "taxable_interest_income": amounts["INT_VAL"] * TAXABLE_INTEREST_FRACTION, + "qualified_dividend_income": dividends * QUALIFIED_DIVIDEND_FRACTION, + "non_qualified_dividend_income": dividends * (1 - QUALIFIED_DIVIDEND_FRACTION), + "long_term_capital_gains_before_response": gains + * LONG_TERM_CAPITAL_GAIN_FRACTION, + "short_term_capital_gains": gains * (1 - LONG_TERM_CAPITAL_GAIN_FRACTION), + } + + +def derive_cps_carried_current_leaves( + selected: SelectedCurrentMoney, + *, + routing: pd.DataFrame, + spm_membership: np.ndarray, + spm_ids: np.ndarray, +) -> CpsCarriedCurrentLeaves: + """Derive the corrected monetary CPS leaves plus ``age`` from selected money.""" + _require(type(selected) is SelectedCurrentMoney, "TYPED_SELECTED_REQUIRED") + rows = selected.person_rows + codes = _routing(routing, rows) + amounts = _amounts(selected, rows) + # The money recipe already contributes zero for a declared-NIU annuity, so + # the pension base is the plain sum of the two restated amounts. + pensions = amounts["PNSN_VAL"] + amounts["ANN_VAL"] + person: dict[str, np.ndarray] = { + "age": codes["A_AGE"].astype("float64"), + **derive_cps_current_predictor_leaves( + {name: amounts[name] for name in CPS_CURRENT_PREDICTOR_MONEY_FIELDS} + ), + "taxable_private_pension_income": pensions * TAXABLE_PENSION_FRACTION, + "tax_exempt_private_pension_income": pensions * (1 - TAXABLE_PENSION_FRACTION), + "taxable_ira_distributions": _ira_distributions(amounts, codes, rows), + "rental_income": amounts["RNT_VAL"], + "farm_operations_income": amounts["FRSE_VAL"], + "unemployment_compensation": amounts["UC_VAL"], + "health_insurance_premiums_without_medicare_part_b": amounts["PHIP_VAL"], + "other_medical_expenses": amounts["PMED_VAL"], + "over_the_counter_health_expenses": amounts["POTC_VAL"], + } + person.update(_social_security(amounts["SS_VAL"], codes)) + person.update(_other_income(amounts["OI_VAL"], codes)) + spm_unit = { + "spm_unit_pre_subsidy_childcare_expenses": _childcare( + amounts["SPM_CHILDCAREXPNS"], spm_membership, spm_ids + ) + } + _require( + tuple(sorted(person)) == CPS_CARRIED_CURRENT_PERSON_LEAVES, "PERSON_LEAF_ROSTER" + ) + _require( + tuple(sorted(spm_unit)) == CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES, + "SPM_UNIT_LEAF_ROSTER", + ) + for values in (*person.values(), *spm_unit.values()): + _require(values.dtype == np.dtype("float64"), "LEAF_DTYPE") + _require(bool(np.isfinite(values).all()), "LEAF_NONFINITE") + for name, values in person.items(): + _require(len(values) == rows, "LEAF_ROW_ALIGNMENT") + del name + return CpsCarriedCurrentLeaves(person, spm_unit) + + +def cps_carried_current_leaf_contract() -> dict: + """Return the recorded derivation contract for these corrected leaves.""" + return { + "schema": CPS_CARRIED_CURRENT_CONTRACT_SCHEMA, + "amount_source": "us_asec_selected_current_money", + "amount_basis": "target_current_2024", + "raw_dollar_alternates": [], + "money_fields": list(CPS_CARRIED_CURRENT_MONEY_FIELDS), + "routing_columns": list(CPS_CARRIED_CURRENT_ROUTING_COLUMNS), + "person_leaves": list(CPS_CARRIED_CURRENT_PERSON_LEAVES), + "spm_unit_leaves": list(CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + "dtype": "float64", + "model_assumptions": { + "taxable_interest_fraction": TAXABLE_INTEREST_FRACTION, + "qualified_dividend_fraction": QUALIFIED_DIVIDEND_FRACTION, + "taxable_pension_fraction": TAXABLE_PENSION_FRACTION, + "long_term_capital_gain_fraction": LONG_TERM_CAPITAL_GAIN_FRACTION, + "annuity_niu_contributes_zero": True, + "signed_losses_retained": ["SEMP_VAL", "RNT_VAL", "FRSE_VAL"], + "social_security_reason_routing": "RESNSS1/RESNSS2 with age-62 fallback", + "social_security_age_fallback": _SOCIAL_SECURITY_RETIREMENT_AGE_FALLBACK, + "other_income_split": "derive_us_alimony_from_asec on OI_VAL/OI_OFF", + "ira_distribution_code": _IRA_DISTRIBUTION_CODE, + "spm_childcare_grain": "spm_unit_repeated_on_person, member-consistent", + }, + "release_eligible": False, + } + + +__all__ = [ + "CPS_CARRIED_CURRENT_CONTRACT_SCHEMA", + "CPS_CARRIED_CURRENT_MONEY_FIELDS", + "CPS_CARRIED_CURRENT_PERSON_LEAVES", + "CPS_CARRIED_CURRENT_ROUTING_COLUMNS", + "CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES", + "CpsCarriedCurrentLeaves", + "CpsCarriedCurrentRefusalError", + "cps_carried_current_leaf_contract", + "derive_cps_carried_current_leaves", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py new file mode 100644 index 000000000..b206f709c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py @@ -0,0 +1,441 @@ +"""Observed ACS income anchors from the retained original source archive. + +Values and projections are descriptive, not source issuers. Consumers retain the +original preparation and requalify after their last relevant I/O. This module +does not decompose income, create tax inputs, or fill any analytical zeros. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import acs_housing_universe_source as housing +from . import acs_person_coverage_authentication as records +from . import graph_full_puf_enrichment as physical +from . import source_csv_builtin +from . import support_provenance as provenance +from . import survey_population_preparation as preparation + +PROTOCOL = "microcosm.us.current-acs-income-anchors.v1" +COLUMNS = ("SERIALNO", "SPORDER", "INTP", "RETP", "ADJINC", "AGEP", "FINTP", "FRETP") +ANCHORS = ( + ("INTP", "FINTP", "property_income", "acs_interest_dividend_rental_income"), + ("RETP", "FRETP", "retirement_income", "acs_retirement_income"), +) +STRING_DTYPE = pd.StringDtype(storage="python", na_value=pd.NA) + + +def require(condition, reason): + if not condition: + raise ValueError("ACS_INCOME_ANCHOR_" + reason) + + +def _literal(value): + require(type(value) is str and len(value) <= 64, "LITERAL_TYPE_OR_BOUND") + return value + + +def parse_anchor( + token: str, *, age: str, adjustment: str, allocation: str, field: str +) -> dict: + """Classify source literals separately from native numeric storage. + + Allocation is provenance, not amount validity. Published domains describe + released records; their disclosure bounds are not latent income bounds. + """ + require(field in ("INTP", "RETP"), "FIELD") + token, age, adjustment, allocation = map( + _literal, (token, age, adjustment, allocation) + ) + require(re.fullmatch(r"[0-9]{1,2}", age, re.ASCII) is not None, "AGE_LITERAL") + parsed_age = int(age) + factor_valid = ( + re.fullmatch(r"[0-9]{1,7}", adjustment, re.ASCII) is not None + and int(adjustment) > 0 + ) + status, amount = "observed", None + if token == "": + status = ( + "outside_universe_blank" if parsed_age < 15 else "missing_source_amount" + ) + elif re.fullmatch(r"-?[0-9]+", token, re.ASCII) is None: + status = "malformed_source_amount" + else: + raw = int(token) + in_domain = ( + raw == 0 or 4 <= raw <= 999999 or (field == "INTP" and -10000 <= raw <= -4) + ) + if not in_domain: + status = "outside_published_domain" + elif parsed_age < 15: + status = "outside_universe_observation" + elif not factor_valid: + status = "invalid_adjustment" + else: + amount = float( + np.float64(raw) * (np.float64(int(adjustment)) / 1_000_000.0) + ) + return { + "amount": amount, + "known": status == "observed", + "status": status, + "allocation_status": { + "0": "not_allocated", + "1": "allocated", + "": "allocation_missing", + }.get(allocation, "allocation_unrecognized"), + "adjustment_known": factor_valid, + "in_income_universe": parsed_age >= 15, + } + + +def _key(row): + require( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", row["SERIALNO"], re.ASCII) is not None, + "SOURCE_HOUSEHOLD_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,2}", row["SPORDER"], re.ASCII) is not None + and 1 <= int(row["SPORDER"]) <= 20, + "SOURCE_PERSON_KEY", + ) + return row["SERIALNO"], int(row["SPORDER"]) + + +def _scan(stream, wanted, selected, *, maximum): + require(source_csv_builtin.capture_csv_reader(csv) is not None, "CSV_BINDING") + header, count = None, 0 + for raw in records._records(stream): + values = records._decode_record(raw, first=header is None) + if header is None: + header = values + require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(COLUMNS) <= set(header), + "SOURCE_HEADER", + ) + positions = [header.index(c) for c in COLUMNS] + continue + count += 1 + require(count <= maximum and len(values) == len(header), "SOURCE_ROW_SHAPE") + row = dict(zip(COLUMNS, (values[p] for p in positions), strict=True)) + require(all(len(v) <= 64 for v in row.values()), "SOURCE_TOKEN_BOUND") + key = _key(row) + if key in wanted: + require(key not in selected, "DUPLICATE_SELECTED_SOURCE_KEY") + selected[key] = row + require(header is not None, "SOURCE_HEADER") + return count + + +def _origins(state, document): + payload = document["origins"]["persons"] + full = pd.DataFrame(payload["rows"], columns=payload["columns"]) + people = state.frame.person + require( + full.person_id.dtype == np.dtype("int64") + and full.person_id.is_unique + and np.array_equal(full.person_id.to_numpy(), people.person_id.to_numpy()), + "ORIGIN_AXIS", + ) + channel = provenance.support_channel_column("person") + native_id = provenance.spine_source_id_column("person") + require( + np.array_equal(full.source.to_numpy(), people[channel].to_numpy()) + and np.array_equal( + full.selected_receiving_person_id.to_numpy(), people[native_id].to_numpy() + ), + "ORIGIN_FRAME_IDENTITY", + ) + origins = full.loc[full.source.eq("acs")].copy().set_index("person_id", drop=True) + require( + len(origins) > 0 + and origins.source_year.eq(2024).all() + and origins.survey_year.eq(2024).all(), + "ORIGIN_PERIOD", + ) + native = state.source_frames[0].person + require( + np.array_equal( + origins.selected_receiving_person_id.to_numpy(), native.person_id.to_numpy() + ), + "ORIGIN_NATIVE_AXIS", + ) + keys = [] + for _, row in origins.iterrows(): + require( + all( + type(v) is str + for v in ( + row.raw_native_household_id, + row.raw_native_person_id, + row.native_line_numeric_original, + ) + ), + "ORIGIN_LITERAL_TYPE", + ) + key = _key( + { + "SERIALNO": row.raw_native_household_id, + "SPORDER": row.native_line_numeric_original, + } + ) + require(row.raw_native_person_id == str(key[1]), "ORIGIN_NATIVE_PERSON_KEY") + keys.append(key) + require(len(set(keys)) == len(keys), "ORIGIN_DUPLICATE") + origins["anchor_source_key"] = keys + return origins + + +def _raw_table(origins, selected): + require(set(selected) == set(origins.anchor_source_key), "SELECTED_SOURCE_ROSTER") + raw = pd.DataFrame( + [selected[k] for k in origins.anchor_source_key], + index=origins.index.copy(), + columns=COLUMNS, + dtype=object, + ) + require( + all( + _key(row) == key + for row, key in zip( + raw.to_dict("records"), origins.anchor_source_key, strict=True + ) + ), + "SOURCE_COORDINATE_CHANGED", + ) + return raw + + +def _native_numbers(tokens): + # Match the existing mapper's numeric-coercion semantics for identity only; + # this operation does not determine source amount knownness. + return pd.to_numeric(pd.Series(tokens, dtype=object), errors="coerce").to_numpy( + dtype=np.float64, na_value=np.nan + ) + + +def _same_number(left, right): + if pd.isna(left): + return bool(pd.isna(right)) + if isinstance(left, (bool, np.bool_)) or not isinstance( + left, (int, float, np.integer, np.floating) + ): + return False + return np.float64(left).view("uint64") == np.float64(right).view("uint64") + + +def _compare_retained(raw, origins, native): + people = native.person + require( + raw.index.equals(origins.index) + and np.array_equal( + origins.selected_receiving_person_id.to_numpy(), people.person_id.to_numpy() + ), + "RETAINED_AXIS", + ) + households = native.table("household").set_index("household_id", drop=False) + require( + households.index.is_unique + and np.array_equal( + origins.selected_receiving_household_id.to_numpy(), + people.person_household_id.to_numpy(), + ), + "RETAINED_HOUSEHOLD_ID", + ) + serials = households.SERIALNO.reindex(people.person_household_id.to_numpy()) + require( + np.array_equal(raw.SERIALNO.to_numpy(), serials.to_numpy()) + and np.array_equal(raw.SPORDER.map(int).to_numpy(), people.SPORDER.to_numpy()), + "RETAINED_SOURCE_ID", + ) + numbers = {c: _native_numbers(raw[c]) for c in ("INTP", "RETP", "ADJINC", "AGEP")} + for column, expected_numbers in numbers.items(): + for literal, stored, expected in zip( + raw[column], people[column], expected_numbers, strict=True + ): + same = ( + stored == literal + if type(stored) is str + else _same_number(stored, expected) + ) + require(bool(same), "RETAINED_RAW_" + column) + adjustment = numbers["ADJINC"] + for column, _flag, _prefix, output in ANCHORS: + require(people[output].dtype == np.dtype("float64"), "RETAINED_ADJUSTED_DTYPE") + expected = numbers[column] * (adjustment / 1_000_000.0) + actual = people[output].to_numpy(copy=False) + require( + np.array_equal(np.isnan(expected), np.isnan(actual)) + and np.array_equal( + expected[~np.isnan(expected)].view("uint64"), + actual[~np.isnan(actual)].view("uint64"), + ), + "RETAINED_ADJUSTED_BITS", + ) + + +def _parsed_table(raw, origins): + result = raw.copy(deep=True) + result["native_person_id"] = origins.selected_receiving_person_id.to_numpy( + copy=True + ) + result["source_year"] = 2024 + result["dollar_year"] = 2024 + for column, flag, prefix, _output in ANCHORS: + values = [ + parse_anchor( + r[column], + age=r["AGEP"], + adjustment=r["ADJINC"], + allocation=r[flag], + field=column, + ) + for r in raw.to_dict("records") + ] + parsed = pd.DataFrame(values, index=raw.index) + for name in parsed: + dtype = ( + "Float64" + if name == "amount" + else ( + bool + if name in ("known", "adjustment_known", "in_income_universe") + else STRING_DTYPE + ) + ) + result[prefix + "_" + name] = pd.array(parsed[name], dtype=dtype) + return result + + +@dataclass(frozen=True) +class QualifiedAcsIncomeAnchors: + """Detached source observations, not a substitute for the preparation.""" + + anchors: pd.DataFrame + projection: bytes + evidence: dict + + +def income_anchor_seal(qualified: QualifiedAcsIncomeAnchors) -> tuple: + require(type(qualified) is QualifiedAcsIncomeAnchors, "QUALIFIED_TYPE") + require( + qualified.projection + == qualified.anchors.reset_index().to_json(orient="table", index=False).encode() + and qualified.evidence["projection_sha256"] + == hashlib.sha256(qualified.projection).hexdigest(), + "PROJECTION_BINDING", + ) + return ( + physical._table_stamp(qualified.anchors), + qualified.projection, + json.dumps( + qualified.evidence, sort_keys=True, separators=(",", ":"), allow_nan=False + ), + ) + + +def _capture_person(root, pin, wanted, total_rows): + """Return literal values from an owned capture, after cleanup I/O completes. + + Caller-supplied pins grant no authority here; the qualifier binds them to + the retained catalogue and rechecks that original owner after this returns. + """ + _role, name, digest, size = pin + selected, count = {}, 0 + with tempfile.TemporaryDirectory( + prefix="microcosm-acs-income-anchor-" + ) as directory: + captured = Path(directory) / name + require( + housing._copy(root / "acs" / name, captured, size, exact_size=size) + == digest, + "ACS_CAPTURE_DIGEST", + ) + with zipfile.ZipFile(captured) as archive: + members, prefix = records._members(archive, "person") + for item in members: + with archive.open(item) as stream: + if item.filename.casefold().startswith(prefix): + count += _scan( + stream, + wanted, + selected, + maximum=total_rows - count, + ) + else: + while stream.read(65536): + pass + require( + count == total_rows and housing._persisted_sha(captured, size) == digest, + "ACS_CAPTURE_CHANGED", + ) + return selected + + +def qualify_current_acs_income_anchors(source_preparation) -> QualifiedAcsIncomeAnchors: + """Read exact original ACS members and recheck the real owner before return.""" + require( + type(source_preparation) + is preparation.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = source_preparation._checked() + state, document = entry[2], json.loads(entry[1]) + origins = _origins(state, document) + owned = preparation.acs_catalogue._lookup(state.catalogues[0]) + catalogue = json.loads(owned.receipt) + pins = tuple(p for p in owned.pins if p[0] == "person") + require( + len(pins) == 1 and catalogue["source_year"] == catalogue["survey_year"] == 2024, + "ACS_PIN_OR_PERIOD", + ) + selected = _capture_person( + state.root, + pins[0], + set(origins.anchor_source_key), + catalogue["counts"]["people"], + ) + raw = _raw_table(origins, selected) + _compare_retained(raw, origins, state.source_frames[0]) + anchors = _parsed_table(raw, origins) + projection = anchors.reset_index().to_json(orient="table", index=False).encode() + evidence = { + "protocol": PROTOCOL, + "preparation_sha256": hashlib.sha256(entry[1]).hexdigest(), + "acs_catalogue_sha256": hashlib.sha256(owned.receipt).hexdigest(), + "acs_person_archive_sha256": pins[0][2], + "selected_persons": len(anchors), + "source_year": 2024, + "dollar_year": 2024, + "income_period": "rolling_12_months", + "projection_sha256": hashlib.sha256(projection).hexdigest(), + "source_admission_issued": False, + "unallocated_observation_claim": False, + "analytical_universe_zeros_supplied": False, + "decomposition_performed": False, + "release_eligible": False, + } + result = QualifiedAcsIncomeAnchors(anchors, projection, evidence) + seal = income_anchor_seal(result) + require(source_preparation._checked() is entry, "PREPARATION_CHANGED") + preparation._pure_final(state) + require( + preparation._ISSUED.get(id(source_preparation)) is entry + and preparation.acs_catalogue._lookup(state.catalogues[0]) is owned, + "FINAL_OWNER", + ) + require(income_anchor_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py new file mode 100644 index 000000000..88f5d32ab --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py @@ -0,0 +1,487 @@ +"""Observed child support received, paid and required-to-pay source answers. + +Obligation is not payment. Returned values are descriptive and issue no source +authority; a consuming host retains/requalifies the actual preparation. +""" + +from __future__ import annotations + +import hashlib +import json +import tempfile +from dataclasses import dataclass +from functools import lru_cache +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as physical +from . import current_asec_income_routing_source as routing + +PROTOCOL = "microcosm.us.current-asec-child-support-source.v1" +AMOUNT_FIELDS = ("CSP_VAL", "CHSP_VAL") +# Width, position, PDF page, printed page, printed universe, zero label. +RESPONSE_ENTRIES = { + "CHELSEW_YN": (1, 711, 52, "6C-31", "All Persons aged 15+", "Niu"), + "CHSP_YN": (1, 717, 52, "6C-31", "CHELSEW_YN", "Niu"), + "CSP_YN": (1, 723, 52, "6C-31", "All Persons aged 15+", "Niu"), +} +RESPONSE_MEANINGS = { + "CHELSEW_YN": "has a child living outside the household", + "CHSP_YN": "required to pay child support; not whether payment occurred", + "CSP_YN": "received child support payments", +} +ALLOCATION_ENTRIES = { + "I_CHELSEWYN": (1, 807, 54, "6C-33", "CHELSEW_YN > 0"), + "I_CHSPVAL": (1, 808, 54, "6C-33", "CHSP_YN = 1"), + "I_CHSPYN": (1, 809, 54, "6C-33", "CHELSEW_YN = 1"), + "I_CSPVAL": (1, 810, 54, "6C-33", "CSP_YN = 1"), + "I_CSPYN": (1, 811, 54, "6C-33", "CSP_YN > 0"), +} +TOPCODE_ENTRIES = { + "TCHSP_VAL": (1, 899, 59, "6C-38", "CHSP_VAL > 0"), + "TCSP_VAL": (1, 901, 59, "6C-38", "CSP_VAL > 0"), +} +READ_COLUMNS = ( + *routing.COORDINATE_COLUMNS, + *AMOUNT_FIELDS, + *RESPONSE_ENTRIES, + *ALLOCATION_ENTRIES, + *TOPCODE_ENTRIES, +) +OBLIGATION_UNIVERSE_NOTE = ( + "CHSP_YN prints bare CHELSEW_YN, whereas I_CHSPYN prints CHELSEW_YN = 1. " + "The bare field is not silently compiled as a general predicate. Positive " + "paid amounts are qualified only on the observed intersection of age15+, " + "CHELSEW_YN=1 and CHSP_YN=1; other routes remain explicit and unresolved." +) + + +def require(condition, reason): + if not condition: + raise ValueError("ASEC_CHILD_SUPPORT_SOURCE_" + reason) + + +@lru_cache(maxsize=1) +def _cached_amount_entries_json(): + payload = ( + resources.files(__package__).joinpath(routing.DOMAINS_RESOURCE).read_bytes() + ) + require(routing._sha(payload) == routing.DOMAINS_SHA256, "DOMAINS_HASH") + entries = {} + for field in json.loads(payload)["fields"]: + if field["name"] not in AMOUNT_FIELDS: + continue + require(field["name"] not in entries, "DOMAIN_FIELD_DUPLICATE:" + field["name"]) + vintages = [ + v + for v in field["vintages"] + if v["income_year"] == routing.CURRENT_INCOME_YEAR + ] + require(len(vintages) == 1, "DOMAIN_VINTAGE:" + field["name"]) + vintage = vintages[0] + require( + vintage["dictionary_spelling"] == field["name"] + and vintage["pdf_sha256"] == routing.DICTIONARY_SHA256 + and vintage["source_url"] == routing.DICTIONARY_URL, + "DICTIONARY_PIN", + ) + routing._require_nonnegative_amount_domain( + field, + vintage, + zero_semantics="niu" + if field["name"] == "CHSP_VAL" + else "none_or_niu_not_distinguishable_from_amount_alone", + valid_minimum=1 if field["name"] == "CHSP_VAL" else 0, + ) + entries[field["name"]] = { + "width": vintage["ascii_length_as_printed"], + "position": vintage["ascii_position_as_printed"], + "pdf_page": vintage["pdf_page_1based"], + "printed_page": vintage["printed_page"], + "universe": vintage["universe_as_printed"], + "domain": field["domain"], + "period": field["period"], + "temporal_authority": field["temporal_authority"], + } + require(set(entries) == set(AMOUNT_FIELDS), "AMOUNT_ROSTER") + return json.dumps(entries, separators=(",", ":")).encode() + + +def amount_entries(): + """Return deeply detached evidence from privately cached immutable bytes.""" + return json.loads(_cached_amount_entries_json()) + + +amount_entries.cache_clear = _cached_amount_entries_json.cache_clear + + +def _domain_agreement(ready): + routing._require_live_amount_domains( + ready, + { + name: ( + entry["domain"]["encoded_range_inclusive"]["minimum"], + entry["domain"]["encoded_range_inclusive"]["maximum"], + entry["domain"]["zero_semantics"], + ) + for name, entry in amount_entries().items() + }, + ) + + +def _amount_pair(token, entry): + bounds = entry["domain"]["encoded_range_inclusive"] + return routing.literal_code( + token, range(bounds["minimum"], bounds["maximum"] + 1), width=entry["width"] + ) + + +def _paid_status(age, elsewhere, obligation, amount): + value, parse_status = amount + if parse_status != "in_printed_range": + return ( + "missing_amount" if parse_status == "missing" else "invalid_amount_literal" + ) + if age < 15: + return ( + "outside_reporting_universe" + if elsewhere[0] == obligation[0] == 0 and value == 0 + else "contradictory_outside_reporting_universe" + ) + if value == 0: + return "declared_niu_amount" + if elsewhere[1] != "in_printed_range": + return "unresolved_child_elsewhere_literal" + if obligation[1] != "in_printed_range": + return ( + "missing_obligation_literal" + if obligation[1] == "missing" + else "invalid_obligation_literal" + ) + if elsewhere[0] != 1: + return "unresolved_child_elsewhere_route" + if obligation[0] != 1: + return "payment_outside_published_obligation_universe" + return "observed_positive_payment" + + +def project_child_support_literals(ordered): + """Descriptive projection only; arbitrary literals grant no authority.""" + require( + type(ordered) is pd.DataFrame and set(READ_COLUMNS) <= set(ordered), "COLUMNS" + ) + ages = ordered.A_AGE.to_numpy(dtype=np.int64) + require(((ages >= 0) & (ages <= 99)).all(), "AGE_RANGE") + out = pd.DataFrame(index=range(len(ordered))) + parsed, codes = {}, {} + for name, entry in amount_entries().items(): + pairs = [_amount_pair(token, entry) for token in ordered[name]] + parsed[name] = pairs + out[name + "_literal"] = pd.array(ordered[name].tolist(), dtype="string") + out[name + "_literal_status"] = pd.array( + [status for _, status in pairs], dtype="string" + ) + out[name + "_published_amount"] = pd.array( + [value for value, _ in pairs], dtype="Float64" + ) + for name, entry in RESPONSE_ENTRIES.items(): + frame, values, statuses = routing._codes_frame( + name, + ordered[name], + routing.RECEIPT_CODE_DOMAIN, + {0: entry[5], 1: "Yes", 2: "No"}, + width=entry[0], + ) + out = pd.concat([out, frame], axis=1) + codes[name] = list(zip(values, statuses, strict=True)) + for name in AMOUNT_FIELDS: + labels = [] + for i, age in enumerate(ages): + value, status = parsed[name][i] + if name == "CHSP_VAL": + label = _paid_status( + age, codes["CHELSEW_YN"][i], codes["CHSP_YN"][i], parsed[name][i] + ) + elif status not in ("missing", "in_printed_range"): + label = "invalid_amount_literal" + else: + kind = ( + "missing" + if value is None + else ("zero" if value == 0 else "nonzero") + ) + label = routing.receipt_status( + bool(age >= 15), + codes["CSP_YN"][i], + kind, + net_measure=False, + zero_is_dollars=False, + ) + labels.append(label) + known = np.array( + [ + label in (*routing.KNOWN_AMOUNT_STATUSES, "observed_positive_payment") + for label in labels + ] + ) + out[name + "_reporting_status"] = pd.array(labels, dtype="string") + out[name + "_amount_known"] = known + out[name + "_amount"] = out[name + "_published_amount"].where(known) + flags = {} + for name, entry in ALLOCATION_ENTRIES.items(): + frame, values, statuses = routing._codes_frame( + name, ordered[name], routing.ALLOCATION_ANNVAL_CODES, width=entry[0] + ) + out = pd.concat([out, frame], axis=1) + flags[name] = (values, statuses) + out["received_allocation_origin"] = routing._allocation_origin( + {name: flags[name] for name in ("I_CSPVAL", "I_CSPYN")}, unflagged=False + ) + out["paid_allocation_origin"] = routing._allocation_origin( + {name: flags[name] for name in ("I_CHELSEWYN", "I_CHSPVAL", "I_CHSPYN")}, + unflagged=False, + ) + for name, entry in TOPCODE_ENTRIES.items(): + frame, _, _ = routing._codes_frame(name, ordered[name], (0, 1), width=entry[0]) + out = pd.concat([out, frame], axis=1) + out["source_age"] = ages + out["obligation_implies_payment"] = False + out["voluntary_payment_absence_known"] = False + return out + + +@dataclass(frozen=True) +class CurrentAsecChildSupportValues: + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def child_support_values_seal(values): + """Physical value seal, including nullable backing storage and exact bits.""" + require(type(values) is CurrentAsecChildSupportValues, "VALUES_TYPE") + digest = hashlib.sha256(PROTOCOL.encode()) + for table in (values.person, values.asec_literals): + require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE_TYPE") + digest.update( + physical._json( + { + "columns": list(table.columns), + "columns_axis": physical.checkpoint._index_spec( + table.columns, label="child support columns" + ), + "index": physical.checkpoint._index_spec( + table.index, label="child support" + ), + } + ) + ) + physical._series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + series = table[column] + if isinstance(series.dtype, pd.Float64Dtype): + # The checkpoint digest supports NumPy floats, but deliberately + # excludes Float64 extension arrays. Keep the nullable tag and + # hash both physical arrays, including values under null masks. + digest.update(physical._json({"dtype": "Float64", "nullable": True})) + physical._series_digest(digest, pd.Series(series.array._data)) + physical._series_digest(digest, pd.Series(series.array._mask)) + else: + physical._series_digest(digest, series) + digest.update(physical._json(values.evidence)) + return digest.hexdigest() + + +def _capture_member(root, pin): + _, member, _, digest, rows, size = pin + with tempfile.TemporaryDirectory(prefix="microcosm-asec-child-support-") as tmp: + path = Path(tmp) / member + identity = routing.coverage._capture( + root / "asec" / member, + path, + size=size, + digest=digest, + budget=[routing.coverage._BODY_MAX], + ) + raw = routing._read_capture(path, rows=rows, columns=READ_COLUMNS, patterns={}) + require( + routing.coverage._identity(path.stat(follow_symlinks=False)) == identity + and routing._file_sha(path) == digest, + "CAPTURE_CHANGED", + ) + return raw + + +def _compare_amount(ready, positions, name, literals): + field = ready.field(name) + entry = amount_entries()[name] + pairs = [_amount_pair(t, entry) for t in literals] + require( + all(s in ("missing", "in_printed_range") for _, s in pairs), "AMOUNT_LITERAL" + ) + valid = field.validity[positions] == 1 + require(np.array_equal(valid, [v is not None for v, _ in pairs]), "AMOUNT_VALIDITY") + expected = np.array( + [np.nan if v is None else v for v, _ in pairs], dtype=np.float64 + ) + require( + np.array_equal( + field.amounts[positions][valid].view("uint64"), + expected[valid].view("uint64"), + ), + "AMOUNT_BITS", + ) + return field + + +def qualify_current_asec_child_support(preparation): + """Borrow the original preparation, capture once, and requalify before return.""" + source = routing.source + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state, native = entry[2], entry[2].native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + _domain_agreement(ready) + header, native_document = json.loads(ready.header), json.loads(issued[1]) + require( + header["target_year"] == routing.CURRENT_INCOME_YEAR + and header["semantic"] == "annual_current_money" + and native_document["source_year"] + == native_document["income_year"] + == routing.CURRENT_INCOME_YEAR + and native_document["survey_year"] == (routing.CURRENT_INCOME_YEAR + 1), + "PERIOD", + ) + pins = [ + p for p in routing.coverage._MEMBER_PINS if p[0] == routing.CURRENT_INCOME_YEAR + ] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "MEMBER_BINDING", + ) + raw = _capture_member(state.root, pins[0]) + positions = np.flatnonzero( + np.asarray(parent.scope.person_years) == routing.CURRENT_INCOME_YEAR + ) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent.frame.person.iloc[positions][parent_name].to_numpy(), + ), + "PARENT_COORDINATE", + ) + fields = { + name: _compare_amount(ready, positions, name, ordered[name]) + for name in AMOUNT_FIELDS + } + basis = project_child_support_literals(ordered) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + for field_name, field in fields.items(): + for name in ("statuses", "validity", "zero_origin"): + basis[field_name + "_parent_" + name] = getattr(field, name)[positions] + selected = state.frame.person.loc[ + state.frame.person[routing.support_channel_column("person")].eq("asec") + ] + native_ids = selected[routing.spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + literals = ordered.copy() + literals.index = basis.index.copy() + evidence = { + "protocol": PROTOCOL, + "dictionary_url": routing.DICTIONARY_URL, + "dictionary_sha256": routing.DICTIONARY_SHA256, + "amount_entries": amount_entries(), + "response_entries": RESPONSE_ENTRIES, + "response_meanings": RESPONSE_MEANINGS, + "obligation_universe_note": OBLIGATION_UNIVERSE_NOTE, + "response_period_note": ( + "CHSP_YN and CHELSEW_YN do not state an independent reference year " + "in their printed entries. Their 2025 source context is preserved; " + "no new current-at-interview or policy eligibility input is issued." + ), + "allocation_entries": ALLOCATION_ENTRIES, + "topcode_entries": TOPCODE_ENTRIES, + "preparation_sha256": routing._sha(entry[1]), + "asec_native_sha256": routing._sha(issued[1]), + "money_header_sha256": routing._sha(ready.header), + "source_member_sha256": digest, + "source_year": routing.CURRENT_INCOME_YEAR, + "survey_year": (routing.CURRENT_INCOME_YEAR + 1), + "complete_source_rows": rows, + "selected_rows": len(out), + "read_columns": READ_COLUMNS, + "obligation_implies_payment": False, + "voluntary_payment_absence_known": False, + "paid_niu_completed_with_zero": False, + "tax_treatment_assigned": False, + "source_admission_issued": False, + "release_eligible": False, + } + # Freeze detached JSON-compatible metadata before its seal; do not retain the + # module's mutable constant dictionaries inside a returned descriptive view. + result = CurrentAsecChildSupportValues( + out, literals, json.loads(json.dumps(evidence)) + ) + seal = child_support_values_seal(result) + require( + preparation._checked() is entry and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require(child_support_values_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py new file mode 100644 index 000000000..12b88afe4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py @@ -0,0 +1,379 @@ +"""Current survey source projection for ASEC sex and household state. + +Reuse the maintained demographic source owner for A_SEX/AXSEX. Add only the +missing GESTFIPS household projection through the original weight owner's +bounded capture/CSV primitives and closed member registry. Returned numerical +values are not source authority; a graph host must re-run this live qualifier +and compare materialized descendants, including after replay. +""" + +from __future__ import annotations + +import csv +import io +import json +import os +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_demographic_source as demographic +from . import asec_original_household_weights as household +from . import source_csv_builtin +from . import survey_population_preparation as preparation_owner +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-asec-demographic-projection.v1" +STATE_COLUMNS = ("H_SEQ", "GESTFIPS") +STATE_CONTRACT = { + "field": "GESTFIPS", + "concept": "State FIPS code", + "entity": "household", + "survey_year": 2025, + "income_year": 2024, + "dictionary_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "dictionary_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "pdf_page_1based": 8, + "printed_page": "6A-1", + "printed_length": 2, + "printed_position": 44, + "printed_range": [1, 56], + "universe": "All Households", + "canonical_mapping": "numeric_identity", + "unassigned_jurisdiction_validation": "separate_geography_gate", + "unknown_policy": "preserve_token_and_unbound_state; never_use_carried_state_as_fallback", +} + + +def _require(condition, reason): + if not condition: + raise ValueError("CURRENT_ASEC_DEMOGRAPHICS_" + reason) + + +def _state_token(token): + """Preserve an unknown literal rather than interpreting it as a state.""" + _require( + type(token) is str and len(token) <= household._MAX_FIELD_CHARS, + "STATE_TOKEN_BOUND", + ) + if token == "": + return None, "missing" + if re.fullmatch(r"[0-9]{1,2}", token, re.ASCII) is None: + return None, "malformed" + code = int(token) + return ( + (code, "in_printed_range") + if 1 <= code <= 56 + else (None, "outside_printed_range") + ) + + +def _read_state_capture(path, pin): + """Low-level literal reader; a caller-supplied pin grants no authority.""" + reader = source_csv_builtin.capture_csv_reader(csv) + _require(reader is not None, "CSV_READER_CHANGED") + result = {} + with open(path, "rb", buffering=0, opener=household._regular_opener) as raw: + before = os.fstat(raw.fileno()) + _require(before.st_size == pin.size_bytes, "CAPTURE_SIZE") + digest = household._DigestReader(raw, pin.size_bytes) + with io.TextIOWrapper( + io.BufferedReader(digest), encoding="utf-8-sig", newline="" + ) as text: + lines = household._RecordLines(text) + records = reader(lines, strict=True) + header = next(records, []) + _require( + bool(header) + and all(header) + and len(header) == len(set(header)) + and set(STATE_COLUMNS) <= set(header), + "STATE_HEADER", + ) + positions = [header.index(c) for c in STATE_COLUMNS] + while True: + lines.characters = 0 + try: + row = next(records) + except StopIteration: + break + _require( + len(row) == len(header) and len(result) < pin.rows, + "STATE_ROW_SHAPE", + ) + key, token = (row[i] for i in positions) + _require( + re.fullmatch(r"[0-9]{1,5}", key, re.ASCII) is not None + and 1 <= int(key) <= 99999, + "HOUSEHOLD_KEY", + ) + native = int(key) + _require(native not in result, "HOUSEHOLD_DUPLICATE_KEY") + code, status = _state_token(token) + result[native] = { + "H_SEQ": key, + "GESTFIPS": token, + "state_code": code, + "status": status, + "member_row_1based": len(result) + 1, + } + _require(len(result) == pin.rows, "STATE_ROW_COUNT") + _require( + digest.count == pin.size_bytes + and digest.digest.hexdigest() == pin.member_sha256, + "STATE_CAPTURE_CHANGED", + ) + after = os.fstat(raw.fileno()) + _require( + all( + getattr(before, k) == getattr(after, k) + for k in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ), + "STATE_CAPTURE_CHANGED", + ) + return result + + +def _load_current_state(member_path, expected_member): + """Closed registry lookup; expected member is retained native-owner evidence.""" + pins = household._registry() + selected = [pin for pin in pins if pin.income_year == 2024] + _require(len(selected) == 1, "CURRENT_STATE_REGISTRY") + pin = selected[0] + _require( + pin.survey_year == 2025 + and pin.canonical_member_id == expected_member["canonical_member_id"] + and pin.member_sha256 == expected_member["member_sha256"] + and pin.archive_sha256 == expected_member["archive_sha256"], + "STATE_NATIVE_MEMBER_BINDING", + ) + registry = household._encode(household._implementation()) + paths = household._member_path_snapshot({2024: member_path}, pins) + with tempfile.TemporaryDirectory(prefix="asec-current-state-") as directory: + capture = Path(directory) / "hhpub25.csv" + _require( + household._capture_owner._snapshot( + paths[2024], capture, size=pin.size_bytes + ) + == pin.member_sha256, + "STATE_SOURCE_SHA256", + ) + rows = _read_state_capture(capture, pin) + _require( + household._encode(household._implementation()) == registry, + "STATE_REGISTRY_CHANGED", + ) + return rows, { + "canonical_member_id": pin.canonical_member_id, + "member_sha256": pin.member_sha256, + "archive_sha256": pin.archive_sha256, + "bytes": pin.size_bytes, + "rows": pin.rows, + } + + +@dataclass(frozen=True) +class CurrentAsecDemographicValues: + """Source-derived selected arrays; no independent authority or launch verdict.""" + + person: pd.DataFrame + household: pd.DataFrame + receipt: bytes + + +def qualify_current_asec_demographics(preparation): + """Reconstruct the real parent sex owner and current household state source. + + The selected current survey is joined after full-source reconstruction. + Missing/unlabelled sex allocation or state codes remain explicitly unknown; + no non-2→male or carried-state fallback is permitted. + """ + _require( + type(preparation) is preparation_owner.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + native = state.native[1] + native_entry = preparation_owner.asec_native._ISSUED.get(id(native)) + _require( + native_entry is not None + and native_entry[0]() is native + and native.payload == native_entry[1], + "NATIVE_ISSUANCE", + ) + parent = native_entry[2].parent + native_receipt = json.loads(native_entry[1]) + _require( + native_receipt["source_year"] == native_receipt["income_year"] == 2024 + and native_receipt["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + observed = demographic.load_authenticated_asec_demographic_source( + parent, + member_paths={ + year: state.root / "asec" / f"pppub{year - 1999}.csv" + for year in (2022, 2023, 2024) + }, + ) + sex_receipt = observed.receipt + _require( + sex_receipt["source_identity"] == parent.source.identity.decode(), + "DEMOGRAPHIC_PARENT_IDENTITY", + ) + fields = native_entry[2].fields.document + _require(len(fields["members"]) == 1, "STATE_MEMBER_ROSTER") + states, state_pin = _load_current_state( + state.root / "asec" / "hhpub25.csv", fields["members"][0] + ) + source_households = { + row["household_id"]: row for row in native_receipt["households"] + } + households = state.frame.table("household") + selected_hh = households.loc[ + households[support_channel_column("household")].eq("asec") + ] + hrows = [] + for stacked, source_hh in selected_hh[ + ["household_id", spine_source_id_column("household")] + ].itertuples(index=False, name=None): + _require(source_hh in source_households, "HOUSEHOLD_NATIVE_JOIN") + current = source_households[source_hh] + year, key = current["native_key"] + _require(year == 2024 and key in states, "HOUSEHOLD_SOURCE_JOIN") + row = states[key] + _require( + row["H_SEQ"] == current["household_fields"]["H_SEQ"], + "HOUSEHOLD_LITERAL_KEY", + ) + hrows.append( + ( + int(stacked), + int(source_hh), + key, + row["GESTFIPS"], + row["state_code"], + row["status"], + ) + ) + htable = pd.DataFrame( + hrows, + columns=( + "household_id", + "native_household_id", + "H_SEQ_integer", + "GESTFIPS", + "state_fips", + "state_status", + ), + ).set_index("household_id", drop=False) + htable["state_fips"] = pd.array(htable.state_fips, dtype="Int64") + htable["state_known"] = htable.state_fips.notna() + full_ids, years = observed.array("person_id"), observed.array("income_year") + _require( + len(set(zip(years, full_ids, strict=True))) == len(full_ids), "SEX_SOURCE_AXIS" + ) + lookup = pd.MultiIndex.from_arrays((years, full_ids)) + person = state.frame.person + selected = person.loc[person[support_channel_column("person")].eq("asec")] + original_ids = selected[spine_source_id_column("person")].to_numpy(dtype=np.int64) + positions = lookup.get_indexer( + pd.MultiIndex.from_arrays( + (np.full(len(selected), 2024, dtype=np.int64), original_ids) + ) + ) + _require( + (positions >= 0).all() and len(set(positions)) == len(positions), + "SEX_NATIVE_JOIN", + ) + ptable = pd.DataFrame( + { + "person_id": selected.person_id.to_numpy(dtype=np.int64), + "native_person_id": original_ids, + }, + index=pd.Index(selected.person_id.to_numpy(), name="person_id"), + ) + for name in ( + "asec_A_SEX", + "asec_AXSEX", + "asec_sex_binding_state", + "asec_sex_allocation_state", + ): + ptable[name] = observed.array(name)[positions] + binding = ptable.asec_sex_binding_state.to_numpy() + _require(np.isin(binding, (0, 1, 2)).all(), "SEX_BINDING_DOMAIN") + ptable["is_female"] = pd.array( + [False if code == 1 else True if code == 2 else None for code in binding], + dtype="boolean", + ) + ptable["sex_known"] = binding != 0 + ptable["sex_universe"] = "All Persons" + ptable["sex_origin"] = np.where( + ptable.asec_sex_allocation_state.eq(2), + "census_allocated", + np.where( + ptable.asec_sex_allocation_state.eq(1), + "source_no_change", + "unresolved_allocation", + ), + ) + receipt = preparation_owner._encode( + { + "protocol": PROTOCOL, + "preparation_sha256": preparation_owner._sha(entry[1]), + "native_population_sha256": preparation_owner._sha(native_entry[1]), + "demographic_source_content_sha256": observed.content_sha256, + "demographic_source_receipt": sex_receipt, + "state_source": state_pin, + "state_contract": STATE_CONTRACT, + "selected_person_ids": ptable.person_id.tolist(), + "selected_household_ids": htable.household_id.tolist(), + "sex_known_persons": int(ptable.sex_known.sum()), + "sex_unknown_persons": int((~ptable.sex_known).sum()), + "state_known_households": int(htable.state_known.sum()), + "state_unknown_households": int((~htable.state_known).sum()), + "sex_observation_year": 2025, + "state_observation_year": 2025, + "income_year": 2024, + "state_is_income_year_residence_claim": False, + "selected_household_state_tokens": htable.loc[ + :, + [ + "household_id", + "native_household_id", + "H_SEQ_integer", + "GESTFIPS", + "state_status", + ], + ].values.tolist(), + "person_projection_sha256": preparation_owner._sha( + ptable.reset_index(drop=True).to_json(orient="table").encode() + ), + "household_projection_sha256": preparation_owner._sha( + htable.reset_index(drop=True).to_json(orient="table").encode() + ), + "source_admission_issued": False, + "release_eligible": False, + } + ) + observed.validate() + preparation_owner._pure_final(state) + _require( + preparation_owner._ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1] + and preparation_owner.asec_native._ISSUED.get(id(native)) is native_entry + and native.payload == native_entry[1] + and native_entry[2].parent is parent, + "FINAL_ISSUANCE", + ) + return CurrentAsecDemographicValues(ptable, htable, receipt) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_dividend_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_dividend_source.py new file mode 100644 index 000000000..8a760bb80 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_dividend_source.py @@ -0,0 +1,519 @@ +"""Current ASEC dividend observations and possible survivor-property routes. + +Source flags are provenance, never donor filters. No survivor amount, tax +treatment, ACS imputation or new source authority is issued by this module. +""" + +from __future__ import annotations + +import hashlib +import json +import tempfile +from dataclasses import dataclass +from functools import lru_cache +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as physical +from . import current_asec_income_routing_source as routing + +PROTOCOL = "microcosm.us.current-asec-dividend-source.v1" +AMOUNT_FIELDS = ("DIV_VAL",) +# Width, position, PDF page, printed page, printed universe, zero wording. +RECEIPT_ENTRIES = { + "DIV_YN": (1, 484, 45, "6C-24", "All Persons aged 15+", "niu"), + "SUR_YN": (1, 664, 50, "6C-29", "All Persons aged 15+", "niu"), +} +SURVIVOR_ENTRIES = { + "SUR_SC1": (2, 648, 50, "6C-29", "SUR_YN = 1"), + "SUR_SC2": (2, 650, 50, "6C-29", "SUR_YN = 1"), +} +SURVIVOR_CODES = { + 0: "none or niu", + 1: "company or union survivor pension", + 2: "federal government", + 3: "US military retirement survivor pension", + 4: "state or local government survivor pension", + 5: "US railroad retirement survivor pension", + 6: "worker compensation survivor", + 7: "black lung", + 8: "regular payments from estates or trusts", + 9: "regular payments from annuities or paid-up life insurance", + 10: "other or don't know", +} +ALLOCATION_ENTRIES = { + "I_DIVVAL": (1, 819, 54, "6C-33", "DIV_YN = 1", "0:9", "See I_ANNVAL"), + "I_DIVYN": (1, 820, 54, "6C-33", "All Persons 15+", "0:1", "See I_ANNVAL"), +} +ALLOCATION_CONFLICT_NOTE = ( + "I_DIVYN prints header range0:1 but references I_ANNVAL values0:9. " + "Both axes are preserved; a code2..9 is not silently admitted or discarded. " + "Flag values do not alter dividend amount knownness or select donors." +) +TOPCODE_ENTRIES = { + "TDIV_VAL": (1, 905, 59, "6C-38", "DIV_VAL > 0"), + "TRNT_VAL": (1, 917, 60, "6C-39", "RNT_VAL > 0"), +} +READ_COLUMNS = ( + *routing.COORDINATE_COLUMNS, + *AMOUNT_FIELDS, + *RECEIPT_ENTRIES, + *SURVIVOR_ENTRIES, + *ALLOCATION_ENTRIES, + *TOPCODE_ENTRIES, +) + + +def require(condition, reason): + if not condition: + raise ValueError("ASEC_DIVIDEND_SOURCE_" + reason) + + +def _decode_amount_entry(data): + fields = [f for f in data["fields"] if f["name"] == "DIV_VAL"] + require(len(fields) == 1, "DOMAIN_FIELD_ROSTER") + field = fields[0] + vintages = [ + v for v in field["vintages"] if v["income_year"] == routing.CURRENT_INCOME_YEAR + ] + require(len(vintages) == 1, "DOMAIN_VINTAGE") + vintage, domain = vintages[0], field["domain"] + require( + vintage["dictionary_spelling"] == "DIV_VAL" + and vintage["survey_year"] == 2025 + and vintage["pdf_sha256"] == routing.DICTIONARY_SHA256 + and vintage["source_url"] == routing.DICTIONARY_URL, + "DICTIONARY_PIN", + ) + require( + field["entity"] == "person" + and domain["encoded_range_inclusive"] == {"minimum": 0, "maximum": 999999} + and domain["valid_dollar_range_inclusive"] == {"minimum": 0, "maximum": 999999} + and domain["negative_dollars_permitted"] is False + and domain["declared_negative_nonmoney_codes"] == [] + and domain["declared_other_missing_codes"] == [] + and domain["declared_niu_codes"] == [0] + and domain["valid_dollar_range_excludes"] == [] + and domain["zero_semantics"] + == "none_or_niu_not_distinguishable_from_amount_alone", + "DOMAIN_UNSUPPORTED", + ) + return ( + vintage["ascii_length_as_printed"], + vintage["ascii_position_as_printed"], + vintage["pdf_page_1based"], + vintage["printed_page"], + domain["encoded_range_inclusive"]["maximum"], + vintage["universe_as_printed"], + domain["zero_semantics"], + ) + + +@lru_cache(maxsize=1) +def _amount_entry(): + payload = ( + resources.files(__package__).joinpath(routing.DOMAINS_RESOURCE).read_bytes() + ) + require(routing._sha(payload) == routing.DOMAINS_SHA256, "DOMAINS_HASH") + # Every cached value is immutable; callers never receive a shared mapping. + return _decode_amount_entry(json.loads(payload)) + + +def amount_entries(): + return {"DIV_VAL": _amount_entry()} + + +def _domain_agreement(ready): + domains = [d for d in ready.bindings.spec.fields if d.name == "DIV_VAL"] + require(len(domains) == 1, "MONEY_DOMAIN_ROSTER") + domain, entry = domains[0], amount_entries()["DIV_VAL"] + require( + domain.entity == "person" + and domain.minimum == 0 + and domain.maximum == entry[4] + and domain.zero_semantics == entry[6], + "MONEY_DOMAIN_DISAGREEMENT", + ) + + +def _survivor_route(age, receipt, slots): + """Clear only resolved non-property source routes; never infer absent amounts.""" + code, status = receipt + values = [pair[0] for pair in slots] + readable = all(pair[1] == "in_printed_range" for pair in slots) + if age < 15: + if status != "in_printed_range" or not readable: + return "unresolved_outside_reporting_universe", None + return ( + "outside_reporting_universe" + if code == 0 and readable and values == [0, 0] + else "contradictory_outside_reporting_universe", + None, + ) + if status != "in_printed_range": + return ( + "missing_receipt_literal" + if status == "missing" + else "unrecognized_receipt_literal", + None, + ) + if code == 0: + return ( + "niu" if readable and values == [0, 0] else "unresolved_niu_route", + None, + ) + if code == 2: + return ( + ("known_nonreceipt", True) + if readable and values == [0, 0] + else ("unresolved_nonreceipt_source_route", None) + ) + if 8 in values: + return "possible_estate_or_trust_route", False + if not readable: + return "unresolved_source_literal", None + if 10 in values: + return "source_type_unspecified", None + if values == [0, 0]: + return "unreported_source_slots", None + return "known_other_survivor_sources", True + + +def _dividend_allocation_conflict(tokens): + # Width1 admits only one decimal digit. Parsing physical syntax against + # 0..9 does not decide which conflicting codebook range is authoritative. + frame, values, statuses = routing._codes_frame("I_DIVYN", tokens, range(10)) + frame["I_DIVYN_literal_status"] = pd.array( + [ + "well_formed" if status == "in_printed_range" else status + for status in statuses + ], + dtype="string", + ) + header = [routing.literal_code(t, (0, 1), width=1)[1] for t in tokens] + frame["I_DIVYN_header_range_status"] = pd.array(header, dtype="string") + frame["I_DIVYN_referenced_values_status"] = pd.array(statuses, dtype="string") + frame["I_DIVYN_codebook_status"] = pd.array( + [ + "literal_unresolved" + if status != "in_printed_range" + else ( + "header_reference_conflict" + if value not in (0, 1) + else "header_reference_agree" + ) + for value, status in zip(values, statuses, strict=True) + ], + dtype="string", + ) + return frame + + +def project_dividend_literals(ordered): + """Descriptive source projection; an arbitrary DataFrame grants no authority.""" + require( + type(ordered) is pd.DataFrame and set(READ_COLUMNS) <= set(ordered), "COLUMNS" + ) + ages = ordered.A_AGE.to_numpy(dtype=np.int64) + require(((ages >= 0) & (ages <= 99)).all(), "AGE_RANGE") + out = pd.DataFrame(index=range(len(ordered))) + entry = amount_entries()["DIV_VAL"] + amounts = [ + routing.literal_code(t, range(entry[4] + 1), width=entry[0]) + for t in ordered.DIV_VAL + ] + out["DIV_VAL_literal"] = pd.array(ordered.DIV_VAL.tolist(), dtype="string") + out["DIV_VAL_published_amount"] = pd.array([v for v, _ in amounts], dtype="Float64") + out["DIV_VAL_literal_status"] = pd.array([s for _, s in amounts], dtype="string") + codes = {} + for name, item in (*RECEIPT_ENTRIES.items(), *SURVIVOR_ENTRIES.items()): + allowed = ( + routing.RECEIPT_CODE_DOMAIN if name in RECEIPT_ENTRIES else SURVIVOR_CODES + ) + labels = ( + {0: "niu", 1: "yes", 2: "no"} if name in RECEIPT_ENTRIES else SURVIVOR_CODES + ) + frame, values, statuses = routing._codes_frame( + name, ordered[name], allowed, labels, width=item[0] + ) + out = pd.concat([out, frame], axis=1) + codes[name] = list(zip(values, statuses, strict=True)) + labels = [] + for age, receipt, (value, status) in zip( + ages, codes["DIV_YN"], amounts, strict=True + ): + kind = "missing" if value is None else ("zero" if value == 0 else "nonzero") + label = routing.receipt_status( + bool(age >= 15), receipt, kind, net_measure=False, zero_is_dollars=False + ) + labels.append( + label + if status in ("missing", "in_printed_range") + else "invalid_amount_literal" + ) + known = np.array([s in routing.KNOWN_AMOUNT_STATUSES for s in labels]) + out["DIV_VAL_reporting_status"] = pd.array(labels, dtype="string") + out["DIV_VAL_amount_known"] = known + out["DIV_VAL_amount"] = out.DIV_VAL_published_amount.where(known) + routes = [ + _survivor_route(age, receipt, (one, two)) + for age, receipt, one, two in zip( + ages, codes["SUR_YN"], codes["SUR_SC1"], codes["SUR_SC2"], strict=True + ) + ] + out["survivor_property_route_status"] = pd.array( + [s for s, _ in routes], dtype="string" + ) + out["survivor_property_route_clear"] = pd.array( + [v for _, v in routes], dtype="boolean" + ) + out["survivor_estate_or_trust_code_present"] = [ + one[0] == 8 or two[0] == 8 + for one, two in zip(codes["SUR_SC1"], codes["SUR_SC2"], strict=True) + ] + out["survivor_unspecified_source_code_present"] = [ + one[0] == 10 or two[0] == 10 + for one, two in zip(codes["SUR_SC1"], codes["SUR_SC2"], strict=True) + ] + flags, _, _ = routing._codes_frame( + "I_DIVVAL", ordered.I_DIVVAL, routing.ALLOCATION_ANNVAL_CODES + ) + out = pd.concat( + [out, flags, _dividend_allocation_conflict(ordered.I_DIVYN)], axis=1 + ) + for name, item in TOPCODE_ENTRIES.items(): + flags, _, _ = routing._codes_frame( + name, + ordered[name], + (0, 1), + {0: "not topcoded", 1: "topcoded"}, + width=item[0], + ) + out = pd.concat([out, flags], axis=1) + out["source_age"] = ages + return out + + +@dataclass(frozen=True) +class CurrentAsecDividendValues: + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def dividend_values_seal(values): + """Physical value seal, including nullable backing storage and exact bits.""" + require(type(values) is CurrentAsecDividendValues, "VALUES_TYPE") + digest = hashlib.sha256(PROTOCOL.encode()) + for table in (values.person, values.asec_literals): + require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE_TYPE") + digest.update( + physical._json( + { + "columns": list(table.columns), + "columns_axis": physical.checkpoint._index_spec( + table.columns, label="dividend columns" + ), + "index": physical.checkpoint._index_spec( + table.index, label="dividend" + ), + } + ) + ) + physical._series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + series = table[column] + if isinstance(series.dtype, pd.Float64Dtype): + # The checkpoint digest supports NumPy floats, but deliberately + # excludes Float64 extension arrays. Keep the nullable tag and + # hash both physical arrays, including values under null masks. + digest.update(physical._json({"dtype": "Float64", "nullable": True})) + physical._series_digest(digest, pd.Series(series.array._data)) + physical._series_digest(digest, pd.Series(series.array._mask)) + else: + physical._series_digest(digest, series) + digest.update(physical._json(values.evidence)) + return digest.hexdigest() + + +def _capture_member(root, pin): + _, member, _, digest, rows, size = pin + with tempfile.TemporaryDirectory(prefix="microcosm-asec-dividend-") as tmp: + path = Path(tmp) / member + identity = routing.coverage._capture( + root / "asec" / member, + path, + size=size, + digest=digest, + budget=[routing.coverage._BODY_MAX], + ) + raw = routing._read_capture(path, rows=rows, columns=READ_COLUMNS, patterns={}) + require( + routing.coverage._identity(path.stat(follow_symlinks=False)) == identity + and routing._file_sha(path) == digest, + "CAPTURE_CHANGED", + ) + return raw + + +def _compare_total(ready, positions, literals): + field = ready.field("DIV_VAL") + entry = amount_entries()["DIV_VAL"] + pairs = [ + routing.literal_code(t, range(entry[4] + 1), width=entry[0]) for t in literals + ] + require( + all(s in ("missing", "in_printed_range") for _, s in pairs), "TOTAL_LITERAL" + ) + valid = field.validity[positions] == 1 + require(np.array_equal(valid, [v is not None for v, _ in pairs]), "TOTAL_VALIDITY") + expected = np.array( + [np.nan if v is None else v for v, _ in pairs], dtype=np.float64 + ) + require( + np.array_equal( + field.amounts[positions][valid].view("uint64"), + expected[valid].view("uint64"), + ), + "TOTAL_BITS", + ) + return field + + +def qualify_current_asec_dividend(preparation): + """Borrow the original preparation, capture once, and requalify before return.""" + source = routing.source + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state, native = entry[2], entry[2].native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + _domain_agreement(ready) + header, native_document = json.loads(ready.header), json.loads(issued[1]) + require( + header["target_year"] == 2024 + and header["semantic"] == "annual_current_money" + and native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "PERIOD", + ) + pins = [p for p in routing.coverage._MEMBER_PINS if p[0] == 2024] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "MEMBER_BINDING", + ) + raw = _capture_member(state.root, pins[0]) + positions = np.flatnonzero(np.asarray(parent.scope.person_years) == 2024) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent.frame.person.iloc[positions][parent_name].to_numpy(), + ), + "PARENT_COORDINATE", + ) + field = _compare_total(ready, positions, ordered.DIV_VAL) + basis = project_dividend_literals(ordered) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + for name in ("statuses", "validity", "zero_origin"): + basis["DIV_VAL_parent_" + name] = getattr(field, name)[positions] + selected = state.frame.person.loc[ + state.frame.person[routing.support_channel_column("person")].eq("asec") + ] + native_ids = selected[routing.spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + literals = ordered.copy() + literals.index = basis.index.copy() + evidence = { + "protocol": PROTOCOL, + "dictionary_url": routing.DICTIONARY_URL, + "dictionary_sha256": routing.DICTIONARY_SHA256, + "amount_entries": amount_entries(), + "receipt_entries": RECEIPT_ENTRIES, + "survivor_entries": SURVIVOR_ENTRIES, + "survivor_codes": SURVIVOR_CODES, + "allocation_entries": ALLOCATION_ENTRIES, + "allocation_conflict_note": ALLOCATION_CONFLICT_NOTE, + "topcode_entries": TOPCODE_ENTRIES, + "preparation_sha256": routing._sha(entry[1]), + "asec_native_sha256": routing._sha(issued[1]), + "money_header_sha256": routing._sha(ready.header), + "source_member_sha256": digest, + "source_year": 2024, + "survey_year": 2025, + "complete_source_rows": rows, + "selected_rows": len(out), + "read_columns": READ_COLUMNS, + "source_flags_used_as_donor_filter": False, + "survivor_amounts_read": False, + "survivor_code8_is_possible_property_scope_only": True, + "survivor_code10_property_clearance_known": False, + "trnt_flag_universe_evaluated": False, + "tax_treatment_assigned": False, + "source_admission_issued": False, + "release_eligible": False, + } + # Freeze detached JSON-compatible metadata before its seal; do not retain the + # module's mutable constant dictionaries inside a returned descriptive view. + result = CurrentAsecDividendValues(out, literals, json.loads(json.dumps(evidence))) + seal = dividend_values_seal(result) + require( + preparation._checked() is entry and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require(dividend_values_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py new file mode 100644 index 000000000..29c406e65 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -0,0 +1,1493 @@ +"""Reporting/routing projection for five current ASEC original income families. + +Pension/annuity, retirement distributions, net property income, farm operations +and other income are the original-channel leaves the current survey graph does +not yet supply. This module qualifies the exact retained 2025 ASEC person member +against the authenticated current-money owner and projects the source literals +those families publish: totals, receipt universes, routing codes and allocation +provenance. + +Nothing here observes a taxable amount. The combined pension total is not split, +account identity is not a taxable fraction, the net property total is not +independently labelled rental, farm losses are not nonfarm self-employment, and +no residual other-income category rule is applied. Returned values are +descriptive: a consuming host requalifies the retained preparation and compares +these values before and after its own I/O. This module issues no source +admission and grants no release eligibility. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +from dataclasses import dataclass +from functools import lru_cache +from importlib import resources +from pathlib import Path +from typing import NamedTuple + +import numpy as np +import pandas as pd + +from . import asec_coverage_authentication as coverage +from . import asec_current_money as money +from . import source_csv_builtin +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-asec-income-routing-source.v1" +DICTIONARY_URL = ( + "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/" + "asec2025_ddl_pub_full.pdf" +) +DICTIONARY_SHA256 = "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f" + +COORDINATE_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE") +COORDINATE_WIDTHS = {"PH_SEQ": 5, "A_LINENO": 2, "A_AGE": 2} +TOKEN_MAX_CHARS = 64 + +# The nine printed money entries are already attested in the packaged current +# money domains artifact, so they are read from it rather than re-typed here. +DOMAINS_RESOURCE = "asec_current_money_domains_v1.json" +DOMAINS_SHA256 = money.RESOURCE_PINS[0] +CURRENT_INCOME_YEAR = 2024 +AMOUNT_FIELDS = ( + "PNSN_VAL", + "ANN_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "RNT_VAL", + "FRSE_VAL", + "OI_VAL", +) + + +@dataclass(frozen=True) +class PrintedAmountEntry: + """One money entry exactly as the pinned domains artifact records it.""" + + name: str + printed_length: int + printed_position: int + printed_page: str + pdf_page_1based: int + encoded_minimum: int + encoded_maximum: int + universe_as_printed: str + values_as_printed: str + negative_dollars_permitted: bool + nonmoney_codes: tuple[int, ...] + zero_semantics: str + + +@lru_cache(maxsize=1) +def _cached_printed_amount_entries(): + """Read the pinned domains artifact once, on demand, never at import.""" + payload = resources.files(__package__).joinpath(DOMAINS_RESOURCE).read_bytes() + require(_sha(payload) == DOMAINS_SHA256, "DOMAINS_RESOURCE_SHA256") + data = json.loads(payload) + entries = {} + for field in data["fields"]: + if field["name"] not in AMOUNT_FIELDS: + continue + require(field["name"] not in entries, "DOMAIN_FIELD_DUPLICATE:" + field["name"]) + domain = field["domain"] + vintages = [ + v for v in field["vintages"] if v["income_year"] == CURRENT_INCOME_YEAR + ] + require(len(vintages) == 1, "DOMAIN_VINTAGE:" + field["name"]) + vintage = vintages[0] + require( + vintage["dictionary_spelling"] == field["name"] + and vintage["pdf_sha256"] == DICTIONARY_SHA256 + and vintage["source_url"] == DICTIONARY_URL, + "DOMAIN_DICTIONARY_PIN:" + field["name"], + ) + entries[field["name"]] = PrintedAmountEntry( + name=field["name"], + printed_length=vintage["ascii_length_as_printed"], + printed_position=vintage["ascii_position_as_printed"], + printed_page=vintage["printed_page"], + pdf_page_1based=vintage["pdf_page_1based"], + encoded_minimum=domain["encoded_range_inclusive"]["minimum"], + encoded_maximum=domain["encoded_range_inclusive"]["maximum"], + universe_as_printed=vintage["universe_as_printed"], + values_as_printed=vintage["values_description"], + negative_dollars_permitted=domain["negative_dollars_permitted"], + nonmoney_codes=tuple(domain["declared_negative_nonmoney_codes"]), + zero_semantics=domain["zero_semantics"], + ) + require(set(entries) == set(AMOUNT_FIELDS), "DOMAIN_FIELD_ROSTER") + return tuple(entries.items()) + + +def printed_amount_entries(): + """Return a detached mapping over privately cached immutable entries.""" + return dict(_cached_printed_amount_entries()) + + +# Retain the existing test/revalidation cache-control surface. +printed_amount_entries.cache_clear = _cached_printed_amount_entries.cache_clear + + +def _require_nonnegative_amount_domain( + field, vintage, *, zero_semantics, valid_minimum +): + """Reject domains unsupported by the sibling unsigned-literal parsers. + + Amount bounds remain artifact-derived. The checks bind the exact semantics + assumed by those parsers rather than silently adapting a later artifact. + """ + name, domain = field["name"], field["domain"] + maximum = domain["encoded_range_inclusive"]["maximum"] + require( + field["entity"] == field["grain"] == "person" + and field["column"] == name + and type(maximum) is int + and maximum > 0 + and domain["encoded_range_inclusive"] == {"minimum": 0, "maximum": maximum} + and field["minimum"] == 0 + and field["maximum"] == maximum + and vintage["range_header"] == domain["encoded_range_inclusive"] + and domain["negative_dollars_permitted"] is False + and domain["declared_negative_nonmoney_codes"] == [] + and domain["declared_other_missing_codes"] == [] + and domain["declared_niu_codes"] == field["declared_niu_codes"] == [0] + and domain["valid_dollar_range_excludes"] == [] + and domain["valid_dollar_range_inclusive"] + == {"minimum": valid_minimum, "maximum": maximum} + and domain["numeric_type"] == "integer_US_dollars_in_public_use_dictionary" + and domain["zero_semantics"] == field["zero_semantics"] == zero_semantics, + "UNSUPPORTED_AMOUNT_DOMAIN:" + name, + ) + + +def _require_live_amount_domains(ready, entries): + """Bind source domain triples to a unique retained MoneyDomain roster.""" + fields = ready.bindings.spec.fields + domains = {d.name: d for d in fields} + require(len(domains) == len(fields), "MONEY_DOMAIN_DUPLICATE") + for name, (minimum, maximum, zero_semantics) in entries.items(): + domain = domains.get(name) + require(domain is not None, "MONEY_DOMAIN_MISSING:" + name) + require( + domain.entity == domain.grain == "person" + and domain.column == name + and domain.minimum == minimum + and domain.maximum == maximum + and domain.zero_semantics == zero_semantics, + "MONEY_DOMAIN_DISAGREEMENT:" + name, + ) + + +# Printed yes/no entries. The zero label is not shared: OI_YN prints +# "none or niu" where the others print "niu", so a zero receipt literal does not +# carry the same meaning across families. +class ReceiptEntry(NamedTuple): + printed_length: int + printed_position: int + pdf_page_1based: int + printed_page: str + universe_as_printed: str + zero_label_as_printed: str + + +RECEIPT_ENTRIES = { + "PEN_YN": ReceiptEntry(1, 570, 47, "6C-26", "All Persons aged 15+", "niu"), + "ANN_YN": ReceiptEntry(1, 444, 44, "6C-23", "All Persons aged 15+", "niu"), + "DST_YN": ReceiptEntry( + 1, + 519, + 46, + "6C-25", + "Persons aged 58 and over (a_age \u2265 58)", + "niu", + ), + "DST_YN_YNG": ReceiptEntry( + 1, 520, 46, "6C-25", "Persons under age 58 (a_age < 58)", "niu" + ), + "RNT_YN": ReceiptEntry(1, 627, 49, "6C-28", "All Persons aged 15+", "niu"), + "FRSE_YN": ReceiptEntry(1, 397, 43, "6C-22", "ERN_YN=1 or FRMOTR=1", "Niu"), + "ERN_YN": ReceiptEntry(1, 381, 43, "6C-22", "WORKYN=1 OR WTEMP=1", "niu"), + "FRMOTR": ReceiptEntry(1, 389, 43, "6C-22", "ERN_OTR = 1", "niu"), + "OI_YN": ReceiptEntry(1, 555, 47, "6C-26", "All Persons aged 15+", "none or niu"), +} +RECEIPT_CODE_DOMAIN = (0, 1, 2) + + +def receipt_codes(field): + """Printed yes/no labels for one entry; the zero label is not shared. + + OI_YN prints "none or niu" where the other entries print "niu"/"Niu", so a + single shared label would attribute a "respondent reported none" reading to + eight fields whose dictionary entry does not support it. + """ + return {0: RECEIPT_ENTRIES[field].zero_label_as_printed, 1: "yes", 2: "no"} + + +# Retirement account identity. Code 4 names a regular IRA; it does not observe +# any taxable fraction of the distribution. +ACCOUNT_CODES = { + 0: "NIU", + 1: "401k account", + 2: "403b account", + 3: "Roth IRA", + 4: "Regular IRA", + 5: "KEOGH plan", + 6: "SEP plan (Simplified Employee Pension)", + 7: "Other type of retirement account", +} +REGULAR_IRA_CODE = 4 + + +class AccountEntry(NamedTuple): + printed_length: int + printed_position: int + pdf_page_1based: int + printed_page: str + universe_as_printed: str + + +ACCOUNT_ENTRIES = { + "DST_SC1": AccountEntry(1, 491, 45, "6C-24", "DST_VAL1 > 0 and a_age \u2265 58"), + "DST_SC1_YNG": AccountEntry(1, 492, 45, "6C-24", "DST_YN_YNG = 1 and a_age < 58"), + "DST_SC2": AccountEntry(1, 493, 45, "6C-24", "DST_VAL2 > 0 and a_age \u2265 58"), + "DST_SC2_YNG": AccountEntry(1, 494, 45, "6C-24", "DST_VAL_YNG > 0 and a_age < 58"), +} + +# OI_OFF, verbatim. Code 20 is the reported alimony category. No residual rule +# maps any other code, including 19 "anything else", onto alimony. +OTHER_INCOME_CATEGORIES = { + 0: "niu", + 1: "social security", + 2: "private pensions", + 3: "afdc", + 4: "other public assistance", + 5: "interest", + 6: "dividends", + 7: "rents or royalties", + 8: "estates or trusts", + 9: "state disability payments (worker's comp)", + 10: "disability payments (own insurance)", + 11: "unemployment compensation", + 12: "strike benefits", + 13: "annuities or paid up insurance policies", + 14: "not income", + 15: "longest job", + 16: "wages or salary", + 17: "nonfarm self-employment", + 18: "farm self-employment", + 19: "anything else", + 20: "alimony", +} +ALIMONY_CATEGORY_CODE = 20 +OTHER_INCOME_CATEGORY_ENTRY = AccountEntry(2, 547, 47, "6C-26", "OI_YN = 1") + +# Published allocation flags. Values 0-9 follow I_ANNVAL; the DST composites +# follow I_INTYN (0, 10, 11); I_DSTSC prints its own 0/1/9 set. I_FRMYN prints +# an empty Values block, so only its (0:9) range header is published and the +# meaning of its codes is not; it is accepted on the printed range alone. +ALLOCATION_ANNVAL_CODES = tuple(range(10)) +ALLOCATION_COMPOSITE_CODES = (0, 10, 11) +ALLOCATION_DSTSC_CODES = (0, 1, 9) +# I_FRMYN's printed Values block is empty; its (0:9) range header is all that is +# published, so no code meaning is claimed for it. +ALLOCATION_PRINTED_RANGE_CODES = tuple(range(10)) +ALLOCATION_CODE_MEANINGS_UNPUBLISHED = frozenset({"I_FRMYN"}) + + +class AllocationEntry(NamedTuple): + printed_length: int + printed_position: int + pdf_page_1based: int + printed_page: str + universe_as_printed: str + codes: tuple + + +ALLOCATION_ENTRIES = { + "I_ANNVAL": AllocationEntry( + 1, 802, 53, "6C-32", "ANN_YN =1", ALLOCATION_ANNVAL_CODES + ), + "I_ANNYN": AllocationEntry( + 1, 803, 53, "6C-32", "ANN_YN > 0", ALLOCATION_ANNVAL_CODES + ), + "I_DSTSC": AllocationEntry( + 1, 821, 55, "6C-34", "DST_YN =1", ALLOCATION_DSTSC_CODES + ), + "I_DSTSCCOMP": AllocationEntry( + 1, + 822, + 55, + "6C-34", + "DST_YN = 1 or DST_YNG_YN = 1", + ALLOCATION_ANNVAL_CODES, + ), + "I_DSTVAL1COMP": AllocationEntry( + 2, 823, 55, "6C-34", "", ALLOCATION_COMPOSITE_CODES + ), + "I_DSTVAL2COMP": AllocationEntry( + 2, 825, 55, "6C-34", "DST_VAL2> 0", ALLOCATION_COMPOSITE_CODES + ), + "I_DSTYNCOMP": AllocationEntry( + 2, 827, 55, "6C-34", "DST_YN > 0", ALLOCATION_COMPOSITE_CODES + ), + "I_ERNYN": AllocationEntry( + 1, 833, 55, "6C-34", "ERN_YN > 0", ALLOCATION_ANNVAL_CODES + ), + "I_FRMYN": AllocationEntry( + 1, 837, 55, "6C-34", "FRMOTR > 0", ALLOCATION_PRINTED_RANGE_CODES + ), + "I_OIVAL": AllocationEntry( + 1, 843, 56, "6C-35", "OI_VAL > 0", ALLOCATION_ANNVAL_CODES + ), + "I_PENYN": AllocationEntry( + 1, 854, 57, "6C-36", "PEN_YN > 0", ALLOCATION_ANNVAL_CODES + ), + "I_RNTVAL": AllocationEntry( + 1, 861, 57, "6C-36", "RNT_VAL > 0", ALLOCATION_ANNVAL_CODES + ), + "I_RNTYN": AllocationEntry( + 1, 862, 57, "6C-36", "RNT_YN > 0", ALLOCATION_ANNVAL_CODES + ), +} + +# Which printed field each published flag names, and the fields for which the +# 2025 dictionary publishes no flag at all. A family holding an unflagged field +# can never report a clean "no allocation": that evidence is simply not printed. +PUBLISHED_ALLOCATION_FLAG_BY_FIELD = { + "ANN_VAL": "I_ANNVAL", + "ANN_YN": "I_ANNYN", + "PEN_YN": "I_PENYN", + "RNT_VAL": "I_RNTVAL", + "RNT_YN": "I_RNTYN", + "OI_VAL": "I_OIVAL", + "ERN_YN": "I_ERNYN", + "FRMOTR": "I_FRMYN", + "DST_SC1": "I_DSTSC", + "DST_SC2": "I_DSTSC", + "DST_VAL1": "I_DSTVAL1COMP", + "DST_VAL2": "I_DSTVAL2COMP", + "DST_YN": "I_DSTYNCOMP", +} +UNFLAGGED_FIELDS = ( + "PNSN_VAL", + "FRSE_VAL", + "FRSE_YN", + "OI_OFF", + "OI_YN", + "DST_VAL1_YNG", + "DST_VAL2_YNG", + "DST_YN_YNG", + "DST_SC1_YNG", + "DST_SC2_YNG", +) +# The DST_SC(2) notation is read here as "the two DST_SC slots", which is an +# inference from the printed parenthesis rather than a printed statement. +AMBIGUOUS_FLAG_COVERAGE = { + "I_DSTSC": "printed label names DST_SC(2); mapping it to both DST_SC1 and " + "DST_SC2 reads the parenthesis as a slot count, which the dictionary does " + "not state", + "I_DSTSCCOMP": "printed label names DST_SC(2) while its printed universe " + "names DST_YN = 1 or DST_YNG_YN = 1; whether it covers the under-58 source " + "codes is unresolved", + "I_FRMYN": "printed Values block is empty, so the meaning of its codes is " + "not published; only the (0:9) range header is", +} + +READ_COLUMNS = ( + *COORDINATE_COLUMNS, + *AMOUNT_FIELDS, + *RECEIPT_ENTRIES, + *ACCOUNT_ENTRIES, + "OI_OFF", + *ALLOCATION_ENTRIES, +) + +# Printed labels that stay separable from any modelled component. +PENSION_TOTAL_SCOPE = ( + "total combined amount of pension income received from all pension sources" +) +NET_PROPERTY_AMOUNT_SCOPE = "income from rent after expenses" +NET_PROPERTY_RECEIPT_SCOPE = ( + "own any land, property, rented to others, or receive income from royalties, " + "roomers or boarders, or from estates or trusts" +) +FARM_AMOUNT_SCOPE = ( + "total amount of farm self-employment earnings (combined amounts in " + "ERN_VAL, if ERN_SRCE=3, and FRM_VAL)" +) + +FAMILIES = ( + "pension_annuity", + "retirement_distribution", + "net_property", + "farm", + "other_income", +) + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_ASEC_INCOME_ROUTING_" + reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _file_sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + while chunk := handle.read(1_048_576): + digest.update(chunk) + return digest.hexdigest() + + +@lru_cache(maxsize=1) +def amount_patterns(): + """Token shapes from the printed width and the printed encoded minimum.""" + entries = printed_amount_entries() + return { + name: re.compile( + ("-?" if entry.encoded_minimum < 0 else "") + + r"[0-9]{1," + + str(entry.printed_length) + + r"}", + re.ASCII, + ) + for name, entry in entries.items() + } + + +def literal_code(token, allowed, *, width): + """Keep an unreadable literal unknown; never recode it to a printed value. + + Missing, malformed and out-of-range literals stay distinguishable from each + other and from a printed code, so a consumer cannot mistake an unresolved + routing answer for an observed one. A literal wider than the printed entry + is malformed, not a zero-padded reading of a shorter code. + """ + require(type(token) is str and len(token) <= TOKEN_MAX_CHARS, "TOKEN_BOUND") + if token == "": + return None, "missing" + if re.fullmatch(r"[0-9]{1," + str(width) + r"}", token, re.ASCII) is None: + return None, "malformed" + value = int(token) + if value not in allowed: + return None, "outside_printed_range" + return value, "in_printed_range" + + +def amount_state(value, status): + """Read the dollar meaning of one cell from the parent's own status axis. + + The authenticated money owner normalizes ANN_VAL's printed -1 to a stored + zero and records DECLARED_NIU. Re-deriving NIU from the stored number would + read that cell as a zero dollar annuity, so the status axis decides. + """ + status = int(status) + if status == money.CodebookStatus.MISSING_NULL: + require(np.isnan(value), "MISSING_STATUS_WITH_AMOUNT") + return "missing", np.nan + require(np.isfinite(value), "AMOUNT_NOT_FINITE") + if status == money.CodebookStatus.DECLARED_NIU: + return "declared_niu", np.nan + return ("zero" if value == 0 else "nonzero"), float(value) + + +# Statuses whose dollar reading is established by the source itself. Every other +# status leaves the canonical amount unknown rather than completing it with zero. +# A recipient zero qualifies only where the printed entry says the zero is valid +# dollars; every entry here except ANN_VAL prints "0 = none or niu", which the +# pinned domains artifact records as not distinguishable from an amount alone. +DOLLAR_ZERO_SEMANTICS = "valid_zero_dollars" +KNOWN_AMOUNT_STATUSES = ( + "known_receipt", + "known_recipient_zero", + "known_nonreceipt", +) + + +def receipt_status(universe, receipt, amount_kind, *, net_measure, zero_is_dollars): + """Classify one family row without inventing receipt, absence or dollars. + + `universe` is True, False, or None when the printed universe itself is not + resolved by the retained literals. + + A zero amount under a yes answer establishes a dollar reading only when the + printed entry says its zero is valid dollars. `net_measure` does not make a + zero known: it only separates a signed net entry's recipient zero, which a + consumer may want to treat differently, from a gross entry's. Both stay + unknown, because both print "0 = none or niu". + """ + code, status = receipt + if universe is None: + return "unresolved_reporting_universe" + if universe is False: + outside = code == 0 and amount_kind in ("zero", "declared_niu", "missing") + return ( + "outside_reporting_universe" + if outside + else "contradictory_outside_reporting_universe" + ) + if status == "missing": + return "missing_receipt_literal" + if status != "in_printed_range": + return "unrecognized_receipt_literal" + if amount_kind == "declared_niu": + return "niu" if code == 0 else "contradictory_declared_niu_amount" + if amount_kind == "missing": + return "missing_amount" + if code == 0: + return "niu" if amount_kind == "zero" else "contradictory_niu_nonzero" + if code == 1: + if amount_kind == "nonzero": + return "known_receipt" + if zero_is_dollars: + return "known_recipient_zero" + return "receipt_with_net_zero" if net_measure else "ambiguous_recipient_zero" + return "known_nonreceipt" if amount_kind == "zero" else "contradictory_no_nonzero" + + +def _classify(amounts, statuses, universes, receipts, *, net_measure, zero_is_dollars): + """Row-wise family classification returning labels and canonical amounts.""" + labels, canonical = [], np.full(len(amounts), np.nan, dtype=np.float64) + kinds, sources = [], np.full(len(amounts), np.nan, dtype=np.float64) + for i, value in enumerate(amounts): + kind, dollars = amount_state(value, statuses[i]) + kinds.append(kind) + sources[i] = dollars + label = receipt_status( + universes[i], + receipts[i], + kind, + net_measure=net_measure, + zero_is_dollars=zero_is_dollars, + ) + labels.append(label) + if label in KNOWN_AMOUNT_STATUSES: + canonical[i] = dollars if label == "known_receipt" else 0.0 + return labels, canonical, kinds, sources + + +def _zero_is_dollars(field): + """Whether the printed entry says its zero is valid dollars, not none/niu.""" + return printed_amount_entries()[field].zero_semantics == DOLLAR_ZERO_SEMANTICS + + +def _codes_frame(prefix, tokens, allowed, labels=None, *, width=1): + """Typed code/label/status columns for one printed routing literal.""" + pairs = [literal_code(t, allowed, width=width) for t in tokens] + frame = pd.DataFrame(index=range(len(tokens))) + frame[prefix + "_literal"] = pd.array(list(tokens), dtype="string") + frame[prefix + "_code"] = pd.array([c for c, _ in pairs], dtype="Int16") + frame[prefix + "_literal_status"] = pd.array([s for _, s in pairs], dtype="string") + if labels is not None: + frame[prefix + "_label"] = pd.array( + [labels[c] if c is not None else pd.NA for c, _ in pairs], dtype="string" + ) + return frame, [c for c, _ in pairs], [s for _, s in pairs] + + +def _age_universe(ages): + return [bool(a >= 15) for a in ages] + + +def _allocation_origin(flags, unflagged): + """Family allocation provenance from published flags only. + + Every published flag here prints a conditional universe (for example + I_RNTVAL is printed for RNT_VAL > 0), and those universes are not evaluated + here. An unpopulated literal is therefore not a defect, and an all-zero + reading is not an assertion of non-allocation: on a non-recipient row the + flag is outside its own printed universe. The zero readings are reported as + exactly that, never as publisher-confirmed absence of allocation. + A readable nonzero code whose meaning is unpublished cannot establish + allocation; a documented allocation flag can still establish it. + """ + origins = [] + count = len(next(iter(flags.values()))[0]) + for i in range(count): + codes = {f: flags[f][0][i] for f in flags} + statuses = [flags[f][1][i] for f in flags] + if any(s not in ("missing", "in_printed_range") for s in statuses): + origins.append("unresolved_allocation_provenance") + elif any( + c is not None and c != 0 + for f, c in codes.items() + if f not in ALLOCATION_CODE_MEANINGS_UNPUBLISHED + ): + origins.append("publisher_allocated") + elif any( + c is not None and c != 0 + for f, c in codes.items() + if f in ALLOCATION_CODE_MEANINGS_UNPUBLISHED + ): + origins.append("allocation_code_meaning_unpublished") + elif any(s == "missing" for s in statuses): + origins.append("allocation_flag_not_populated") + elif unflagged: + origins.append("published_flags_all_zero_with_unflagged_fields") + else: + origins.append("published_flags_all_zero") + return pd.array(origins, dtype="string") + + +def _pension_annuity(raw, ages): + """Separate pension and annuity totals; the combined total is never split.""" + out = pd.DataFrame(index=range(len(ages))) + universe = _age_universe(ages) + for prefix, amount_field, receipt_field in ( + ("pension", "PNSN_VAL", "PEN_YN"), + ("annuity", "ANN_VAL", "ANN_YN"), + ): + printed = receipt_codes(receipt_field) + frame, codes, statuses = _codes_frame( + "pension_annuity_" + prefix + "_receipt", + raw[receipt_field], + printed, + printed, + ) + labels, canonical, kinds, sources = _classify( + raw["amounts"][amount_field], + raw["statuses"][amount_field], + universe, + list(zip(codes, statuses, strict=True)), + net_measure=False, + zero_is_dollars=_zero_is_dollars(amount_field), + ) + out = pd.concat([out, frame], axis=1) + out["pension_annuity_" + prefix + "_source_total"] = sources + out["pension_annuity_" + prefix + "_amount_kind"] = pd.array( + kinds, dtype="string" + ) + out["pension_annuity_" + prefix + "_reporting_status"] = pd.array( + labels, dtype="string" + ) + out["pension_annuity_" + prefix + "_known_amount"] = canonical + out["pension_annuity_source_reporting_universe"] = pd.array( + universe, dtype="boolean" + ) + out["pension_annuity_combined_total_scope"] = pd.array( + [PENSION_TOTAL_SCOPE] * len(ages), dtype="string" + ) + out["pension_annuity_private_share_applied"] = pd.array( + [False] * len(ages), dtype="boolean" + ) + out["pension_annuity_taxable_amount_known"] = pd.array( + [False] * len(ages), dtype="boolean" + ) + out["pension_annuity_allocation_origin"] = _allocation_origin( + {f: raw["allocations"][f] for f in ("I_PENYN", "I_ANNVAL", "I_ANNYN")}, + unflagged=True, + ) + out["pension_annuity_pension_total_has_published_flag"] = pd.array( + [False] * len(ages), dtype="boolean" + ) + return out + + +def _retirement_distribution(raw, ages): + """Preserve every printed slot, the age route and the unresolved tax share.""" + rows = len(ages) + out = pd.DataFrame(index=range(rows)) + route = ["age58_and_over" if a >= 58 else "under_age58" for a in ages] + out["retirement_distribution_route"] = pd.array(route, dtype="string") + slots = ( + ("slot1", "DST_SC1", "DST_VAL1", "age58_and_over"), + ("slot2", "DST_SC2", "DST_VAL2", "age58_and_over"), + ("slot1_young", "DST_SC1_YNG", "DST_VAL1_YNG", "under_age58"), + ("slot2_young", "DST_SC2_YNG", "DST_VAL2_YNG", "under_age58"), + ) + slot_codes, slot_amounts, slot_kinds, slot_status = {}, {}, {}, {} + for name, code_field, amount_field, slot_route in slots: + prefix = "retirement_distribution_" + name + "_account" + frame, codes, statuses = _codes_frame( + prefix, raw[code_field], ACCOUNT_CODES, ACCOUNT_CODES + ) + out = pd.concat([out, frame], axis=1) + values = raw["amounts"][amount_field] + slot_statuses = raw["statuses"][amount_field] + kinds = [amount_state(v, slot_statuses[i])[0] for i, v in enumerate(values)] + out["retirement_distribution_" + name + "_amount"] = values + out["retirement_distribution_" + name + "_applicable"] = pd.array( + [r == slot_route for r in route], dtype="boolean" + ) + statuses_out = [] + for i in range(rows): + if route[i] != slot_route: + statuses_out.append( + "off_route" + if kinds[i] in ("zero", "missing") + else "off_route_nonzero" + ) + elif codes[i] is None: + statuses_out.append("unresolved_slot_account") + elif kinds[i] == "missing": + statuses_out.append("missing_slot_amount") + elif codes[i] == 0: + statuses_out.append( + "niu_slot" + if kinds[i] == "zero" + else "contradictory_niu_slot_amount" + ) + else: + statuses_out.append( + "known_slot" if kinds[i] == "nonzero" else "ambiguous_slot_zero" + ) + out["retirement_distribution_" + name + "_slot_status"] = pd.array( + statuses_out, dtype="string" + ) + slot_codes[name] = codes + slot_amounts[name] = values + slot_kinds[name] = kinds + slot_status[name] = statuses_out + applicable = { + "age58_and_over": ("slot1", "slot2"), + "under_age58": ("slot1_young", "slot2_young"), + } + totals = np.full(rows, np.nan, dtype=np.float64) + total_status = np.full(rows, int(money.CodebookStatus.AMOUNT_NONZERO), dtype="u1") + ira = np.full(rows, np.nan, dtype=np.float64) + ira_slots, ambiguity, offroute = [], [], [] + for i in range(rows): + names = applicable[route[i]] + others = [n for n, *_ in slots if n not in names] + offroute.append(any(slot_kinds[n][i] == "nonzero" for n in others)) + if all(slot_kinds[n][i] != "missing" for n in names): + totals[i] = float(sum(slot_amounts[n][i] for n in names)) + if totals[i] == 0: + total_status[i] = int(money.CodebookStatus.ZERO_NONE_OR_NIU) + ambiguity.append( + any( + slot_codes[n][i] not in (None, 0) and slot_kinds[n][i] == "zero" + for n in names + ) + ) + # Both axes must be resolved: an unreadable account code could itself be + # a regular IRA, and a declared account whose amount is a "none or niu" + # zero does not observe a zero dollar distribution from that account. + if all(slot_status[n][i] in ("niu_slot", "known_slot") for n in names): + matched = [n for n in names if slot_codes[n][i] == REGULAR_IRA_CODE] + ira[i] = float(sum(slot_amounts[n][i] for n in matched)) + ira_slots.append(len(matched)) + else: + ira_slots.append(None) + # Both printed recipiency literals are retained. The route selects which one + # applies; the other stays inspectable, because an answered off-route + # recipiency is a source contradiction rather than a missing answer. + route_codes = {} + for suffix, field in (("58", "DST_YN"), ("young", "DST_YN_YNG")): + frame, codes, statuses = _codes_frame( + "retirement_distribution_receipt_" + suffix, + raw[field], + receipt_codes(field), + receipt_codes(field), + ) + out = pd.concat([out, frame], axis=1) + route_codes[field] = (codes, statuses) + offroute_receipt = [ + ( + route_codes["DST_YN_YNG" if route[i] == "age58_and_over" else "DST_YN"][0][ + i + ] + not in (0, None) + ) + for i in range(rows) + ] + out["retirement_distribution_offroute_receipt"] = pd.array( + offroute_receipt, dtype="boolean" + ) + receipt_tokens = [ + raw["DST_YN"][i] if route[i] == "age58_and_over" else raw["DST_YN_YNG"][i] + for i in range(rows) + ] + frame, codes, statuses = _codes_frame( + "retirement_distribution_receipt", + receipt_tokens, + receipt_codes("DST_YN"), + receipt_codes("DST_YN"), + ) + out = pd.concat([out, frame], axis=1) + # The two printed DST universes name only the age-58 split. Unlike the four + # age-universe families here (PEN_YN, ANN_YN, RNT_YN, OI_YN) they print no + # 15+ floor, and unlike the farm family they are not gated on other + # literals either, so whether a person under 15 is inside them is a source + # question left unresolved. + universes = [True if a >= 15 else None for a in ages] + labels, canonical, _, _ = _classify( + totals, + np.where(np.isnan(totals), money.CodebookStatus.MISSING_NULL, total_status), + universes, + list(zip(codes, statuses, strict=True)), + net_measure=False, + zero_is_dollars=_zero_is_dollars("DST_VAL1"), + ) + # An answered off-route recipiency or off-route dollars contradict the route + # this row is on, and an applicable slot whose declared account carries a + # "none or niu" zero leaves the composition unresolved. Neither may end in a + # known amount, however the route's own receipt literal reads. + for i in range(rows): + if offroute[i] or offroute_receipt[i]: + labels[i] = "contradictory_offroute_evidence" + canonical[i] = np.nan + elif ambiguity[i]: + labels[i] = "unresolved_slot_composition" + canonical[i] = np.nan + out["retirement_distribution_source_total"] = totals + out["retirement_distribution_reporting_status"] = pd.array(labels, dtype="string") + out["retirement_distribution_known_amount"] = canonical + out["retirement_distribution_regular_ira_amount"] = ira + out["retirement_distribution_regular_ira_slots"] = pd.array(ira_slots, dtype="Int8") + out["retirement_distribution_slot_zero_ambiguity"] = pd.array( + ambiguity, dtype="boolean" + ) + out["retirement_distribution_offroute_nonzero"] = pd.array( + offroute, dtype="boolean" + ) + out["retirement_distribution_source_reporting_universe"] = pd.array( + universes, dtype="boolean" + ) + out["retirement_distribution_taxable_amount_known"] = pd.array( + [False] * rows, dtype="boolean" + ) + out["retirement_distribution_allocation_origin"] = _allocation_origin( + { + f: raw["allocations"][f] + for f in ( + "I_DSTSC", + "I_DSTSCCOMP", + "I_DSTVAL1COMP", + "I_DSTVAL2COMP", + "I_DSTYNCOMP", + ) + }, + unflagged=True, + ) + out["retirement_distribution_route_has_published_flag"] = pd.array( + [r == "age58_and_over" for r in route], dtype="boolean" + ) + return out + + +def _net_property(raw, ages): + """Signed net property total; receipt scope is wider than the amount scope.""" + rows = len(ages) + universe = _age_universe(ages) + frame, codes, statuses = _codes_frame( + "net_property_receipt", + raw["RNT_YN"], + receipt_codes("RNT_YN"), + receipt_codes("RNT_YN"), + ) + labels, canonical, kinds, sources = _classify( + raw["amounts"]["RNT_VAL"], + raw["statuses"]["RNT_VAL"], + universe, + list(zip(codes, statuses, strict=True)), + net_measure=True, + zero_is_dollars=_zero_is_dollars("RNT_VAL"), + ) + out = frame + out["net_property_source_total"] = sources + out["net_property_amount_kind"] = pd.array(kinds, dtype="string") + out["net_property_reporting_status"] = pd.array(labels, dtype="string") + out["net_property_known_amount"] = canonical + out["net_property_is_net_loss"] = pd.array( + [bool(np.isfinite(v) and v < 0) for v in sources], dtype="boolean" + ) + out["net_property_source_reporting_universe"] = pd.array(universe, dtype="boolean") + out["net_property_amount_scope"] = pd.array( + [NET_PROPERTY_AMOUNT_SCOPE] * rows, dtype="string" + ) + out["net_property_receipt_scope"] = pd.array( + [NET_PROPERTY_RECEIPT_SCOPE] * rows, dtype="string" + ) + out["net_property_component_split_known"] = pd.array( + [False] * rows, dtype="boolean" + ) + out["net_property_allocation_origin"] = _allocation_origin( + {f: raw["allocations"][f] for f in ("I_RNTVAL", "I_RNTYN")}, unflagged=False + ) + return out + + +def _farm(raw, ages): + """Farm universe comes from ERN_YN/FRMOTR evidence, never from age alone.""" + rows = len(ages) + out = pd.DataFrame(index=range(rows)) + universe_codes = {} + for field in ("ERN_YN", "FRMOTR"): + frame, codes, _ = _codes_frame( + "farm_" + field.lower(), + raw[field], + receipt_codes(field), + receipt_codes(field), + ) + out = pd.concat([out, frame], axis=1) + universe_codes[field] = codes + universe = [] + for i in range(rows): + pair = (universe_codes["ERN_YN"][i], universe_codes["FRMOTR"][i]) + if 1 in pair: + universe.append(True) + elif all(c is not None for c in pair): + universe.append(False) + else: + universe.append(None) + frame, codes, statuses = _codes_frame( + "farm_receipt", + raw["FRSE_YN"], + receipt_codes("FRSE_YN"), + receipt_codes("FRSE_YN"), + ) + out = pd.concat([out, frame], axis=1) + labels, canonical, kinds, sources = _classify( + raw["amounts"]["FRSE_VAL"], + raw["statuses"]["FRSE_VAL"], + universe, + list(zip(codes, statuses, strict=True)), + net_measure=True, + zero_is_dollars=_zero_is_dollars("FRSE_VAL"), + ) + out["farm_source_total"] = sources + out["farm_amount_kind"] = pd.array(kinds, dtype="string") + out["farm_reporting_status"] = pd.array(labels, dtype="string") + out["farm_known_amount"] = canonical + out["farm_is_net_loss"] = pd.array( + [bool(np.isfinite(v) and v < 0) for v in sources], dtype="boolean" + ) + out["farm_source_reporting_universe"] = pd.array(universe, dtype="boolean") + out["farm_amount_scope"] = pd.array([FARM_AMOUNT_SCOPE] * rows, dtype="string") + out["farm_is_nonfarm_self_employment"] = pd.array([False] * rows, dtype="boolean") + out["farm_allocation_origin"] = _allocation_origin( + {f: raw["allocations"][f] for f in ("I_ERNYN", "I_FRMYN")}, unflagged=True + ) + out["farm_total_has_published_flag"] = pd.array([False] * rows, dtype="boolean") + return out + + +def _other_income(raw, ages): + """Reported category 20 is observed alimony; no residual rule creates one.""" + rows = len(ages) + universe = _age_universe(ages) + frame, codes, statuses = _codes_frame( + "other_income_receipt", + raw["OI_YN"], + receipt_codes("OI_YN"), + receipt_codes("OI_YN"), + ) + category, category_codes, category_statuses = _codes_frame( + "other_income_category", + raw["OI_OFF"], + OTHER_INCOME_CATEGORIES, + OTHER_INCOME_CATEGORIES, + width=OTHER_INCOME_CATEGORY_ENTRY.printed_length, + ) + labels, canonical, kinds, sources = _classify( + raw["amounts"]["OI_VAL"], + raw["statuses"]["OI_VAL"], + universe, + list(zip(codes, statuses, strict=True)), + net_measure=False, + zero_is_dollars=_zero_is_dollars("OI_VAL"), + ) + out = pd.concat([frame, category], axis=1) + out["other_income_source_total"] = sources + out["other_income_amount_kind"] = pd.array(kinds, dtype="string") + out["other_income_reporting_status"] = pd.array(labels, dtype="string") + out["other_income_known_amount"] = canonical + out["other_income_source_reporting_universe"] = pd.array(universe, dtype="boolean") + routing = [] + for i in range(rows): + code, status = category_codes[i], category_statuses[i] + # OI_OFF is printed for OI_YN = 1, and OI_YN for persons aged 15+, so a + # row outside that universe carries no in-universe category either. + if universe[i] is None: + routing.append("unresolved_reporting_universe_routing") + elif universe[i] is False: + routing.append("outside_reporting_universe_routing") + elif status == "missing": + routing.append("missing_category_literal") + elif code is None: + routing.append("unrecognized_category_literal") + elif codes[i] is None: + # The category is printed but the receipt literal cannot be read, so + # the pair is unresolved and the category is not a reported receipt. + routing.append("unresolved_receipt_routing") + elif codes[i] == 1 and code == 0: + routing.append("receipt_without_category") + elif codes[i] in (0, 2) and code != 0: + routing.append("category_without_receipt") + elif code == 0: + routing.append("niu_category") + else: + routing.append("reported_category") + out["other_income_routing_status"] = pd.array(routing, dtype="string") + out["other_income_is_reported_alimony"] = pd.array( + [ + bool(c == ALIMONY_CATEGORY_CODE and r == "reported_category") + for c, r in zip(category_codes, routing, strict=True) + ], + dtype="boolean", + ) + out["other_income_residual_rule_applied"] = pd.array( + [False] * rows, dtype="boolean" + ) + out["other_income_allocation_origin"] = _allocation_origin( + {"I_OIVAL": raw["allocations"]["I_OIVAL"]}, unflagged=True + ) + out["other_income_category_has_published_flag"] = pd.array( + [False] * rows, dtype="boolean" + ) + return out + + +def _read_capture(path, *, rows, columns=None, patterns=None): + """Bounded literal reader; its path and row count establish no authority. + + Private source extensions may choose an explicit literal roster and amount + grammars. The original routing qualifier retains its unchanged defaults. + """ + columns = READ_COLUMNS if columns is None else tuple(columns) + patterns = amount_patterns() if patterns is None else dict(patterns) + require( + len(set(columns)) == len(columns) + and set(COORDINATE_COLUMNS) <= set(columns) + and set(patterns) <= set(columns), + "READ_COLUMN_CONTRACT", + ) + reader = source_csv_builtin.capture_csv_reader(csv) + require(reader is not None, "CSV_READER_CHANGED") + records, keys, coordinates = [], set(), set() + with Path(path).open("r", encoding="utf-8-sig", newline="") as handle: + stream = reader(handle, strict=True) + header = next(stream, []) + require( + bool(header) + and all(header) + and len(header) == len(set(header)) + and set(columns) <= set(header), + "HEADER", + ) + positions = [header.index(c) for c in columns] + for row in stream: + require(len(row) == len(header) and len(records) < rows, "ROW_SHAPE") + record = dict(zip(columns, (row[i] for i in positions), strict=True)) + key = record["PERIDNUM"] + require(re.fullmatch(r"[0-9]{22}", key, re.ASCII) is not None, "PERSON_KEY") + for name, width in COORDINATE_WIDTHS.items(): + require( + re.fullmatch( + r"[0-9]{1," + str(width) + r"}", record[name], re.ASCII + ) + is not None, + "COORDINATE:" + name, + ) + pair = (int(record["PH_SEQ"]), int(record["A_LINENO"])) + require( + min(pair) > 0 and pair not in coordinates and key not in keys, + "DUPLICATE_OR_INVALID_COORDINATE", + ) + for name, pattern in patterns.items(): + require( + record[name] == "" or pattern.fullmatch(record[name]) is not None, + "AMOUNT_TOKEN:" + name, + ) + for name in columns: + if name in COORDINATE_COLUMNS or name in patterns: + continue + require(len(record[name]) <= TOKEN_MAX_CHARS, "TOKEN_BOUND:" + name) + records.append(record) + keys.add(key) + coordinates.add(pair) + require(len(records) == rows, "ROW_COUNT") + return pd.DataFrame(records, columns=columns).set_index("PERIDNUM", drop=False) + + +def amount_observations(field, positions, literals): + """Join literal validity; missing backing storage never becomes a zero. + + The retained parent requires complete ready money before issuing a + preparation. This keeps that prerequisite separate from the literal join so + a missing field would still be represented correctly if that scope grows. + """ + require(type(field) is money.MoneyField, "MONEY_FIELD_TYPE") + require(field.name in AMOUNT_FIELDS, "MONEY_FIELD_NAME") + positions = np.asarray(positions) + require( + positions.dtype == np.dtype("int64") and positions.ndim == 1, "MONEY_POSITIONS" + ) + tokens = tuple(literals) + entry = printed_amount_entries()[field.name] + pattern = amount_patterns()[field.name] + require( + len(tokens) == len(positions) + and all(type(t) is str and (t == "" or pattern.fullmatch(t)) for t in tokens), + "AMOUNT_TOKEN", + ) + valid = field.validity[positions] == 1 + require( + np.array_equal(valid, np.array([t != "" for t in tokens])), + "CURRENT_AMOUNT_SOURCE_VALIDITY", + ) + amounts = field.amounts[positions].copy() + statuses = field.statuses[positions].copy() + literal_values = np.asarray([float(t) if t else np.nan for t in tokens]) + minimum, maximum = entry.encoded_minimum, entry.encoded_maximum + niu_codes = entry.nonmoney_codes + niu = statuses == money.CodebookStatus.DECLARED_NIU + require(not (niu & ~valid).any(), "NIU_WITHOUT_VALIDITY") + # A printed NIU code is stored as a normalized zero under DECLARED_NIU. Its + # literal must still be one of that entry's printed non-dollar codes. + require( + np.array_equal(amounts[valid & niu], np.zeros(int((valid & niu).sum()))) + and all(v in niu_codes for v in literal_values[valid & niu]), + "CURRENT_AMOUNT_NIU_IDENTITY", + ) + direct = valid & ~niu + require( + np.array_equal(amounts[direct], literal_values[direct]), + "CURRENT_AMOUNT_SOURCE_IDENTITY", + ) + amounts[~valid] = np.nan + inside = amounts[direct] + require( + ((inside >= minimum) & (inside <= maximum)).all() + and (inside == np.floor(inside)).all(), + "AMOUNT_PRINTED_DOMAIN:" + field.name, + ) + return amounts, statuses + + +def _domain_agreement(ready): + """Bind printed entries to the live current-money domain contract.""" + _require_live_amount_domains( + ready, + { + name: (entry.encoded_minimum, entry.encoded_maximum, entry.zero_semantics) + for name, entry in printed_amount_entries().items() + }, + ) + + +@dataclass(frozen=True) +class CurrentAsecIncomeRoutingValues: + """Descriptive transport. Constructing this grants no source authority.""" + + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def project_income_routing(raw, ages): + """Pure per-family projection over already-validated literal arrays.""" + ages = np.asarray(ages, dtype=np.float64) + require( + np.isfinite(ages).all() + and ((ages >= 0) & (ages <= 99) & (ages == np.floor(ages))).all(), + "AGE_DOMAIN", + ) + require( + set(raw["amounts"]) == set(AMOUNT_FIELDS) + and set(raw["statuses"]) == set(AMOUNT_FIELDS), + "AMOUNT_ARRAYS", + ) + require( + set(raw["allocations"]) == set(ALLOCATION_ENTRIES) + and all( + len(v) == 2 and len(v[0]) == len(v[1]) == len(ages) + for v in raw["allocations"].values() + ), + "ALLOCATION_ARRAYS", + ) + for name, values in raw["amounts"].items(): + values = np.asarray(values) + statuses = np.asarray(raw["statuses"][name]) + require( + values.dtype == np.dtype("float64") + and values.ndim == 1 + and len(values) == len(ages) + and not np.isinf(values).any() + and statuses.dtype == np.dtype("u1") + and len(statuses) == len(ages) + and np.isin(statuses, [int(c) for c in money.CodebookStatus]).all(), + "AMOUNT_ARRAY_CONTRACT:" + name, + ) + for name in (*RECEIPT_ENTRIES, *ACCOUNT_ENTRIES, "OI_OFF"): + tokens = raw.get(name) + require( + type(tokens) is list + and len(tokens) == len(ages) + and all(type(t) is str and len(t) <= TOKEN_MAX_CHARS for t in tokens), + "ROUTING_TOKEN_CONTRACT:" + name, + ) + parts = [ + _pension_annuity(raw, ages), + _retirement_distribution(raw, ages), + _net_property(raw, ages), + _farm(raw, ages), + _other_income(raw, ages), + ] + out = pd.concat([p.reset_index(drop=True) for p in parts], axis=1) + out.insert(0, "source_age", ages) + require(len(set(out.columns)) == len(out.columns), "PROJECTION_COLUMN_COLLISION") + return out + + +def _raw_arrays(ordered, ready, positions): + allocations = {} + for name, entry in ALLOCATION_ENTRIES.items(): + pairs = [ + literal_code(t, entry.codes, width=entry.printed_length) + for t in ordered[name].tolist() + ] + allocations[name] = ( + [c for c, _ in pairs], + [s for _, s in pairs], + ) + observed = { + name: amount_observations(ready.field(name), positions, ordered[name]) + for name in AMOUNT_FIELDS + } + raw = { + "amounts": {k: v[0] for k, v in observed.items()}, + "statuses": {k: v[1] for k, v in observed.items()}, + "allocations": allocations, + } + for name in (*RECEIPT_ENTRIES, *ACCOUNT_ENTRIES, "OI_OFF"): + raw[name] = ordered[name].tolist() + return raw + + +def qualify_current_asec_income_routing(preparation): + """Capture the exact current source member retained by the original owner.""" + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "MONEY_PERIOD", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + _domain_agreement(ready) + pins = [p for p in coverage._MEMBER_PINS if p[0] == 2024] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and retained[0]["member"] == member + and retained[0]["archive_sha256"] == archive + and retained[0]["member_sha256"] == digest + and retained[0]["rows"] == rows + and retained[0]["member_bytes"] == size, + "NATIVE_MEMBER_BINDING", + ) + with tempfile.TemporaryDirectory(prefix="microcosm-current-income-routing-") as tmp: + captured = Path(tmp) / member + identity = coverage._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + budget=[coverage._BODY_MAX], + ) + raw_member = _read_capture(captured, rows=rows) + require( + coverage._identity(captured.stat(follow_symlinks=False)) == identity + and _file_sha(captured) == digest, + "CAPTURE_CHANGED", + ) + positions = np.flatnonzero(np.asarray(parent.scope.person_years) == 2024) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows + and len(set(keys)) == rows + and set(keys) == set(raw_member.index), + "COMPLETE_CURRENT_SOURCE_JOIN", + ) + ordered = raw_member.loc[keys] + parent_people = parent.frame.person.iloc[positions] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent_people[parent_name].to_numpy(), + ), + "PARENT_COORDINATE_IDENTITY", + ) + ages = ordered.A_AGE.to_numpy(dtype="float64") + basis = project_income_routing(_raw_arrays(ordered, ready, positions), ages) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + for name in AMOUNT_FIELDS: + field = ready.field(name) + basis["amount_status_" + name] = field.statuses[positions] + basis["amount_validity_" + name] = field.validity[positions] + basis["zero_origin_" + name] = field.zero_origin[positions] + people = state.frame.person + channel = people[support_channel_column("person")] + selected = people.loc[channel.eq("asec")] + native_ids = selected[spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + literals = ordered.copy() + literals.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + evidence = { + "protocol": PROTOCOL, + "dictionary": { + "url": DICTIONARY_URL, + "sha256": DICTIONARY_SHA256, + "amount_entries_source": { + "resource": DOMAINS_RESOURCE, + "sha256": DOMAINS_SHA256, + "income_year": CURRENT_INCOME_YEAR, + }, + "amount_entries": { + name: { + "printed_length": e.printed_length, + "printed_position": e.printed_position, + "printed_page": e.printed_page, + "pdf_page_1based": e.pdf_page_1based, + "encoded_range_inclusive": [e.encoded_minimum, e.encoded_maximum], + "universe_as_printed": e.universe_as_printed, + "values_as_printed": e.values_as_printed, + "negative_dollars_permitted": e.negative_dollars_permitted, + "declared_negative_nonmoney_codes": list(e.nonmoney_codes), + "zero_semantics": e.zero_semantics, + } + for name, e in printed_amount_entries().items() + }, + "receipt_entries": {k: v._asdict() for k, v in RECEIPT_ENTRIES.items()}, + "account_entries": {k: v._asdict() for k, v in ACCOUNT_ENTRIES.items()}, + "other_income_category_entry": OTHER_INCOME_CATEGORY_ENTRY._asdict(), + "allocation_entries": { + k: {**v._asdict(), "codes": list(v.codes)} + for k, v in ALLOCATION_ENTRIES.items() + }, + "fields_without_published_allocation_flag": list(UNFLAGGED_FIELDS), + "published_allocation_flag_by_field": dict( + PUBLISHED_ALLOCATION_FLAG_BY_FIELD + ), + "ambiguous_allocation_flag_coverage": dict(AMBIGUOUS_FLAG_COVERAGE), + "printed_universe_questions": { + "DST_VAL1": "printed universe is DST_SC1 = 1 although the label names " + "the source-1 distribution amount; retained verbatim", + "DST_SC2_YNG": "printed universe names DST_VAL_YNG, which has no " + "dictionary entry; retained verbatim", + "I_DSTVAL1COMP": "printed universe line is empty", + "DST_YN": "printed universes name only the a_age 58 split; they " + "print no 15+ floor as the four age-universe families here do, " + "and they are not gated on other literals as the farm family is, " + "so coverage below age 15 is unresolved", + }, + "printed_scope_and_code_questions": { + "RNT_YN": "printed question scope: " + NET_PROPERTY_RECEIPT_SCOPE, + "RNT_VAL": "printed question scope: " + NET_PROPERTY_AMOUNT_SCOPE, + "FRSE_VAL": "printed label: " + FARM_AMOUNT_SCOPE, + "PNSN_VAL": "printed label: " + PENSION_TOTAL_SCOPE, + "OI_YN": "printed zero label is 'none or niu' where PEN_YN, " + "ANN_YN, DST_YN and RNT_YN print 'niu'", + "DST_SC1": "gated on DST_VAL1 > 0 and a_age \u2265 58 while " + "DST_SC1_YNG is gated on DST_YN_YNG = 1 and a_age < 58; the two " + "routes are not symmetric", + "FRSE_YN": "printed universe is ERN_YN=1 or FRMOTR=1, so the farm " + "family prints no age floor at all", + }, + }, + "preparation_sha256": _sha(entry[1]), + "asec_native_sha256": _sha(issued[1]), + "money_header_sha256": _sha(ready.header), + "source_member_sha256": digest, + "read_columns": list(READ_COLUMNS), + "complete_current_source_rows": rows, + "declared_niu_normalized_to_zero": [ + name + for name, entry in printed_amount_entries().items() + if entry.nonmoney_codes + ], + "joined_person_years": [CURRENT_INCOME_YEAR], + "restatement_note": ( + "the money owner restates non-2024 cohorts to the pinned price basis, " + "so the literal identity join is only taken on the 2024 rows" + ), + "selected_rows": len(out), + "acs_channel_rows": int(channel.eq("acs").sum()), + "families": list(FAMILIES), + "asec_income_year": 2024, + "asec_interview_year": 2025, + "policy": ( + "separate_source_totals_retained; ambiguous_NIU_missing_contradictions_" + "retained_unknown; no_default_zero_completion" + ), + "pension_private_share_applied": False, + "pension_taxable_amount_known": False, + "retirement_distribution_taxable_amount_known": False, + "net_property_component_split_known": False, + "farm_mapped_to_nonfarm_self_employment": False, + "other_income_residual_rule_applied": False, + "acs_components_modeled": False, + "observed_taxable_amount_claim": False, + "unallocated_observation_claim": False, + "source_admission_issued": False, + "release_eligible": False, + "projection_sha256": _sha(out.to_json(orient="table").encode()), + "literals_sha256": _sha(literals.to_json(orient="table").encode()), + } + require( + preparation._checked()[1] == entry[1] and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require( + _sha(out.to_json(orient="table").encode()) == evidence["projection_sha256"] + and _sha(literals.to_json(orient="table").encode()) + == evidence["literals_sha256"], + "FINAL_PROJECTION_CHANGED", + ) + return CurrentAsecIncomeRoutingValues(out, literals, evidence) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py new file mode 100644 index 000000000..e1c42112a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py @@ -0,0 +1,492 @@ +"""Source observations of ordinary and retirement-account ASEC interest. + +No tax treatment, model, component balancing or source issuer lives here. +The consuming host retains and requalifies the actual preparation around I/O. +""" + +from __future__ import annotations + +import hashlib +import json +import tempfile +from dataclasses import dataclass +from functools import lru_cache +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as physical +from . import current_asec_income_routing_source as routing + +PROTOCOL = "microcosm.us.current-asec-interest-source.v1" +AMOUNT_FIELDS = ("INT_VAL", "TRDINT_VAL", "RINT_VAL1", "RINT_VAL2") +# Width, position, PDF page, printed page, maximum, universe, zero wording. +# INT_VAL is read from the already pinned current-money domain resource. +ADDITIONAL_AMOUNT_ENTRIES = { + "TRDINT_VAL": (5, 665, 50, "6C-29", 99999, "INT_YN = 1", "dollar value"), + "RINT_VAL1": (6, 608, 49, "6C-28", 999999, "RINT_SC1 > 0", "none or niu"), + "RINT_VAL2": (6, 614, 49, "6C-28", 999999, "RINT_SC2 > 0", "none or niu"), +} +RECEIPT_ENTRIES = { + "INT_YN": (1, 543, 46, "6C-25", "All Persons aged 15+", "niu"), + "RINT_YN": (1, 620, 49, "6C-28", "All Persons aged 15+", "niu"), +} +ACCOUNT_ENTRIES = { + "RINT_SC1": (1, 606, 49, "6C-28", "RINT_YN = 1"), + "RINT_SC2": (1, 607, 49, "6C-28", "RINT_YN = 1"), +} +# These published component and composite flags have different code systems. +ALLOCATION_ENTRIES = { + "I_INTVAL": (2, 838, 56, "6C-35", "INT_VAL> 0", (0, 11, 12, 13, 14, 15)), + "I_INTYN": (2, 840, 56, "6C-35", "INT_YN > 0", routing.ALLOCATION_COMPOSITE_CODES), + "I_RINTSC": (1, 857, 57, "6C-36", "RINT_SC1 > 0", routing.ALLOCATION_ANNVAL_CODES), + "I_RINTVAL1": ( + 1, + 858, + 57, + "6C-36", + "RINT_VAL1 > 0", + routing.ALLOCATION_ANNVAL_CODES, + ), + "I_RINTVAL2": ( + 1, + 859, + 57, + "6C-36", + "RINT_VAL2 > 0", + routing.ALLOCATION_ANNVAL_CODES, + ), + "I_RINTYN": (1, 860, 57, "6C-36", "RINT_YN > 0", routing.ALLOCATION_ANNVAL_CODES), +} +TOPCODE_ENTRIES = { + "TRINT_VAL1": (1, 915, 60, "6C-39", "RINT_VAL1 > 0"), + "TRINT_VAL2": (1, 916, 60, "6C-39", "RINT_VAL2 > 0"), + "TTRDINT_VAL": (1, 918, 60, "6C-39", "TRDINT_VAL > 0"), +} +READ_COLUMNS = ( + *routing.COORDINATE_COLUMNS, + *AMOUNT_FIELDS, + *RECEIPT_ENTRIES, + *ACCOUNT_ENTRIES, + *ALLOCATION_ENTRIES, + *TOPCODE_ENTRIES, +) + + +def require(condition, reason): + if not condition: + raise ValueError("ASEC_INTEREST_SOURCE_" + reason) + + +@lru_cache(maxsize=1) +def _cached_amount_entries(): + payload = ( + resources.files(__package__).joinpath(routing.DOMAINS_RESOURCE).read_bytes() + ) + require(routing._sha(payload) == routing.DOMAINS_SHA256, "DOMAINS_HASH") + fields = [f for f in json.loads(payload)["fields"] if f["name"] == "INT_VAL"] + require(len(fields) == 1, "DOMAIN_FIELD_ROSTER:INT_VAL") + field = fields[0] + vintages = [ + v for v in field["vintages"] if v["income_year"] == routing.CURRENT_INCOME_YEAR + ] + require(len(vintages) == 1, "DOMAIN_VINTAGE:INT_VAL") + vintage = vintages[0] + require( + vintage["dictionary_spelling"] == "INT_VAL" + and vintage["pdf_sha256"] == routing.DICTIONARY_SHA256 + and vintage["source_url"] == routing.DICTIONARY_URL, + "DICTIONARY_PIN", + ) + routing._require_nonnegative_amount_domain( + field, + vintage, + zero_semantics="none_or_niu_not_distinguishable_from_amount_alone", + valid_minimum=0, + ) + return tuple( + { + "INT_VAL": ( + vintage["ascii_length_as_printed"], + vintage["ascii_position_as_printed"], + vintage["pdf_page_1based"], + vintage["printed_page"], + field["domain"]["encoded_range_inclusive"]["maximum"], + vintage["universe_as_printed"], + field["domain"]["zero_semantics"], + ), + **ADDITIONAL_AMOUNT_ENTRIES, + }.items() + ) + + +def amount_entries(): + """Return a detached mapping over privately cached immutable tuples.""" + return dict(_cached_amount_entries()) + + +amount_entries.cache_clear = _cached_amount_entries.cache_clear + + +def _domain_agreement(ready): + entry = amount_entries()["INT_VAL"] + routing._require_live_amount_domains(ready, {"INT_VAL": (0, entry[4], entry[6])}) + + +def _slot_status(age, receipt, account, amount): + code, status = account + value, amount_status = amount + receipt_code, receipt_state = receipt + # Reuse the shared universe/receipt classifier before refining account slots. + kind = "missing" if value is None else ("zero" if value == 0 else "nonzero") + base = routing.receipt_status( + bool(age >= 15), receipt, kind, net_measure=False, zero_is_dollars=False + ) + if age < 15: + return base if code == 0 else "contradictory_outside_reporting_universe" + if receipt_state != "in_printed_range": + return base + if amount_status != "in_printed_range": + return ( + "missing_amount" if amount_status == "missing" else "invalid_amount_literal" + ) + if status != "in_printed_range": + return ( + "missing_account_literal" + if status == "missing" + else "invalid_account_literal" + ) + if receipt_code != 1: + return base if code == 0 else "account_without_receipt" + if code == 0: + return "unreported_account_slot" if value == 0 else "amount_without_account" + return base + + +def project_interest_literals(ordered): + """Descriptive projection; a DataFrame argument grants no source authority.""" + require( + type(ordered) is pd.DataFrame and set(READ_COLUMNS) <= set(ordered), "COLUMNS" + ) + out = pd.DataFrame(index=range(len(ordered))) + ages = ordered.A_AGE.to_numpy(dtype=np.int64) + require(((ages >= 0) & (ages <= 99)).all(), "AGE_RANGE") + parsed = {} + for name, entry in amount_entries().items(): + pairs = [ + routing.literal_code(t, range(entry[4] + 1), width=entry[0]) + for t in ordered[name] + ] + parsed[name] = pairs + out[name + "_published_amount"] = pd.array( + [v for v, _ in pairs], dtype="Float64" + ) + out[name + "_literal_status"] = pd.array([s for _, s in pairs], dtype="string") + codes = {} + for name in (*RECEIPT_ENTRIES, *ACCOUNT_ENTRIES): + allowed = ( + routing.RECEIPT_CODE_DOMAIN + if name in RECEIPT_ENTRIES + else routing.ACCOUNT_CODES + ) + labels = ( + {0: RECEIPT_ENTRIES[name][5], 1: "yes", 2: "no"} + if name in RECEIPT_ENTRIES + else routing.ACCOUNT_CODES + ) + frame, values, statuses = routing._codes_frame( + name, ordered[name], allowed, labels + ) + out = pd.concat([out, frame], axis=1) + codes[name] = list(zip(values, statuses, strict=True)) + for name in AMOUNT_FIELDS: + labels = [] + for i, age in enumerate(ages): + value, status = parsed[name][i] + if name.startswith("RINT_VAL"): + label = _slot_status( + age, + codes["RINT_YN"][i], + codes["RINT_SC" + name[-1]][i], + parsed[name][i], + ) + else: + kind = ( + "missing" + if value is None + else ("zero" if value == 0 else "nonzero") + ) + label = routing.receipt_status( + bool(age >= 15), + codes["INT_YN"][i], + kind, + net_measure=False, + zero_is_dollars=name == "TRDINT_VAL", + ) + if status not in ("missing", "in_printed_range"): + label = "invalid_amount_literal" + # Unlike INT_VAL, TRDINT_VAL prints 'dollar value', not 'none or niu'. + if name == "TRDINT_VAL" and label == "known_recipient_zero": + label = "observed_zero_component" + labels.append(label) + known = np.array( + [ + s in (*routing.KNOWN_AMOUNT_STATUSES, "observed_zero_component") + for s in labels + ] + ) + out[name + "_reporting_status"] = pd.array(labels, dtype="string") + out[name + "_amount_known"] = known + out[name + "_amount"] = out[name + "_published_amount"].where(known) + for name in ACCOUNT_ENTRIES: + out[name + "_account_known"] = [ + bool(age >= 15 and receipt[0] == 1 and code is not None and code > 0) + for age, receipt, (code, _) in zip( + ages, codes["RINT_YN"], codes[name], strict=True + ) + ] + flags = {} + for name, entry in ALLOCATION_ENTRIES.items(): + frame, values, statuses = routing._codes_frame( + name, ordered[name], entry[5], width=entry[0] + ) + out = pd.concat([out, frame], axis=1) + flags[name] = (values, statuses) + out["allocation_origin"] = routing._allocation_origin( + flags, ("TRDINT_VAL", "RINT_SC2") + ) + for name, entry in TOPCODE_ENTRIES.items(): + frame, _, _ = routing._codes_frame(name, ordered[name], (0, 1), width=entry[0]) + out = pd.concat([out, frame], axis=1) + published = [out[name + "_published_amount"] for name in AMOUNT_FIELDS] + out["published_components_sum"] = published[1] + published[2] + published[3] + out["combined_minus_published_components"] = ( + published[0] - out.published_components_sum + ) + out["component_discrepancy_known"] = out.combined_minus_published_components.notna() + out["all_component_amounts_observed"] = out[ + [name + "_amount_known" for name in AMOUNT_FIELDS[1:]] + ].all(axis=1) + out["source_age"] = ages + return out + + +@dataclass(frozen=True) +class CurrentAsecInterestValues: + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def interest_values_seal(values): + """Physical value seal, including nullable backing storage and exact bits.""" + require(type(values) is CurrentAsecInterestValues, "VALUES_TYPE") + digest = hashlib.sha256(PROTOCOL.encode()) + for table in (values.person, values.asec_literals): + require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE_TYPE") + digest.update( + physical._json( + { + "columns": list(table.columns), + "columns_axis": physical.checkpoint._index_spec( + table.columns, label="interest columns" + ), + "index": physical.checkpoint._index_spec( + table.index, label="interest" + ), + } + ) + ) + physical._series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + series = table[column] + if isinstance(series.dtype, pd.Float64Dtype): + # The checkpoint digest supports NumPy floats, but deliberately + # excludes Float64 extension arrays. Keep the nullable tag and + # hash both physical arrays, including values under null masks. + digest.update(physical._json({"dtype": "Float64", "nullable": True})) + physical._series_digest(digest, pd.Series(series.array._data)) + physical._series_digest(digest, pd.Series(series.array._mask)) + else: + physical._series_digest(digest, series) + digest.update(physical._json(values.evidence)) + return digest.hexdigest() + + +def _capture_member(root, pin): + _, member, _, digest, rows, size = pin + with tempfile.TemporaryDirectory(prefix="microcosm-asec-interest-") as tmp: + path = Path(tmp) / member + identity = routing.coverage._capture( + root / "asec" / member, + path, + size=size, + digest=digest, + budget=[routing.coverage._BODY_MAX], + ) + raw = routing._read_capture(path, rows=rows, columns=READ_COLUMNS, patterns={}) + require( + routing.coverage._identity(path.stat(follow_symlinks=False)) == identity + and routing._file_sha(path) == digest, + "CAPTURE_CHANGED", + ) + return raw + + +def _compare_total(ready, positions, literals): + field = ready.field("INT_VAL") + entry = amount_entries()["INT_VAL"] + pairs = [ + routing.literal_code(t, range(entry[4] + 1), width=entry[0]) for t in literals + ] + require( + all(s in ("missing", "in_printed_range") for _, s in pairs), "TOTAL_LITERAL" + ) + valid = field.validity[positions] == 1 + require(np.array_equal(valid, [v is not None for v, _ in pairs]), "TOTAL_VALIDITY") + expected = np.array( + [np.nan if v is None else v for v, _ in pairs], dtype=np.float64 + ) + require( + np.array_equal( + field.amounts[positions][valid].view("uint64"), + expected[valid].view("uint64"), + ), + "TOTAL_BITS", + ) + return field + + +def qualify_current_asec_interest(preparation): + """Borrow the original preparation, capture once, and requalify before return.""" + source = routing.source + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state, native = entry[2], entry[2].native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + _domain_agreement(ready) + header, native_document = json.loads(ready.header), json.loads(issued[1]) + require( + header["target_year"] == 2024 + and header["semantic"] == "annual_current_money" + and native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "PERIOD", + ) + pins = [p for p in routing.coverage._MEMBER_PINS if p[0] == 2024] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "MEMBER_BINDING", + ) + raw = _capture_member(state.root, pins[0]) + positions = np.flatnonzero(np.asarray(parent.scope.person_years) == 2024) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent.frame.person.iloc[positions][parent_name].to_numpy(), + ), + "PARENT_COORDINATE", + ) + field = _compare_total(ready, positions, ordered.INT_VAL) + basis = project_interest_literals(ordered) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + for name in ("statuses", "validity", "zero_origin"): + basis["INT_VAL_parent_" + name] = getattr(field, name)[positions] + selected = state.frame.person.loc[ + state.frame.person[routing.support_channel_column("person")].eq("asec") + ] + native_ids = selected[routing.spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + literals = ordered.copy() + literals.index = basis.index.copy() + evidence = { + "protocol": PROTOCOL, + "dictionary_url": routing.DICTIONARY_URL, + "dictionary_sha256": routing.DICTIONARY_SHA256, + "amount_entries": amount_entries(), + "receipt_entries": RECEIPT_ENTRIES, + "account_entries": ACCOUNT_ENTRIES, + "account_codes": routing.ACCOUNT_CODES, + "allocation_entries": ALLOCATION_ENTRIES, + "topcode_entries": TOPCODE_ENTRIES, + "fields_without_individual_allocation_flag": ["TRDINT_VAL", "RINT_SC2"], + "preparation_sha256": routing._sha(entry[1]), + "asec_native_sha256": routing._sha(issued[1]), + "money_header_sha256": routing._sha(ready.header), + "source_member_sha256": digest, + "source_year": 2024, + "survey_year": 2025, + "complete_source_rows": rows, + "selected_rows": len(out), + "read_columns": READ_COLUMNS, + "component_discrepancy_is_diagnostic_only": True, + "unreported_slot_imputed_zero": False, + "tax_treatment_assigned": False, + "source_admission_issued": False, + "release_eligible": False, + } + # Freeze detached JSON-compatible metadata before its seal; do not retain the + # module's mutable constant dictionaries inside a returned descriptive view. + result = CurrentAsecInterestValues(out, literals, json.loads(json.dumps(evidence))) + seal = interest_values_seal(result) + require( + preparation._checked() is entry and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require(interest_values_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py new file mode 100644 index 000000000..b04d0c95f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py @@ -0,0 +1,430 @@ +"""Pure, descriptive ASEC property donor basis; never a source authority. + +The caller retains the preparation and qualified-source owners around I/O. +These tables establish neither a source seal nor a tax treatment. Original +household design weights describe coverage; they do not alter observations. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from math import fsum + +import numpy as np +import pandas as pd + +from .property_income_constants import PROPERTY_COMPONENTS, PROPERTY_REPORTED_TOTAL + +# Modeling bridge exclusions, not alternative source codebooks. +OTHER_PROPERTY_CATEGORIES = frozenset({5, 6, 7, 8}) +OTHER_UNSPECIFIED_CATEGORY = 19 + + +@dataclass(frozen=True) +class AsecPropertyBasis: + """Detached descriptive outputs. Constructing or copying grants no authority.""" + + person: pd.DataFrame + provenance: pd.DataFrame + exclusions: pd.DataFrame + summary: pd.DataFrame + + +def _require(condition, reason): + if not condition: + raise ValueError("ASEC_PROPERTY_BASIS_" + reason) + + +def _axis(frame): + _require(type(frame) is pd.DataFrame, "TABLE") + _require( + frame.index.name == "person_id" + and frame.index.dtype == np.dtype("int64") + and frame.index.is_unique + and frame.columns.is_unique, + "PERSON_AXIS", + ) + _require({"native_person_id", "source_age"} <= set(frame), "IDENTITY_COLUMNS") + native = frame.native_person_id + _require( + native.dtype == np.dtype("int64") and native.is_unique, + "NATIVE_AXIS", + ) + age = _numeric(frame.source_age, "AGE") + _require( + np.isfinite(age).all() + and ((age >= 0) & (age <= 99) & (age == np.floor(age))).all(), + "AGE", + ) + return age + + +def _numeric(series, label): + _require( + pd.api.types.is_numeric_dtype(series.dtype) + and not pd.api.types.is_bool_dtype(series.dtype) + and not pd.api.types.is_complex_dtype(series.dtype), + "NUMERIC:" + label, + ) + values = series.to_numpy(dtype=np.float64, na_value=np.nan, copy=True) + _require(not np.isinf(values).any(), "NONFINITE:" + label) + return values + + +def _text(frame, name): + _require(name in frame, "COLUMN:" + name) + series = frame[name] + _require( + series.notna().all() and all(type(v) is str for v in series), + "STATUS:" + name, + ) + return series.to_numpy(copy=True) + + +def _code(frame, name): + _require(name + "_code" in frame, "COLUMN:" + name) + code = _numeric(frame[name + "_code"], name) + status = _text(frame, name + "_literal_status") + _require(((np.isnan(code)) | (code == np.floor(code))).all(), "CODE:" + name) + return code, status == "in_printed_range" + + +def _amount(frame, name, *, signed=False, ordinary=False): + for suffix in ("_amount", "_amount_known", "_reporting_status"): + _require(name + suffix in frame, "COLUMN:" + name + suffix) + amount = _numeric(frame[name + "_amount"], name) + known = frame[name + "_amount_known"] + _require( + pd.api.types.is_bool_dtype(known.dtype) and known.notna().all(), + "KNOWNNESS:" + name, + ) + known = known.to_numpy(dtype=bool, copy=True) + status = _text(frame, name + "_reporting_status") + allowed = np.isin(status, ["known_receipt", "known_nonreceipt"]) + if ordinary: + allowed |= status == "observed_zero_component" + _require( + np.array_equal(known, np.isfinite(amount)) + and np.array_equal(known, allowed) + and (signed or (amount[known] >= 0).all()) + and (amount[status == "known_nonreceipt"] == 0).all(), + "AMOUNT_CONTRACT:" + name, + ) + return amount + + +def _retirement(interest): + receipt, readable = _code(interest, "RINT_YN") + slots = [] + unused = [] + nonreceipt = [] + for n in (1, 2): + name = f"RINT_VAL{n}" + amount = _amount(interest, name) + code, valid = _code(interest, f"RINT_SC{n}") + published = _numeric(interest[name + "_published_amount"], name) + literal = _text(interest, name + "_literal_status") + status = _text(interest, name + "_reporting_status") + active = ( + valid + & (code >= 1) + & (code <= 7) + & (amount > 0) + & (status == "known_receipt") + ) + structural = ( + valid + & (code == 0) + & (published == 0) + & (literal == "in_printed_range") + & (status == "unreported_account_slot") + & np.isnan(amount) + ) + no = ( + valid + & (code == 0) + & (published == 0) + & (amount == 0) + & (literal == "in_printed_range") + & (status == "known_nonreceipt") + ) + slots.append((amount, active)) + unused.append(structural) + nonreceipt.append(no) + yes = readable & (receipt == 1) + resolved_yes = ( + yes + & (slots[0][1] | slots[1][1]) + & (slots[0][1] | unused[0]) + & (slots[1][1] | unused[1]) + ) + resolved_no = readable & (receipt == 2) & nonreceipt[0] & nonreceipt[1] + total = np.full(len(interest), np.nan) + total[resolved_no] = 0 + with np.errstate(over="raise", invalid="raise"): + total[resolved_yes] = ( + np.where(unused[0], 0, slots[0][0])[resolved_yes] + + np.where(unused[1], 0, slots[1][0])[resolved_yes] + ) + derived = [u & resolved_yes for u in unused] + count = derived[0].astype(np.int8) + derived[1].astype(np.int8) + label = np.full(len(interest), "unresolved_retirement_interest", dtype=object) + label[resolved_no] = "known_nonreceipt_sum" + label[resolved_yes] = "declared_account_sum" + label[count > 0] = "declared_account_sum_with_unused_slot_zero" + return total, count, label, derived + + +def _other_route(routing): + receipt, receipt_ok = _code(routing, "other_income_receipt") + category, category_ok = _code(routing, "other_income_category") + status = _text(routing, "other_income_reporting_status") + route = _text(routing, "other_income_routing_status") + reported = ( + receipt_ok & category_ok & (receipt == 1) & (route == "reported_category") + ) + possible = reported & np.isin(category, tuple(OTHER_PROPERTY_CATEGORIES)) + clear = ( + receipt_ok + & category_ok + & (receipt == 2) + & (category == 0) + & (status == "known_nonreceipt") + & (route == "niu_category") + ) | ( + reported + & (category >= 1) + & (category <= 20) + & ~np.isin(category, (*OTHER_PROPERTY_CATEGORIES, OTHER_UNSPECIFIED_CATEGORY)) + ) + return clear, possible + + +def _survivor_route(dividend, age): + # The qualifier already owns this route's source interpretation. Check the + # lightweight declared code/status agreement, never reread its source here. + receipt, receipt_ok = _code(dividend, "SUR_YN") + first, first_ok = _code(dividend, "SUR_SC1") + second, second_ok = _code(dividend, "SUR_SC2") + readable = first_ok & second_ok + yes = receipt_ok & (receipt == 1) + possible = yes & ((first == 8) | (second == 8)) + clear = (receipt_ok & (receipt == 2) & readable & (first == 0) & (second == 0)) | ( + yes + & readable + & ((first > 0) | (second > 0)) + & (first >= 0) + & (first <= 9) + & (first != 8) + & (second >= 0) + & (second <= 9) + & (second != 8) + ) + clear &= age >= 15 + possible &= age >= 15 + declared = dividend.survivor_property_route_clear + _require( + pd.api.types.is_bool_dtype(declared.dtype) + and np.array_equal(clear, declared.fillna(False).to_numpy(dtype=bool)) + and np.array_equal( + possible, declared.eq(False).fillna(False).to_numpy(dtype=bool) + ), + "SURVIVOR_ROUTE_AGREEMENT", + ) + # SRVS_VAL includes unedited third/fourth sources absent from SUR_SC1/2 + # (2025 dictionary, PDF49 / printed6C-28). Visible-slot clearance cannot + # establish full property scope for a person reporting survivor income. + # Retain that descriptive clearance, but only known nonreceipt can clear + # the first donor bridge until the additional-source scope is qualified. + full_clear = clear & receipt_ok & (receipt == 2) + extra_sources_unresolved = yes & (age >= 15) + return clear, possible, full_clear, extra_sources_unresolved + + +def _weights(index, membership, weights): + _require( + type(membership) is pd.Series + and membership.index.equals(index) + and membership.index.name == index.name + and membership.dtype == np.dtype("int64"), + "HOUSEHOLD_MEMBERSHIP", + ) + _require( + type(weights) is pd.Series + and weights.index.dtype == np.dtype("int64") + and weights.index.name == "household_id" + and weights.index.is_unique + and set(membership) == set(weights.index), + "DESIGN_WEIGHT_MEMBERSHIP", + ) + values = _numeric(weights, "DESIGN_WEIGHTS") + _require(np.isfinite(values).all() and (values >= 0).all(), "DESIGN_WEIGHTS") + return weights.loc[membership].to_numpy(dtype=np.float64, copy=True) + + +def build_asec_property_basis( + *, + interest: pd.DataFrame, + income_routing: pd.DataFrame, + dividend: pd.DataFrame, + original_household_membership: pd.Series, + original_household_design_weights: pd.Series, +) -> AsecPropertyBasis: + """Compose exactly aligned *original*, qualified ASEC person descriptions. + + No sorting, inner joins, dropping, imputation or source qualification occurs. + A jointly permuted input is valid; a permutation of one source alone refuses. + The two masks distinguish reported-total eligibility from complete, balanced + joint-component fit eligibility. Neither requests a second aggregate model. + All mass summaries use explicitly supplied original household design weights. + """ + age = _axis(interest) + for frame in (income_routing, dividend): + other_age = _axis(frame) + _require(frame.index.equals(interest.index), "PERSON_ALIGNMENT") + _require( + frame.native_person_id.equals(interest.native_person_id), "NATIVE_ALIGNMENT" + ) + _require(np.array_equal(age, other_age), "AGE_ALIGNMENT") + design = _weights( + interest.index, original_household_membership, original_household_design_weights + ) + ordinary = _amount(interest, "TRDINT_VAL", ordinary=True) + reported_interest = _amount(interest, "INT_VAL") + dividends = _amount(dividend, "DIV_VAL") + div_receipt, div_readable = _code(dividend, "DIV_YN") + _require( + ( + ~np.isfinite(dividends) + | ( + div_readable + & ( + ((div_receipt == 1) & (dividends > 0)) + | ((div_receipt == 2) & (dividends == 0)) + ) + ) + ).all(), + "DIVIDEND_RECEIPT_AGREEMENT", + ) + property_amount = _numeric(income_routing.net_property_known_amount, "RNT_VAL") + property_status = _text(income_routing, "net_property_reporting_status") + _require( + np.array_equal( + np.isfinite(property_amount), + np.isin(property_status, ["known_receipt", "known_nonreceipt"]), + ) + and (property_amount[property_status == "known_nonreceipt"] == 0).all() + and (property_amount[property_status == "known_receipt"] != 0).all(), + "PROPERTY_KNOWNNESS", + ) + retirement, count, derivation, derived_slots = _retirement(interest) + other_clear, other_possible = _other_route(income_routing) + ( + survivor_visible_clear, + survivor_possible, + survivor_clear, + survivor_extra_unresolved, + ) = _survivor_route(dividend, age) + person = interest[["native_person_id", "source_age"]].copy(deep=True) + person["original_household_id"] = original_household_membership.to_numpy(copy=True) + person["original_household_design_weight"] = design + person["survivor_visible_routes_clear"] = survivor_visible_clear + person["survivor_full_scope_clear"] = survivor_clear + for name, amount in zip( + PROPERTY_COMPONENTS, + (ordinary, retirement, dividends, property_amount), + strict=True, + ): + person[name] = amount + with np.errstate(over="raise", invalid="raise"): + person[PROPERTY_REPORTED_TOTAL] = ( + reported_interest + dividends + property_amount + ) + person["property_component_sum"] = ( + ordinary + retirement + dividends + property_amount + ) + person["interest_component_discrepancy"] = reported_interest - ( + ordinary + retirement + ) + person["reported_minus_component_total"] = ( + person[PROPERTY_REPORTED_TOTAL].to_numpy() + - person.property_component_sum.to_numpy() + ) + person["retirement_interest_derivation"] = pd.array(derivation, dtype="string") + person["retirement_structural_zero_slot_count"] = count + for n, values in enumerate(derived_slots, 1): + person[f"retirement_slot{n}_derived_unused_zero"] = values + exclusions = pd.DataFrame( + { + "under15": age < 15, + "reported_total_unknown": person[PROPERTY_REPORTED_TOTAL].isna(), + "ordinary_interest_unknown": np.isnan(ordinary), + "retirement_interest_unknown": np.isnan(retirement), + "dividends_unknown": np.isnan(dividends), + "property_receipts_unknown": np.isnan(property_amount), + "interest_discrepancy_unknown": person.interest_component_discrepancy.isna(), + "interest_discrepancy_nonzero": person.interest_component_discrepancy.notna() + & person.interest_component_discrepancy.ne(0), + "other_income_possible_property": other_possible, + "other_income_route_unresolved": ~other_clear & ~other_possible, + "survivor_possible_property": survivor_possible, + "survivor_route_unresolved": ~survivor_visible_clear & ~survivor_possible, + "survivor_additional_sources_unresolved": survivor_extra_unresolved, + }, + index=interest.index, + ) + person["reported_total_eligible"] = ( + (age >= 15) + & np.isfinite(person[PROPERTY_REPORTED_TOTAL]) + & other_clear + & survivor_clear + ) + person["joint_component_fit_eligible"] = ~exclusions.any(axis=1) + # Detached qualified columns preserve source statuses, amounts, allocation + # and disclosure flags. Prefixes make disagreements visible without choosing + # one qualifier's value or attributing a discrepancy to disclosure treatment. + provenance = pd.concat( + [ + frame.copy(deep=True).add_prefix(prefix + ".") + for prefix, frame in ( + ("interest", interest), + ("income_routing", income_routing), + ("dividend", dividend), + ) + ], + axis=1, + ) + masks = {"all": np.ones(len(person), dtype=bool)} + masks.update( + { + name: person[name].to_numpy() + for name in ("reported_total_eligible", "joint_component_fit_eligible") + } + ) + masks["excluded_joint_component_fit"] = ~masks["joint_component_fit_eligible"] + masks.update( + {"excluded:" + name: exclusions[name].to_numpy() for name in exclusions} + ) + rows = [] + for name, mask in masks.items(): + households = original_household_membership.iloc[np.flatnonzero(mask)].unique() + with np.errstate(over="raise", invalid="raise"): + mass = fsum(design[mask]) + household_mass = fsum( + original_household_design_weights.loc[households].to_numpy( + dtype=np.float64 + ) + ) + _require(np.isfinite(mass) and np.isfinite(household_mass), "MASS_OVERFLOW") + rows.append((name, int(mask.sum()), len(households), mass, household_mass)) + summary = pd.DataFrame( + rows, + columns=[ + "selection", + "person_count", + "household_count", + "design_weighted_person_mass", + "union_household_design_mass", + ], + ).set_index("selection") + return AsecPropertyBasis(person, provenance, exclusions, summary) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py new file mode 100644 index 000000000..3113b96da --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py @@ -0,0 +1,842 @@ +"""Pure retirement candidate ledger, with explicit measurement assumptions. + +A descriptive table or weight Series grants no source authority. The host owns +complete original-source admission. Candidate bounds are neither fiscal amounts +nor identified ACS labels; aggregate differences are never allocated. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from math import fsum +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from . import current_asec_income_routing_source as routing +from . import current_asec_retirement_detail_source as detail + +PROTOCOL = "microcosm.us.asec-retirement-candidate-basis.v1" +# These bridge route sets are decisions, distinct from the source code labels. +CANDIDATE_CODES = MappingProxyType( + { + "pension": frozenset(range(1, 7)), + "disability": frozenset(range(2, 6)), + "survivor": frozenset(range(1, 5)), + } +) +RAILROAD_CODES = MappingProxyType({"pension": 7, "disability": 6, "survivor": 5}) +FAMILIES = ( + ("pension", "PEN", "PNSN_VAL"), + ("disability", "DIS", "DSAB_VAL"), + ("survivor", "SUR", "SRVS_VAL"), +) +DISTRIBUTION_SLOTS = ( + ("slot1", "DST_VAL1", False), + ("slot2", "DST_VAL2", False), + ("slot1_young", "DST_VAL1_YNG", True), + ("slot2_young", "DST_VAL2_YNG", True), +) +OTHER_RETIREMENT_CANDIDATES = frozenset((2, 13)) +OTHER_UNSPECIFIED = 19 + + +def _require(condition, reason): + if not condition: + raise ValueError("ASEC_RETIREMENT_BASIS_" + reason) + + +@dataclass(frozen=True) +class RetirementCandidateAssumptions: + """Required choices; no constructor defaults or observed-treatment claims.""" + + pension_annuity_regularity: str + disability_pension_eligibility: str + survivor_annuity_overlap: str + withdrawal_regularity_netting: str + aggregate_accounting: str + + def __post_init__(self): + allowed = { + "pension_annuity_regularity": ("unresolved", "assume_regular"), + "disability_pension_eligibility": ("unresolved", "assume_qualifying"), + "survivor_annuity_overlap": ("unresolved",), + "withdrawal_regularity_netting": ("unresolved",), + "aggregate_accounting": ("exact_visible_balance_only",), + } + _require( + all( + type(getattr(self, k)) is str and getattr(self, k) in v + for k, v in allowed.items() + ), + "ASSUMPTIONS", + ) + + def to_bytes(self) -> bytes: + self.__post_init__() + return ( + json.dumps( + {"protocol": PROTOCOL, **asdict(self)}, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ).encode() + + +@dataclass(frozen=True) +class AsecRetirementBasis: + """Detached descriptive tables and immutable assumption bytes; no issuer.""" + + person: pd.DataFrame + slots: pd.DataFrame + provenance: pd.DataFrame + exclusions: pd.DataFrame + summary: pd.DataFrame + assumptions_payload: bytes + + +def _numeric(series, label): + _require( + pd.api.types.is_numeric_dtype(series.dtype) + and not pd.api.types.is_bool_dtype(series.dtype) + and not pd.api.types.is_complex_dtype(series.dtype), + "NUMERIC:" + label, + ) + values = series.to_numpy(dtype="float64", na_value=np.nan, copy=True) + _require(not np.isinf(values).any(), "INFINITE:" + label) + return values + + +def _text(frame, name): + _require(name in frame, "COLUMN:" + name) + values = frame[name].to_numpy(copy=True) + _require(all(type(v) is str for v in values), "TEXT:" + name) + return values + + +def _axis(frame): + _require( + type(frame) is pd.DataFrame + and frame.columns.is_unique + and frame.index.dtype == np.dtype("int64") + and frame.index.name == "person_id" + and frame.index.is_unique, + "PERSON_AXIS", + ) + _require({"native_person_id", "source_age"} <= set(frame), "IDENTITY_COLUMNS") + _require( + frame.native_person_id.dtype == np.dtype("int64") + and frame.native_person_id.is_unique, + "NATIVE_AXIS", + ) + age = _numeric(frame.source_age, "AGE") + _require( + np.isfinite(age).all() + and ((age >= 0) & (age <= 99) & (age == np.floor(age))).all(), + "AGE", + ) + return age + + +def _equal(left, right, label): + _require(np.array_equal(left, right, equal_nan=True), "AGREEMENT:" + label) + + +def _codes(frame, prefix, domain): + values = _numeric(frame[prefix + "_code"], prefix) + status = _text(frame, prefix + "_literal_status") + valid = status == "in_printed_range" + _require( + np.array_equal(valid, np.isfinite(values)) + and np.isin(values[valid], tuple(domain)).all(), + "CODE:" + prefix, + ) + return values, status + + +def _published(frame): + values = {} + for name, entry in detail.amount_entries().items(): + amount = _numeric(frame[name + "_published_amount"], name) + status = _text(frame, name + "_literal_status") + valid = status == "in_printed_range" + _require( + np.array_equal(valid, np.isfinite(amount)) + and ( + (amount[valid] >= entry.encoded_minimum) + & (amount[valid] <= entry.encoded_maximum) + & (amount[valid] == np.floor(amount[valid])) + ).all(), + "PUBLISHED:" + name, + ) + values[name] = amount + return values + + +def _known_routing(frame, prefix): + status = _text(frame, prefix + "_reporting_status") + amount = _numeric(frame[prefix + "_known_amount"], prefix) + known = np.isin(status, routing.KNOWN_AMOUNT_STATUSES) + _require( + np.array_equal(known, np.isfinite(amount)) + and (amount[known] >= 0).all() + and ( + amount[np.isin(status, ("known_nonreceipt", "known_recipient_zero"))] == 0 + ).all(), + "KNOWN_ROUTING:" + prefix, + ) + return amount, status + + +def _reference_agreement(d, r, published, ages): + for family, field, receipt_prefix in ( + ("pension", "PNSN_VAL", "pension_annuity_pension_receipt"), + ("annuity", "ANN_VAL", "pension_annuity_annuity_receipt"), + ): + prefix = "pension_annuity_" + family + receipt, receipt_status = _codes(r, receipt_prefix, routing.RECEIPT_CODE_DOMAIN) + if family == "pension": + other, other_status = _codes(d, "PEN_YN", routing.RECEIPT_CODE_DOMAIN) + _equal(receipt, other, "PEN_YN") + _require(np.array_equal(receipt_status, other_status), "PEN_YN_STATUS") + amount, statuses = _known_routing(r, prefix) + source = published[field].copy() + kinds = np.where( + np.isnan(source), + "missing", + np.where( + source == -1, "declared_niu", np.where(source == 0, "zero", "nonzero") + ), + ) + source[kinds == "declared_niu"] = np.nan + _equal(source, _numeric(r[prefix + "_source_total"], prefix), field) + expected = [ + routing.receipt_status( + bool(age >= 15), + (None if np.isnan(code) else int(code), literal), + kind, + net_measure=False, + zero_is_dollars=routing._zero_is_dollars(field), + ) + for age, code, literal, kind in zip( + ages, receipt, receipt_status, kinds, strict=True + ) + ] + _require(np.array_equal(statuses, expected), "ROUTING_STATUS:" + field) + _equal( + amount, + np.where(np.isin(expected, routing.KNOWN_AMOUNT_STATUSES), source, np.nan), + "KNOWN:" + field, + ) + for slot, field, _ in DISTRIBUTION_SLOTS: + _equal( + published[field], + _numeric(r["retirement_distribution_" + slot + "_amount"], slot), + field, + ) + + +def _route(family, code): + if not np.isfinite(code): + return "unresolved" + if code == 0: + return "unused_or_nonreceipt" + if code in CANDIDATE_CODES[family]: + return "candidate" + if code == RAILROAD_CODES[family]: + return "railroad" + if family == "disability" and code in (1, 7, 8, 9): + return "other_compensation" + if family == "survivor" and code == 8: + return "property" + if family == "survivor" and code == 9: + return "annuity_overlap" + return "unresolved" + + +def _detail_slots(d, published, ages): + result = {family: [] for family, *_ in FAMILIES} + for family, prefix, _ in FAMILIES: + receipt, receipt_literal = _codes( + d, prefix + "_YN", routing.RECEIPT_CODE_DOMAIN + ) + for slot in (1, 2): + field = prefix + "_VAL" + str(slot) + code, literal = _codes( + d, prefix + "_SC" + str(slot), detail.SOURCE_CODES[prefix] + ) + statuses = _text(d, field + "_reporting_status") + amount = _numeric(d[field + "_amount"], field) + known = d[field + "_amount_known"] + _require( + pd.api.types.is_bool_dtype(known.dtype) and known.notna().all(), + "KNOWN_MASK:" + field, + ) + expected = [ + detail._slot_status( + age, + (None if np.isnan(rec) else int(rec), recstat), + (None if np.isnan(c) else int(c), cstat), + (None if np.isnan(v) else int(v), vs), + ) + for age, rec, recstat, c, cstat, v, vs in zip( + ages, + receipt, + receipt_literal, + code, + literal, + published[field], + _text(d, field + "_literal_status"), + strict=True, + ) + ] + _require(np.array_equal(statuses, expected), "SLOT_STATUS:" + field) + expected_known = np.isin(statuses, routing.KNOWN_AMOUNT_STATUSES) + _require( + np.array_equal(known.to_numpy(dtype=bool), expected_known), + "KNOWN_MASK:" + field, + ) + _equal( + amount, + np.where(expected_known, published[field], np.nan), + "SLOT_AMOUNT:" + field, + ) + for i in range(len(d)): + structural = ( + statuses[i] == "unreported_source_slot" + and code[i] == 0 + and published[field][i] == 0 + ) + result[family].append( + { + "position": i, + "family": family, + "slot": slot, + "amount_field": field, + "source_code": code[i], + "source_literal_status": literal[i], + "published_amount": published[field][i], + "reporting_status": statuses[i], + "source_known_amount": amount[i], + "source_amount_known": bool(expected_known[i]), + "route": _route(family, code[i]), + "applicable": bool(ages[i] >= 15), + "structural_zero_comparison": bool(structural), + } + ) + return result + + +def _comparison(d, published, family, total, first, second): + difference = published[total] - published[first] - published[second] + suffix = "main_slots" if family == "distribution" else "visible_slots" + _equal( + difference, + _numeric(d[family + "_total_minus_" + suffix], family), + "COMPARISON:" + family, + ) + return difference + + +def _family_row(family, receipt, total, difference, slots, age, assumptions): + lower, upper = np.nan, np.nan + route_amounts = { + route: fsum( + s["source_known_amount"] + for s in slots + if s["route"] == route and s["source_amount_known"] + ) + for route in ( + "candidate", + "railroad", + "property", + "annuity_overlap", + "other_compensation", + "unresolved", + ) + } + usable = all( + s["source_amount_known"] or s["structural_zero_comparison"] for s in slots + ) + no = ( + receipt == 2 + and total == 0 + and all(s["reporting_status"] == "known_nonreceipt" for s in slots) + ) + contradiction = ( + any("contradictory" in s["reporting_status"] for s in slots) + or (np.isfinite(difference) and difference < 0) + or (receipt == 2 and np.isfinite(total) and total != 0) + ) + if age < 15: + status = ( + "contradictory_outside_reporting_universe" + if contradiction or (np.isfinite(total) and total != 0) + else "unresolved_outside_reporting_universe" + if not np.isfinite(total) + or any(s["reporting_status"] != "outside_reporting_universe" for s in slots) + else "outside_reporting_universe" + ) + elif contradiction: + status = "contradictory_accounting" + elif no: + lower = upper = 0.0 + status = "known_nonreceipt" + elif ( + not usable + or not np.isfinite(total) + or not np.isfinite(difference) + or receipt != 1 + or total <= 0 + ): + status = "unresolved_source_accounting" + elif difference > 0: + status = "additional_scope_unresolved" + elif family == "survivor": + status = "additional_survivor_scope_unresolved" + else: + excluded = route_amounts["railroad"] + route_amounts["other_compensation"] + upper = total - excluded + _require( + upper >= 0 + and upper == route_amounts["candidate"] + route_amounts["unresolved"], + "CANDIDATE_ACCOUNTING", + ) + assumed = ( + assumptions.pension_annuity_regularity == "assume_regular" + if family == "pension" + else assumptions.disability_pension_eligibility == "assume_qualifying" + ) + lower = route_amounts["candidate"] if assumed else 0.0 + status = ( + "route_outside_only" + if upper == 0 + else "candidate_under_assumptions" + if assumed + else "candidate_regularity_or_scope_unresolved" + ) + return { + family + "_candidate_lower": lower, + family + "_candidate_upper": upper, + family + "_status": status, + family + "_accounting_difference": difference, + **{ + family + "_observed_" + route + "_subtotal": amount + for route, amount in route_amounts.items() + }, + } + + +def _distributions(r, published, ages, differences): + total, status = _known_routing(r, "retirement_distribution") + receipt, receipt_literal = _codes( + r, "retirement_distribution_receipt", routing.RECEIPT_CODE_DOMAIN + ) + codes = { + slot: _codes( + r, "retirement_distribution_" + slot + "_account", routing.ACCOUNT_CODES + ) + for slot, *_ in DISTRIBUTION_SLOTS + } + raw_status = { + slot: _text(r, "retirement_distribution_" + slot + "_slot_status") + for slot, *_ in DISTRIBUTION_SLOTS + } + offroute_receipts = { + name: _codes( + r, "retirement_distribution_receipt_" + name, routing.RECEIPT_CODE_DOMAIN + ) + for name in ("young", "58") + } + roster, rows = [], [] + for i, age in enumerate(ages): + known_composition, account_amounts, sum_applicable = ( + True, + {code: 0.0 for code in routing.ACCOUNT_CODES if code != 0}, + 0.0, + ) + offroute = False + for slot_number, (slot, field, young) in enumerate(DISTRIBUTION_SLOTS, 1): + applicable = bool((age < 58) == young) + declared = r["retirement_distribution_" + slot + "_applicable"].iloc[i] + _require( + isinstance(declared, (bool, np.bool_)) and bool(declared) == applicable, + "DISTRIBUTION_AGE_ROUTE", + ) + code, literal = codes[slot][0][i], codes[slot][1][i] + value = published[field][i] + active = ( + applicable + and literal == "in_printed_range" + and code > 0 + and np.isfinite(value) + and value > 0 + and raw_status[slot][i] == "known_slot" + ) + niu = ( + applicable + and code == 0 + and literal == "in_printed_range" + and value == 0 + and raw_status[slot][i] == "niu_slot" + ) + if applicable: + known_composition &= bool(active or niu) + sum_applicable += value + if active: + account_amounts[int(code)] += value + else: + offroute |= not bool( + np.isfinite(value) + and value == 0 + and literal == "in_printed_range" + and code == 0 + ) + roster.append( + { + "position": i, + "family": "distribution", + "slot": slot_number, + "amount_field": field, + "source_code": code, + "source_literal_status": literal, + "published_amount": value, + "reporting_status": raw_status[slot][i], + "source_known_amount": value if active else np.nan, + "source_amount_known": bool(active), + "route": "withdrawal_regularity_netting_unresolved" + if active + else "unused_or_unresolved", + "applicable": applicable, + "structural_zero_comparison": bool(niu), + } + ) + other = "young" if age >= 58 else "58" + other_receipt, other_literal = offroute_receipts[other] + offroute |= not bool( + other_literal[i] == "in_printed_range" and other_receipt[i] == 0 + ) + if np.isfinite(total[i]) and known_composition: + _require(total[i] == sum_applicable, "DISTRIBUTION_TOTAL") + eligible = ( + age >= 15 + and known_composition + and not offroute + and np.isfinite(total[i]) + and receipt_literal[i] == "in_printed_range" + and receipt[i] in (1, 2) + ) + if eligible: + _require( + (receipt[i] == 1 and total[i] > 0) + or (receipt[i] == 2 and total[i] == 0), + "DISTRIBUTION_RECEIPT", + ) + candidate_status = ( + "known_nonreceipt" + if eligible and total[i] == 0 + else "regularity_netting_unresolved" + if eligible + else "unresolved_source_composition" + ) + if age >= 58 and (not np.isfinite(differences[i]) or differences[i] != 0): + eligible = False + candidate_status = ( + "unresolved_source_accounting" + if not np.isfinite(differences[i]) + else "contradictory_accounting" + if differences[i] < 0 + else "additional_scope_unresolved" + ) + rows.append( + { + "distribution_candidate_lower": 0.0 if eligible else np.nan, + "distribution_candidate_upper": total[i] if eligible else np.nan, + "distribution_status": candidate_status, + "distribution_source_known_amount": total[i], + "distribution_source_reporting_status": status[i], + "distribution_account_composition_known": bool( + known_composition and not offroute + ), + **{ + "distribution_account_" + str(c) + "_amount": v + if known_composition and not offroute and age >= 15 + else np.nan + for c, v in account_amounts.items() + }, + } + ) + return rows, roster + + +def _other_income(r, age): + amount, status = _known_routing(r, "other_income") + receipt, _ = _codes(r, "other_income_receipt", routing.RECEIPT_CODE_DOMAIN) + category, _ = _codes(r, "other_income_category", routing.OTHER_INCOME_CATEGORIES) + route = _text(r, "other_income_routing_status") + no = ( + (receipt == 2) + & (category == 0) + & (amount == 0) + & (status == "known_nonreceipt") + & (route == "niu_category") + ) + yes = ( + (receipt == 1) + & (category > 0) + & (amount > 0) + & (status == "known_receipt") + & (route == "reported_category") + ) + clear = (age >= 15) & ( + no + | (yes & ~np.isin(category, (*OTHER_RETIREMENT_CANDIDATES, OTHER_UNSPECIFIED))) + ) + return ( + amount, + clear, + yes & np.isin(category, tuple(OTHER_RETIREMENT_CANDIDATES)), + yes & (category == OTHER_UNSPECIFIED), + yes & (category == 1), + yes & (category == 8), + ) + + +def _summary(index, membership, weights, masks): + _require( + type(membership) is pd.Series + and membership.index.equals(index) + and membership.index.name == "person_id" + and membership.dtype == np.dtype("int64"), + "HOUSEHOLD_MEMBERSHIP", + ) + _require( + type(weights) is pd.Series + and weights.index.dtype == np.dtype("int64") + and weights.index.name == "household_id" + and weights.index.is_unique + and set(weights.index) == set(membership), + "DESIGN_MEMBERSHIP", + ) + values = _numeric(weights, "DESIGN_WEIGHTS") + _require(np.isfinite(values).all() and (values >= 0).all(), "DESIGN_WEIGHTS") + design = weights.loc[membership].to_numpy(dtype="float64") + rows = [] + for name, mask in masks.items(): + households = membership.iloc[np.flatnonzero(mask)].unique() + person_mass = fsum(design[mask]) + household_mass = fsum(weights.loc[households].to_numpy(dtype="float64")) + _require( + np.isfinite(person_mass) and np.isfinite(household_mass), "DESIGN_MASS" + ) + rows.append( + (name, int(mask.sum()), len(households), person_mass, household_mass) + ) + return pd.DataFrame( + rows, + columns=( + "selection", + "person_count", + "household_count", + "design_weighted_person_mass", + "union_household_design_mass", + ), + ).set_index("selection") + + +def build_asec_retirement_basis( + *, + retirement_detail: pd.DataFrame, + income_routing: pd.DataFrame, + original_household_membership: pd.Series, + original_household_design_weights: pd.Series, + assumptions: RetirementCandidateAssumptions, +) -> AsecRetirementBasis: + """Describe aligned original ASEC source evidence; perform no I/O or fitting. + + Every bound is a conditional candidate accounting result, never a fiscal + label. Positive survivor receipt and nonzero aggregate differences remain + unresolved. DESIGN Series custody and complete-person admission stay external. + """ + _require(type(assumptions) is RetirementCandidateAssumptions, "ASSUMPTIONS_TYPE") + payload = assumptions.to_bytes() + d, r = retirement_detail, income_routing + age = _axis(d) + other_age = _axis(r) + _require( + d.index.equals(r.index) and d.native_person_id.equals(r.native_person_id), + "SOURCE_ALIGNMENT", + ) + _equal(age, other_age, "AGE") + _require(len(d) > 0, "EMPTY_CANDIDATE_SCOPE") + published = _published(d) + _reference_agreement(d, r, published, age) + records = _detail_slots(d, published, age) + by_person = {family: [[] for _ in range(len(d))] for family in records} + for family, entries in records.items(): + for entry in entries: + by_person[family][entry["position"]].append(entry) + differences = { + family: _comparison(d, published, family, total, first, second) + for family, total, first, second in detail.COMPARISONS + } + distribution, distribution_records = _distributions( + r, published, age, differences["distribution"] + ) + annuity, annuity_status = _known_routing(r, "pension_annuity_annuity") + other_amount, other_clear, oi_candidate, oi_unspecified, oi_ss, oi_property = ( + _other_income(r, age) + ) + receipts = { + family: _codes(d, prefix + "_YN", routing.RECEIPT_CODE_DOMAIN)[0] + for family, prefix, _ in FAMILIES + } + rows = [] + for i in range(len(d)): + row = { + "native_person_id": int(d.native_person_id.iloc[i]), + "source_age": int(age[i]), + } + for family, _, total in FAMILIES: + slots = by_person[family][i] + row.update( + _family_row( + family, + receipts[family][i], + published[total][i], + differences[family][i], + slots, + age[i], + assumptions, + ) + ) + known_annuity = age[i] >= 15 and np.isfinite(annuity[i]) + row.update( + annuity_candidate_lower=( + annuity[i] + if assumptions.pension_annuity_regularity == "assume_regular" + else 0.0 + ) + if known_annuity + else np.nan, + annuity_candidate_upper=annuity[i] if known_annuity else np.nan, + annuity_status=annuity_status[i], + other_income_reported_amount=other_amount[i], + other_income_scope_clear=bool(other_clear[i]), + distribution_accounting_difference=differences["distribution"][i], + **distribution[i], + ) + families = ("pension", "disability", "survivor", "annuity", "distribution") + available = ( + age[i] >= 15 + and other_clear[i] + and all( + np.isfinite(row[f + "_candidate_lower"]) + and np.isfinite(row[f + "_candidate_upper"]) + for f in families + ) + ) + lower = ( + fsum(row[f + "_candidate_lower"] for f in families) if available else np.nan + ) + upper = ( + fsum(row[f + "_candidate_upper"] for f in families) if available else np.nan + ) + row.update( + retirement_candidate_lower=lower, + retirement_candidate_upper=upper, + candidate_interval_available=bool(available), + candidate_point_under_assumptions=bool(available and lower == upper), + candidate_basis_eligible=bool(available), + point_identified=False, + fiscal_outputs_produced=False, + ) + rows.append(row) + person = pd.DataFrame(rows, index=d.index.copy()) + all_records = [ + s.copy() for family in records.values() for s in family + ] + distribution_records + for item in all_records: + i = item.pop("position") + item["person_id"] = int(d.index[i]) + item["native_person_id"] = int(d.native_person_id.iloc[i]) + item["source_age"] = int(age[i]) + slots = ( + pd.DataFrame(all_records) + .sort_values(["person_id", "family", "slot"], kind="stable") + .reset_index(drop=True) + ) + exclusions = pd.DataFrame( + { + "under15": age < 15, + "survivor_additional_sources_unresolved": (age >= 15) + & (receipts["survivor"] == 1), + **{ + family + "_reported_" + route: person[ + family + "_observed_" + route + "_subtotal" + ].to_numpy() + > 0 + for family in records + for route in ( + "railroad", + "property", + "annuity_overlap", + "other_compensation", + "unresolved", + ) + }, + "other_income_scope_unresolved": ~other_clear, + "other_income_retirement_candidate": oi_candidate, + "other_income_unspecified": oi_unspecified, + "other_income_ss_overlap": oi_ss, + "other_income_property_overlap": oi_property, + "survivor_annuity_overlap": ( + person.survivor_observed_annuity_overlap_subtotal.to_numpy() > 0 + ) + & (annuity > 0), + **{ + family + "_candidate_unresolved": ~np.isfinite( + person[family + "_candidate_upper"].to_numpy() + ) + for family in ( + "pension", + "disability", + "survivor", + "annuity", + "distribution", + ) + }, + **{ + family + "_aggregate_nonzero_or_unknown": ~np.isfinite(diff) + | (diff != 0) + for family, diff in differences.items() + }, + }, + index=d.index.copy(), + ) + provenance = pd.concat( + [ + d.copy(deep=True).add_prefix("retirement_detail."), + r.copy(deep=True).add_prefix("income_routing."), + ], + axis=1, + ) + masks = { + "all": np.ones(len(d), dtype=bool), + **{ + name: person[name].to_numpy() + for name in ( + "candidate_basis_eligible", + "candidate_interval_available", + "candidate_point_under_assumptions", + ) + }, + **{"diagnostic:" + name: exclusions[name].to_numpy() for name in exclusions}, + } + summary = _summary( + d.index, original_household_membership, original_household_design_weights, masks + ) + return AsecRetirementBasis(person, slots, provenance, exclusions, summary, payload) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py new file mode 100644 index 000000000..1fa6292bd --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py @@ -0,0 +1,614 @@ +"""Qualified ASEC pension, disability and survivor source details. + +Published amounts and component comparisons are observations, not a complete +ACS retirement bridge. No regularity, taxability, overlap allocation or source +admission is inferred. A host retains and requalifies the actual preparation. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import tempfile +from dataclasses import asdict, dataclass +from functools import lru_cache +from pathlib import Path +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as physical +from . import current_asec_dividend_source as dividend +from . import current_asec_income_routing_source as routing + +PROTOCOL = "microcosm.us.current-asec-retirement-detail-source.v1" +REFERENCE_FIELDS = ( + "PNSN_VAL", + "ANN_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", +) +# Exact dictionary entries not supplied by the routing owner's domain mapping. +# Name, width, position, physical PDF page, printed page, maximum, universe. +DETAIL_AMOUNT_ENTRIES = ( + ("PEN_VAL1", 6, 558, 47, "6C-26", 999999, "PEN_SC1 > 0"), + ("PEN_VAL2", 6, 564, 47, "6C-26", 999999, "PEN_SC2 > 0"), + ("DIS_VAL1", 6, 465, 44, "6C-23", 999999, "DIS_SC1 > 0"), + ("DIS_VAL2", 6, 471, 44, "6C-23", 999999, "DIS_SC2 > 0"), + ("SUR_VAL1", 6, 652, 50, "6C-29", 999999, "SUR_YN = 1"), + ("SUR_VAL2", 6, 658, 50, "6C-29", 999999, "SUR_YN = 1"), + ("DSAB_VAL", 6, 485, 45, "6C-24", 999999, "DIS_VAL1 > 0 OR DIS_VAL2 > 0"), + ("DBTN_VAL", 7, 452, 44, "6C-23", 9999999, "DST_VAL1 > 0 OR DST_VAL2 > 0"), + ("SRVS_VAL", 6, 628, 49, "6C-28", 999999, "SUR_YN = 1"), +) +AMOUNT_FIELDS = (*REFERENCE_FIELDS, *(entry[0] for entry in DETAIL_AMOUNT_ENTRIES)) +RETAINED_MONEY_FIELDS = (*REFERENCE_FIELDS, "DIS_VAL1", "DIS_VAL2") +INDEPENDENT_LITERAL_FIELDS = tuple( + n for n in AMOUNT_FIELDS if n not in RETAINED_MONEY_FIELDS +) +RECEIPT_ENTRIES = MappingProxyType( + { + "PEN_YN": routing.RECEIPT_ENTRIES["PEN_YN"], + "SUR_YN": routing.ReceiptEntry(*dividend.RECEIPT_ENTRIES["SUR_YN"]), + "DIS_YN": routing.ReceiptEntry( + 1, 477, 45, "6C-24", "All Persons aged 15+", "niu" + ), + "DIS_CS": routing.ReceiptEntry( + 1, 459, 44, "6C-23", "All Persons aged 15+", "niu" + ), + "DIS_HP": routing.ReceiptEntry( + 1, 460, 44, "6C-23", "All Persons aged 15+", "niu" + ), + } +) +PENSION_CODES = MappingProxyType( + { + 0: "niu", + 1: "Company pension", + 2: "Union pension", + 3: "Federal government pension", + 4: "State government pension", + 5: "Local government pension", + 6: "US Military pension", + 7: "US Railroad Retirement", + 8: "Other", + } +) +DISABILITY_CODES = MappingProxyType( + { + 0: "NIU", + 1: "worker's compensation", + 2: "company or union disability", + 3: "federal government disability", + 4: "US military retirement disability", + 5: "state or local government employee disability", + 6: "US railroad retirement disability", + 7: "accident or disability insurance", + 8: "blacklung miners disability", + 9: "state temporary sickness", + 10: "other or don't know", + } +) +# Source code width, position, physical page, printed page, printed universe. +SOURCE_ENTRIES = MappingProxyType( + { + "PEN_SC1": (1, 556, 47, "6C-26", "PEN_YN = 1"), + "PEN_SC2": (1, 557, 47, "6C-26", "PEN_VAL2 > 0"), + "DIS_SC1": (2, 461, 44, "6C-23", "DIS_YN = 1"), + "DIS_SC2": (2, 463, 44, "6C-23", "DIS_YN = 1"), + **dividend.SURVIVOR_ENTRIES, + } +) +SOURCE_CODES = MappingProxyType( + {"PEN": PENSION_CODES, "DIS": DISABILITY_CODES, "SUR": dividend.SURVIVOR_CODES} +) +# Position, physical page, printed page, universe, supported published values. +ALLOCATION_ENTRIES = ( + ("I_PENSC1", 850, 56, "6C-35", "PEN_SC1 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_PENSC2", 851, 56, "6C-35", "PEN_SC2 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_PENVAL1", 852, 57, "6C-36", "PEN_VAL1 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_PENVAL2", 853, 57, "6C-36", "PEN_VAL2 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_DISCS", 812, 54, "6C-33", "DIS_CS > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_DISHP", 813, 54, "6C-33", "DIS_HP > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_DISSC1", 814, 54, "6C-33", "DIS_SC1 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_DISSC2", 815, 54, "6C-33", "DIS_SC2 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_DISVL1", 816, 54, "6C-33", "DIS_VAL1 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_DISVL2", 817, 54, "6C-33", "DIS_VAL2 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_DISYN", 818, 54, "6C-33", "DIS_YN > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_SURSC1", 873, 58, "6C-37", "SUR_SC1 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_SURSC2", 874, 58, "6C-37", "SUR_SC2 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_SURVL1", 875, 58, "6C-37", "SUR_VAL1 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_SURVL2", 876, 58, "6C-37", "SURV_VAL2 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_SURYN", 877, 58, "6C-37", "SUR_YN > 0", routing.ALLOCATION_ANNVAL_CODES), +) +TOPCODE_ENTRIES = ( + ("TPEN_VAL1", 913, 60, "6C-39", "PEN_VAL1 > 0"), + ("TPEN_VAL2", 914, 60, "6C-39", "PEN_VAL2 > 0"), + ("TDISVAL1", 903, 59, "6C-38", "DIS_VAL1 > 0"), + ("TDISVAL2", 904, 59, "6C-38", "DIS_VAL2 > 0"), + ("TSURVAL1", 670, 50, "6C-29", "SUR_VAL1 > 0"), + ("TSURVAL2", 671, 50, "6C-29", "SUR_VAL2 > 0"), +) +READ_COLUMNS = ( + *routing.COORDINATE_COLUMNS, + *AMOUNT_FIELDS, + *RECEIPT_ENTRIES, + *SOURCE_ENTRIES, + *(e[0] for e in ALLOCATION_ENTRIES), + *(e[0] for e in TOPCODE_ENTRIES), +) +COMPARISONS = ( + ("pension", "PNSN_VAL", "PEN_VAL1", "PEN_VAL2"), + ("disability", "DSAB_VAL", "DIS_VAL1", "DIS_VAL2"), + ("survivor", "SRVS_VAL", "SUR_VAL1", "SUR_VAL2"), + ("distribution", "DBTN_VAL", "DST_VAL1", "DST_VAL2"), +) + + +def require(condition, reason): + if not condition: + raise ValueError("ASEC_RETIREMENT_DETAIL_" + reason) + + +@lru_cache(maxsize=1) +def _cached_amount_entries(): + reference = routing.printed_amount_entries() + entries = [(name, reference[name]) for name in REFERENCE_FIELDS] + for ( + name, + width, + position, + page, + printed, + maximum, + universe, + ) in DETAIL_AMOUNT_ENTRIES: + entries.append( + ( + name, + routing.PrintedAmountEntry( + name, + width, + position, + printed, + page, + 0, + maximum, + universe, + "0 = none or niu; 1-" + str(maximum) + " = income amount", + False, + (), + "none_or_niu_not_distinguishable_from_amount_alone", + ), + ) + ) + require(len(entries) == len(set(n for n, _ in entries)), "DUPLICATE_AMOUNT") + return tuple(entries) + + +def amount_entries(): + """Detached mapping over immutable, source-pinned amount entries.""" + return dict(_cached_amount_entries()) + + +amount_entries.cache_clear = _cached_amount_entries.cache_clear + + +def _domain_agreement(ready): + entries = amount_entries() + routing._require_live_amount_domains( + ready, + { + name: ( + entries[name].encoded_minimum, + entries[name].encoded_maximum, + entries[name].zero_semantics, + ) + for name in RETAINED_MONEY_FIELDS + }, + ) + # A future expanded money roster must add real retained-bit comparisons; + # it may not silently leave those new fields on the independent path. + require( + not ( + set(INDEPENDENT_LITERAL_FIELDS) + & {f.name for f in ready.bindings.spec.fields} + ), + "UNREVIEWED_RETAINED_DOMAIN", + ) + + +def _amount_pair(token, entry): + require(type(token) is str and len(token) <= routing.TOKEN_MAX_CHARS, "TOKEN_BOUND") + if token == "": + return None, "missing" + pattern = ( + ("-?" if entry.encoded_minimum < 0 else "") + + r"[0-9]{1," + + str(entry.printed_length) + + "}" + ) + if re.fullmatch(pattern, token, re.ASCII) is None: + return None, "malformed" + value = int(token) + if not entry.encoded_minimum <= value <= entry.encoded_maximum: + return None, "outside_printed_range" + return value, "in_printed_range" + + +def _slot_status(age, receipt, source_code, amount): + value, status = amount + if status not in ("missing", "in_printed_range"): + return "invalid_amount_literal" + kind = "missing" if value is None else "zero" if value == 0 else "nonzero" + baseline = routing.receipt_status( + bool(age >= 15), receipt, kind, net_measure=False, zero_is_dollars=False + ) + if age < 15: + if ( + receipt[1] != "in_printed_range" + or source_code[1] != "in_printed_range" + or value is None + ): + return "unresolved_outside_reporting_universe" + return ( + "outside_reporting_universe" + if receipt[0] == source_code[0] == 0 and value == 0 + else "contradictory_outside_reporting_universe" + ) + if receipt[1] != "in_printed_range" or value is None: + return baseline + code, code_status = source_code + if code_status != "in_printed_range": + return "unresolved_source_slot" + if receipt[0] == 1 and code == 0: + return "unreported_source_slot" if value == 0 else "contradictory_source_slot" + if receipt[0] in (0, 2) and code != 0: + return "contradictory_source_slot" + return baseline + + +def project_retirement_detail_literals(ordered): + """Preserve literals/knownness; aggregate comparisons do not allocate dollars.""" + require( + type(ordered) is pd.DataFrame + and ordered.columns.is_unique + and set(READ_COLUMNS) <= set(ordered), + "COLUMNS", + ) + age_pairs = [routing.literal_code(t, range(100), width=2) for t in ordered.A_AGE] + require(all(s == "in_printed_range" for _, s in age_pairs), "AGE_LITERAL") + ages = np.array([v for v, _ in age_pairs], dtype=np.int64) + out = pd.DataFrame(index=range(len(ordered))) + parsed, codes = {}, {} + for name, entry in amount_entries().items(): + pairs = [_amount_pair(t, entry) for t in ordered[name]] + parsed[name] = pairs + out[name + "_literal"] = pd.array(ordered[name].tolist(), dtype="string") + out[name + "_literal_status"] = pd.array([s for _, s in pairs], dtype="string") + out[name + "_published_amount"] = pd.array( + [v for v, _ in pairs], dtype="Float64" + ) + for name, entry in RECEIPT_ENTRIES.items(): + frame, values, statuses = routing._codes_frame( + name, + ordered[name], + routing.RECEIPT_CODE_DOMAIN, + {0: entry.zero_label_as_printed, 1: "yes", 2: "no"}, + width=entry.printed_length, + ) + out = pd.concat([out, frame], axis=1) + codes[name] = list(zip(values, statuses, strict=True)) + for name, entry in SOURCE_ENTRIES.items(): + domain = SOURCE_CODES[name[:3]] + frame, values, statuses = routing._codes_frame( + name, ordered[name], domain, domain, width=entry[0] + ) + out = pd.concat([out, frame], axis=1) + codes[name] = list(zip(values, statuses, strict=True)) + for family in SOURCE_CODES: + for slot in (1, 2): + name = family + "_VAL" + str(slot) + labels = [ + _slot_status(age, receipt, code, amount) + for age, receipt, code, amount in zip( + ages, + codes[family + "_YN"], + codes[family + "_SC" + str(slot)], + parsed[name], + strict=True, + ) + ] + known = np.array( + [s in routing.KNOWN_AMOUNT_STATUSES for s in labels], dtype=bool + ) + out[name + "_reporting_status"] = pd.array(labels, dtype="string") + out[name + "_amount_known"] = known + out[name + "_amount"] = out[name + "_published_amount"].where(known) + for prefix, total, first, second in COMPARISONS: + suffix = "main_slots" if prefix == "distribution" else "visible_slots" + # Only a difference of readable published numbers, including raw NIU + # zeros. A zero difference is not analytic completeness or absence. + out[prefix + "_total_minus_" + suffix] = ( + out[total + "_published_amount"] + - out[first + "_published_amount"] + - out[second + "_published_amount"] + ) + for name, _, _, _, _, allowed in ALLOCATION_ENTRIES: + frame, _, _ = routing._codes_frame(name, ordered[name], allowed, width=1) + out = pd.concat([out, frame], axis=1) + for name, *_ in TOPCODE_ENTRIES: + frame, _, _ = routing._codes_frame(name, ordered[name], (0, 1), width=1) + out = pd.concat([out, frame], axis=1) + out["source_age"] = ages + out["survivor_visible_slots_exhaustive"] = False + out["taxability_assigned"] = False + out["regularity_assigned"] = False + out["acs_retirement_component_assigned"] = False + return out + + +@dataclass(frozen=True) +class CurrentAsecRetirementDetailValues: + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def retirement_detail_values_seal(values): + """Physical value seal, including nullable backing storage and exact bits.""" + require(type(values) is CurrentAsecRetirementDetailValues, "VALUES_TYPE") + digest = hashlib.sha256(PROTOCOL.encode()) + for table in (values.person, values.asec_literals): + require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE_TYPE") + digest.update( + physical._json( + { + "columns": list(table.columns), + "columns_axis": physical.checkpoint._index_spec( + table.columns, label="retirement detail columns" + ), + "index": physical.checkpoint._index_spec( + table.index, label="retirement detail" + ), + } + ) + ) + physical._series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + series = table[column] + if isinstance(series.dtype, pd.Float64Dtype): + # The checkpoint digest supports NumPy floats, but deliberately + # excludes Float64 extension arrays. Keep the nullable tag and + # hash both physical arrays, including values under null masks. + digest.update(physical._json({"dtype": "Float64", "nullable": True})) + physical._series_digest(digest, pd.Series(series.array._data)) + physical._series_digest(digest, pd.Series(series.array._mask)) + else: + physical._series_digest(digest, series) + digest.update(physical._json(values.evidence)) + return digest.hexdigest() + + +def _capture_member(root, pin): + _, member, _, digest, rows, size = pin + with tempfile.TemporaryDirectory(prefix="microcosm-asec-retirement-detail-") as tmp: + path = Path(tmp) / member + identity = routing.coverage._capture( + root / "asec" / member, + path, + size=size, + digest=digest, + budget=[routing.coverage._BODY_MAX], + ) + raw = routing._read_capture(path, rows=rows, columns=READ_COLUMNS, patterns={}) + require( + routing.coverage._identity(path.stat(follow_symlinks=False)) == identity + and routing._file_sha(path) == digest, + "CAPTURE_CHANGED", + ) + return raw + + +def _compare_amount(ready, positions, name, literals): + field = ready.field(name) + entry = amount_entries()[name] + pairs = [_amount_pair(t, entry) for t in literals] + require( + all(s in ("missing", "in_printed_range") for _, s in pairs), "AMOUNT_LITERAL" + ) + valid = field.validity[positions] == 1 + require(np.array_equal(valid, [v is not None for v, _ in pairs]), "AMOUNT_VALIDITY") + expected = np.array( + [np.nan if v is None else v for v, _ in pairs], dtype=np.float64 + ) + if name in REFERENCE_FIELDS: + routing.amount_observations(field, positions, literals) + # ANN's published -1 is retained as +0 with DECLARED_NIU by the money owner. + # Normalize only that declared non-dollar code for this comparison; the + # separate literal/published projection below keeps the original -1. + niu = valid & np.isin(expected, entry.nonmoney_codes) + require( + np.array_equal( + field.statuses[positions] == routing.money.CodebookStatus.DECLARED_NIU, + niu, + ), + "AMOUNT_NIU_STATUS", + ) + expected[niu] = 0.0 + require( + np.array_equal( + field.amounts[positions][valid].view("uint64"), + expected[valid].view("uint64"), + ), + "AMOUNT_BITS", + ) + return field + + +def qualify_current_asec_retirement_detail(preparation): + """Borrow the original preparation, capture once, and requalify before return.""" + source = routing.source + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state, native = entry[2], entry[2].native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + _domain_agreement(ready) + header, native_document = json.loads(ready.header), json.loads(issued[1]) + require( + header["target_year"] == routing.CURRENT_INCOME_YEAR + and header["semantic"] == "annual_current_money" + and native_document["source_year"] + == native_document["income_year"] + == routing.CURRENT_INCOME_YEAR + and native_document["survey_year"] == (routing.CURRENT_INCOME_YEAR + 1), + "PERIOD", + ) + pins = [ + p for p in routing.coverage._MEMBER_PINS if p[0] == routing.CURRENT_INCOME_YEAR + ] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "MEMBER_BINDING", + ) + raw = _capture_member(state.root, pins[0]) + positions = np.flatnonzero( + np.asarray(parent.scope.person_years) == routing.CURRENT_INCOME_YEAR + ) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent.frame.person.iloc[positions][parent_name].to_numpy(), + ), + "PARENT_COORDINATE", + ) + fields = { + name: _compare_amount(ready, positions, name, ordered[name]) + for name in RETAINED_MONEY_FIELDS + } + basis = project_retirement_detail_literals(ordered) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + for field_name, field in fields.items(): + for name in ("statuses", "validity", "zero_origin"): + basis[field_name + "_parent_" + name] = getattr(field, name)[positions] + selected = state.frame.person.loc[ + state.frame.person[routing.support_channel_column("person")].eq("asec") + ] + native_ids = selected[routing.spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + literals = ordered.copy() + literals.index = basis.index.copy() + evidence = { + "protocol": PROTOCOL, + "dictionary_url": routing.DICTIONARY_URL, + "dictionary_sha256": routing.DICTIONARY_SHA256, + "amount_entries": { + name: asdict(entry) for name, entry in amount_entries().items() + }, + "receipt_entries": { + name: entry._asdict() for name, entry in RECEIPT_ENTRIES.items() + }, + "source_entries": dict(SOURCE_ENTRIES), + "source_codes": {name: dict(values) for name, values in SOURCE_CODES.items()}, + "allocation_entries": ALLOCATION_ENTRIES, + "allocation_entry_width": 1, + "allocation_header_range": "0:9; supported value domains retained separately", + "topcode_entries": TOPCODE_ENTRIES, + "topcode_entry_width": 1, + "retained_money_fields": RETAINED_MONEY_FIELDS, + "independently_qualified_literal_fields": INDEPENDENT_LITERAL_FIELDS, + "comparison_scope": "raw readable published amounts, including none/NIU zeros; not a decomposition", + "survivor_total_scope": "SUR_VAL1/2 edited amounts plus unedited sources 3 and 4", + "distribution_total_scope": "printed DBTN_VAL formula uses DST_VAL1 + DST_VAL2, not YNG slots", + "annuity_distribution_reference_scope": "raw retained observations only; canonical receipt/route interpretation remains with income routing", + "universe_notes": [ + "I_SURVL2 prints SURV_VAL2; preserve spelling rather than invent a compiler predicate", + "DIS_CS describes leaving work for health; DIS_HP also includes limited work", + "Regular IRA is an account type, not evidence of regular withdrawals", + ], + "preparation_sha256": routing._sha(entry[1]), + "asec_native_sha256": routing._sha(issued[1]), + "money_header_sha256": routing._sha(ready.header), + "source_member_sha256": digest, + "source_year": routing.CURRENT_INCOME_YEAR, + "survey_year": (routing.CURRENT_INCOME_YEAR + 1), + "complete_source_rows": rows, + "selected_rows": len(out), + "read_columns": READ_COLUMNS, + "taxability_assigned": False, + "regularity_assigned": False, + "acs_retirement_component_assigned": False, + "aggregate_discrepancy_allocated": False, + "under15_completed_with_zero": False, + "source_admission_issued": False, + "release_eligible": False, + } + # Freeze detached JSON-compatible metadata before its seal; do not retain the + # module's mutable constant dictionaries inside a returned descriptive view. + result = CurrentAsecRetirementDetailValues( + out, literals, json.loads(json.dumps(evidence)) + ) + seal = retirement_detail_values_seal(result) + require( + preparation._checked() is entry and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require(retirement_detail_values_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_unemployment_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_unemployment_source.py new file mode 100644 index 000000000..7eb0d827c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_unemployment_source.py @@ -0,0 +1,347 @@ +"""Receipt-universe projection from the retained current ASEC original. + +UC_VAL's numeric zero is not itself evidence of nonreceipt. This module keeps +the source amount separate from a canonical amount whose reporting basis is +known. Detached projection values grant no source authority: consuming hosts +requalify the retained preparation and compare these values before/after I/O. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_coverage_authentication as coverage +from . import asec_current_money as money +from . import source_csv_builtin +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-asec-unemployment-source.v1" +READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "UC_VAL", "UC_YN") +DICTIONARY = { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "pdf_page_1based": 50, + "printed_page": "6C-29", + "receipt_field": "UC_YN", + "receipt_codes": {"0": "niu", "1": "yes", "2": "no"}, + "receipt_universe": "All Persons aged 15+", + "amount_field": "UC_VAL", + "amount_universe": "UC_YN = 1", + "zero_code": "none or niu", +} + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_ASEC_UNEMPLOYMENT_" + reason) + + +def reporting_basis(amount, age, receipt_tokens): + """Classify invented or observed arrays without issuing source authority. + + Invalid/missing receipt literals are unresolved, rather than recoded to no. + Contradictions remain inspectable source observations, excluded from the + modeled donor and canonical amount. Age only gates the printed universe; + it never creates an income zero. + """ + amounts = np.asarray(amount) + ages = np.asarray(age) + tokens = tuple(receipt_tokens) + require( + amounts.dtype == np.dtype("float64") + and ages.dtype == np.dtype("float64") + and amounts.ndim == ages.ndim == 1 + and len(amounts) == len(ages) == len(tokens), + "ARRAY_CONTRACT", + ) + require( + np.isfinite(ages).all() + and ((ages >= 0) & (ages <= 99) & (ages == np.floor(ages))).all(), + "AGE_DOMAIN", + ) + require( + all(type(t) is str and len(t) <= 64 for t in tokens) + and not np.isinf(amounts).any() + and ( + (amounts[np.isfinite(amounts)] >= 0) + & (amounts[np.isfinite(amounts)] <= 99999) + ).all(), + "AMOUNT_OR_TOKEN_DOMAIN", + ) + result = pd.DataFrame({"source_amount": amounts.copy(), "receipt_literal": tokens}) + result["source_reporting_universe"] = ages >= 15 + result["receipt_code_known"] = [t in ("0", "1", "2") for t in tokens] + result["receipt_code"] = pd.array( + [int(t) if t in ("0", "1", "2") else pd.NA for t in tokens], dtype="Int8" + ) + canonical = np.full(len(amounts), np.nan, dtype=np.float64) + labels = [] + for i, (value, years, token) in enumerate(zip(amounts, ages, tokens, strict=True)): + if years < 15: + label = ( + "outside_reporting_universe" + if token == "0" and (np.isnan(value) or value == 0) + else "contradictory_outside_reporting_universe" + ) + elif token == "": + label = "missing_receipt_literal" + elif token not in ("0", "1", "2"): + label = "unrecognized_receipt_literal" + elif np.isnan(value): + label = "missing_amount" + elif token == "0": + label = "niu" if value == 0 else "contradictory_niu_positive" + elif token == "1": + label = "known_receipt" if value > 0 else "ambiguous_recipient_zero" + if value > 0: + canonical[i] = value + else: + label = "known_nonreceipt" if value == 0 else "contradictory_no_positive" + if value == 0: + canonical[i] = value + labels.append(label) + result["reporting_status"] = labels + result["canonical_amount"] = canonical + result["canonical_amount_known"] = np.isfinite(canonical) + return result + + +def _read_capture(path, *, rows): + """Bounded literal reader; its path and row count do not establish authority.""" + reader = source_csv_builtin.capture_csv_reader(csv) + require(reader is not None, "CSV_READER_CHANGED") + records, keys, coordinates = [], set(), set() + with Path(path).open("r", encoding="utf-8-sig", newline="") as handle: + stream = reader(handle, strict=True) + header = next(stream, []) + require( + bool(header) + and all(header) + and len(header) == len(set(header)) + and set(READ_COLUMNS) <= set(header), + "HEADER", + ) + positions = [header.index(c) for c in READ_COLUMNS] + for row in stream: + require(len(row) == len(header) and len(records) < rows, "ROW_SHAPE") + record = dict(zip(READ_COLUMNS, (row[i] for i in positions), strict=True)) + key = record["PERIDNUM"] + require(re.fullmatch(r"[0-9]{22}", key, re.ASCII) is not None, "PERSON_KEY") + for name, width in (("PH_SEQ", 5), ("A_LINENO", 2), ("A_AGE", 2)): + require( + re.fullmatch(r"[0-9]{1," + str(width) + "}", record[name], re.ASCII) + is not None, + "COORDINATE:" + name, + ) + pair = (int(record["PH_SEQ"]), int(record["A_LINENO"])) + require( + min(pair) > 0 and pair not in coordinates and key not in keys, + "DUPLICATE_OR_INVALID_COORDINATE", + ) + require( + record["UC_VAL"] == "" + or re.fullmatch(r"[0-9]{1,5}", record["UC_VAL"], re.ASCII) is not None, + "AMOUNT_TOKEN", + ) + require(len(record["UC_YN"]) <= 64, "RECEIPT_TOKEN_BOUND") + records.append(record) + keys.add(key) + coordinates.add(pair) + require(len(records) == rows, "ROW_COUNT") + return pd.DataFrame(records).set_index("PERIDNUM", drop=False) + + +def _file_sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + while chunk := handle.read(1_048_576): + digest.update(chunk) + return digest.hexdigest() + + +def _amount_observations(field, positions, literals): + """Join literal validity; missing backing storage never becomes a survey zero. + + Today's retained parent requires complete ready money before issuing a + preparation. This helper keeps that prerequisite separate from the literal + join, so a missing field is represented correctly if that scope later grows. + """ + require(type(field) is money.MoneyField, "MONEY_FIELD_TYPE") + positions = np.asarray(positions) + require( + positions.dtype == np.dtype("int64") and positions.ndim == 1, "MONEY_POSITIONS" + ) + tokens = tuple(literals) + require( + len(tokens) == len(positions) + and all( + type(t) is str and (t == "" or re.fullmatch(r"[0-9]{1,5}", t, re.ASCII)) + for t in tokens + ), + "AMOUNT_TOKEN", + ) + valid = field.validity[positions] == 1 + require( + np.array_equal(valid, np.array([t != "" for t in tokens])), + "CURRENT_AMOUNT_SOURCE_VALIDITY", + ) + amounts = field.amounts[positions].copy() + literal_values = np.asarray([float(t) if t else np.nan for t in tokens]) + require( + np.array_equal(amounts[valid], literal_values[valid]), + "CURRENT_AMOUNT_SOURCE_IDENTITY", + ) + amounts[~valid] = np.nan + return amounts + + +@dataclass(frozen=True) +class CurrentAsecUnemploymentValues: + person: pd.DataFrame + evidence: dict + + +def qualify_current_asec_unemployment(preparation): + """Capture the exact current source member retained by the original owner.""" + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "MONEY_PERIOD", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + pins = [p for p in coverage._MEMBER_PINS if p[0] == 2024] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and retained[0]["member"] == member + and retained[0]["archive_sha256"] == archive + and retained[0]["member_sha256"] == digest + and retained[0]["rows"] == rows + and retained[0]["member_bytes"] == size, + "NATIVE_MEMBER_BINDING", + ) + with tempfile.TemporaryDirectory(prefix="microcosm-current-uc-") as tmp: + captured = Path(tmp) / member + identity = coverage._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + budget=[coverage._BODY_MAX], + ) + raw = _read_capture(captured, rows=rows) + require( + coverage._identity(captured.stat(follow_symlinks=False)) == identity + and _file_sha(captured) == digest, + "CAPTURE_CHANGED", + ) + positions = np.flatnonzero(np.asarray(parent.scope.person_years) == 2024) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_CURRENT_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + parent_people = parent.frame.person.iloc[positions] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent_people[parent_name].to_numpy(), + ), + "PARENT_COORDINATE_IDENTITY", + ) + field = ready.field("UC_VAL") + amounts = _amount_observations(field, positions, ordered.UC_VAL) + basis = reporting_basis( + amounts, ordered.A_AGE.to_numpy(dtype="float64"), ordered.UC_YN + ) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + basis["amount_status"] = field.statuses[positions] + basis["amount_validity"] = field.validity[positions] + basis["zero_origin"] = field.zero_origin[positions] + people = state.frame.person + selected = people.loc[people[support_channel_column("person")].eq("asec")] + native_ids = selected[spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + evidence = { + "protocol": PROTOCOL, + "dictionary": json.loads(json.dumps(DICTIONARY)), + "preparation_sha256": hashlib.sha256(entry[1]).hexdigest(), + "asec_native_sha256": hashlib.sha256(issued[1]).hexdigest(), + "money_header_sha256": hashlib.sha256(ready.header).hexdigest(), + "source_member_sha256": digest, + "read_columns": list(READ_COLUMNS), + "complete_current_source_rows": rows, + "selected_rows": len(out), + "projection_sha256": hashlib.sha256( + out.to_json(orient="table").encode() + ).hexdigest(), + "policy": "known_yes_positive_or_no_zero_only; ambiguous_NIU_missing_contradictions_retained_unknown", + "unallocated_observation_claim": False, + "source_admission_issued": False, + "release_eligible": False, + } + require( + preparation._checked()[1] == entry[1] and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require( + hashlib.sha256(out.to_json(orient="table").encode()).hexdigest() + == evidence["projection_sha256"], + "FINAL_PROJECTION_CHANGED", + ) + return CurrentAsecUnemploymentValues(out, evidence) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_completion_routing.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_completion_routing.py new file mode 100644 index 000000000..1765a6aa1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_completion_routing.py @@ -0,0 +1,714 @@ +"""Pure completion diagnostics over original source descriptions; no authority. + +The retaining country host must qualify sources and check the complete parent +around relevant I/O. This module diagnoses descriptions, never issues that +authority, fits a model, or assigns an amount to a population. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from math import fsum + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, WeightKind +from microcosm.graph import ArtifactOutput, ArtifactType + +from . import current_property_income_sources as sources +from . import support_provenance as provenance +from .property_income_constants import PROPERTY_COMPONENTS + +PROTOCOL = "microcosm.us.property-completion-routing.v1" +PROPERTY_COMPLETION_TYPE = ArtifactType("microcosm.us.property_completion_routing", 1) +_COMPONENT_FIELDS = ( + (PROPERTY_COMPONENTS[0], "TRDINT_VAL", "ordinary_interest_known"), + (PROPERTY_COMPONENTS[2], "DIV_VAL", "dividends_known"), +) +# Routing-policy categories, not alternative source codebooks. Raw source +# statuses remain visible and an unreviewed new status refuses classification. +_ASEC_STATUSES = frozenset( + { + "known_receipt", + "known_nonreceipt", + "observed_zero_component", + "outside_reporting_universe", + "contradictory_outside_reporting_universe", + "missing_receipt_literal", + "unrecognized_receipt_literal", + "missing_amount", + "invalid_amount_literal", + "niu", + "contradictory_niu_nonzero", + "contradictory_no_nonzero", + "ambiguous_recipient_zero", + } +) +_ACS_STATUSES = frozenset( + { + "observed", + "missing_source_amount", + "malformed_source_amount", + "outside_published_domain", + "invalid_adjustment", + "outside_universe_blank", + "outside_universe_observation", + } +) +_LITERAL_STATUSES = frozenset( + {"missing", "malformed", "outside_printed_range", "in_printed_range"} +) + + +@dataclass(frozen=True) +class PropertyCompletionRouting: + """Detached diagnostics. This value is not a source or parent admission.""" + + person: pd.DataFrame + components: pd.DataFrame + reasons: pd.DataFrame + clones: pd.DataFrame + summary: pd.DataFrame + payload: bytes + + +def property_completion_artifact_output() -> ArtifactOutput: + """Declaration for future source-projection wiring; not an attached node.""" + return ArtifactOutput("completion_routing", PROPERTY_COMPLETION_TYPE) + + +def _require(value, reason): + if not value: + raise ValueError("PROPERTY_COMPLETION_" + reason) + + +def _ids(values, reason): + _require(values.dtype == np.dtype("int64"), reason) + return values.to_numpy(copy=True) + + +def _axis(table, expected, reason): + _require( + type(table) is pd.DataFrame + and table.columns.is_unique + and table.index.name == "person_id" + and table.index.dtype == np.dtype("int64") + and table.index.is_unique + and table.index.equals(expected.index), + reason, + ) + _require( + table.native_person_id.dtype == np.dtype("int64") + and table.native_person_id.equals(expected.native_person_id), + reason + "_NATIVE", + ) + + +def _source_error(status, literal=None): + return ( + status.startswith("contradictory_") + or status + in { + "invalid_amount_literal", + "unrecognized_receipt_literal", + "malformed_source_amount", + "outside_published_domain", + "invalid_adjustment", + } + or literal in {"malformed", "outside_printed_range"} + ) + + +def _origin_agreement(qualified, origins): + columns = ["person_id", "native_person_id", "source"] + _require( + qualified.shared_predictors.origins[columns].equals(origins[columns]), + "SHARED_ORIGIN_IDENTITY", + ) + document = json.loads(qualified.origin_document) + records = document["persons"] + table = pd.DataFrame(records["rows"], columns=records["columns"]) + _require( + np.array_equal(_ids(table.person_id, "DOCUMENT_PERSON_ID"), origins.index) + and np.array_equal( + _ids(table.selected_receiving_person_id, "DOCUMENT_NATIVE_ID"), + origins.native_person_id, + ) + and np.array_equal(table.source, origins.source), + "DOCUMENT_ORIGIN_IDENTITY", + ) + for frame in (qualified.source_frame, qualified.shared_predictors.source_frame): + _require( + np.array_equal( + _ids( + frame.person[provenance.spine_source_id_column("person")], + "SOURCE_NATIVE_ID", + ), + origins.native_person_id, + ) + and np.array_equal( + frame.person[provenance.support_channel_column("person")].astype(str), + origins.source, + ), + "SOURCE_FRAME_ORIGIN_IDENTITY", + ) + + +def _asec_components(qualified, origins): + tables = ( + qualified.asec_interest_values.person, + qualified.asec_dividend_values.person, + ) + rows = [] + for table, (component, field, _) in zip(tables, _COMPONENT_FIELDS, strict=True): + _axis(table, origins, "ASEC_AXIS") + statuses = table[field + "_reporting_status"] + literals = table[field + "_literal_status"] + _require( + statuses.notna().all() and set(statuses) <= _ASEC_STATUSES, "ASEC_STATUS" + ) + _require( + literals.notna().all() and set(literals) <= _LITERAL_STATUSES, + "LITERAL_STATUS", + ) + known = table[field + "_amount_known"] + amount = table[field + "_amount"].to_numpy(dtype=np.float64, na_value=np.nan) + _require(known.dtype == np.dtype("bool"), "KNOWNNESS_DTYPE") + expected = statuses.isin( + ("known_receipt", "known_nonreceipt", "observed_zero_component") + ) + if field == "DIV_VAL": + _require( + not statuses.eq("observed_zero_component").any(), "DIVIDEND_ZERO_STATUS" + ) + _require( + known.equals(expected.rename(known.name)) + and np.array_equal(known, np.isfinite(amount)) + and not np.isinf(amount).any() + and not (amount < 0).any(), + "COMPONENT_KNOWNNESS", + ) + basis = qualified.donor_basis.person[component].to_numpy(dtype=np.float64) + _require( + np.array_equal(np.isnan(basis), np.isnan(amount)) + and np.array_equal( + basis[known].view("uint64"), amount[known].view("uint64") + ), + "BASIS_COMPONENT_AGREEMENT", + ) + published = table[field + "_published_amount"].to_numpy( + dtype=np.float64, na_value=np.nan + ) + _require( + np.array_equal( + published[known].view("uint64"), amount[known].view("uint64") + ), + "PUBLISHED_COMPONENT_AGREEMENT", + ) + for original, value, is_known, status, literal in zip( + table.index, amount, known, statuses, literals, strict=True + ): + derived = status == "known_nonreceipt" + _require(not derived or value == 0, "NONRECEIPT_NONZERO") + rows.append( + ( + int(original), + component, + float(value), + bool(is_known), + "derived" + if derived + else ("observed" if is_known else "unresolved"), + status, + literal, + "qualified_known_nonreceipt" + if derived + else ( + "observed_component_zero" if is_known and value == 0 else "none" + ), + ) + ) + return rows + + +def _clones(frame, origins): + _require(type(frame) is Frame, "CLONE_FRAME_TYPE") + table = frame.person + ids = _ids(table.person_id, "CLONE_ID_DTYPE") + original = _ids( + table[provenance.support_source_id_column("person")], "CLONE_ORIGIN_DTYPE" + ) + native = _ids( + table[provenance.spine_source_id_column("person")], "CLONE_NATIVE_DTYPE" + ) + role = _ids( + table[provenance.support_clone_index_column("person")], "CLONE_ROLE_DTYPE" + ) + source = table[provenance.support_channel_column("person")].astype(str).to_numpy() + _require( + table.person_id.is_unique and np.isin(original, origins.index).all(), + "CLONE_COVERAGE", + ) + lookup = origins.reindex(original) + _require( + np.array_equal(native, lookup.native_person_id) + and np.array_equal(source, lookup.source), + "CLONE_SOURCE_IDENTITY", + ) + result = pd.DataFrame( + { + "person_id": ids, + "original_person_id": original, + "native_person_id": native, + "source": source, + "clone_index": role, + } + ) + _require( + np.isin(role, (0, 1)).all() + and not result.duplicated(["original_person_id", "clone_index"]).any() + and result.groupby("original_person_id").size().eq(2).all() + and set(original) == set(origins.index), + "WHOLE_CLONE_PAIRS", + ) + return result.set_index("person_id") + + +def _original_design(frame, origins): + _require( + type(frame) is Frame and frame.schema.person_entity == "person", "SOURCE_FRAME" + ) + _require( + np.array_equal(_ids(frame.person.person_id, "SOURCE_ID_DTYPE"), origins.index), + "SOURCE_FRAME_AXIS", + ) + weights = frame.weights_for("household") + _require( + weights.kind is WeightKind.DESIGN + and frame.resolve_weights("person").kind is WeightKind.DESIGN, + "ORIGINAL_DESIGN_KIND", + ) + households = _ids(frame.table("household").household_id, "HOUSEHOLD_ID_DTYPE") + membership = _ids(frame.person.person_household_id, "MEMBERSHIP_DTYPE") + _require( + len(set(households)) == len(households) + and np.isin(membership, households).all(), + "HOUSEHOLD_MEMBERSHIP", + ) + values = np.array(weights.values, dtype=np.float64, copy=True) + _require(np.isfinite(values).all() and (values >= 0).all(), "DESIGN_VALUES") + series = pd.Series(values, index=pd.Index(households, name="household_id")) + return membership, series, series.loc[membership].to_numpy(copy=True) + + +def _table_document(table): + def cell(value): + if value is pd.NA or value is None: + return None + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, float): + if np.isnan(value): + return None + _require(np.isfinite(value), "PAYLOAD_NONFINITE") + _require(type(value) in (str, int, float, bool), "PAYLOAD_SCALAR") + return value + + # itertuples preserves integer scalar types; iterrows can coerce IDs >2**53. + return { + "columns": list(table.columns), + "index": [cell(v) for v in table.index], + "index_name": table.index.name, + "rows": [ + [cell(v) for v in row] for row in table.itertuples(index=False, name=None) + ], + } + + +def build_property_completion_routing( + qualified: sources.QualifiedPropertyIncomeSources, clone_frame: Frame +) -> PropertyCompletionRouting: + """Diagnose original component availability; never consume native I/O or fit. + + The source descriptions and the supplied Frame/weights are not admissions. + An integrating host retains and requalifies their real issuers around I/O. + """ + _require( + type(qualified) is sources.QualifiedPropertyIncomeSources, + "QUALIFIED_DESCRIPTION_TYPE", + ) + before = sources.property_income_sources_seal(qualified) + clone_before = sources._frame_seal(clone_frame) + origins = qualified.origins + _require( + type(origins) is pd.DataFrame + and origins.index.name == "person_id" + and origins.index.dtype == np.dtype("int64") + and origins.index.is_unique + and origins.columns.is_unique + and len(origins) > 0, + "ORIGIN_AXIS", + ) + _ids(origins.native_person_id, "NATIVE_ID_DTYPE") + _require( + np.array_equal(_ids(origins.person_id, "ORIGIN_ID_DTYPE"), origins.index) + and set(origins.source) <= {"acs", "asec"}, + "ORIGIN_IDENTITY", + ) + _origin_agreement(qualified, origins) + asec = origins.loc[origins.source.eq("asec")] + acs = origins.loc[origins.source.eq("acs")] + for table in (qualified.donor_basis.person, qualified.asec_routing_values.person): + _axis(table, asec, "ASEC_BASIS_AXIS") + anchors = qualified.acs_anchor_values.anchors + _axis(anchors, acs, "ACS_AXIS") + ages = pd.Series(index=origins.index, dtype="float64") + ages.loc[asec.index] = qualified.asec_interest_values.person.source_age + ages.loc[acs.index] = pd.to_numeric(anchors.AGEP, errors="coerce") + _require( + np.isfinite(ages).all() + and ages.between(0, 99).all() + and np.equal(ages, np.floor(ages)).all(), + "SOURCE_AGE", + ) + for table in ( + qualified.asec_dividend_values.person, + qualified.asec_routing_values.person, + qualified.donor_basis.person, + ): + _require( + np.array_equal(table.source_age, ages.loc[asec.index]), + "SOURCE_AGE_AGREEMENT", + ) + components = _asec_components(qualified, asec) + anchor_status = anchors.property_income_status + _require( + anchor_status.notna().all() and set(anchor_status) <= _ACS_STATUSES, + "ACS_STATUS", + ) + known_anchor = anchors.property_income_known + anchor_values = anchors.property_income_amount.to_numpy( + dtype=np.float64, na_value=np.nan + ) + _require( + known_anchor.dtype == np.dtype("bool") + and np.array_equal(known_anchor, anchor_status.eq("observed")) + and np.array_equal(known_anchor, np.isfinite(anchor_values)) + and np.array_equal( + anchors.property_income_in_income_universe, ages.loc[acs.index] >= 15 + ), + "ACS_KNOWNNESS_OR_UNIVERSE", + ) + for original in acs.index: + for component, _, _ in _COMPONENT_FIELDS: + components.append( + ( + int(original), + component, + np.nan, + False, + "unresolved", + "component_not_separately_observed", + "not_separately_observed", + "none", + ) + ) + components = pd.DataFrame( + components, + columns=[ + "original_person_id", + "component", + "amount", + "known", + "origin", + "reporting_status", + "literal_status", + "zero_basis", + ], + ) + order = {int(original): i for i, original in enumerate(origins.index)} + components["_order"] = components.original_person_id.map(order) + components = ( + components.sort_values(["_order", "component"], kind="stable") + .drop(columns="_order") + .reset_index(drop=True) + ) + membership, weights, person_design = _original_design( + qualified.source_frame, origins + ) + clones = _clones(clone_frame, origins) + person = origins[["native_person_id", "source"]].copy(deep=True) + person["source_age"] = ages.astype("int64") + person["original_household_id"] = membership + person["original_household_design_weight"] = person_design + person["anchor_known"] = False + person["anchor_amount"] = np.nan + person["anchor_status"] = "not_an_acs_aggregate_anchor" + person["anchor_origin"] = "unresolved" + person.loc[acs.index, "anchor_known"] = known_anchor.to_numpy() + person.loc[acs.index, "anchor_amount"] = anchor_values + person.loc[acs.index, "anchor_status"] = anchor_status.to_numpy() + person.loc[acs.index[known_anchor], "anchor_origin"] = "observed" + reasons = pd.DataFrame( + False, + index=origins.index, + columns=[ + "under15", + "source_error", + "contradictory_evidence", + "positive_outside_universe", + "missing_source_evidence", + "ambiguous_zero", + "niu", + "ordinary_interest_unknown", + "dividends_unknown", + ], + ) + reasons["under15"] = ages < 15 + for component, field, known_name in _COMPONENT_FIELDS: + selected = ( + components.loc[components.component.eq(component)] + .set_index("original_person_id") + .reindex(origins.index) + ) + person[known_name] = selected.known.to_numpy() + reasons[known_name.replace("_known", "_unknown")] = ~person[known_name] + status, literal = selected.reporting_status, selected.literal_status + reasons["source_error"] |= np.array( + [ + _source_error(status_value, literal_value) + for status_value, literal_value in zip(status, literal, strict=True) + ] + ) + reasons["contradictory_evidence"] |= status.str.startswith("contradictory_") + reasons["missing_source_evidence"] |= status.str.startswith("missing_") + reasons["ambiguous_zero"] |= status.eq("ambiguous_recipient_zero") + reasons["niu"] |= status.eq("niu") + table = ( + qualified.asec_interest_values.person + if field == "TRDINT_VAL" + else qualified.asec_dividend_values.person + ) + published = table[field + "_published_amount"].to_numpy( + dtype=np.float64, na_value=np.nan + ) + reasons.loc[asec.index, "positive_outside_universe"] |= ( + ages.loc[asec.index] < 15 + ) & (published > 0) + reasons.loc[acs.index, "source_error"] |= np.array( + [_source_error(s) for s in anchor_status], dtype=bool + ) + reasons.loc[acs.index, "missing_source_evidence"] |= anchor_status.eq( + "missing_source_amount" + ) + reasons.loc[acs.index, "positive_outside_universe"] |= ( + ages.loc[acs.index] < 15 + ) & (pd.to_numeric(anchors.INTP, errors="coerce") > 0) + reasons.loc[acs.index, "contradictory_evidence"] |= anchor_status.eq( + "outside_universe_observation" + ) + for name in qualified.donor_basis.exclusions: + column = qualified.donor_basis.exclusions[name] + _require( + column.index.equals(asec.index) and column.dtype == np.dtype("bool"), + "JOINT_EXCLUSION_AXIS", + ) + reasons["joint:" + name] = False + reasons.loc[asec.index, "joint:" + name] = column.to_numpy() + routes = [] + for original in origins.index: + if reasons.loc[original, "under15"]: + route = "unsupported_under15_measurement" + elif reasons.loc[original, "source_error"]: + route = "source_review_required" + elif ( + person.loc[original, "ordinary_interest_known"] + and person.loc[original, "dividends_known"] + ): + route = "carry_known_components" + elif person.loc[original, "source"] == "acs": + route = ( + "existing_acs_anchor_decomposition" + if person.loc[original, "anchor_known"] + else "acs_anchor_completion_review" + ) + else: + route = "asec_component_completion_review" + routes.append(route) + person["completion_route"] = routes + masks = {"all": np.ones(len(person), dtype=bool)} + masks.update( + { + "route:" + r: person.completion_route.eq(r).to_numpy() + for r in sorted(set(routes)) + } + ) + masks.update({"reason:" + c: reasons[c].to_numpy() for c in reasons}) + summary_rows = [] + for name, mask in masks.items(): + households = np.unique(membership[mask]) + mass = fsum(person_design[mask]) + household_mass = fsum(weights.loc[households]) + _require( + np.isfinite(mass) and np.isfinite(household_mass), "DESIGN_MASS_OVERFLOW" + ) + summary_rows.append( + (name, int(mask.sum()), len(households), mass, household_mass) + ) + summary = pd.DataFrame( + summary_rows, + columns=[ + "selection", + "person_count", + "household_count", + "design_weighted_person_mass", + "union_household_design_mass", + ], + ).set_index("selection") + tables = { + "person": person, + "components": components, + "reasons": reasons, + "clones": clones, + "summary": summary, + } + payload = json.dumps( + { + "protocol": PROTOCOL, + "source_authority": False, + "complete_parent_authority": False, + "amounts_assigned": False, + "model_executed": False, + "source_input": "descriptive_qualified_values; host_custody_required", + "stage": "before_property_models", + "tables": {k: _table_document(v) for k, v in tables.items()}, + }, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + _require( + sources.property_income_sources_seal(qualified) == before + and sources._frame_seal(clone_frame) == clone_before, + "INPUTS_CHANGED", + ) + return PropertyCompletionRouting( + person, components, reasons, clones, summary, payload + ) + + +# Explicit publication vocabulary. A new source/route reason needs review here +# before it can enter a public receipt; never export arbitrary summary labels. +_PUBLIC_ROUTES = frozenset( + { + "unsupported_under15_measurement", + "source_review_required", + "carry_known_components", + "existing_acs_anchor_decomposition", + "acs_anchor_completion_review", + "asec_component_completion_review", + } +) +_PUBLIC_REASONS = frozenset( + { + "under15", + "source_error", + "contradictory_evidence", + "positive_outside_universe", + "missing_source_evidence", + "ambiguous_zero", + "niu", + "ordinary_interest_unknown", + "dividends_unknown", + "joint:under15", + "joint:reported_total_unknown", + "joint:ordinary_interest_unknown", + "joint:retirement_interest_unknown", + "joint:dividends_unknown", + "joint:property_receipts_unknown", + "joint:interest_discrepancy_unknown", + "joint:interest_discrepancy_nonzero", + "joint:other_income_possible_property", + "joint:other_income_route_unresolved", + "joint:survivor_possible_property", + "joint:survivor_route_unresolved", + "joint:survivor_additional_sources_unresolved", + } +) +_PUBLIC_MEASURES = ( + "person_count", + "household_count", + "design_weighted_person_mass", + "union_household_design_mass", +) + + +def property_completion_public_summary(value: PropertyCompletionRouting) -> dict: + """Allowlisted aggregate description only; private row payload stays local. + + DESIGN measures describe original support, not calibrated representation or + independently disclosive proof. This function does not grant source custody. + """ + _require(type(value) is PropertyCompletionRouting, "PUBLIC_TYPE") + table = value.summary + labels = ( + {"all"} + | {"route:" + r for r in _PUBLIC_ROUTES} + | {"reason:" + r for r in _PUBLIC_REASONS} + ) + _require( + type(table) is pd.DataFrame + and table.index.is_unique + and table.index.name == "selection" + and set(table.index) <= labels + and "all" in table.index + and tuple(table.columns) == _PUBLIC_MEASURES, + "PUBLIC_ROSTER", + ) + rows = [] + for label, row in zip(table.index, table.itertuples(index=False), strict=True): + counts, masses = row[:2], row[2:] + _require( + all( + isinstance(n, (int, np.integer)) + and not isinstance(n, (bool, np.bool_)) + and n >= 0 + for n in counts + ) + and all( + isinstance(n, (float, np.floating)) and np.isfinite(n) and n >= 0 + for n in masses + ), + "PUBLIC_VALUES", + ) + rows.append( + { + "selection": label, + **dict( + zip( + _PUBLIC_MEASURES, + ( + int(counts[0]), + int(counts[1]), + float(masses[0]), + float(masses[1]), + ), + strict=True, + ) + ), + } + ) + return { + "protocol": PROTOCOL, + "stage": "before_property_models", + "amounts_assigned": False, + "model_executed": False, + "source_authority": False, + "complete_parent_authority": False, + "support": "original_household_DESIGN; descriptive_only", + "routes_mutually_exclusive": True, + "reasons_overlap": True, + "summary": rows, + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_income_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_income_sources.py new file mode 100644 index 000000000..5eccce15c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_income_sources.py @@ -0,0 +1,396 @@ +"""Qualify original survey property branches without fitting or issuing authority.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.fit import model_input +from microcosm.frame import WeightKind +from microcosm.graph import Population + +from . import current_acs_income_anchor_source as acs +from . import current_asec_dividend_source as dividend +from . import current_asec_income_routing_source as routing +from . import current_asec_interest_source as interest +from . import current_survey_predictors as shared +from . import graph_full_puf_enrichment as physical +from .current_asec_property_basis import AsecPropertyBasis, build_asec_property_basis +from .property_income_constants import PROPERTY_COMPONENTS, PROPERTY_REPORTED_TOTAL + +PROTOCOL = "microcosm.us.current-property-income-sources.v1" + + +def require(condition, reason): + if not condition: + raise ValueError("PROPERTY_INCOME_SOURCES_" + reason) + + +@dataclass(frozen=True) +class QualifiedPropertyIncomeSources: + """Descriptive values only; the caller must retain and requalify actual owners.""" + + shared_predictors: shared.QualifiedSurveyPredictors + acs_anchor_values: acs.QualifiedAcsIncomeAnchors + asec_interest_values: interest.CurrentAsecInterestValues + asec_routing_values: routing.CurrentAsecIncomeRoutingValues + asec_dividend_values: dividend.CurrentAsecDividendValues + donor_basis: AsecPropertyBasis + source_frame: object + donor_frame: object | None + donor_columns: pd.DataFrame + recipient_frame: object | None + recipient_columns: pd.DataFrame + recipient_matrix: bytes | None + origins: pd.DataFrame + origin_document: bytes + recipient_diagnostics: pd.DataFrame + projection: bytes + evidence: dict + + +def _routing_seal(value): + require(type(value) is routing.CurrentAsecIncomeRoutingValues, "ROUTING_TYPE") + # The legacy routing qualifier exposes no physical-seal helper. Preserve + # its complete detached descriptions, including nullable backing storage. + return ( + physical._table_stamp(value.person), + physical._table_stamp(value.asec_literals), + shared.codec.encode_json(value.evidence), + ) + + +def _source_seals(values): + predictor, anchors, interest_values, routing_values, dividend_values = values + return ( + shared._qualified_seal(predictor), + acs.income_anchor_seal(anchors), + interest.interest_values_seal(interest_values), + _routing_seal(routing_values), + dividend.dividend_values_seal(dividend_values), + ) + + +def _frame_seal(frame): + return ( + None + if frame is None + else physical._population_stamp( + Population.from_frame(frame, "property_income_sources.description") + ) + ) + + +def property_income_sources_seal(value): + """Complete descriptive physical seal, not source authentication or a token.""" + require(type(value) is QualifiedPropertyIncomeSources, "VALUES_TYPE") + return ( + _source_seals( + ( + value.shared_predictors, + value.acs_anchor_values, + value.asec_interest_values, + value.asec_routing_values, + value.asec_dividend_values, + ) + ), + tuple( + _frame_seal(frame) + for frame in (value.source_frame, value.donor_frame, value.recipient_frame) + ), + tuple( + physical._table_stamp(table) + for table in ( + value.donor_basis.person, + value.donor_basis.provenance, + value.donor_basis.exclusions, + value.donor_basis.summary, + value.donor_columns, + value.recipient_columns, + value.origins, + value.recipient_diagnostics, + ) + ), + value.recipient_matrix, + value.origin_document, + value.projection, + shared.codec.encode_json(value.evidence), + ) + + +def _same_axis(frame, origins, reason): + require( + frame.index.equals(origins.index) + and frame.index.name == origins.index.name + and frame.native_person_id.equals(origins.native_person_id), + reason, + ) + + +def _compose(values, origin_document): + """Pure assembly of already-qualified values; no source authority from args.""" + predictor, anchors, interest_values, routing_values, dividend_values = values + frame = predictor.source_frame + people = frame.person + origins = predictor.origins.copy(deep=True) + ids = pd.Index(people.person_id.to_numpy(copy=True), name="person_id") + require( + ids.equals(origins.index) + and np.array_equal(origins.person_id.to_numpy(), ids.to_numpy()) + and set(origins.source) == {"asec", "acs"}, + "ORIGIN_AXIS", + ) + raw_origins = json.loads(origin_document)["persons"] + full_origins = pd.DataFrame(raw_origins["rows"], columns=raw_origins["columns"]) + require( + np.array_equal(full_origins.person_id.to_numpy(), ids.to_numpy()) + and np.array_equal(full_origins.source.to_numpy(), origins.source.to_numpy()) + and np.array_equal( + full_origins.selected_receiving_person_id.to_numpy(), + origins.native_person_id.to_numpy(), + ), + "FULL_ORIGIN_AXIS", + ) + donor_origins = origins.loc[origins.source.eq("asec")] + recipient_origins = origins.loc[origins.source.eq("acs")] + for selected in ( + interest_values.person, + routing_values.person, + dividend_values.person, + ): + _same_axis(selected, donor_origins, "ASEC_SOURCE_AXIS") + _same_axis(anchors.anchors, recipient_origins, "ACS_SOURCE_AXIS") + require( + predictor.donor_columns.index.equals(donor_origins.index) + and np.array_equal( + predictor.donor_frame.person.person_id.to_numpy(), + donor_origins.index.to_numpy(), + ), + "DONOR_FEATURE_AXIS", + ) + donor_source = predictor.donor_frame + require( + donor_source.weights_for("household").kind is WeightKind.DESIGN + and donor_source.resolve_weights("person").kind is WeightKind.DESIGN, + "ORIGINAL_DESIGN_WEIGHTS", + ) + membership = pd.Series( + donor_source.person.person_household_id.to_numpy(copy=True), + index=donor_origins.index, + name="household_id", + ) + household_ids = pd.Index( + donor_source.table("household").household_id.to_numpy(copy=True), + name="household_id", + ) + design_weights = pd.Series( + donor_source.weights_for("household").values.copy(), index=household_ids + ) + basis = build_asec_property_basis( + interest=interest_values.person, + income_routing=routing_values.person, + dividend=dividend_values.person, + original_household_membership=membership, + original_household_design_weights=design_weights, + ) + require( + np.array_equal( + basis.person.original_household_design_weight.to_numpy(), + donor_source.resolve_weights("person").values, + ), + "PERSON_DESIGN_WEIGHT_MAPPING", + ) + features = shared.feature_columns(predictor.demographic_conditioning) + donor_columns = predictor.donor_columns.loc[:, list(features)].copy(deep=True) + for name in (PROPERTY_REPORTED_TOTAL, *PROPERTY_COMPONENTS): + donor_columns[name] = basis.person[name].to_numpy(copy=True) + donor_mask = basis.person.joint_component_fit_eligible.to_numpy(dtype=bool) + donor_columns = donor_columns.loc[donor_mask].copy(deep=True) + donor_frame = donor_source.select(donor_mask) if donor_mask.any() else None + decoded = model_input.decode_recipient_matrix(predictor.matrix) + require( + decoded.entity == "person" + and tuple(decoded.features.columns) == features + and decoded.features.index.equals(recipient_origins.index), + "RECIPIENT_FEATURE_AXIS", + ) + original_anchors = anchors.anchors + diagnostics = original_anchors.copy(deep=True) + adult = original_anchors.property_income_in_income_universe.to_numpy(dtype=bool) + known = original_anchors.property_income_known.to_numpy(dtype=bool) + amounts = original_anchors.property_income_amount.to_numpy( + dtype=np.float64, na_value=np.nan + ) + require(np.array_equal(known, np.isfinite(amounts)), "ACS_ANCHOR_KNOWNNESS") + diagnostics["excluded_under15"] = ~adult + diagnostics["excluded_unknown_anchor"] = ~known + diagnostics["eligible_recipient"] = adult & known + recipient_mask = adult & known + recipient_columns = decoded.features.copy(deep=True) + recipient_columns[PROPERTY_REPORTED_TOTAL] = amounts + recipient_columns = recipient_columns.loc[recipient_mask].copy(deep=True) + frame_mask = np.zeros(len(frame.person), dtype=bool) + frame_mask[np.flatnonzero(origins.source.eq("acs"))] = recipient_mask + recipient_frame = frame.select(frame_mask) if frame_mask.any() else None + matrix = ( + model_input.encode_recipient_matrix( + recipient_columns, + entity="person", + entity_ids=recipient_columns.index.to_numpy(dtype=" 0 and pair not in coordinates and selected[0] not in seen, + "COORDINATE_DUPLICATE", + ) + coordinates.add(pair) + seen.add(selected[0]) + records.append(record) + require(len(records) == rows, "ROW_COUNT") + return pd.DataFrame(records, columns=READ_COLUMNS).set_index("PERIDNUM", drop=False) + + +def _codes(tokens, allowed): + values = np.full(len(tokens), np.nan, dtype=np.float64) + for i, token in enumerate(tokens): + require(type(token) is str, "CODE_TOKEN_TYPE") + if token != "": + require(re.fullmatch(r"[0-9]+", token, re.ASCII) is not None, "CODE_TOKEN") + require(int(token) in allowed, "CODE_DOMAIN") + values[i] = int(token) + return values + + +def _allocation_labels(raw): + total = _codes(raw.I_SSVAL, {0, 11, 12, 13, 14, 15}) + receipt = _codes(raw.I_SSYN, {0, 10, 11}) + reason = _codes(raw.RESNSSA, set(range(10))) + return pd.DataFrame( + { + "amount_allocation_code": total, + "recipiency_allocation_code": receipt, + "reason_allocation_code": reason, + "allocation_origin": [ + "unresolved_allocation_provenance" + if not np.isfinite([a, b, c]).all() + else "publisher_allocated" + if any(v != 0 for v in (a, b, c)) + else "publisher_no_allocation" + for a, b, c in zip(total, receipt, reason, strict=True) + ], + }, + index=raw.index, + ) + + +@dataclass(frozen=True) +class CurrentSocialSecurityProjection: + """Computed data, requalified by the host before execution and replay.""" + + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def qualify_current_social_security(preparation): + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + acs_owned = source.acs_native._owned(state.native[0]) + acs_receipt = json.loads(acs_owned.payload) + require(acs_receipt["vintage"] == 2024, "ACS_NATIVE_PERIOD") + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "MONEY_PERIOD", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + selected_pins = [pin for pin in coverage._MEMBER_PINS if pin[0] == 2024] + require(len(selected_pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = selected_pins[0] + native_sources = issued[2].coverage.receipt["sources"] + retained = [s for s in native_sources if s["source_year"] == year] + require( + len(retained) == 1 + and retained[0]["member"] == member + and retained[0]["archive_sha256"] == archive + and retained[0]["member_sha256"] == digest + and retained[0]["rows"] == rows + and retained[0]["member_bytes"] == size, + "NATIVE_MEMBER_BINDING", + ) + with tempfile.TemporaryDirectory(prefix="microcosm-current-ss-") as tmp: + captured = Path(tmp) / member + identity = coverage._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + # _capture's budget bounds its encoded coverage projection, not + # CSV bytes. Short valid source rows can expand in that format. + # Use the source owner's existing closed projection bound; the + # exact captured CSV size and digest remain separate checks. + budget=[coverage._BODY_MAX], + ) + raw = _read_capture(captured, rows=rows) + require( + coverage._identity(captured.stat(follow_symlinks=False)) == identity, + "CAPTURE_CHANGED", + ) + require(_file_sha(captured) == digest, "CAPTURE_DIGEST") + + scope = parent.scope + current_positions = np.flatnonzero(np.asarray(scope.person_years) == 2024) + keys = np.asarray(scope.person_native_keys)[current_positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_CURRENT_SOURCE_JOIN", + ) + ordered = raw.loc[keys].copy() + pp = parent.frame.person.iloc[current_positions] + for native_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[native_name].astype("int64").to_numpy(), + pp[parent_name].to_numpy(), + ), + "PARENT_COORDINATE_IDENTITY", + ) + field = ready.field("SS_VAL") + require((field.validity[current_positions] == 1).all(), "CURRENT_AMOUNT_UNKNOWN") + amount = field.amounts[current_positions].copy() + require( + amount.dtype == np.dtype("float64") + and np.isfinite(amount).all() + and (amount >= 0).all(), + "CURRENT_AMOUNT_DOMAIN", + ) + literal_amount = _codes(ordered.SS_VAL, range(100000)) + require(np.array_equal(literal_amount, amount), "CURRENT_AMOUNT_SOURCE_IDENTITY") + reason_1 = _codes(ordered.RESNSS1, range(9)) + reason_2 = _codes(ordered.RESNSS2, range(9)) + recipiency = _codes(ordered.SS_YN, {0, 1, 2}) + report_amount, components, allowed, labels = basis_owner.asec_reporting_basis( + amount, + ordered.A_AGE.to_numpy(dtype=np.float64), + recipiency, + reason_1, + reason_2, + ) + allocation = _allocation_labels(ordered) + + people = state.frame.person + require( + people.person_id.dtype == np.dtype("int64") + and people.person_id.is_unique + and (people.person_id >= 0).all(), + "PERSON_AXIS", + ) + channel = people[support_channel_column("person")] + require(set(channel) == {"acs", "asec"}, "CHANNEL_ROSTER") + out = pd.DataFrame(index=pd.Index(people.person_id, name="person_id")) + out["native_person_id"] = people[spine_source_id_column("person")].to_numpy() + out["source"] = channel.to_numpy() + out["social_security_source_total"] = np.nan + out["source_reporting_universe"] = False + out["source_reporting_unit"] = "person_report_record" + for c in basis_owner.COMPONENTS: + out[c] = np.nan + out["allowed_" + c] = True + out["basis_origin"] = "unresolved" + out["allocation_origin"] = "unresolved_allocation_provenance" + original_ids = np.asarray(scope.person_ids)[current_positions] + lookup = {int(pid): i for i, pid in enumerate(original_ids)} + asec_ids = out.index[out.source.eq("asec")] + require( + all(int(v) in lookup for v in out.loc[asec_ids, "native_person_id"]), + "SELECTED_NATIVE_JOIN", + ) + take = np.asarray([lookup[int(v)] for v in out.loc[asec_ids, "native_person_id"]]) + require(len(set(take)) == len(take), "SELECTED_NATIVE_DUPLICATE") + out.loc[asec_ids, "social_security_source_total"] = report_amount[take] + out.loc[asec_ids, "source_reporting_universe"] = ( + ordered.A_AGE.to_numpy(dtype=np.float64)[take] >= 15 + ) + out.loc[asec_ids, "source_reporting_unit"] = ( + "person_report_may_combine_family_payments" + ) + for j, c in enumerate(basis_owner.COMPONENTS): + out.loc[asec_ids, c] = components[take, j] + out.loc[asec_ids, "allowed_" + c] = allowed[take, j] + out.loc[asec_ids, "basis_origin"] = np.asarray(labels)[take] + out.loc[asec_ids, "allocation_origin"] = allocation.allocation_origin.to_numpy()[ + take + ] + + acs = people.loc[channel.eq("acs")] + require( + {"SSP", "AGEP", "ADJINC", "acs_social_security_income"} <= set(acs), + "ACS_SOURCE_COLUMNS", + ) + raw_ss = _numeric(acs.SSP) + adjusted = _numeric(acs.acs_social_security_income) + age = _numeric(acs.AGEP) + factor = _numeric(acs.ADJINC) / 1_000_000 + require( + np.isfinite(age).all() + and (age == np.floor(age)).all() + and ((age >= 0) & (age <= 99)).all(), + "ACS_AGE", + ) + require(np.isfinite(factor).all() and (factor > 0).all(), "ACS_PRICE_FACTOR") + eligible = age >= 15 + out.loc[acs.person_id, "source_reporting_universe"] = eligible + require( + np.isnan(raw_ss[~eligible]).all() and np.isnan(adjusted[~eligible]).all(), + "ACS_OUTSIDE_REPORTING_UNIVERSE_OBSERVATION", + ) + require( + np.isfinite(raw_ss[eligible]).all() + and ( + (raw_ss[eligible] == 0) + | ((raw_ss[eligible] >= 4) & (raw_ss[eligible] <= 55000)) + ).all() + and (raw_ss[eligible] == np.floor(raw_ss[eligible])).all(), + "ACS_SSP_DOMAIN", + ) + require( + np.array_equal(adjusted[eligible], raw_ss[eligible] * factor[eligible]), + "ACS_SSP_ADJUSTMENT_IDENTITY", + ) + # Below-age-15 SSP is explicitly out of the question universe. Keep it + # unknown here; a separate reviewed reporting-unit convention is needed. + out.loc[acs.person_id, "social_security_source_total"] = adjusted + zero_ids = acs.person_id.to_numpy()[eligible & (raw_ss == 0)] + out.loc[zero_ids, list(basis_owner.COMPONENTS)] = 0.0 + out.loc[zero_ids, ["allowed_" + c for c in basis_owner.COMPONENTS]] = False + out.loc[zero_ids, "basis_origin"] = "known_total_zero" + out.loc[acs.person_id.to_numpy()[eligible & (raw_ss > 0)], "basis_origin"] = ( + "acs_combined_positive_requires_model" + ) + out.loc[acs.person_id.to_numpy()[~eligible], "basis_origin"] = ( + "acs_below15_outside_reporting_universe" + ) + evidence = { + "protocol": PROTOCOL, + "dictionary": json.loads(json.dumps(DICTIONARY)), + "preparation_sha256": _sha(entry[1]), + "asec_native_sha256": _sha(issued[1]), + "acs_native_sha256": _sha(acs_owned.payload), + "acs_survey_year": 2024, + "money_header_sha256": _sha(ready.header), + "source_member_sha256": digest, + "read_columns": list(READ_COLUMNS), + "complete_current_source_rows": rows, + "asec_income_year": 2024, + "asec_interview_year": 2025, + "price_basis_year": 2024, + "acs_income_window": "rolling_prior_12_months_at_2024_interview", + "acs_price_adjustment": "SSP*(ADJINC/1000000)", + "acs_dictionary": { + "url": "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf", + "sha256": "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff", + "ssp_pdf_page_1based": 45, + "fssp_pdf_page_1based": 130, + }, + "acs_allocation_flag": "FSSP_not_in_current_native_projection; provenance_unresolved", + "asec_component_mapping": "unique_reported_component_only; no priority or age fallback", + "knownness": "unknown components and outside-universe SSP retained as unknown", + "reporting_grain": "source person report; ASEC may combine family payments", + "individual_beneficiary_assignment_claim": False, + "below15_convention": "both surveys retain unknown beneficiary totals; ASEC zero literal is NIU", + "observed_component_amounts_claim": False, + "source_admission_issued": False, + "release_eligible": False, + } + require( + preparation._checked()[1] == entry[1] and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + return CurrentSocialSecurityProjection(out, ordered, evidence) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py new file mode 100644 index 000000000..78dd8e4fa --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py @@ -0,0 +1,459 @@ +"""Qualified current-money targets and source-keyed clone attachment values. + +The closed family has one UC route and one conditional health-cost route. +Values are derived from live retained owners; this module issues no authority. +""" + +from __future__ import annotations + +from dataclasses import dataclass, fields + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input +from microcosm.frame import WeightKind + +from . import current_asec_unemployment_source as unemployment +from . import current_survey_predictors as predictors +from . import graph_full_puf_enrichment as physical +from . import graph_survey_puf55 as parent_host +from . import support_provenance as provenance + +PROTOCOL = "microcosm.us.current-survey-amount-family.v1" +SEED = 579 + + +@dataclass(frozen=True) +class AmountGroup: + key: str + fields: tuple[tuple[str, str], ...] + + @property + def targets(self): + return tuple("survey_amount_target_" + raw for raw, _ in self.fields) + + +GROUPS = ( + AmountGroup("unemployment", (("UC_VAL", "unemployment_compensation"),)), + AmountGroup( + "health_costs", + ( + ("PHIP_VAL", "health_insurance_premiums_without_medicare_part_b"), + ("PMED_VAL", "other_medical_expenses"), + ("POTC_VAL", "over_the_counter_health_expenses"), + ), + ), +) +UC_REPORT_COLUMNS = ( + "source_amount", + "receipt_literal", + "source_reporting_universe", + "receipt_code_known", + "receipt_code", + "reporting_status", + "canonical_amount_known", + "amount_status", + "amount_validity", + "zero_origin", +) + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_AMOUNTS_" + reason) + + +def selected_groups(names): + require( + type(names) is tuple + and bool(names) + and len(set(names)) == len(names) + and all(type(n) is str for n in names), + "GROUP_NAMES", + ) + result = tuple(g for g in GROUPS if g.key in names) + require(tuple(g.key for g in result) == names, "GROUP_ROSTER_OR_ORDER") + return result + + +@dataclass(frozen=True) +class GroupValues: + spec: AmountGroup + donor_frame: object + donor_columns: pd.DataFrame + matrix: bytes + keep: np.ndarray + + +@dataclass(frozen=True) +class QualifiedSurveyAmounts: + projection: bytes + evidence: dict + source_frame: object + origins: pd.DataFrame + native: pd.DataFrame + reports: pd.DataFrame + features: tuple[str, ...] + groups: tuple[GroupValues, ...] + + +def seal(value): + require(type(value) is QualifiedSurveyAmounts, "QUALIFIED_TYPE") + return ( + value.projection, + codec.encode_json(value.evidence), + value.features, + physical._population_stamp( + parent_host.population_ops.Population.from_frame( + value.source_frame, "survey_amounts.source" + ) + ), + *( + physical._table_stamp(t) + for t in (value.origins, value.native, value.reports) + ), + tuple( + ( + g.spec, + g.matrix, + g.keep.dtype.str, + g.keep.tobytes(), + physical._table_stamp(g.donor_columns), + physical._population_stamp( + parent_host.population_ops.Population.from_frame( + g.donor_frame, "survey_amounts.donor" + ) + ), + ) + for g in value.groups + ), + ) + + +def origin_digest(origins): + """Bind the original source axis and columns even when both name person_id.""" + axis = pd.Index(predictors._ids(origins.index.to_series(index=origins.index))) + columns = ( + origins.reset_index(drop=True).to_json(orient="table", index=False).encode() + ) + return codec.sha( + codec.encode_json( + { + "index_name": origins.index.name, + "index_sha256": codec.sha(axis.to_numpy(dtype="= 15 + require(keep.any(), "NO_QUALIFIED_DONORS:" + spec.key) + require(recipient.any(), "NO_QUALIFIED_RECIPIENTS:" + spec.key) + donor_frame = base.source_frame.select(keep) + require( + donor_frame.resolve_weights("person").kind is WeightKind.DESIGN, + "DONOR_DESIGN_WEIGHTS", + ) + donor_columns = features.loc[ids[keep]].copy() + for target, output in zip(spec.targets, outputs, strict=True): + donor_columns[target] = native.loc[ids[keep], output].to_numpy() + require( + np.array_equal( + donor_frame.person.person_id.to_numpy(), ids[keep].to_numpy() + ), + "DONOR_AXIS", + ) + recipient_matrix = model_input.encode_recipient_matrix( + features.loc[ids[recipient]], + entity="person", + entity_ids=ids[recipient].to_numpy(dtype="= 0).all(), + "DRAW_AXIS_OR_VALUES", + ) + require( + qualified.origins.loc[draw.index, "source"].eq("acs").all(), "DRAW_SOURCE" + ) + for target, (_, output) in zip( + group.spec.targets, group.spec.fields, strict=True + ): + completed.loc[draw.index, output] = draw[target].to_numpy() + all_columns = pd.concat([completed, qualified.reports], axis=1) + require(not set(all_columns) & set(people), "ATTACH_OWNERSHIP_COLLISION") + index = pd.Index(ids, name="person_id") + return { + ("person", name): pd.Series( + all_columns[name].reindex(original).array.copy(), index=index, name=name + ) + for name in all_columns + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py new file mode 100644 index 000000000..f463b6412 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py @@ -0,0 +1,313 @@ +"""Qualified observed geography and stable household draw keys for both surveys. + +This additive, values-only projection borrows current authenticated sources. +It does not assign a location, mutate a Frame, or issue source or Population +authority. Hosts must requalify the projection before execution and replay. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass + +import pandas as pd + +from microcosm.graph.population import dtype_for_token + +from . import current_asec_demographics as demographics +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-survey-geography.v1" +COLUMNS = ( + "survey_geography_origin_key", + "survey_observed_state", + "survey_observed_puma", +) +MAX_HOUSEHOLDS = 64 * 1024**2 // 128 +MAX_RECEIPT_BYTES = 64 * 1024 + + +def _require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_GEOGRAPHY_" + reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _origin_key(row): + channel = row["source"] + _require(type(channel) is str and channel in {"acs", "asec"}, "SOURCE") + _require( + type(row["source_year"]) is int + and row["source_year"] == 2024 + and type(row["survey_year"]) is int + and row["survey_year"] == (2024 if channel == "acs" else 2025), + "SOURCE_PERIOD", + ) + raw = row["raw_native_id"] + _require(type(raw) is str and 0 < len(raw) <= 128, "NATIVE_KEY") + # Preserve the source literal, including ASEC's leading zeroes. Receiving + # integer ids are coordinates only and never enter the keyed draw stream. + return json.dumps( + [channel, row["source_year"], row["survey_year"], raw], + ensure_ascii=True, + separators=(",", ":"), + allow_nan=False, + ) + + +def _project(origins, households, acs, asec): + """Pure alignment from just-qualified values, with no authority of its own.""" + _require( + type(origins) is list + and 0 < len(origins) <= MAX_HOUSEHOLDS + and len(origins) == len(households), + "HOUSEHOLD_COUNT", + ) + _require( + str(households.household_id.dtype) == "int64" + and households.household_id.is_unique + and households.household_id.ge(0).all(), + "HOUSEHOLD_AXIS", + ) + channel_column = support_channel_column("household") + native_column = spine_source_id_column("household") + _require(set(households[channel_column]) == {"acs", "asec"}, "CHANNEL_ROSTER") + _require( + {"household_id", "SERIALNO", "ST", "PUMA", "puma_geoid", "puma"} <= set(acs) + and acs.household_id.is_unique, + "ACS_SOURCE_COLUMNS", + ) + _require( + { + "household_id", + "native_household_id", + "H_SEQ_integer", + "GESTFIPS", + "state_fips", + "state_status", + "state_known", + } + <= set(asec) + and asec.household_id.is_unique, + "ASEC_SOURCE_COLUMNS", + ) + acs_by_id = acs.set_index("household_id", drop=False) + asec_by_id = asec.set_index("household_id", drop=False) + origin_by_id, keys = {}, set() + for row in origins: + _require(type(row) is dict, "ORIGIN_RECORD") + household_id = row["household_id"] + _require( + type(household_id) is int + and household_id >= 0 + and household_id not in origin_by_id + and type(row["selected_receiving_household_id"]) is int + and row["selected_receiving_household_id"] >= 0, + "ORIGIN_COORDINATE", + ) + key = _origin_key(row) + _require(key not in keys, "ORIGIN_DUPLICATE") + keys.add(key) + origin_by_id[household_id] = (row, key) + _require(set(origin_by_id) == set(households.household_id), "ORIGIN_ROSTER") + rows, used_acs, used_asec = [], set(), set() + for household_id, channel, native_id in households[ + ["household_id", channel_column, native_column] + ].itertuples(index=False, name=None): + row, key = origin_by_id[household_id] + _require( + channel == row["source"] + and native_id == row["selected_receiving_household_id"], + "RECEIVING_ORIGIN_JOIN", + ) + puma = None + if channel == "acs": + _require(native_id in acs_by_id.index, "ACS_NATIVE_JOIN") + original = acs_by_id.loc[native_id] + _require(original.SERIALNO == row["raw_native_id"], "ACS_LITERAL_KEY") + state, local_puma = original.ST, original.PUMA + _require( + type(state) is str + and re.fullmatch(r"[0-9]{2}", state, re.ASCII) is not None + and int(state) > 0 + and type(local_puma) is str + and re.fullmatch(r"[0-9]{5}", local_puma, re.ASCII) is not None + and int(local_puma) > 0, + "ACS_LITERAL_GEOGRAPHY", + ) + puma = state + local_puma + _require( + original.puma_geoid == puma and original.puma == puma, + "ACS_PUMA_IDENTITY", + ) + used_acs.add(native_id) + else: + _require(household_id in asec_by_id.index, "ASEC_RECEIVING_JOIN") + original = asec_by_id.loc[household_id] + raw = row["raw_native_id"] + _require( + re.fullmatch(r"[0-9]+", raw, re.ASCII) is not None + and int(raw) == original.H_SEQ_integer + and native_id == original.native_household_id, + "ASEC_LITERAL_KEY", + ) + code, status = demographics._state_token(original.GESTFIPS) + _require( + original.state_status == status + and bool(original.state_known) == (code is not None) + and ( + pd.isna(original.state_fips) + if code is None + else original.state_fips == code + ), + "ASEC_STATE_IDENTITY", + ) + state = None if code is None else str(code).zfill(2) + used_asec.add(household_id) + rows.append((int(household_id), key, state, puma)) + _require( + used_acs == set(acs.household_id) and used_asec == set(asec.household_id), + "COMPLETE_NATIVE_ROSTER", + ) + return tuple(rows) + + +def _projection_digest(household): + """Stream bounded row encodings; the receipt never contains source keys.""" + _require( + tuple(household.columns) == COLUMNS + and household.index.name == "household_id" + and str(household.index.dtype) == "int64" + and household.index.is_unique + and 0 < len(household) <= MAX_HOUSEHOLDS + and all(household[c].dtype == dtype_for_token("string") for c in COLUMNS), + "PROJECTION_STORAGE", + ) + digest = hashlib.sha256() + for row in household.itertuples(index=True, name=None): + values = [int(row[0]), *(None if pd.isna(v) else v for v in row[1:])] + encoded = source._encode(values, maximum=4096) + digest.update(len(encoded).to_bytes(4, "big")) + digest.update(encoded) + return digest.hexdigest() + + +def _check_asec_projection(values, receipt): + """Bind the detached household values to their producer's exact receipt.""" + _require(type(receipt) is bytes and values.receipt == receipt, "ASEC_RECEIPT") + document = json.loads(receipt) + _require( + _sha(values.household.reset_index(drop=True).to_json(orient="table").encode()) + == document.get("household_projection_sha256"), + "ASEC_PROJECTION_DIGEST", + ) + + +@dataclass(frozen=True) +class CurrentSurveyGeographyValues: + """Detached values; retained output bytes grant no downstream admission.""" + + household: pd.DataFrame + receipt: bytes + + +def qualify_current_survey_geography(preparation): + """Read current source geography, aligned to selected receiving households.""" + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + origins = json.loads(entry[1])["origins"]["households"] + acs_native, acs_frame = state.native[0], state.source_frames[0] + source.acs_native.verify_acs_native_coverage(acs_native, acs_frame) + acs_owned = source.acs_native._owned(acs_native) + acs_payload = acs_owned.payload + _require(json.loads(acs_payload)["vintage"] == 2024, "ACS_NATIVE_PERIOD") + asec = demographics.qualify_current_asec_demographics(preparation) + asec_receipt = asec.receipt + _check_asec_projection(asec, asec_receipt) + projected = _project( + origins, + state.frame.table("household"), + acs_frame.table("household"), + asec.household, + ) + table = pd.DataFrame( + { + column: pd.array( + [row[i + 1] for row in projected], dtype=dtype_for_token("string") + ) + for i, column in enumerate(COLUMNS) + }, + index=pd.Index( + [row[0] for row in projected], name="household_id", dtype="int64" + ), + ) + projection_sha256 = _projection_digest(table) + receipt = source._encode( + { + "protocol": PROTOCOL, + "preparation_sha256": _sha(entry[1]), + "acs_native_sha256": _sha(acs_payload), + "asec_demographic_sha256": _sha(asec_receipt), + "projection_sha256": projection_sha256, + "columns": list(COLUMNS), + "households": len(table), + "acs_households": len(acs_frame.table("household")), + "asec_households": len(asec.household), + "state_unknown_households": int(table[COLUMNS[1]].isna().sum()), + "puma_observed_households": int(table[COLUMNS[2]].notna().sum()), + "draw_identity": ["source", "source_year", "survey_year", "raw_native_id"], + "draw_key_encoding": "compact ASCII JSON array; source literal unchanged", + "acs_survey_year": 2024, + "asec_survey_year": 2025, + "asec_income_year": 2024, + "state_is_income_year_residence_claim": False, + "observed_state": "ACS ST; current ASEC GESTFIPS; two-digit strings", + "observed_puma": "ACS ST+PUMA; ASEC remains unknown", + "unknown_state": "preserved; no carried-state fallback", + "unassigned_jurisdiction_validation": "separate geography gate", + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + }, + maximum=MAX_RECEIPT_BYTES, + ) + result = CurrentSurveyGeographyValues(table, receipt) + # Complete owner I/O before pure comparisons. A callback during the last + # source check must not mutate a retained input or detached output unnoticed. + source.acs_native.verify_acs_native_coverage(acs_native, acs_frame) + final_entry = preparation._checked() + _require( + final_entry is entry + and source._ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1] + and source.acs_native._ISSUED.get(acs_native) is acs_owned + and acs_native.payload == acs_payload + and acs_owned.frame is acs_frame + and asec.receipt == asec_receipt, + "FINAL_ISSUANCE", + ) + source._pure_final(state) + _check_asec_projection(asec, asec_receipt) + _require( + _project( + origins, + state.frame.table("household"), + acs_frame.table("household"), + asec.household, + ) + == projected + and result.receipt == receipt + and _projection_digest(result.household) == projection_sha256, + "FINAL_PROJECTION", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py new file mode 100644 index 000000000..066a13553 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py @@ -0,0 +1,245 @@ +"""Interview-date coverage recodes; detached values grant no source authority. + +Only literal source yes/no codes determine coverage. Allocation is separate +provenance, and a broader ACS item never becomes a narrower canonical flag. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from . import support_provenance as provenance +from .cps_carried import US_REPORTED_COVERAGE_PERSON_INPUTS + +PROTOCOL = "microcosm.us.current-survey-health-coverage.v1" +STRING_DTYPE = pd.StringDtype(storage="python", na_value=pd.NA) + + +@dataclass(frozen=True) +class CoverageField: + output: str + asec: str + acs: str | None + acs_gap: str | None = None + + @property + def columns(self): + return (self.output, self.output + "__known", self.output + "__source_status") + + +# Separate NOW_CAID from the broader NOW_MCAID legacy mapping. These are +# source recodes, not benefit eligibility or prior-calendar-year coverage. +FIELDS = ( + CoverageField("has_esi", "NOW_GRP", "HINS1"), + CoverageField( + "has_marketplace_health_coverage_at_interview", + "NOW_MRK", + None, + "direct_purchase_marketplace_split_unobserved", + ), + CoverageField( + "has_non_marketplace_direct_purchase_health_coverage_at_interview", + "NOW_NONM", + None, + "direct_purchase_marketplace_split_unobserved", + ), + CoverageField( + "has_medicaid_health_coverage_at_interview", + "NOW_CAID", + None, + "means_tested_program_split_unobserved", + ), + CoverageField( + "has_other_means_tested_health_coverage_at_interview", + "NOW_OTHMT", + None, + "means_tested_program_split_unobserved", + ), + CoverageField( + "has_tricare_health_coverage_at_interview", + "NOW_MIL", + None, + "military_coverage_type_unresolved", + ), + CoverageField( + "has_champva_health_coverage_at_interview", + "NOW_CHAMPVA", + None, + "champva_not_separately_observed", + ), + CoverageField( + "has_va_health_coverage_at_interview", + "NOW_VACARE", + None, + "va_champva_recode_scope_unresolved", + ), + CoverageField( + "has_indian_health_service_coverage_at_interview", "NOW_IHSFLG", "HINS7" + ), +) +ASEC_VALUE_COLUMNS = tuple(f.asec for f in FIELDS) + ("NOW_MCAID",) +ASEC_ALLOCATION_COLUMNS = tuple("I_" + c for c in ASEC_VALUE_COLUMNS) +ACS_VALUE_COLUMNS = tuple("HINS" + str(i) for i in range(1, 8)) +ACS_ALLOCATION_COLUMNS = tuple("FHINS" + str(i) + "P" for i in range(1, 8)) +ACS_EDIT_COLUMNS = ("FHINS3C", "FHINS4C", "FHINS5C") +RAW_COLUMNS = ( + *ASEC_VALUE_COLUMNS, + *ASEC_ALLOCATION_COLUMNS, + *ACS_VALUE_COLUMNS, + *ACS_ALLOCATION_COLUMNS, + *ACS_EDIT_COLUMNS, +) +SOURCE_PREFIX = "health_source_" + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_HEALTH_" + reason) + + +def _field(name): + require( + set(f.output for f in FIELDS) == set(US_REPORTED_COVERAGE_PERSON_INPUTS), + "PROFILE_ROSTER", + ) + matches = [f for f in FIELDS if f.output == name] + require(len(matches) == 1, "FIELD") + return matches[0] + + +def recode(tokens, allocations, *, survey): + """Keep missing, unrecognized, allocated and reported source states apart. + + This classifier accepts literal strings only. A valid source yes/no value + survives unknown allocation provenance, without claiming a raw response. + Neither blank nor an out-of-dictionary code means no coverage. + """ + require(survey in ("asec", "acs"), "SURVEY") + values, flags = tuple(tokens), tuple(allocations) + require( + len(values) == len(flags) + and all(type(x) is str and len(x) <= 64 for x in (*values, *flags)), + "LITERAL_CONTRACT", + ) + allocation_labels = ( + {"0": "reported", "1": "hotdeck", "2": "logical", "3": "whole_unit"} + if survey == "asec" + else {"0": "not_allocated", "1": "allocated"} + ) + canonical, known, status = [], [], [] + for token, allocation in zip(values, flags, strict=True): + valid = token in ("1", "2") + canonical.append(token == "1" if valid else pd.NA) + known.append(valid) + if not valid: + status.append( + "missing_source_value" if token == "" else "unrecognized_source_value" + ) + else: + method = allocation_labels.get(allocation, "allocation_unknown") + status.append(("source_yes_" if token == "1" else "source_no_") + method) + return pd.DataFrame( + { + "value": pd.array(canonical, dtype="boolean"), + "known": np.asarray(known, dtype=bool), + "status": pd.array(status, dtype=STRING_DTYPE), + } + ) + + +def recode_field(raw, name): + """Transform one field on the full original-person axis, retaining gaps.""" + spec = _field(name) + require( + type(raw) is pd.DataFrame + and raw.index.is_unique + and raw.index.name == "person_id" + and {"source", *RAW_COLUMNS} <= set(raw) + and raw.source.isin(("acs", "asec")).all(), + "RAW_AXIS", + ) + result = pd.DataFrame(index=raw.index.copy()) + result[spec.output] = pd.array([pd.NA] * len(raw), dtype="boolean") + result[spec.output + "__known"] = False + result[spec.output + "__source_status"] = pd.array( + [pd.NA] * len(raw), dtype=STRING_DTYPE + ) + for survey, raw_name in (("asec", spec.asec), ("acs", spec.acs)): + mask = raw.source.eq(survey) + if raw_name is None: + result.loc[mask, spec.output + "__source_status"] = ( + "semantic_gap:" + spec.acs_gap + ) + continue + flag_name = "I_" + raw_name if survey == "asec" else "F" + raw_name + "P" + part = recode(raw.loc[mask, raw_name], raw.loc[mask, flag_name], survey=survey) + for output, value in zip(spec.columns, part, strict=True): + result.loc[mask, output] = part[value].array + return result + + +def source_columns(raw): + """Namespace literal observations so they cannot impersonate output leaves.""" + require(raw.index.name == "person_id" and raw.index.is_unique, "RAW_AXIS") + result = pd.DataFrame(index=raw.index.copy()) + for name in RAW_COLUMNS: + require(all(x is None or type(x) is str for x in raw[name]), "RAW_LITERAL_TYPE") + result[SOURCE_PREFIX + name] = pd.array(raw[name], dtype=STRING_DTYPE) + for name in ("source", "source_year", "survey_year"): + result[SOURCE_PREFIX + name] = ( + pd.array(raw[name], dtype=STRING_DTYPE) + if name == "source" + else raw[name].array.copy() + ) + return result + + +def attach_columns(origins, receiving, columns): + """Fan original observations out to exactly their unchanged two clones. + + The owning host authenticates origins and source columns before calling. + Existing fields are never overwritten, including another family's leaves. + """ + require( + type(origins) is pd.DataFrame + and type(columns) is pd.DataFrame + and origins.index.is_unique + and origins.index.name == "person_id" + and columns.index.equals(origins.index) + and columns.columns.is_unique, + "ATTACH_SOURCE_AXIS", + ) + people = receiving.person + ids = people.person_id + original = people[provenance.support_source_id_column("person")].to_numpy() + clone = people[provenance.support_clone_index_column("person")].to_numpy() + native = people[provenance.spine_source_id_column("person")].to_numpy() + channel = people[provenance.support_channel_column("person")].to_numpy() + require( + ids.dtype == np.dtype("int64") + and ids.is_unique + and original.dtype == clone.dtype == native.dtype == np.dtype("int64") + and len(original) == 2 * len(origins) + and set(original) == set(origins.index), + "CLONE_SOURCE_ROSTER", + ) + expected = origins.reindex(original) + require( + np.array_equal(native, expected.native_person_id.to_numpy()) + and np.array_equal(channel, expected.source.to_numpy()), + "CLONE_SOURCE_IDENTITY", + ) + pairs = pd.MultiIndex.from_arrays((original, clone)) + require(pairs.is_unique and np.isin(clone, (0, 1)).all(), "CLONE_PAIR") + require(not set(columns) & set(people), "ATTACH_OWNERSHIP_COLLISION") + return { + ("person", name): pd.Series( + columns[name].reindex(original).array.copy(), + index=pd.Index(ids.to_numpy(), name="person_id"), + name=name, + ) + for name in columns + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py new file mode 100644 index 000000000..f9b31c825 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py @@ -0,0 +1,343 @@ +"""Health-code projection from the retained original ACS and ASEC sources. + +The preparation issues source identity. This module only borrows it, captures +its pinned original members and returns descriptive values. A host must retain +that owner and requalify after its last relevant I/O before returning output. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import acs_housing_universe_source as housing +from . import acs_person_coverage_authentication as records +from . import asec_coverage_authentication as asec +from . import current_survey_health_coverage as health +from . import source_csv_builtin +from . import survey_population_preparation as source + +require = health.require +ASEC_KEYS = ("PERIDNUM", "PH_SEQ", "A_LINENO") +ACS_KEYS = ("SERIALNO", "SPORDER") +ASEC_COLUMNS = (*ASEC_KEYS, *health.ASEC_VALUE_COLUMNS, *health.ASEC_ALLOCATION_COLUMNS) +ACS_COLUMNS = ( + *ACS_KEYS, + *health.ACS_VALUE_COLUMNS, + *health.ACS_ALLOCATION_COLUMNS, + *health.ACS_EDIT_COLUMNS, +) + + +def _key(row, survey): + if survey == "asec": + require( + re.fullmatch(r"[0-9]{22}", row["PERIDNUM"], re.ASCII) is not None, + "ASEC_PERSON_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,5}", row["PH_SEQ"], re.ASCII) is not None, + "ASEC_HOUSEHOLD_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,2}", row["A_LINENO"], re.ASCII) is not None + and 1 <= int(row["A_LINENO"]) <= 20 + and int(row["PH_SEQ"]) > 0, + "ASEC_LINE_KEY", + ) + return (int(row["PH_SEQ"]), row["PERIDNUM"], int(row["A_LINENO"])) + require( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", row["SERIALNO"], re.ASCII) is not None, + "ACS_HOUSEHOLD_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,2}", row["SPORDER"], re.ASCII) is not None + and 1 <= int(row["SPORDER"]) <= 20, + "ACS_LINE_KEY", + ) + return (row["SERIALNO"], str(int(row["SPORDER"])), int(row["SPORDER"])) + + +def _scan(stream, *, survey, wanted, selected, maximum): + """Exhaust a byte-bounded literal member; selected records remain strings. + + This helper grants no authority to a file, row count or caller key list. + A complete authenticated source/roster is bound by the enclosing qualifier. + """ + require( + survey in ("asec", "acs") + and source_csv_builtin.capture_csv_reader(csv) is not None, + "SCANNER_OR_CSV_BINDING", + ) + columns = ASEC_COLUMNS if survey == "asec" else ACS_COLUMNS + header, count = None, 0 + for raw in records._records(stream): + values = records._decode_record(raw, first=header is None) + if header is None: + header = values + require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(columns) <= set(header), + "SOURCE_HEADER", + ) + positions = [header.index(c) for c in columns] + continue + count += 1 + require(count <= maximum and len(values) == len(header), "SOURCE_ROW_SHAPE") + row = dict(zip(columns, (values[p] for p in positions), strict=True)) + key = _key(row, survey) + require(all(len(row[c]) <= 64 for c in columns), "SOURCE_TOKEN_BOUND") + if key in wanted: + require(key not in selected, "DUPLICATE_SELECTED_SOURCE_KEY") + selected[key] = row + require(header is not None, "SOURCE_HEADER") + return count + + +def _origins(preparation_frame, document): + payload = document["origins"]["persons"] + original = pd.DataFrame(payload["rows"], columns=payload["columns"]) + required = { + "person_id", + "source", + "source_year", + "survey_year", + "raw_native_household_id", + "raw_native_person_id", + "native_line_numeric_original", + "selected_receiving_person_id", + } + require( + required <= set(original) + and original.person_id.dtype == np.dtype("int64") + and original.person_id.is_unique, + "ORIGIN_ROSTER", + ) + original = original.set_index("person_id", drop=True) + people = preparation_frame.person + require( + np.array_equal(original.index.to_numpy(), people.person_id.to_numpy()) + and np.array_equal( + original.source.to_numpy(), + people[health.provenance.support_channel_column("person")].to_numpy(), + ) + and np.array_equal( + original.selected_receiving_person_id.to_numpy(), + people[health.provenance.spine_source_id_column("person")].to_numpy(), + ), + "ORIGIN_FRAME_IDENTITY", + ) + require( + original.source.isin(("acs", "asec")).all() + and original.source_year.eq(2024).all() + and original.survey_year.eq( + original.source.map({"acs": 2024, "asec": 2025}) + ).all(), + "ORIGIN_PERIOD", + ) + original["native_person_id"] = original.selected_receiving_person_id + keys = {} + for pid, row in original.iterrows(): + household = row.raw_native_household_id + require( + type(household) is str + and type(row.raw_native_person_id) is str + and type(row.native_line_numeric_original) is str, + "ORIGIN_COORDINATE_TYPE", + ) + line = row.native_line_numeric_original + if row.source == "asec": + key = _key( + { + "PH_SEQ": household, + "PERIDNUM": row.raw_native_person_id, + "A_LINENO": line, + }, + "asec", + ) + else: + key = _key({"SERIALNO": household, "SPORDER": line}, "acs") + require(row.raw_native_person_id == key[1], "ORIGIN_PERSON_KEY") + require((row.source, key) not in keys, "DUPLICATE_ORIGIN") + keys[row.source, key] = pid + return original, keys + + +def _combine(origins, keys, selected): + require(set(selected) == {"acs", "asec"}, "SOURCE_ARMS") + require( + all(set(selected[s]) == {k for arm, k in keys if arm == s} for s in selected), + "SELECTED_SOURCE_ROSTER", + ) + raw = origins.loc[:, ["source", "source_year", "survey_year"]].copy() + for column in health.RAW_COLUMNS: + raw[column] = pd.Series([None] * len(raw), index=raw.index, dtype=object) + for (survey, key), pid in keys.items(): + row = selected[survey][key] + require(_key(row, survey) == key, "SOURCE_COORDINATE_CHANGED") + for column in health.RAW_COLUMNS: + if column in row: + raw.at[pid, column] = row[column] + return raw + + +@dataclass(frozen=True) +class QualifiedSurveyHealthCoverage: + """A detached projection, not a substitute for the retained source owner.""" + + source_frame: object + origins: pd.DataFrame + raw: pd.DataFrame + projection: bytes + evidence: dict + + +def qualify_current_survey_health(preparation): + """Re-read pinned originals without native Frame or engine reconstruction. + + This entry point does read original microdata when called by an authorized + build. Tests can exercise the same path with privately pinned invented bytes. + """ + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + document = json.loads(entry[1]) + origins, keys = _origins(state.frame, document) + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "ASEC_NATIVE_ISSUANCE", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "ASEC_NATIVE_PERIOD", + ) + pin = tuple(p for p in asec._MEMBER_PINS if p[0] == 2024) + require(len(pin) == 1, "ASEC_PIN_ROSTER") + year, member, archive_digest, digest, rows, size = pin[0] + retained = tuple( + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ) + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive_digest), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "ASEC_MEMBER_BINDING", + ) + acs_owned = source.acs_catalogue._lookup(state.catalogues[0]) + acs_doc = json.loads(acs_owned.receipt) + person_pins = tuple(p for p in acs_owned.pins if p[0] == "person") + require( + len(person_pins) == 1 + and acs_doc["source_year"] == acs_doc["survey_year"] == 2024, + "ACS_PIN_OR_PERIOD", + ) + _role, acs_name, acs_digest, acs_size = person_pins[0] + selected = {"asec": {}, "acs": {}} + with tempfile.TemporaryDirectory(prefix="microcosm-health-source-") as temporary: + directory = Path(temporary) + captured = directory / member + identity = asec._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + budget=[asec._BODY_MAX], + ) + with captured.open("rb") as stream: + count = _scan( + stream, + survey="asec", + wanted={k for s, k in keys if s == "asec"}, + selected=selected["asec"], + maximum=rows, + ) + require( + count == rows + and asec._identity(captured.stat(follow_symlinks=False)) == identity + and housing._persisted_sha(captured, size) == digest, + "ASEC_CAPTURE_CHANGED", + ) + captured = directory / acs_name + require( + housing._copy( + state.root / "acs" / acs_name, captured, acs_size, exact_size=acs_size + ) + == acs_digest, + "ACS_CAPTURE_DIGEST", + ) + count = 0 + with zipfile.ZipFile(captured) as archive: + members, prefix = records._members(archive, "person") + for item in members: + with archive.open(item) as stream: + if item.filename.casefold().startswith(prefix): + count += _scan( + stream, + survey="acs", + wanted={k for s, k in keys if s == "acs"}, + selected=selected["acs"], + maximum=acs_doc["counts"]["people"] - count, + ) + else: + while stream.read(65536): + pass # Exhaust auxiliary members so their CRC is checked. + require( + count == acs_doc["counts"]["people"] + and housing._persisted_sha(captured, acs_size) == acs_digest, + "ACS_CAPTURE_CHANGED", + ) + raw = _combine(origins, keys, selected) + projection = raw.reset_index().to_json(orient="table", index=False).encode() + evidence = { + "protocol": health.PROTOCOL, + "preparation_sha256": hashlib.sha256(entry[1]).hexdigest(), + "asec_native_sha256": hashlib.sha256(issued[1]).hexdigest(), + "acs_catalogue_sha256": hashlib.sha256(acs_owned.receipt).hexdigest(), + "asec_person_member_sha256": digest, + "acs_person_archive_sha256": acs_digest, + "asec_coverage_observation_year": 2025, + "acs_coverage_observation_year": 2024, + "projection_sha256": hashlib.sha256(projection).hexdigest(), + "selected_rows": len(raw), + "source_admission_issued": False, + "unallocated_observation_claim": False, + "release_eligible": False, + } + # The original issuer checks source bytes after the last capture/cleanup I/O. + require(preparation._checked() is entry, "PREPARATION_CHANGED") + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and source.acs_catalogue._lookup(state.catalogues[0]) is acs_owned, + "FINAL_OWNER", + ) + return QualifiedSurveyHealthCoverage( + state.frame, origins, raw, projection, evidence + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py new file mode 100644 index 000000000..44190a51b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py @@ -0,0 +1,639 @@ +"""Source-qualified current ASEC to ACS financial predictor preparation. + +This is a survey-multispine operation before PUF enrichment. The live retained +preparation authenticates source observations; serialized evidence never issues +source authority. Financial ACS leaves are modeled from three current ASEC +money totals, with the maintained split assumptions applied after the draws. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, fields + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input +from microcosm.frame import WeightKind + +from . import acs_income_universe as universe +from . import cps_carried_current as leaves +from . import current_survey_geography as observed_geography +from . import graph_puf_diagnostic_consumer as host +from . import support_provenance as provenance + +source = host.survey_source +FEATURES = ( + "survey_predictor_age", + "survey_predictor_employment_income", + "survey_predictor_self_employment_income", +) +DEMOGRAPHIC_FEATURES = ( + *FEATURES, + "survey_predictor_is_female", + "survey_predictor_state_fips", +) +TARGETS = ("survey_current_INT_VAL", "survey_current_DIV_VAL", "survey_current_CAP_VAL") +MONEY_FIELDS = leaves.CPS_CURRENT_PREDICTOR_MONEY_FIELDS +OUTPUTS = ( + *leaves.CPS_CURRENT_PREDICTOR_PERSON_LEAVES, + "tax_exempt_interest_income", +) +PROTOCOL = "microcosm.us.current-survey-predictor-completion.v1" +PHASE = "survey_multispine_current_financial_completion" +SEED = 578 + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_PREDICTOR_" + reason) + + +def feature_columns(demographic_conditioning=False): + require(type(demographic_conditioning) is bool, "DEMOGRAPHIC_OPTION") + return DEMOGRAPHIC_FEATURES if demographic_conditioning else FEATURES + + +def _check_demographic_values(values, receipt): + """Detached projections must still match their source producer's receipt.""" + require( + type(values) is observed_geography.demographics.CurrentAsecDemographicValues + and type(receipt) is bytes + and values.receipt == receipt, + "DEMOGRAPHIC_RECEIPT", + ) + document = json.loads(receipt) + for entity in ("person", "household"): + table = getattr(values, entity) + require( + codec.sha(table.reset_index(drop=True).to_json(orient="table").encode()) + == document[f"{entity}_projection_sha256"], + "DEMOGRAPHIC_PROJECTION_CHANGED", + ) + + +def _demographic_features(preparation, entry): + """Bind selected source sex and observation-time state, without fallback. + + The maintained ASEC owner retains unknown literals and allocation status. + This opt-in model refuses those unknowns rather than completing them. State + uses the shared literal geography join, not an assigned block's state. + """ + state = entry[2] + values = observed_geography.demographics.qualify_current_asec_demographics( + preparation + ) + receipt = values.receipt + _check_demographic_values(values, receipt) + document = json.loads(receipt) + require( + document["preparation_sha256"] == codec.sha(entry[1]) + and document["sex_observation_year"] == 2025 + and document["state_observation_year"] == 2025 + and document["income_year"] == 2024, + "DEMOGRAPHIC_PERIOD", + ) + acs_frame = state.source_frames[0] + source.acs_native.verify_acs_native_coverage(state.native[0], acs_frame) + acs_payload = source.acs_native._owned(state.native[0]).payload + require(json.loads(acs_payload)["vintage"] == 2024, "ACS_DEMOGRAPHIC_PERIOD") + # This pure operation verifies literal household keys and the complete + # selected source roster using the just-qualified ASEC household projection. + households = observed_geography._project( + json.loads(entry[1])["origins"]["households"], + state.frame.table("household"), + acs_frame.table("household"), + values.household, + ) + people = state.frame.person + index = pd.Index(_ids(people.person_id), name="person_id") + asec = people[provenance.support_channel_column("person")].eq("asec") + native_ids = people[provenance.spine_source_id_column("person")] + require( + values.person.index.is_unique + and set(values.person.index) == set(index[asec]) + and np.array_equal( + values.person.reindex(index[asec]).native_person_id.to_numpy(), + native_ids.loc[asec].to_numpy(), + ), + "DEMOGRAPHIC_ASEC_PERSON_JOIN", + ) + asec_person = values.person.reindex(index[asec]) + require( + asec_person.sex_known.all() and asec_person.is_female.notna().all(), + "DEMOGRAPHIC_UNKNOWN", + ) + original = acs_frame.person.set_index("person_id", drop=False) + require( + original.index.is_unique + and set(original.index) == set(native_ids.loc[~asec]) + and {"SEX", "is_female"} <= set(original), + "DEMOGRAPHIC_ACS_PERSON_JOIN", + ) + acs_person = original.reindex(native_ids.loc[~asec].to_numpy()) + sex = _numeric(acs_person.SEX) + require( + np.isin(sex, (1, 2)).all() + and pd.api.types.is_bool_dtype(acs_person.is_female.dtype) + and acs_person.is_female.notna().all() + and np.array_equal(acs_person.is_female.to_numpy(), sex == 2), + "ACS_SEX_IDENTITY", + ) + female = np.empty(len(index), dtype=np.float64) + female[asec] = asec_person.is_female.to_numpy(dtype=np.float64) + female[~asec] = (sex == 2).astype(np.float64) + states = {row[0]: row[2] for row in households} + selected_states = people.person_household_id.map(states) + require(selected_states.notna().all(), "DEMOGRAPHIC_UNKNOWN") + features = pd.DataFrame( + { + DEMOGRAPHIC_FEATURES[-2]: female, + DEMOGRAPHIC_FEATURES[-1]: selected_states.to_numpy(dtype=np.float64), + }, + index=index, + ) + evidence = { + "asec_projection_sha256": codec.sha(receipt), + "acs_native_sha256": codec.sha(acs_payload), + "asec_sex_and_state_observation_year": 2025, + "acs_sex_and_state_observation_year": 2024, + "state_is_income_year_residence_claim": False, + "asec_allocated_sex": "retained_when_source_owner_binding_is_known", + "acs_sex_allocation_provenance": "unresolved; no_unallocated_value_claim", + "unknown_policy": "refuse_opted_in_fit; no_fill_or_carried_fallback", + "state_encoding": "numeric_source_FIPS_split_predictor; not_geographic_distance", + "state_domain": "source_dictionary_codes; not_atomic_geography_admission", + "features_sha256": codec.sha(features.to_numpy(dtype="= 0).all()) and len(set(result)) == len(result), "ID_AXIS") + return result + + +def _acs_earnings(frame): + """Verify both native adjustment identities before the named NIU operator.""" + person = frame.person + mask = person[provenance.support_channel_column("person")].eq("acs") + selected = person.loc[mask] + require(len(selected) > 0, "ACS_EMPTY") + require(selected.source_year.astype(str).eq("2024").all(), "ACS_PERIOD") + age = _numeric(selected.age) + raw_age = _numeric(selected.AGEP) + require( + np.array_equal(age, raw_age) + and ((age >= 0) & (age <= 99) & (age == np.floor(age))).all(), + "ACS_AGE_IDENTITY", + ) + adjustment = _numeric(selected.ADJINC, nullable=True) + require(np.isfinite(adjustment).all() and (adjustment > 0).all(), "ACS_ADJINC") + for output, raw in universe.ACS_PUMS_EARNINGS_SOURCE_COLUMNS.items(): + require(raw in selected and output in selected, "ACS_EARNINGS_COLUMNS") + original = _numeric(selected[raw], nullable=True) + carried = _numeric(selected[output], nullable=True) + observed = ~np.isnan(original) + require(not (observed & (age < 15)).any(), "ACS_OUTSIDE_UNIVERSE_OBSERVED") + require( + np.array_equal(observed, ~np.isnan(carried)), "ACS_EARNINGS_MISSINGNESS" + ) + if raw == "WAGP": + require((original[observed] >= 0).all(), "ACS_WAGE_DOMAIN") + expected = original * (adjustment / 1_000_000.0) + require( + np.array_equal( + expected[observed].view("uint64"), carried[observed].view("uint64") + ), + "ACS_EARNINGS_ADJUSTMENT", + ) + require(not ((age >= 15) & ~observed).any(), "ACS_ELIGIBLE_EARNINGS_UNKNOWN") + applied = universe.apply_acs_pums_earnings_universe_zeros( + frame, + person_scope=mask, + boundary="current survey financial predictor preparation", + ) + return applied.frame, applied.receipt + + +@dataclass(frozen=True) +class QualifiedSurveyPredictors: + """Computed values; callers requalify live owners at materialization/replay.""" + + projection: bytes + matrix: bytes + source_frame: object + donor_frame: object + donor_columns: pd.DataFrame + native_money: pd.DataFrame + origins: pd.DataFrame + evidence: dict + demographic_conditioning: bool = False + geography_config_payload: bytes | None = None + geography_validation: bytes | None = None + + +def _qualified_seal(value): + """Seal the actual returned values across the last source/support borrow.""" + require(type(value) is QualifiedSurveyPredictors, "QUALIFIED_VALUES_TYPE") + geography = host.survey_budget.geography + tables = [] + for table in (value.donor_columns, value.native_money, value.origins): + require(type(table) is pd.DataFrame, "QUALIFIED_TABLE_TYPE") + metadata = codec.encode_json( + { + "columns": list(table.columns), + "column_axis": [ + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + ], + "index_axis": [ + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + ], + "shape": list(table.shape), + } + ) + # Every series below is exactly len(table) rows, so the whole-series + # slice selects the same rows in the same order, byte for byte. + parts = tuple( + (str(series.dtype), geography._storage_parts(series, slice(None))) + for series in ( + pd.Series(table.index.to_numpy(copy=False)), + *(table[c] for c in table), + ) + ) + tables.append((metadata, parts)) + return ( + value.projection, + value.matrix, + value.demographic_conditioning, + value.geography_config_payload, + value.geography_validation, + codec.encode_json(value.evidence), + tuple( + geography._population_stamp( + host.survey_budget.Population.from_frame( + frame, "survey_predictors.detached_values" + ) + ) + for frame in (value.source_frame, value.donor_frame) + ), + tuple(tables), + ) + + +def qualify_current_survey_predictors( + preparation, + allocated_population, + clone_population, + *, + demographic_conditioning=False, + geography_config=None, +): + """Read the actual full-parent money owner once, then seal all retained state. + + ASEC donors retain original household design weights, before allocation or + cloning. No legacy raw nominal dollar column can stand in for current money. + The source periods are distinct: ASEC 2025 interview/2024 annual income, + restated to 2024 price basis; ACS 2024 rolling prior-12-month amounts adjusted + by ADJINC. This model uses that explicitly declared temporal harmonization; + it does not claim the observation windows are equivalent. + """ + predictors = feature_columns(demographic_conditioning) + config_payload = host.survey_budget._config_payload(geography_config) + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + view = source.CheckedSurveyPopulationView( + entry[1], state.context, state.frame, state.plan, json.loads(entry[1]) + ) + _, allocation, geography_binding = host.survey_budget._initial( + view, + allocated_population, + clone_population, + preparation=preparation, + geography_config=geography_config, + _with_geography_binding=True, + ) + seals = tuple( + host.survey_budget._population_identity(p) + for p in (allocated_population, clone_population) + ) + native_entry = source.asec_native._ISSUED.get(id(state.native[1])) + require( + native_entry is not None + and native_entry[0]() is state.native[1] + and state.native[1].payload == native_entry[1], + "NATIVE_ISSUANCE", + ) + parent = native_entry[2].parent + demographics = ( + _demographic_features(preparation, entry) if demographic_conditioning else None + ) + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "CURRENT_MONEY_PERIOD", + ) + corrected, universe_receipt = _acs_earnings(state.frame) + people = corrected.person + pids = _ids(people.person_id) + channels = people[provenance.support_channel_column("person")].astype(str) + require(set(channels) == {"asec", "acs"}, "SOURCE_CHANNELS") + asec = channels.eq("asec").to_numpy() + acs = ~asec + native_ids = people[provenance.spine_source_id_column("person")].to_numpy( + dtype=np.int64 + ) + require( + set(native_ids[asec]) == set(state.source_frames[1].person.person_id), + "SELECTED_ASEC_ROSTER", + ) + scope = parent.scope + positions = {pid: i for i, pid in enumerate(scope.person_ids)} + require(len(positions) == len(scope.person_ids), "PARENT_ROSTER") + require(all(int(pid) in positions for pid in native_ids[asec]), "PARENT_MEMBER") + take = np.array([positions[int(pid)] for pid in native_ids[asec]], dtype=np.int64) + require(all(scope.person_years[int(i)] == 2024 for i in take), "CURRENT_COHORT") + index = pd.Index(pids, name="person_id") + money = pd.DataFrame(np.nan, index=index, columns=MONEY_FIELDS, dtype=np.float64) + field_evidence = {} + for name in MONEY_FIELDS: + field = ready.field(name) + domain = next(d for d in parent.spec.fields if d.name == name) + require(domain.entity == "person", "MONEY_ENTITY") + amounts = field.amounts[take] + valid = field.validity[take] + require( + (valid == 1).all() and np.isfinite(amounts).all(), "CURRENT_MONEY_UNKNOWN" + ) + money.loc[pids[asec], name] = amounts + field_evidence[name] = { + "domain": {f.name: getattr(domain, f.name) for f in fields(domain)}, + "amount_sha256": codec.sha(amounts.astype(" bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _registry_json(registry: TargetRegistry) -> str: + return _json( + {"country": registry.country, "specs": [asdict(s) for s in registry]} + ).decode() + + +def _validate_registry(registry: TargetRegistry) -> None: + if registry.country != "us" or not len(registry): + raise ValueError("A nonempty US demographic registry is required.") + scopes = set() + for spec in registry: + metadata = spec.metadata + table = metadata.get("table") + if table in RESERVED_FAMILIES or spec.family in RESERVED_FAMILIES: + raise ValueError("Income and tenure tables are reserved holdouts.") + if table not in {"S0101", "B01001"} or spec.family != f"acs.{table}": + raise ValueError( + "Only explicit ACS S0101/B01001 demographic families are supported." + ) + cell = re.fullmatch( + r"S0101_C01_(\d{3})" if table == "S0101" else r"B01001_(\d{3})", spec.name + ) + if cell is None or not 1 <= int(cell[1]) <= (19 if table == "S0101" else 49): + raise ValueError( + "A supported count cell must identify each demographic target." + ) + if ( + set(metadata) != _METADATA + or metadata["role"] != "calibration" + or metadata["geography"] != "0100000US" + or metadata["universe"] != "population" + or metadata["evidence_scope"] not in {"invented", "source_documented"} + or re.fullmatch(r"[a-f0-9]{64}", metadata["reference_sha256"]) is None + or spec.entity != "household" + or spec.period != "2024" + or spec.signed + or spec.value < 0 + or spec.filter is not None + or spec.tolerance is not None + or (spec.se is not None and not np.isfinite(spec.se)) + ): + raise ValueError( + "Demographic target scope must be explicit national population counts for 2024." + ) + if spec.hierarchy is None or spec.hierarchy != ( + age_module.expected_demographic_hierarchy( + table, + variable=spec.name, + label=spec.hierarchy.target.label, + geography=metadata["geography"], + ) + ): + raise ValueError( + "Each demographic target must carry exactly its own national " + "Census ACS calibration hierarchy." + ) + scopes.add(metadata["evidence_scope"]) + if len(scopes) != 1: + raise ValueError( + "Invented and source-documented evidence scopes cannot be mixed." + ) + + +def _registry_from_json(text: str) -> TargetRegistry: + if not isinstance(text, str) or len(text.encode()) > 1_048_576: + raise ValueError("A bounded canonical registry declaration is required.") + document = json.loads(text) + if not isinstance(document, dict) or set(document) != {"country", "specs"}: + raise ValueError("Unsupported registry declaration shape.") + specs = [] + for raw in document["specs"]: + if not isinstance(raw, dict): + raise ValueError("Unsupported registry declaration shape.") + hierarchy = raw.get("hierarchy") + if hierarchy is not None: + raw = {**raw, "hierarchy": CalibrationHierarchy.from_dict(hierarchy)} + specs.append(TargetSpec(**raw)) + registry = TargetRegistry(specs, country=document["country"]) + if text != _registry_json(registry): + raise ValueError( + "Registry declaration must have its canonical typed representation." + ) + _validate_registry(registry) + return registry + + +def _solver_options(params) -> dict: + if ( + set(params) != _PARAMS + or params["mass"] != "free" + or params["weight_anchor"] != "design" + ): + raise ValueError( + "The development calibration requires free mass and an original design cap." + ) + if type(params["epochs"]) is not int or params["epochs"] < 1: + raise ValueError("epochs must be a positive integer.") + for key, minimum in ( + ("learning_rate", 0), + ("max_weight_ratio", 1), + ("max_initial_weight_ratio", 1), + ): + value = params[key] + if ( + type(value) not in {int, float} + or not np.isfinite(value) + or value < minimum + or (key == "learning_rate" and value == 0) + ): + raise ValueError(f"Unsupported {key}.") + return { + **{key: params[key] for key in ("epochs", "learning_rate", "mass")}, + "max_weight_ratio": params["max_initial_weight_ratio"], + } + + +def demographic_calibration_node( + registry: TargetRegistry, + *, + base: str, + node_id: str = "national.demographic_calibration", + epochs: int, + learning_rate: float, + max_weight_ratio: float, + max_initial_weight_ratio: float, +) -> Node: + """Freeze a caller's explicit registry and declare its actual count inputs. + + Source citations/digests are declarations here, not independently verified + acquisitions or permission to repurpose held-out evidence. Activating a + genuine registry remains a separate, predeclared build decision. + + max_initial_weight_ratio bounds the solver relative to incoming importance + weights. max_weight_ratio separately guards the result relative to original + design weights in the executor. These anchors need not be equal. The kernel + cannot inspect undeclared design weights; a violated design cap refuses the + graph result rather than silently changing the solver's answer. + """ + frozen = _registry_from_json(_registry_json(registry)) + params = dict( + registry=_registry_json(frozen), + epochs=epochs, + learning_rate=learning_rate, + max_weight_ratio=max_weight_ratio, + max_initial_weight_ratio=max_initial_weight_ratio, + mass="free", + weight_anchor="design", + ) + _solver_options(params) + return Node( + id=node_id, + kernel=DemographicCalibrationKernel.ref, + inputs=(Slice("household", tuple(dict.fromkeys(s.measure for s in frozen))),), + params=params, + base=base, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + artifact_outputs=_OUTPUTS, + ) + + +class DemographicCalibrationKernel(KernelBase): + """Call the real solver and persist its complete v6 diagnostic artifact.""" + + ref = "us.demographic_calibration@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.REWEIGHT, + consumes_se=False, + dependencies=("numpy", "pandas", "scipy", "torch"), + ) + + def implementation_hash(self) -> str: + return hashlib.sha256( + _json( + { + "adapter": source_hash( + sys.modules[__name__], + age_module, + registry_module, + diagnostics_module, + attribution_module, + solve_module, + kernels_module, + monetary_binding_module, + protocol_module, + dependencies=self.capabilities.dependencies, + ), + "solver": kernels_module.CALIBRATE_ADAM.implementation_hash(), + } + ) + ).hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + registry = _registry_from_json(context.params["registry"]) + options = _solver_options(context.params) + expected = demographic_calibration_node( + registry, + base=context.node.base, + node_id=context.node.id, + **{ + k: context.params[k] + for k in ( + "epochs", + "learning_rate", + "max_weight_ratio", + "max_initial_weight_ratio", + ) + }, + ) + if context.node.normative() != expected.normative(): + raise ValueError("Calibration node differs from its complete declaration.") + if context.weights["household"].kind is not WeightKind.IMPORTANCE: + raise ValueError("Calibration requires the declared importance weights.") + if not np.all(context.weights["household"].values > 0): + raise ValueError("Calibration requires strictly positive incoming weights.") + table = context.tables["household"] + for spec in registry: + dtype = table[spec.measure].dtype + if not is_numeric_dtype(dtype) or is_complex_dtype(dtype): + raise ValueError("Demographic measures require numeric count columns.") + values = table[spec.measure].to_numpy(dtype=float, na_value=np.nan) + if not np.all( + np.isfinite(values) & (values >= 0) & (values == np.floor(values)) + ): + raise ValueError( + "Demographic measures must be nonnegative finite integer counts." + ) + frame = kernels_module._frame_from_context(context, "household") + result = calibrate( + frame, + registry.to_target_set(), + weight_entity="household", + method="adam", + seed=0, + **options, + ) + if result.skipped: + raise ValueError("Demographic calibration cannot skip declared targets.") + anchors = { + "solver_weight_anchor": "incoming_importance", + "solver_max_weight_ratio": context.params["max_initial_weight_ratio"], + "executor_weight_anchor": "original_design", + "executor_max_weight_ratio": context.params["max_weight_ratio"], + } + payload = _json( + diagnostics_payload(result, target_registry=registry, build=anchors) + ) + return KernelResult( + weights=result.frame.weights_for("household"), + artifacts={"diagnostics": payload}, + receipt={ + "scope": "national_demographic_calibration_development", + "evidence_scope": registry.specs[0].metadata["evidence_scope"], + "registry_sha256": hashlib.sha256( + context.params["registry"].encode() + ).hexdigest(), + "registry_version": registry.version, + "diagnostics_sha256": hashlib.sha256(payload).hexdigest(), + "reserved_families": sorted(RESERVED_FAMILIES), + "consumes_standard_errors": False, + **anchors, + "release_eligible": False, + "national_validation": "not_evaluated", + "congressional_district_validation": "not_evaluated", + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py index dc7e69826..a1e157806 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py @@ -52,6 +52,7 @@ from __future__ import annotations +from collections.abc import Mapping from importlib.resources import files import numpy as np @@ -69,6 +70,9 @@ SourceRuntimeError, run_source_stage, ) +from microcosm.build.us_runtime._person_signal_summary import ( + validate_person_signal_summary, +) from microcosm.frame import Frame from microcosm.frame.units import US_SCHEMA @@ -78,6 +82,11 @@ "US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS", "US_ELIGIBILITY_INPUTS_STAGE_NAME", "derive_us_eligibility_inputs_from_manifest", + "prepare_us_eligibility_person", + "us_eligibility_inputs_person_carries_signal", + "us_eligibility_inputs_gate_from_summary", + "us_eligibility_inputs_person_gate", + "us_eligibility_inputs_person_summary", "us_eligibility_inputs_signal_gate", "us_eligibility_inputs_stage_spec", "us_eligibility_inputs_summary", @@ -273,6 +282,108 @@ def _disabled_carries_signal(person: pd.DataFrame) -> bool: return values.nunique() > 1 +def us_eligibility_inputs_person_carries_signal(person: pd.DataFrame) -> bool: + """Return the complete legacy pass-through decision on the real person table. + + All five outputs must exist, and observed disability must vary. Preserve + the current partial-null rule; other output variation, raw source columns + and weight resolution belong to separate operations and are not inferred. + """ + have_all = all( + column in person.columns for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS + ) + return have_all and _disabled_carries_signal(person) + + +def _validated_eligibility_weights( + person: pd.DataFrame, weights: np.ndarray +) -> np.ndarray: + """Return ``weights`` as a float64 array aligned 1:1 with ``person``.""" + + array = np.asarray(weights, dtype=np.float64) + if array.ndim != 1 or len(array) != len(person): + raise ValueError( + "US eligibility-inputs weights must be a 1-D array aligned 1:1 " + f"with the person table ({len(person)} row(s)); got shape " + f"{array.shape}." + ) + if not np.isfinite(array).all(): + raise ValueError("US eligibility-inputs weights must be finite.") + if (array < 0.0).any(): + raise ValueError("US eligibility-inputs weights must be nonnegative.") + return array + + +def prepare_us_eligibility_person( + person: pd.DataFrame, + weights: np.ndarray, + *, + seed: int, + time_period: int, +) -> pd.DataFrame: + """Derive measured ASEC eligibility inputs onto a copy of ``person``. + + The deterministic table-helper behind :func:`with_us_eligibility_inputs`: + it always runs the ``eligibility_inputs`` source-manifest stage over + ``person`` and ``weights`` and returns the aligned result. It does not + decide whether the surface already carries signal — that + idempotence/pass-through decision belongs to the Frame wrapper. + + Args: + person: The actual person table, carrying the raw ASEC source + columns this stage reads (see + ``US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS``), in its own + index and row order. + weights: Person weights aligned 1:1 with ``person``'s rows (same + length and row order; need not be reindexed by ``person_id``). + seed: Build-wide imputation seed threaded to the source-stage + runtime (the derivation itself is deterministic). + time_period: The dataset's time period. + + Returns: + A copy of ``person`` with all five eligibility columns attached + (``is_disabled``, ``is_blind``, and + ``is_full_time_college_student`` as ``bool``; + ``own_children_in_household`` and ``veterans_benefits`` as + ``float64``), in ``person``'s original index and row order. + + Raises: + ValueError: If ``weights`` does not align 1:1 with ``person``, is + not finite/nonnegative, or the stage output does not cover + every person. + SourceRuntimeError: If required raw ASEC column(s) are missing or + malformed. + """ + + weight_values = _validated_eligibility_weights(person, weights) + stage_person = person.copy(deep=True) + stage_person[_PERSON_WEIGHT_COLUMN] = weight_values + output = run_source_stage( + us_eligibility_inputs_stage_spec(), + tables={"person": stage_person}, + operation_handlers={ + "derive_eligibility_inputs": derive_us_eligibility_inputs_from_manifest, + }, + config=SourceRuntimeConfig(seed=int(seed), target_year=int(time_period)), + ) + aligned = output.set_index("person_id").reindex(person["person_id"]) + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: + if aligned[column].isna().any(): + raise ValueError( + f"US eligibility-inputs stage output does not cover every " + f"person for {column!r}." + ) + + result = person.copy(deep=True) + bool_columns = ("is_disabled", "is_blind", "is_full_time_college_student") + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: + if column in bool_columns: + result[column] = aligned[column].to_numpy(dtype=bool) + else: + result[column] = aligned[column].to_numpy(dtype=np.float64) + return result + + def with_us_eligibility_inputs(frame: Frame, *, seed: int, time_period: int) -> Frame: """Run the ``eligibility_inputs`` manifest stage over a US frame. @@ -302,37 +413,17 @@ def with_us_eligibility_inputs(frame: Frame, *, seed: int, time_period: int) -> if frame.schema != US_SCHEMA: raise ValueError("US eligibility inputs require the US schema.") person = frame.table("person") - have_all = all( - column in person.columns for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS - ) - if have_all and _disabled_carries_signal(person): + if us_eligibility_inputs_person_carries_signal(person): return frame - stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values - output = run_source_stage( - us_eligibility_inputs_stage_spec(), - tables={"person": stage_person}, - operation_handlers={ - "derive_eligibility_inputs": derive_us_eligibility_inputs_from_manifest, - }, - config=SourceRuntimeConfig(seed=int(seed), target_year=int(time_period)), + new_person = prepare_us_eligibility_person( + person, + frame.resolve_weights("person").values, + seed=seed, + time_period=time_period, ) - aligned = output.set_index("person_id").reindex(person["person_id"]) - for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: - if aligned[column].isna().any(): - raise ValueError( - f"US eligibility-inputs stage output does not cover every " - f"person for {column!r}." - ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} - bool_columns = ("is_disabled", "is_blind", "is_full_time_college_student") - for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: - if column in bool_columns: - tables["person"][column] = aligned[column].to_numpy(dtype=bool) - else: - tables["person"][column] = aligned[column].to_numpy(dtype=np.float64) + tables["person"] = new_person return Frame( tables, frame.schema, @@ -343,15 +434,31 @@ def with_us_eligibility_inputs(frame: Frame, *, seed: int, time_period: int) -> ) -def us_eligibility_inputs_summary(frame: Frame) -> dict[str, object]: - """Weighted eligibility-share summary for gates and release manifests.""" +def us_eligibility_inputs_person_summary( + person: pd.DataFrame, weights: np.ndarray +) -> dict[str, object]: + """Weighted eligibility-share summary for gates and release manifests. - person = frame.table("person") - weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) - total_weight = float(weights.sum()) + The real-person-table counterpart of :func:`us_eligibility_inputs_summary`, + for callers (e.g. graph adapters) that hold a person table and an + explicit weight vector without a :class:`~microcosm.frame.Frame`. + + Args: + person: The actual person table, already carrying the five + eligibility columns. + weights: Person weights aligned 1:1 with ``person``'s rows. + + Returns: + The same summary payload as :func:`us_eligibility_inputs_summary`. + """ + + weight_values = _validated_eligibility_weights(person, weights) + total_weight = float(weight_values.sum()) def _share(mask: np.ndarray) -> float: - return float(weights[mask].sum()) / total_weight if total_weight > 0 else 0.0 + return ( + float(weight_values[mask].sum()) / total_weight if total_weight > 0 else 0.0 + ) disabled = person["is_disabled"].astype(bool).to_numpy() student = person["is_full_time_college_student"].astype(bool).to_numpy() @@ -377,31 +484,51 @@ def _share(mask: np.ndarray) -> float: } -def us_eligibility_inputs_signal_gate(frame: Frame) -> GateResult: - """Require the eligibility surface to carry plausible distributions. +def us_eligibility_inputs_summary(frame: Frame) -> dict[str, object]: + """Weighted eligibility-share summary for gates and release manifests.""" - Fails when a column is missing or constant, or when the weighted - disabled, full-time-student, parent, or veterans'-payment shares leave - their plausibility bands — each of which reproduces (or inverts) the - everyone-defaults failure of microcosm #244. + return us_eligibility_inputs_person_summary( + frame.table("person"), frame.resolve_weights("person").values + ) + + +def us_eligibility_inputs_gate_from_summary( + summary: Mapping[str, object], +) -> GateResult: + """Check eligibility-input plausibility bands and signal from a summary. + + The pure decision core of :func:`us_eligibility_inputs_signal_gate`, + factored out so graph adapters can reuse the incumbent checks — same + bands, order, and meaning — against a summary computed off the real + person table (see :func:`us_eligibility_inputs_person_summary`) + without a :class:`~microcosm.frame.Frame`. Assumes the caller has + already confirmed the five output columns are present; missing columns + are a separate failure mode (see + :func:`us_eligibility_inputs_person_gate`). + + Raises: + ValueError: If required fields/counts are missing, measurements are + malformed, or supplied bands differ from the registered policy. """ - person = frame.table("person") + validate_person_signal_summary( + summary, + family="eligibility", + outputs=US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS, + share_bands={ + "disabled_share": ("disabled_share_band", _DISABLED_SHARE_BAND), + "full_time_college_student_share": ( + "full_time_student_share_band", + _FULL_TIME_STUDENT_SHARE_BAND, + ), + "parent_share": ("parent_share_band", _PARENT_SHARE_BAND), + "veterans_benefits_share": ( + "veterans_benefits_share_band", + _VETERANS_BENEFITS_SHARE_BAND, + ), + }, + ) failures: list[str] = [] - missing = [ - column - for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS - if column not in person.columns - ] - if missing: - return GateResult( - name="eligibility_inputs_signal", - passed=False, - failures=(f"person columns missing: {missing}.",), - details={"missing": missing}, - ) - - summary = us_eligibility_inputs_summary(frame) for column, count in summary["unique_counts"].items(): if count < 2: failures.append( @@ -434,3 +561,58 @@ def us_eligibility_inputs_signal_gate(frame: Frame) -> GateResult: failures=tuple(failures), details=summary, ) + + +def us_eligibility_inputs_person_gate( + person: pd.DataFrame, weights: np.ndarray +) -> GateResult: + """Require the eligibility surface to carry plausible distributions. + + The real-person-table counterpart of + :func:`us_eligibility_inputs_signal_gate`, composing + :func:`us_eligibility_inputs_person_summary` and + :func:`us_eligibility_inputs_gate_from_summary` exactly as the Frame + wrapper does. + """ + + missing = [ + column + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS + if column not in person.columns + ] + if missing: + return GateResult( + name="eligibility_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + summary = us_eligibility_inputs_person_summary(person, weights) + return us_eligibility_inputs_gate_from_summary(summary) + + +def us_eligibility_inputs_signal_gate(frame: Frame) -> GateResult: + """Require the eligibility surface to carry plausible distributions. + + Fails when a column is missing or constant, or when the weighted + disabled, full-time-student, parent, or veterans'-payment shares leave + their plausibility bands — each of which reproduces (or inverts) the + everyone-defaults failure of microcosm #244. + """ + + person = frame.table("person") + missing = [ + column + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS + if column not in person.columns + ] + if missing: + return GateResult( + name="eligibility_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + + summary = us_eligibility_inputs_summary(frame) + return us_eligibility_inputs_gate_from_summary(summary) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_leaf_policy.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_leaf_policy.py new file mode 100644 index 000000000..b50944904 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_leaf_policy.py @@ -0,0 +1,255 @@ +"""Explicit fiscal input policy, without source or complete-parent authority. + +Documents describe reviewed intent, not verified approval. No assumptions are +enabled in a US candidate by this module. A dynamic trace of one population +cannot prove universal inactivity, so inactive entries are unsupported. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass + +import numpy as np + +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.canonical import canonical_json + +MAX_POLICY_BYTES = 1024**2 +_POLICY_KIND = "microcosm.us.fiscal_leaf_policy" +_PRODUCER_FIELDS = {"kind", "entity", "producer"} +_ASSUMPTION_FIELDS = { + "kind", + "entity", + "value", + "interpretation", + "period", + "roots", + "affected_roots", + "affected_programs", + "rationale", + "reviewer", + "source_issue", + "reform_sensitivity", +} + + +def _require(condition, reason): + if not condition: + raise ValueError("FISCAL_LEAF_POLICY_" + reason) + + +def _text(value): + return type(value) is str and bool(value.strip()) + + +def _names(value): + return ( + type(value) is list + and bool(value) + and all(_text(item) for item in value) + and len(set(value)) == len(value) + ) + + +def _literal(value): + return ( + type(value) is bool + or (type(value) is int and -(2**63) <= value < 2**63) + or (type(value) is float and math.isfinite(value)) + ) + + +@dataclass(frozen=True) +class FiscalLeafPolicy: + """Exact bytes and digest; descriptive only, never a parent-admission token.""" + + payload: bytes + sha256: str + + +def load_fiscal_leaf_policy( + payload: bytes, *, expected_sha256: str +) -> FiscalLeafPolicy: + """Verify caller-supplied policy bytes against an explicitly reviewed pin. + + No file or network access. An entry's reviewer/issue text is provenance, + not authenticated approval and not permission to run assumptions. + """ + result = FiscalLeafPolicy(payload, expected_sha256) + fiscal_leaf_policy_document(result) + return result + + +def fiscal_leaf_policy_document(policy: FiscalLeafPolicy) -> dict: + """Recheck the pin and schema, returning a detached parsed document.""" + _require(type(policy) is FiscalLeafPolicy, "TYPE") + _require( + type(policy.payload) is bytes and 0 < len(policy.payload) <= MAX_POLICY_BYTES, + "BOUND", + ) + _require( + type(policy.sha256) is str + and hashlib.sha256(policy.payload).hexdigest() == policy.sha256, + "FINGERPRINT", + ) + try: + raw = json.loads(policy.payload) + canonical = canonical_json(raw) + except (ValueError, TypeError, UnicodeError, RecursionError): + raise ValueError("FISCAL_LEAF_POLICY_JSON") from None + _require(canonical == policy.payload, "CANONICAL") + _require( + type(raw) is dict + and set(raw) + == {"schema_version", "artifact_kind", "period", "roots", "entries"}, + "FIELDS", + ) + _require( + type(raw["schema_version"]) is int + and raw["schema_version"] == 1 + and raw["artifact_kind"] == _POLICY_KIND, + "SCHEMA", + ) + _require( + type(raw["period"]) is int and raw["period"] > 0 and _names(raw["roots"]), + "SCOPE", + ) + _require( + type(raw["entries"]) is dict and all(_text(name) for name in raw["entries"]), + "ENTRIES", + ) + for entry in raw["entries"].values(): + _require(type(entry) is dict, "ENTRY") + kind = entry.get("kind") + _require(kind != "inactive", "INACTIVE_UNSUPPORTED") + _require(kind in ("producer", "assumption"), "ENTRY_KIND") + _require(entry.get("entity") in US_SCHEMA.entities, "ENTITY") + if kind == "producer": + _require( + set(entry) == _PRODUCER_FIELDS and _text(entry["producer"]), + "PRODUCER_FIELDS", + ) + continue + _require(set(entry) == _ASSUMPTION_FIELDS, "ASSUMPTION_FIELDS") + _require(_literal(entry["value"]), "LITERAL_UNSUPPORTED") + _require( + entry["interpretation"] in ("baseline_behavior", "scenario_parameter"), + "INTERPRETATION", + ) + _require( + type(entry["period"]) is int + and entry["period"] == raw["period"] + and entry["roots"] == raw["roots"], + "ASSUMPTION_SCOPE", + ) + _require( + _names(entry["affected_roots"]) + and set(entry["affected_roots"]) <= set(raw["roots"]) + and _names(entry["affected_programs"]), + "AFFECTED_SCOPE", + ) + _require( + all( + _text(entry[key]) + for key in ( + "rationale", + "reviewer", + "source_issue", + "reform_sensitivity", + ) + ), + "REVIEW_SCOPE", + ) + return raw + + +def classify_fiscal_leaf_policy(policy, *, period, roots, leaves): + """Bind a policy exactly to the independently derived static input closure.""" + raw = fiscal_leaf_policy_document(policy) + _require(raw["period"] == period, "POLICY_PERIOD") + _require(raw["roots"] == list(roots), "POLICY_ROOTS") + _require(set(leaves) <= set(raw["entries"]), "UNPOLICIED_MODEL_LEAF") + _require(set(raw["entries"]) <= set(leaves), "STALE_MODEL_LEAF") + _require( + all(entry["entity"] == leaves[name] for name, entry in raw["entries"].items()), + "LEAF_ENTITY", + ) + return raw + + +def fiscal_assumption_engine_records(document, *, metadata, defaults): + """Validate literal types and retain actual engine metadata/default records. + + The caller derives metadata/defaults from the engine pinned in the fiscal + contract. Defaults are recorded, never selected by reference. Enum/string + assumptions require an additional reviewed representation and are refused. + """ + records = {} + for name, entry in document["entries"].items(): + if entry["kind"] != "assumption": + continue + item = metadata[name] + _require( + item["name"] == name and item["entity"] == entry["entity"], "ENGINE_ENTITY" + ) + value = entry["value"] + _require( + (item["dtype"] == "bool" and type(value) is bool) + or (item["dtype"] == "int" and type(value) is int) + or (item["dtype"] == "float" and type(value) in (int, float)), + "ENGINE_LITERAL_TYPE", + ) + default = {"available": name in defaults} + if name in defaults: + _require(_literal(defaults[name]), "ENGINE_DEFAULT_UNSUPPORTED") + default["value"] = defaults[name] + records[name] = {"metadata": dict(item), "default": default} + return records + + +def validate_fiscal_leaf_policy_population( + frame: Frame, policy: FiscalLeafPolicy +) -> None: + """Refuse any incumbent assumption column in an actual complete Frame. + + The country host must supply its independently admitted complete parent, + before projection and again around cache/export I/O. Passing a projection + is not absence proof. This function does not issue or authenticate a Frame. + Even wholly or partially unknown incumbent columns count as collisions. + """ + raw = fiscal_leaf_policy_document(policy) + _require(frame.schema == US_SCHEMA, "POPULATION_SCHEMA") + supplied = { + column for entity in frame.entities for column in frame.table(entity).columns + } + for name, entry in raw["entries"].items(): + if entry["kind"] == "assumption": + _require(name not in supplied, "ASSUMPTION_COLUMN_COLLISION:" + name) + + +def private_fiscal_assumption_frame(frame: Frame, policy: FiscalLeafPolicy) -> Frame: + """Prepare detached engine-only tables; not called by the fiscal kernel. + + Future host integration must separately admit the complete parent, bind + policy plus engine records, and call the collision validator around I/O. + This pure helper is not an assumption-enabled graph execution path. + """ + validate_fiscal_leaf_policy_population(frame, policy) + tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + tables.update({name: frame.link(name).copy(deep=True) for name in frame.links}) + for name, entry in fiscal_leaf_policy_document(policy)["entries"].items(): + if entry["kind"] == "assumption": + tables[entry["entity"]][name] = np.full( + frame.n(entry["entity"]), entry["value"] + ) + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + strata=frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py index b839aea4e..bf8c84ab5 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py @@ -3052,6 +3052,15 @@ def _direct_reference_from_fact( signed=_numeric_value(fact) < 0, metadata=metadata, hierarchy=_us_hierarchy_seed(source_name, family), + # These explicitly mapped CBO levels are publisher projections. Keep + # their assertion intact without admitting projections for other inputs. + assertion_policy=( + "allow_source_projection" + if source_name == "cbo" + and measure_id == "projected_amount" + and mapping is not None + else "observed_only" + ), ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py new file mode 100644 index 000000000..4efdda196 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py @@ -0,0 +1,800 @@ +"""Complete canonical PUF enrichment boundaries; no source admission is issued. + +An upstream source owner supplies interpreted, period-aligned canonical donor +columns and a complete survey recipient. This module does not read raw PUF, +reinterpret E19200, choose missing-value aliases, or manufacture source evidence. +It reuses the maintained ordered QRF protocol and person/tax-unit finalizer. +Graph hosts must authenticate actual typed edges and retain their independent +full-population replay checks. These value checks do not replace either duty. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input, qrf, qrf_target +from microcosm.fit.graph_legacy_apply_matrix import decode_matrix_apply_state +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import Frame + +from . import puf_support as support + +PERSON_OUTPUTS = tuple(support.PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS) +TAX_UNIT_OUTPUTS = tuple(support.PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS) +TARGETS = (*PERSON_OUTPUTS, *TAX_UNIT_OUTPUTS) +PREDICTORS = tuple(support.PUF_TAX_DETAIL_DEFAULT_PREDICTORS) +PHASE = "us_full_canonical_puf_enrichment" +PUF59_PREDICTORS = ( + "puf_2015_filing_status_code", + "puf_2015_capped_return_size", + *PREDICTORS[2:], +) +SURVEY_SS_TOTAL_PREDICTOR = "puf_conditioning_social_security_total" +SURVEY_SS_COMPONENTS = tuple(support.PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS) +PUF55_SURVEY_SS_PREDICTORS = (*PUF59_PREDICTORS, SURVEY_SS_TOTAL_PREDICTOR) +PUF55_SURVEY_SS_PERSON_OUTPUTS = tuple( + target for target in PERSON_OUTPUTS if target not in SURVEY_SS_COMPONENTS +) + + +# Compatibility constants above continue to describe FULL65. PUF59 keeps the +# observed person home_mortgage_interest; only these detailed tax-unit fields +# are reserved for the independent downstream SCF producer in that profile. +SCF_MORTGAGE_OUTPUTS = ( + "first_home_mortgage_balance", + "second_home_mortgage_balance", + "first_home_mortgage_interest", + "second_home_mortgage_interest", + "first_home_mortgage_origination_year", + "second_home_mortgage_origination_year", +) +PUF59_TAX_UNIT_OUTPUTS = tuple( + target for target in TAX_UNIT_OUTPUTS if target not in SCF_MORTGAGE_OUTPUTS +) + + +class PufOutputProfile(Enum): + """Closed output contracts; arbitrary target subsets are not profiles.""" + + FULL65 = "full65" + PUF59 = "puf59" + PUF55_SURVEY_SS = "puf55_survey_ss" + PUF55_SURVEY_SS_NO_TOTAL = "puf55_survey_ss_no_total" + + @property + def predictors(self): + if self is PUF55_SURVEY_SS: + return PUF55_SURVEY_SS_PREDICTORS + return PREDICTORS if self is FULL65 else PUF59_PREDICTORS + + @property + def source_predictors(self): + # Independently measured source leaves, never derived here from generic + # filing status or actual recipient membership. The six monetary + # predictors retain their original ordered canonical arithmetic aliases. + if self is PUF55_SURVEY_SS: + return (*self.predictors[:2], SURVEY_SS_TOTAL_PREDICTOR) + return () if self is FULL65 else self.predictors[:2] + + @property + def donor_auxiliary_columns(self): + # Explicit capacity for donor Boolean incidence validation. Return-only + # source owners declare the bound (1 for 0/1 return incidence); person + # tables supply their membership capacity. This is not a physical person + # count observed in a return, nor a predictor or QRF conditioning input. + return () if self is FULL65 else ("puf_person_incidence_capacity",) + + @property + def person_outputs(self): + if self in (PUF55_SURVEY_SS, PUF55_SURVEY_SS_NO_TOTAL): + return PUF55_SURVEY_SS_PERSON_OUTPUTS + return PERSON_OUTPUTS + + @property + def tax_unit_outputs(self): + return TAX_UNIT_OUTPUTS if self is FULL65 else PUF59_TAX_UNIT_OUTPUTS + + @property + def targets(self): + return (*self.person_outputs, *self.tax_unit_outputs) + + @property + def phase(self): + # Legacy kernel parameter schemas are exact: phase carries the profile + # identity into every train/apply key and receipt without widening them. + # FULL65 retains its existing declarations and cache identities. + return PHASE if self is FULL65 else PHASE + "." + self.value + + +FULL65 = PufOutputProfile.FULL65 +PUF59 = PufOutputProfile.PUF59 +PUF55_SURVEY_SS = PufOutputProfile.PUF55_SURVEY_SS +PUF55_SURVEY_SS_NO_TOTAL = PufOutputProfile.PUF55_SURVEY_SS_NO_TOTAL + + +def require_puf_output_profile(profile): + """Require an explicit enum member; never infer a profile from missing data.""" + _require(type(profile) is PufOutputProfile, "PUF_OUTPUT_PROFILE") + return profile + + +def _require(condition, code): + if not condition: + raise ValueError(code) + + +def _numeric(values, *, label, boolean=False, nullable=False): + """Validate physical values before conversion; never parse strings as money.""" + series = pd.Series(values, copy=False) + if not nullable: + _require(not series.isna().any(), f"PUF_UNKNOWN:{label}") + observed = series.dropna() + if boolean: + valid = observed.map(lambda v: isinstance(v, (bool, np.bool_))) + else: + valid = observed.map( + lambda v: ( + not isinstance(v, (bool, np.bool_)) + and isinstance(v, (int, float, np.integer, np.floating)) + ) + ) + _require(bool(valid.all()), f"PUF_PHYSICAL_TYPE:{label}") + if not boolean: + # Do not let integer source identities/amounts silently lose bits at + # the explicit float64 model boundary. + _require( + all( + not isinstance(v, (int, np.integer)) or abs(int(v)) <= 2**53 + for v in observed + ), + f"PUF_FLOAT64_INTEGER_RANGE:{label}", + ) + numeric = series.to_numpy(dtype=np.float64, na_value=np.nan) + _require( + bool(np.isfinite(numeric[~series.isna().to_numpy()]).all()), + f"PUF_NONFINITE:{label}", + ) + return numeric + + +def _profile_source_values(table, *, profile): + """Validate the explicit measured leaves without implementing their producer. + + Physical/knownness checks and strict source qualification remain separate. + These are representation domains only; filing-class disclosure caps and + widow(er) coarsening belong to the independent source measurement receipt. + """ + if not profile.source_predictors: + return () + measured = tuple( + _numeric(table[column], label=column) for column in profile.source_predictors + ) + status, size = measured[:2] + _require( + bool(np.isin(status, [1, 2, 3, 4]).all()), "PUF_PROFILE_FILING_STATUS_DOMAIN" + ) + _require( + bool(((size >= 1) & (size == np.floor(size))).all()), + "PUF_PROFILE_RETURN_SIZE_DOMAIN", + ) + if profile is PUF55_SURVEY_SS: + _require(bool((measured[2] >= 0).all()), "PUF_PROFILE_SOCIAL_SECURITY_DOMAIN") + return measured + + +def _ids(values, label): + series = pd.Series(values, copy=False) + _require(series.dtype == np.dtype("int64"), f"PUF_ID_DTYPE:{label}") + result = series.to_numpy(copy=True) + _require(bool((result > 0).all()), f"PUF_ID_DOMAIN:{label}") + return result + + +def _known(values, known, columns, label): + _require( + isinstance(known, pd.DataFrame) + and known.index.equals(values.index) + and tuple(known.columns) == tuple(columns), + f"PUF_KNOWNNESS_AXIS:{label}", + ) + for column in columns: + mask = known[column] + _numeric(mask, label=f"{label}.{column}.known", boolean=True) + _require(bool(mask.all()), f"PUF_UNKNOWN:{label}.{column}") + + +def canonical_full_puf_donor( + person, + tax_unit, + *, + person_known, + tax_unit_known, + person_targets_at_tax_unit=(), + profile=FULL65, +): + """Reduce canonical columns after upstream interpretation, with no imputation. + + Some person destinations are observed only as return totals, for example + self-employed pension contributions. Such targets must be explicitly listed + in ``person_targets_at_tax_unit`` and supplied on the tax-unit table. A + destination's eventual grain never changes the declared donor grain. + Knownness covers every consumed nonstructural field, including true zeros. + Raw-to-canonical judgments belong upstream. PUF59 does not consume or + derive the six detailed mortgage fields reserved for the SCF producer. + When all person destinations are return totals, ``person=None`` avoids + inventing donor persons. FULL65 retains its canonical return-table + ``tax_unit_person_count`` input. PUF59 instead requires the source-declared + ``puf_person_incidence_capacity`` and its knownness (1 for return-level + 0/1 QBI incidence). Person-table PUF59 donors use membership as capacity; + neither case labels the bound as an observed physical return person count. + """ + profile = require_puf_output_profile(profile) + _require( + type(person_targets_at_tax_unit) is tuple + and len(set(person_targets_at_tax_unit)) == len(person_targets_at_tax_unit) + and set(person_targets_at_tax_unit) <= set(profile.person_outputs), + "PUF_DONOR_GRAIN_DECLARATION", + ) + pcols = tuple( + c for c in profile.person_outputs if c not in person_targets_at_tax_unit + ) + return_only = person is None + _require( + not return_only + or ( + person_targets_at_tax_unit == profile.person_outputs + and person_known is None + ), + "PUF_RETURN_ONLY_GRAIN", + ) + return_capacity_column = ( + profile.donor_auxiliary_columns[0] + if profile.donor_auxiliary_columns + else "tax_unit_person_count" + ) + tcols = ( + "weight", + "filing_status_code", + *((return_capacity_column,) if return_only else ()), + *person_targets_at_tax_unit, + *profile.tax_unit_outputs, + *profile.source_predictors, + ) + tables = [(tax_unit, ("tax_unit_id", *tcols), "tax_unit")] + if not return_only: + tables.append((person, ("person_id", "person_tax_unit_id", *pcols), "person")) + for table, required, label in tables: + _require( + isinstance(table, pd.DataFrame) + and table.index.is_unique + and table.columns.is_unique + and set(required) <= set(table.columns), + f"PUF_DONOR_COLUMNS:{label}", + ) + _require( + (return_only or not set(person_targets_at_tax_unit) & set(person.columns)) + and not set(pcols) & set(tax_unit.columns), + "PUF_DONOR_GRAIN_COLLISION", + ) + tids = _ids(tax_unit.tax_unit_id, "tax_unit") + _require( + len(tids) > 0 and len(set(tids)) == len(tids), + "PUF_DONOR_MEMBERSHIP", + ) + if not return_only: + pids = _ids(person.person_id, "person") + links = _ids(person.person_tax_unit_id, "person_tax_unit_id") + _require( + len(set(pids)) == len(pids) and set(links) == set(tids), + "PUF_DONOR_MEMBERSHIP", + ) + _known(person, person_known, pcols, "person") + _known(tax_unit, tax_unit_known, tcols, "tax_unit") + # Validate EVERY source cell before groupby can skip a missing member. + if not return_only: + normalized = pd.DataFrame(index=person.index) + for column in pcols: + normalized[column] = _numeric( + person[column], + label=f"person.{column}", + boolean=column in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS, + ) + grouped = normalized.groupby(links, sort=False).sum(min_count=1).reindex(tids) + count = pd.Series(links).value_counts(sort=False).reindex(tids).to_numpy() + else: + count = _numeric(tax_unit[return_capacity_column], label=return_capacity_column) + _require( + bool(((count >= 1) & (count == np.floor(count))).all()), + "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN" + if profile.donor_auxiliary_columns + else "PUF_PERSON_COUNT_DOMAIN", + ) + donor = pd.DataFrame(index=tax_unit.index) + for column in (*person_targets_at_tax_unit, *profile.tax_unit_outputs): + donor[column] = _numeric(tax_unit[column], label=f"tax_unit.{column}") + for column in pcols: + donor[column] = _numeric(grouped[column], label=f"reduced.{column}") + status = _numeric(tax_unit.filing_status_code, label="filing_status_code") + _require(bool(np.isin(status, [1, 2, 3, 4, 5]).all()), "PUF_FILING_STATUS_DOMAIN") + for column in ( + c + for c in profile.person_outputs + if c in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + ): + values = donor[column].to_numpy() + _require( + bool( + ((values >= 0) & (values <= count) & (values == np.floor(values))).all() + ), + f"PUF_BOOLEAN_COUNT_DOMAIN:{column}", + ) + for column in ( + c + for c in profile.tax_unit_outputs + if c in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS + ): + values = donor[column].to_numpy() + _require( + bool( + ( + (values == 0) + | ( + (values >= 1000) + & (values <= 9999) + & (values == np.floor(values)) + ) + ).all() + ), + f"PUF_YEAR_DOMAIN:{column}", + ) + # PUF59 matches the survey's total Schedule C measurement. The QBI owner + # partitions that total between ordinary and SSTB destinations, so neither + # component alone represents the predictor. FULL65 keeps its historical + # ordinary-component alias for compatibility with accepted prior evidence. + schedule_c = donor.self_employment_income_before_lsr.to_numpy() + if profile is not FULL65: + schedule_c = ( + schedule_c + donor.sstb_self_employment_income_before_lsr.to_numpy() + ) + # These are canonical arithmetic aliases, after complete cell validation. + feature_values = ( + status, + count.astype(np.float64), + donor.employment_income_before_lsr.to_numpy(), + schedule_c, + donor.taxable_interest_income.to_numpy(), + donor.qualified_dividend_income.to_numpy() + + donor.non_qualified_dividend_income.to_numpy(), + donor.short_term_capital_gains.to_numpy(), + donor.long_term_capital_gains_before_response.to_numpy(), + ) + # PUF59 first two features are already source-measured. Never label generic + # status/membership arithmetic as the PUF2015 disclosure measurement. + if profile.source_predictors: + measured = _profile_source_values(tax_unit, profile=profile) + feature_values = ( + *measured[:2], + *feature_values[2:], + *measured[2:], + ) + for name, values in zip(profile.predictors, feature_values, strict=True): + donor[name] = _numeric(values, label=name) + donor["weight"] = _numeric(tax_unit.weight, label="weight") + weights = donor.weight.to_numpy() + _require( + bool((weights >= 0).all()) and np.isfinite(weights.sum()) and weights.sum() > 0, + "PUF_DONOR_WEIGHT_DOMAIN", + ) + for column in profile.donor_auxiliary_columns: + donor[column] = count.astype(np.float64) + return donor.loc[ + :, + [ + *profile.predictors, + *profile.targets, + "weight", + *profile.donor_auxiliary_columns, + ], + ].copy() + + +def _recipient_matrix(frame, *, predictor_known, profile=FULL65): + profile = require_puf_output_profile(profile) + tax_unit, person = frame.table("tax_unit"), frame.table("person") + mask = support.puf_tax_detail_clone_mask(tax_unit, entity="tax_unit") + ids = _ids(tax_unit.loc[mask, "tax_unit_id"], "recipient") + selected_index = pd.Index(ids, name="tax_unit_id") + _require(len(ids) > 0, "PUF_NO_RECIPIENTS") + _require( + isinstance(predictor_known, pd.DataFrame) + and predictor_known.index.equals(selected_index), + "PUF_RECIPIENT_KNOWNNESS_AXIS", + ) + _known(predictor_known, predictor_known, profile.predictors, "recipient_predictor") + person_mask = person.person_tax_unit_id.isin(ids) + # The maintained resolver below owns ACS source applicability and its exact + # age-15 universe zeros. Screen physical numeric types first, since pandas + # numeric conversion otherwise accepts strings, booleans or datetime values. + for name in profile.predictors: + plan = support._strict_predictor_source_plan( + name, tax_unit=tax_unit, person=person + ) + if name in profile.source_predictors: + _require( + plan.entity == "tax_unit" and plan.columns == (name,), + "PUF_PROFILE_PREDICTOR_SOURCE:" + name, + ) + if plan.source_column == "filing_status_code" or plan.entity == "derived": + continue + table = person if plan.entity == "person" else tax_unit + rows = person_mask if plan.entity == "person" else mask + for column in plan.columns: + if column in table: + _numeric( + table.loc[rows, column], + label=f"recipient.{plan.entity}.{column}", + nullable=True, + ) + features, universe = support._strict_recipient_predictor_surface( + frame, mask, profile.predictors, person_outputs=profile.person_outputs + ) + selected = features.loc[mask, list(profile.predictors)].copy() + selected.index = selected_index + selected = selected.astype("float64") + _profile_source_values(selected, profile=profile) + return ( + model_input.encode_recipient_matrix( + selected, entity="tax_unit", entity_ids=ids + ), + universe, + ) + + +@dataclass(frozen=True) +class FullPufInputs: + """Selected model surfaces, excluding nonmodel donor validation columns.""" + + donor: pd.DataFrame + donor_frame: Frame + matrix: bytes + recipient_universe: Mapping[str, object] + profile: PufOutputProfile = FULL65 + + +def _validated_model_donor(donor, *, profile): + """Select the exact validated donor surface; issue no source/model authority. + + This is the existing values check, including auxiliary incidence capacity. + The receiving host must still authenticate source/period construction and + check the maintained donor Frame conversion without permitting coercion. + """ + _require( + isinstance(donor, pd.DataFrame) + and tuple(donor.columns) + == ( + *profile.predictors, + *profile.targets, + "weight", + *profile.donor_auxiliary_columns, + ), + "PUF_FULL_DONOR_ROSTER", + ) + for column in donor: + _numeric(donor[column], label=f"donor.{column}") + _profile_source_values(donor, profile=profile) + model_counts = donor[profile.predictors[1]].to_numpy() + counts = ( + donor[profile.donor_auxiliary_columns[0]].to_numpy() + if profile.donor_auxiliary_columns + else model_counts + ) + weights = donor["weight"].to_numpy() + _require( + len(donor) > 0 + and donor.index.is_unique + and bool(((counts >= 1) & (counts == np.floor(counts))).all()) + and bool(np.isin(donor[profile.predictors[0]], [1, 2, 3, 4, 5]).all()) + and bool((weights >= 0).all()) + and np.isfinite(weights.sum()) + and weights.sum() > 0, + "PUF_DONOR_MODEL_DOMAIN", + ) + for column in ( + c + for c in profile.person_outputs + if c in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + ): + values = donor[column].to_numpy() + _require( + bool( + ( + (values >= 0) & (values <= counts) & (values == np.floor(values)) + ).all() + ), + f"PUF_BOOLEAN_COUNT_DOMAIN:{column}", + ) + for column in ( + c + for c in profile.tax_unit_outputs + if c in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS + ): + values = donor[column].to_numpy() + _require( + bool( + ( + (values == 0) + | ( + (values >= 1000) + & (values <= 9999) + & (values == np.floor(values)) + ) + ).all() + ), + f"PUF_YEAR_DOMAIN:{column}", + ) + model_donor = donor.loc[:, [*profile.predictors, *profile.targets, "weight"]].copy() + return model_donor + + +def prepare_full_puf_inputs(frame, donor, *, predictor_known, profile=FULL65): + """Prepare selected surfaces without imputing absent source values. + + PUF59's canonical donor carries ``puf_person_incidence_capacity`` solely + for Boolean incidence validation. Return-only capacity is source declared; + person-table capacity comes from membership. The disclosure-capped return- + size predictor cannot serve as that bound. Only model predictors, selected + targets and weight reach the maintained donor Frame machinery. + """ + profile = require_puf_output_profile(profile) + model_donor = _validated_model_donor(donor, profile=profile) + matrix, universe = _recipient_matrix( + frame, predictor_known=predictor_known, profile=profile + ) + # Every input is now physically numeric, finite, and known. The maintained + # helper constructs the actual weighted donor Frame and repeats its strict + # source checks; its historical fillna operation has no missing values to fill. + inputs = support.prepare_us_puf_tax_detail_chain_inputs( + frame, + model_donor, + predictors=profile.predictors, + person_outputs=profile.person_outputs, + tax_unit_outputs=profile.tax_unit_outputs, + require_complete_recipient_predictors=True, + ) + _require( + inputs.donor.equals(model_donor) + and inputs.recipient_predictor_universe == universe, + "PUF_PREPARATION_CHANGED_CANONICAL_INPUT", + ) + return FullPufInputs(inputs.donor, inputs.donor_frame, matrix, universe, profile) + + +def full_puf_train_apply_nodes( + *, + donor_population, + recipient_population, + matrix_producer, + seed, + n_estimators, + zero_atol, + prefix="puf_full", + profile=FULL65, +): + """Declare every selected real fit/draw with ordered raw predecessors.""" + profile = require_puf_output_profile(profile) + fits = legacy_qrf_train_nodes( + prefix + ".fit", + population=donor_population, + entity="tax_unit", + predictors=profile.predictors, + targets=profile.targets, + seed=seed, + n_estimators=n_estimators, + zero_atol=zero_atol, + phase=profile.phase, + ) + applies = legacy_qrf_apply_matrix_nodes( + prefix + ".apply", + population=recipient_population, + fit_nodes=fits, + matrix_producer=matrix_producer, + seed=seed, + phase=profile.phase, + ) + return fits, applies + + +def decode_full_puf_draws( + *, + matrix, + matrix_producer_key, + raw_draws, + apply_state, + training_state, + seed, + profile=FULL65, +): + """Check typed payloads against the selected complete model/raw history. + + The exact ordered target roster is intrinsic to fitted model and application + states. Even the common first target has a different model identity across + profiles; truncated FULL65 artifacts cannot stand in for PUF59 artifacts. + """ + profile = require_puf_output_profile(profile) + _require(codec._hash(matrix_producer_key), "PUF_MATRIX_PRODUCER_KEY") + prepared = model_input.decode_recipient_matrix(matrix) + packet = decode_matrix_apply_state(apply_state) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + training, fitted = codec.read_training(training_state) + fitted_values = fitted.to_dict() + applied_values = chain.to_dict() + _require( + prepared.entity == chain.entity == "tax_unit" + and tuple(prepared.features.columns) + == tuple(chain.predictors) + == profile.predictors + and tuple(chain.targets) == tuple(chain.completed_targets) == profile.targets + and tuple(fitted_values["completed_targets"]) == profile.targets + and all( + fitted_values[key] == applied_values[key] + for key in ( + "predictors", + "targets", + "entity", + "weight_kind", + "weight_sha256", + "model_config", + "donor_index", + ) + ) + and application["models"] == training["models"] + and application["seed"] == seed + and chain.recipient_index == qrf._index_identity(prepared.features.index), + "PUF_FULL_CHAIN_IDENTITY", + ) + _require( + packet["matrix_sha256"] == codec.sha(matrix) + and packet["matrix_producer_key"] == matrix_producer_key, + "PUF_FULL_MATRIX_BINDING", + ) + _require( + isinstance(raw_draws, Mapping) and tuple(raw_draws) == profile.targets, + "PUF_RAW_ROSTER", + ) + _require( + application["raw_targets"] + == [ + {"target": target, "sha256": codec.sha(raw_draws[target])} + for target in profile.targets + ], + "PUF_RAW_HISTORY", + ) + return pd.DataFrame( + { + target: codec.read_raw_target( + raw_draws[target], target=target, index=prepared.features.index + ) + for target in profile.targets + }, + index=prepared.features.index, + ) + + +def _check_fitted_model_donor(donor_frame, *, training_state, last_model, profile): + """Bind complete consumed donor values to an upstream-trusted final model. + + The caller must establish actual producer authority before this trusted + decoder is reached. A caller-selected hash or public receipt is insufficient. + This reuses the existing whole-chain value check and grants no admission. + """ + # The final fitted target consumed every preceding donor outcome. Binding + # its actual trusted producer bytes therefore checks the entire donor value + # surface, including fields used only by sparsity/tail finalization. A donor + # with the same index/weights but altered values cannot adjust these draws. + training, fitted_state = codec.read_training(training_state) + last = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + last_model, expected_sha256=training["models"][-1]["sha256"] + ) + _require( + last.target == profile.targets[-1] + and last.next_training_state == fitted_state + and last.training_id == training["models"][-1]["training_id"], + "PUF_FINAL_MODEL_BINDING", + ) + resolved = qrf._resolve_qrf_fit_input( + donor_frame, list(profile.predictors), list(profile.targets), "design" + ) + qrf.RegimeGatedQRF._validate_chain_donor(last.training_state._chain(), resolved) + _require( + last.donor_sha256 + == qrf_target._consumed_values_sha256( + resolved.table, (*profile.predictors, *profile.targets) + ), + "PUF_DONOR_CONSUMED_BYTES", + ) + + +def finalize_full_puf( + frame, + donor, + *, + predictor_known, + matrix, + matrix_producer_key, + raw_draws, + apply_state, + training_state, + last_model, + seed, + profile=FULL65, +): + """Use actual finalizer judgments only after the complete raw chain validates. + + Returns a candidate Frame plus tail-cap/universe evidence. A graph placement + owner must expose every changed person/tax-unit column, attach only on the + PUF masks, and run the existing full-population replay verifier afterwards. + This function itself never creates a live Population/source qualification. + """ + profile = require_puf_output_profile(profile) + inputs = prepare_full_puf_inputs( + frame, donor, predictor_known=predictor_known, profile=profile + ) + _require(matrix == inputs.matrix, "PUF_RECIPIENT_MATRIX_CHANGED") + raw = decode_full_puf_draws( + matrix=matrix, + matrix_producer_key=matrix_producer_key, + raw_draws=raw_draws, + apply_state=apply_state, + training_state=training_state, + seed=seed, + profile=profile, + ) + _check_fitted_model_donor( + inputs.donor_frame, + training_state=training_state, + last_model=last_model, + profile=profile, + ) + mask = support.puf_tax_detail_clone_mask(frame.table("tax_unit"), entity="tax_unit") + # Explicit adapter between separate graph entity-ID and pandas row indexes; + # matrix recomputation above proves the ordered IDs before this relabeling. + raw.index = frame.table("tax_unit").index[mask] + caps = [] + result = support.finalize_us_puf_tax_detail_predictions( + frame, + inputs.donor, + raw.copy(deep=True), + person_outputs=profile.person_outputs, + tax_unit_outputs=profile.tax_unit_outputs, + tail_bound_diagnostics=caps, + absent_cells=support.PUF_ABSENT_CELLS_PRESERVE_NULLS, + ) + return result, { + "scope": profile.phase, + "output_profile": profile.value, + "predictor_order": list(profile.predictors), + "donor_auxiliary_columns": list(profile.donor_auxiliary_columns), + "person_target_count": len(profile.person_outputs), + "tax_unit_target_count": len(profile.tax_unit_outputs), + "target_count": len(profile.targets), + "target_order": list(profile.targets), + "recipient_universe": dict(inputs.recipient_universe), + "tail_bounds": caps, + "matrix_sha256": codec.sha(matrix), + "raw_target_sha256": { + target: codec.sha(raw_draws[target]) for target in profile.targets + }, + "source_admission_issued": False, + "release_eligible": False, + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py new file mode 100644 index 000000000..2929f225e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py @@ -0,0 +1,698 @@ +"""Typed ACS source evidence and exact whole-household graph selection. + +Only CREATE authenticates source bytes. Selection keeps the original evidence as +an ancestor and emits selected identities/positions; it never mints source +authority or asserts calibrated representation. +""" + +from __future__ import annotations + +import json +import struct +from dataclasses import InitVar, dataclass, replace +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Slice, + SourceRef, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + +from . import acs_housing_universe_source as source +from .graph_context import ( + US_FRAME_CONTEXT_TYPE, + encode_us_frame_context, + us_frame_from_context, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .graph_sources import frame_column_declarations + +ACS_HU_PAYLOAD_MAX_BYTES = 64 * 1024**2 +ACS_HU_HEADER_MAX_BYTES = 65536 +MAGIC = bytes.fromhex("4d43414353485502") +US_ACS_HOUSING_UNIVERSE_TYPE = ArtifactType("microcosm.us.acs_housing_universe", 2) +US_ACS_HOUSING_PREPARATION_TYPE = ArtifactType( + "microcosm.us.acs_housing_preparation", 2 +) +US_ACS_HOUSING_SELECTION_TYPE = ArtifactType("microcosm.us.acs_housing_selection", 2) +CREATE_NODE = "acs_housing_create" +SELECT_NODE = "acs_housing_select" +SOURCE_NAME = "acs_housing_source" +PHASE = "acs_housing_universe_graph_v2" +_TOKEN = object() +_CODES = ( + "interview_scope", + "physical_unit", + "household_kind", + "tenure_subtype", + "occupied_hu", + "hu_tenure_class", + "unresolved_reasons", + "TEN_valid", +) +_HEADER_FIELDS = frozenset( + { + "format", + "release_eligible", + "source_receipt_bytes", + "projection_bytes", + "household_rows", + "person_rows", + "code_columns", + "body_bytes", + "source_receipt_sha256", + "projection_sha256", + "prepared_receipt_sha256", + "frame_context_sha256", + "frame_sha256", + "definition_sha256", + "implementation_sha256", + } +) +_ARTIFACT_OUTPUTS = ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("prepared_receipt", US_ACS_HOUSING_PREPARATION_TYPE), + ArtifactOutput("housing_universe", US_ACS_HOUSING_UNIVERSE_TYPE), +) +_PREPARED_FIELDS = frozenset( + { + "format", + "release_eligible", + "source_receipt_sha256", + "projection_sha256", + "implementation_sha256", + "same_snapshot_frame_and_observations", + "full_source_frame_before_selection", + "full_source_lexical_projection", + "native_selection_before_person_accumulation", + "selection_kind", + "requested_serialnos", + "full_source_inclusion_probability", + "pre_promotion_frame_sha256", + "frame_sha256", + "dtype_transitions", + "entity_rows", + "weight_kind", + "HU_columns", + } +) + + +def _require(value, code): + source._require(value, "GRAPH_ACS_" + code) + + +@dataclass(frozen=True) +class BoundACSHousingEvidence: + """Immutable graph transport, explicitly not AuthenticatedACSHousingSource.""" + + header_json: bytes + source_receipt_json: bytes + projection_json: bytes + codes: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "BOUND_CONSTRUCTOR") + + +def _verify_preparation(prepared, receipt, household, frame_context): + """Check the v2 construction claim without minting source authority.""" + _require( + set(prepared) == _PREPARED_FIELDS + and prepared["format"] == "microcosm.acs_housing_preparation.v2" + and prepared["release_eligible"] is False + and prepared["HU_columns"] == "artifact_only" + and prepared["weight_kind"] == "design" + and prepared["same_snapshot_frame_and_observations"] is True + and prepared["full_source_lexical_projection"] is True + and prepared["full_source_inclusion_probability"] is None, + "PREPARED_SCHEMA", + ) + requested = prepared["requested_serialnos"] + subset = requested is not None + _require( + not subset + or ( + type(requested) is list + and bool(requested) + and all(type(key) is str for key in requested) + and len(set(requested)) == len(requested) + ), + "PREPARED_SELECTION", + ) + chosen = tuple(sorted(household.SERIALNO)) + _require( + prepared["full_source_frame_before_selection"] is (not subset) + and prepared["native_selection_before_person_accumulation"] is subset + and prepared["selection_kind"] + == ("engineering_exact_keys" if subset else "all") + and receipt["selection"] == ("exact_serialnos" if subset else "all") + and (not subset or tuple(sorted(requested)) == chosen) + and receipt["selected_serialnos_sha256"] == source._sha(canonical_json(chosen)), + "PREPARED_SELECTION", + ) + context = source._parse_json(frame_context, ACS_HU_PAYLOAD_MAX_BYTES) + rows = prepared["entity_rows"] + entities = context.get("entities") + _require( + type(rows) is dict + and type(entities) is dict + and set(rows) == set(entities) == set(US_SCHEMA.entities) + and all( + type(rows[entity]) is int + and rows[entity] >= 0 + and type(entities[entity]) is dict + and type(entities[entity].get("rows")) is int + and rows[entity] == entities[entity]["rows"] + for entity in US_SCHEMA.entities + ), + "PREPARED_CONTEXT", + ) + + +def encode_acs_housing_evidence(prepared): + _require( + type(prepared) is source.PreparedACSHousingPopulation + and type(prepared.source) is source.AuthenticatedACSHousingSource, + "PREPARED_TYPE", + ) + receipt = source._parse_json(prepared.receipt_json, source.ACS_HU_RECEIPT_MAX_BYTES) + observation = prepared.source + projection = source._parse_json( + observation.projection_json, ACS_HU_PAYLOAD_MAX_BYTES + ) + _h, _p, _t, _lines, _pw, codes = source._tables(projection) + source.verify_acs_frame_projection(prepared.frame, projection) + _require( + receipt["source_receipt_sha256"] == source._sha(observation.receipt_json) + and receipt["projection_sha256"] == source._sha(observation.projection_json) + and receipt["frame_sha256"] == source.frame_content_sha256(prepared.frame), + "PREPARED_BINDING", + ) + context = encode_us_frame_context(prepared.frame) + _verify_preparation( + receipt, + source._parse_json(observation.receipt_json, source.ACS_HU_RECEIPT_MAX_BYTES), + _h, + context, + ) + native = b"".join(codes[n].to_numpy(dtype="uint8").tobytes() for n in _CODES) + body = observation.receipt_json + observation.projection_json + native + header = { + "format": "microcosm.acs_housing_graph_evidence.v2", + "release_eligible": False, + "source_receipt_bytes": len(observation.receipt_json), + "projection_bytes": len(observation.projection_json), + "household_rows": len(_h), + "person_rows": len(_p), + "code_columns": list(_CODES), + "body_bytes": len(body), + "source_receipt_sha256": source._sha(observation.receipt_json), + "projection_sha256": source._sha(observation.projection_json), + "prepared_receipt_sha256": source._sha(prepared.receipt_json), + "frame_context_sha256": source._sha(context), + "frame_sha256": receipt["frame_sha256"], + "definition_sha256": source._definition()[1], + "implementation_sha256": source._implementation(), + } + encoded = canonical_json(header) + _require(len(encoded) <= ACS_HU_HEADER_MAX_BYTES, "HEADER_SIZE") + payload = MAGIC + struct.pack(" 0 + and nhouse > 0, + "SECTION_SIZE", + ) + _require( + header["body_bytes"] == nreceipt + nprojection + nhouse * len(_CODES) + and len(payload) == start + size + header["body_bytes"] + 32, + "BODY_SIZE", + ) + cursor = start + size + receipt_bytes = payload[cursor : cursor + nreceipt] + projection_bytes = payload[cursor + nreceipt : cursor + nreceipt + nprojection] + codes = payload[cursor + nreceipt + nprojection : -32] + receipt = source._parse_json(receipt_bytes, source.ACS_HU_RECEIPT_MAX_BYTES) + projection = source._parse_json(projection_bytes, ACS_HU_PAYLOAD_MAX_BYTES) + prepared = source._parse_json(prepared_receipt, source.ACS_HU_RECEIPT_MAX_BYTES) + household, person, _typed, _lines, _pw, derived = source._tables(projection) + _require( + source._sha(prepared_receipt) == header["prepared_receipt_sha256"] + and source._sha(frame_context) == header["frame_context_sha256"], + "CONTEXT_BINDING", + ) + _verify_preparation(prepared, receipt, household, frame_context) + _require( + source._sha(receipt_bytes) + == header["source_receipt_sha256"] + == prepared["source_receipt_sha256"] + and source._sha(projection_bytes) + == header["projection_sha256"] + == prepared["projection_sha256"] + == receipt["projection_sha256"], + "SOURCE_BINDING", + ) + _require( + header["definition_sha256"] + == receipt["definition_sha256"] + == source._definition()[1] + and header["implementation_sha256"] + == prepared["implementation_sha256"] + == receipt["implementation_sha256"] + == source._implementation(), + "IMPLEMENTATION_BINDING", + ) + _require(header["frame_sha256"] == prepared["frame_sha256"], "FRAME_BINDING") + # Coverage of the pins is otherwise only transitive, through the + # implementation hash over the archive manifest. Comparing them here + # makes the evidence say what it rests on. + _require( + receipt["format"] == "microcosm.acs_housing_universe_source.v1" + and receipt["release_eligible"] is False + and receipt["vintage"] == 2024 + and receipt["archives"] + == [ + {"role": r, "filename": n, "sha256": d, "bytes": s} + for r, n, d, s in source._pins() + ], + "SOURCE_PINS", + ) + _require( + len(household) == nhouse == receipt["selected_counts"]["households"] + and len(person) + == header["person_rows"] + == receipt["selected_counts"]["persons"], + "ROW_COUNTS", + ) + _require( + codes + == b"".join( + derived[name].to_numpy(dtype="uint8").tobytes() for name in _CODES + ), + "NATIVE_CODES", + ) + return BoundACSHousingEvidence( + raw_header, receipt_bytes, projection_bytes, codes, _token=_TOKEN + ) + except source.ACSHousingSourceError: + raise + except ( + ValueError, + TypeError, + KeyError, + OverflowError, + struct.error, + AttributeError, + ): + raise source.ACSHousingSourceError("GRAPH_ACS_TRANSPORT_CONTRACT") from None + + +def _artifact(context, name, expected_type): + value = context.artifacts.get(name) + _require(value is not None and value.type == expected_type, "ARTIFACT_TYPE") + declarations = [item for item in context.node.artifact_inputs if item.name == name] + _require( + len(declarations) == 1 + and value.key + == opaque_artifact_key(value.producer_key, declarations[0].artifact), + "ARTIFACT_KEY", + ) + return value + + +def _keys(value): + if value is None: + return None + _require( + isinstance(value, (tuple, list)) + and bool(value) + and all(type(v) is str for v in value) + and len(set(value)) == len(value), + "SELECTION_KEYS", + ) + return tuple(value) + + +def _slices(columns): + # The compiler does not treat implicit entity IDs as owned data columns. + # Structural-only groups are reconstructed from declared person membership + # and checked against the parent's typed ID digest, never read as fake cells. + return tuple( + Slice(entity, tuple(c.column for c in columns if c.entity == entity)) + for entity in US_SCHEMA.entities + if any(c.entity == entity for c in columns) + ) + + +def _complete_structural_tables(context, frame_context): + document = json.loads(frame_context) + tables = dict(context.tables) + _require("person" in tables, "PERSON_VIEW") + for group in US_SCHEMA.group_entities: + if group in tables: + continue + id_column = US_SCHEMA.entity_id_column(group) + _require( + document["entities"][group]["columns"] == [id_column], + "UNDECLARED_GROUP_CELLS", + ) + membership = tables["person"][US_SCHEMA.membership_column(group)] + _require(membership.dtype == np.dtype("int64"), "MEMBERSHIP_DTYPE") + tables[group] = pd.DataFrame({id_column: np.unique(membership.to_numpy())}) + # us_frame_from_context checks every reconstructed group's actual IDs + # against the original producer's ordered-ID digest before Frame creation. + return replace(context, tables=tables) + + +class _Kernel(KernelBase): + def implementation_hash(self): + return implementation_hash(source.ACS_HU_STAGE) + + +class ACSHousingCreateKernel(_Kernel): + ref = "us.acs_housing.create@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.CREATE, + dependencies=STAGE_DEPENDENCIES[source.ACS_HU_STAGE], + ) + + def __init__(self, snapshot_root): + self.snapshot_root = Path(snapshot_root) + + def run(self, context: KernelContext): + node = context.node + _require( + node.kernel == self.ref + and node.structural is StructuralDelta.CREATE + and tuple(node.sources) == (SOURCE_NAME,) + and not node.inputs + and not node.artifact_inputs + and node.artifact_outputs == _ARTIFACT_OUTPUTS, + "CREATE_DECLARATION", + ) + _require( + set(context.params) == {"phase", "serialnos"} + and context.params["phase"] == PHASE, + "PARAMS", + ) + prepared = source.prepare_acs_housing_population( + context.sources[SOURCE_NAME], + snapshot_root=self.snapshot_root, + serialnos=_keys(context.params["serialnos"]), + ) + _require( + frame_column_declarations(prepared.frame) == node.outputs, + "COLUMN_INVENTORY", + ) + payload = encode_acs_housing_evidence(prepared) + return KernelResult( + frame=prepared.frame, + artifacts={ + "frame_context": encode_us_frame_context(prepared.frame), + "prepared_receipt": prepared.receipt_json, + "housing_universe": payload, + }, + receipt={ + "phase": PHASE, + "prepared": prepared.receipt, + "housing_universe_sha256": source._sha(payload), + "implementation": implementation_manifest(source.ACS_HU_STAGE), + "release_eligible": False, + }, + ) + + +class ACSHousingSelectKernel(_Kernel): + ref = "us.acs_housing.select@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.FILTER, + dependencies=STAGE_DEPENDENCIES[source.ACS_HU_STAGE], + ) + + def run(self, context: KernelContext): + node = context.node + _require( + node.kernel == self.ref + and node.structural is StructuralDelta.FILTER + and node.mass == "declared" + and not node.outputs + and not node.sources, + "SELECT_DECLARATION", + ) + _require( + set(context.params) == {"phase", "serialnos"} + and context.params["phase"] == PHASE, + "PARAMS", + ) + _require( + set(context.artifacts) + == {"frame_context", "prepared_receipt", "housing_universe"} + and node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ACS_HOUSING_SELECTION_TYPE), + ), + "SELECT_ARTIFACTS", + ) + values = [ + _artifact(context, n, t) + for n, t in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("prepared_receipt", US_ACS_HOUSING_PREPARATION_TYPE), + ("housing_universe", US_ACS_HOUSING_UNIVERSE_TYPE), + ) + ] + _require( + len({v.producer_key for v in values}) == 1 + and len({a.producer for a in node.artifact_inputs}) == 1 + and next(iter(node.artifact_inputs)).producer == node.base, + "SAME_PRODUCER", + ) + bound = bind_acs_housing_evidence( + values[2].payload, + prepared_receipt=values[1].payload, + frame_context=values[0].payload, + ) + frame = us_frame_from_context( + _complete_structural_tables(context, values[0].payload) + ) + _require( + tuple(node.inputs) == _slices(frame_column_declarations(frame)), + "COMPLETE_FRAME_DECLARATION", + ) + header = json.loads(bound.header_json) + _require( + source.frame_content_sha256(frame) == header["frame_sha256"], "FRAME_CELLS" + ) + projection = json.loads(bound.projection_json) + source.verify_acs_frame_projection(frame, projection) + selected_projection, chosen, _counts = source._select( + projection, _keys(context.params["serialnos"]) + ) + household = frame.table("household") + household_keep = household.SERIALNO.isin(chosen).to_numpy() + ids = household.household_id.to_numpy()[household_keep] + keep = frame.person.person_household_id.isin(ids) + _require(bool(keep.any()), "EMPTY_SELECTION") + selected = frame.select(keep) + source.verify_acs_frame_projection(selected, selected_projection) + positions = { + "person": np.flatnonzero(keep).astype("int64"), + "household": np.flatnonzero(household_keep).astype("int64"), + } + _require( + selected.weights_for("household").values.tobytes() + == frame.weights_for("household").values[positions["household"]].tobytes(), + "SELECTED_WEIGHT_BYTES", + ) + before_mass, after_mass = frame.stratum_mass(), selected.stratum_mass() + native_households = dict( + zip( + selected.table("household").household_id, + selected.table("household").SERIALNO, + strict=True, + ) + ) + selection = { + "format": "microcosm.acs_housing_selection.v2", + "release_eligible": False, + "representative": False, + "source_authority": "original_CREATE_only", + "producer_key": values[0].producer_key, + "source_payload_sha256": source._sha(values[2].payload), + "source_prepared_receipt_sha256": source._sha(values[1].payload), + "source_frame_context_sha256": source._sha(values[0].payload), + "selected_frame_sha256": source.frame_content_sha256(selected), + "selected_frame_context_sha256": source._sha( + encode_us_frame_context(selected) + ), + "selected_projection_sha256": source._sha( + canonical_json(selected_projection) + ), + "selected_serialnos_sha256": source._sha(canonical_json(chosen)), + "positions": {e: a.tolist() for e, a in positions.items()}, + "frame_identity_scope": "actual_CREATE_frame", + "native_keys": { + "household": selected.table("household").SERIALNO.tolist(), + "person": [ + [native_households[household_id], int(line)] + for household_id, line in selected.person[ + ["person_household_id", "SPORDER"] + ].itertuples(index=False, name=None) + ], + }, + "ordered_ids": { + e: selected.table(e)[US_SCHEMA.entity_id_column(e)].tolist() + for e in US_SCHEMA.entities + }, + "weight_kind": "design", + "mass_policy": "declared_no_normalization", + } + return KernelResult( + keep=pd.Series( + keep.to_numpy(dtype=bool), + index=pd.Index(frame.person.person_id.to_numpy(), name="person_id"), + dtype=bool, + ), + artifacts={ + "frame_context": encode_us_frame_context(selected), + "selection": canonical_json(selection), + }, + receipt={ + "phase": PHASE, + "implementation": implementation_manifest(source.ACS_HU_STAGE), + "selection_sha256": source._sha(canonical_json(selection)), + "release_eligible": False, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + "stratum_before": { + str(k): float(v) for k, v in before_mass.items() + }, + "stratum_after": {str(k): float(v) for k, v in after_mass.items()}, + }, + }, + ) + + +def acs_housing_graph(columns, *, serialnos=None, selected_serialnos=None): + """Explicit seed source selection followed by exact further household pruning.""" + columns = tuple(columns) + _keys(serialnos) + _keys(selected_serialnos) + create = Node( + id=CREATE_NODE, + kernel=ACSHousingCreateKernel.ref, + structural=StructuralDelta.CREATE, + sources=(SOURCE_NAME,), + outputs=columns, + params={"phase": PHASE, "serialnos": serialnos}, + artifact_outputs=_ARTIFACT_OUTPUTS, + ) + select = Node( + id=SELECT_NODE, + kernel=ACSHousingSelectKernel.ref, + structural=StructuralDelta.FILTER, + mass="declared", + base=CREATE_NODE, + inputs=_slices(columns), + params={"phase": PHASE, "serialnos": selected_serialnos}, + artifact_inputs=tuple( + ArtifactInput(a.name, CREATE_NODE, a.name, a.type) + for a in _ARTIFACT_OUTPUTS + ), + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ACS_HOUSING_SELECTION_TYPE), + ), + ) + return Graph( + country="us", + sources=(SourceRef(SOURCE_NAME, codec=source.ACS_HU_CODEC),), + nodes=(create, select), + ) + + +def acs_housing_registry(*, snapshot_root): + registry = KernelRegistry() + registry.register(ACSHousingCreateKernel(snapshot_root)) + registry.register(ACSHousingSelectKernel()) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py new file mode 100644 index 000000000..addadd581 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py @@ -0,0 +1,665 @@ +"""Typed PAW transport and reported-income accounting, never source authority. + +The six results are source observations and an accounting residual. They are +not simulated SSI/TANF, ACS other income, or a completed eight-category bridge. +""" + +from __future__ import annotations + +import struct +from dataclasses import InitVar, dataclass + +import numpy as np +import pandas as pd + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Numeric, + Owned, + Slice, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from . import asec_income_observations as source +from . import graph_context +from .asec_current_money import MoneyRefusalError, _json, _parse, _require, _sha +from .asec_current_money_selection import ( + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + US_ASEC_SELECTION_TYPE, + SelectedCurrentMoney, + decode_selected_current_money, + encode_selected_current_money, +) +from .graph_context import US_FRAME_CONTEXT_TYPE +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) + +US_ASEC_INCOME_OBSERVATIONS_TYPE = ArtifactType( + "microcosm.us.asec_income_observations", 1 +) +US_ASEC_REPORTED_INCOME_TYPE = ArtifactType("microcosm.us.asec_reported_income", 3) +STAGE = "asec_prepared_v3" +REPORTED_INCOME_NODE = "asec_prepared.reported_income" +RESULT_COLUMNS = tuple( + "asec_reported_" + name + "_2024_price" + for name in ( + "wage_income", + "net_self_employment_income", + "ssi", + "cash_public_assistance", + "total_income", + "income_unallocated", + ) +) +CONTRIBUTING_FIELDS = ("WSAL_VAL", "SEMP_VAL", "FRSE_VAL", "SSI_VAL", "PTOTVAL") +COORDINATES = ("person_id", "income_year", "A_AGE") +EVIDENCE_COLUMNS = tuple( + f"{name}.{axis}" + for name in CONTRIBUTING_FIELDS + for axis in ("status", "validity", "zero_origin") +) + ( + "PAW_VAL.status", + "PAW_VAL.validity", + "PAW_VAL.zero_origin", + "PAW_VAL.source_encoding_class", +) +ACCOUNTING_COLUMNS = COORDINATES + RESULT_COLUMNS + EVIDENCE_COLUMNS +ACCOUNTING_DTYPES = tuple([" BoundIncomeObservations: + """Admit graph transport only against the independently issued CREATE receipt.""" + try: + _require( + type(payload) is bytes + and len(source.MAGIC) + 4 + 32 + < len(payload) + <= source.INCOME_PAYLOAD_MAX_BYTES, + "BOUND_INCOME_SIZE", + ) + binding = prepared_receipt["income_observations"] + _require( + set(binding) == _BINDING_KEYS + and prepared_receipt["schema"] == "microcosm.us.asec_prepared_receipt.v3" + and prepared_receipt["source_kind"] == "us_asec_prepared_current_money_v3" + and prepared_receipt["release_eligible"] is False + and binding["kind"] == source.ARTIFACT_KIND + and binding["columns"] == list(source.COLUMNS), + "BOUND_INCOME_PREPARATION", + ) + _require(_sha(payload) == binding["payload_sha256"], "BOUND_INCOME_PAYLOAD") + _require( + payload.startswith(source.MAGIC) + and _sha(payload[:-32]) == payload[-32:].hex(), + "BOUND_INCOME_FRAMING", + ) + length = struct.unpack_from("= 0).all()) + and len(np.unique(positions)) == len(positions) + and (len(positions) < 2 or bool((np.diff(positions) > 0).all())) + and _sha(positions.tobytes()) == person_positions_sha256, + "ACCOUNTING_SOURCE_POSITIONS", + ) + _require( + np.array_equal(income_years, income.array("income_year")[positions]), + "ACCOUNTING_SOURCE_COHORTS", + ) + _require( + all(selected.entity_of(name) == "person" for name in CONTRIBUTING_FIELDS), + "ACCOUNTING_FIELD_ENTITY", + ) + w = selected.field("WSAL_VAL").amounts + e = selected.field("SEMP_VAL").amounts + selected.field("FRSE_VAL").amounts + s = selected.field("SSI_VAL").amounts + p = income.array("PAW_VAL_2024_price")[positions] + t = selected.field("PTOTVAL").amounts + u = t - (((w + e) + s) + p) + results = (w, e, s, p, t, u) + _require(all(np.isfinite(a).all() for a in results), "ACCOUNTING_NONFINITE") + age = income.array("A_AGE")[positions] + nominal, yn = income.array("PAW_VAL")[positions], income.array("PAW_YN")[positions] + buffers = [ + a.astype(" KernelResult: + from . import graph_asec_prepared as bridge + + bridge._phase(context) + node = context.node + _require( + node.kernel == self.ref + and node.structural is StructuralDelta.NONE + and not node.sources, + "ACCOUNTING_NODE", + ) + _require( + node.inputs == (Slice("person", ("source_year",)),) + and node.outputs == reported_income_declarations(), + "ACCOUNTING_DECLARATIONS", + ) + _require( + set(context.artifacts) + == { + "frame_context", + "selection", + "selected_current_money", + "prepared_receipt", + "income_observations", + }, + "ACCOUNTING_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == ( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + "ACCOUNTING_ARTIFACT_OUTPUTS", + ) + values = { + name: bridge._artifact(context, name, kind) + for name, kind in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("selection", US_ASEC_SELECTION_TYPE), + ("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ) + } + selection_producer = bridge._same_producer( + context, ("frame_context", "selection", "selected_current_money") + ) + source_producer = bridge._same_producer( + context, ("prepared_receipt", "income_observations") + ) + document = bridge._document(values["frame_context"]) + person = context.tables["person"] + bridge._check_identity(document, "person", person) + _require( + not set(RESULT_COLUMNS) & set(document["entities"]["person"]["columns"]), + "OWNED_LEAF_INCUMBENT", + ) + prepared = _parse(values["prepared_receipt"].payload) + selection = _parse(values["selection"].payload) + _require( + canonical_json(prepared) == values["prepared_receipt"].payload + and canonical_json(selection) == values["selection"].payload + and selection["schema"] == bridge.SELECTION_SCHEMA, + "ACCOUNTING_RECEIPTS", + ) + _require( + selection["parent"]["producer_key"] == source_producer + and selection["parent"]["prepared_receipt_sha256"] + == _sha(values["prepared_receipt"].payload), + "ACCOUNTING_SELECTION_PARENT", + ) + for entity, declared in document["entities"].items(): + _require( + all( + selection["entities"][entity][key] == declared[key] + for key in ("rows", "ordered_ids_sha256") + ), + "ACCOUNTING_SELECTION_IDENTITY", + ) + selected = decode_selected_current_money( + values["selected_current_money"].payload, + expected_parent_header_sha256=prepared["money_header_sha256"], + expected_parent_content_sha256=prepared["money_content_sha256"], + expected_prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + expected_selection_sha256=_sha(values["selection"].payload), + ) + _require( + selected.header_data["person_identity_sha256"] + == document["entities"]["person"]["ordered_ids_sha256"] + and selected.header_data["household_identity_sha256"] + == document["entities"]["household"]["ordered_ids_sha256"], + "ACCOUNTING_MONEY_COORDINATES", + ) + income = bind_income_observations( + values["income_observations"].payload, prepared_receipt=prepared + ) + _require( + person.source_year.dtype == np.dtype("int64"), + "ACCOUNTING_SOURCE_YEAR_DTYPE", + ) + result = derive_reported_income( + selected, + income, + person_ids=person.person_id.to_numpy(), + income_years=person.source_year.to_numpy(), + person_positions_sha256=selection["person_positions_sha256"], + prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + selection_sha256=_sha(values["selection"].payload), + selected_money_sha256=_sha(values["selected_current_money"].payload), + income_payload_sha256=_sha(values["income_observations"].payload), + source_producer_key=source_producer, + selection_producer_key=selection_producer, + frame_context_sha256=_sha(values["frame_context"].payload), + ) + index = pd.Index(person.person_id.to_numpy(), name="person_id") + columns = { + ("person", name): pd.Series( + result.array(name), index=index, dtype="float64" + ) + for name in RESULT_COLUMNS + } + document["entities"]["person"]["columns"].extend(RESULT_COLUMNS) + payload = encode_reported_income(result) + return KernelResult( + columns=columns, + artifacts={ + "reported_income": payload, + "frame_context": canonical_json(document), + }, + receipt={ + "phase": bridge.ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(STAGE), + "accounting_sha256": _sha(payload), + "arithmetic_order": ARITHMETIC_ORDER, + "person_rows": len(person), + "release_eligible": False, + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py new file mode 100644 index 000000000..db6a84363 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py @@ -0,0 +1,1136 @@ +"""The five-node ASEC slice: source, selection, leaves, engine and reported income. + +One directory source holds the reviewed restoration inputs. The CREATE kernel +prepares the whole authenticated population from it; a FILTER kernel draws a +seeded whole-household engineering sample and slices the money body to exactly +the selected coordinates; a pure kernel derives the corrected monetary CPS +leaves from that slice alone; and a final kernel runs the real PolicyEngine-US +adapter for the four outputs whose complete input closure this slice produces. + +Two boundaries are load-bearing. The authenticated ``ReadyCurrentMoney`` never +leaves the CREATE process: what travels is its canonical encoded body, typed as +a subset artifact that no ``type(x) is ReadyCurrentMoney`` check accepts. And +engine outputs are formula-owned, so they are retained in an evaluation +artifact and never written back as population cells. + +Nothing here is a release, a calibration, a representative sample, or a tax or +benefit score. Every receipt says so. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence + +import numpy as np +import pandas as pd + +from microcosm.build.frame_sampling import EXACT_COUNT_RULE, sample_frame_households +from microcosm.frame import US_SCHEMA, EntitySchema, Frame, Weights +from microcosm.frame.adapters.policyengine_us import ( + PolicyEngineUSEngine, + PolicyEngineUSVariableMetadataIndex, +) +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from . import asec_housing_status_source as housing_source +from . import graph_context +from .asec_current_money import _sha +from .asec_current_money_graph_resources import load_graph_current_money_consumers +from .asec_current_money_selection import ( + US_ASEC_CURRENT_MONEY_BODY_TYPE, + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + US_ASEC_SELECTION_TYPE, + decode_selected_current_money, + encode_selected_current_money, + parse_current_money_body, + select_current_money, +) +from .asec_engine_evaluation import ( + ADMITTED_ROOTS, + BLOCKED_ENGINE_OUTPUTS, + US_ASEC_ENGINE_EVALUATION_TYPE, + admit_engine_outputs, + encode_engine_evaluation, + engine_runtime_identity, + materialize_engine_outputs, +) +from .asec_prepared_source import ( + PREPARED_SOURCE_FILES, + PREPARED_SOURCE_KIND, + prepare_asec_current_money_population, +) +from .cps_carried_current import ( + CPS_CARRIED_CURRENT_PERSON_LEAVES, + CPS_CARRIED_CURRENT_ROUTING_COLUMNS, + CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES, + cps_carried_current_leaf_contract, + derive_cps_carried_current_leaves, +) +from .graph_asec_income import ( + REPORTED_INCOME_NODE, + RESULT_COLUMNS, + US_ASEC_INCOME_OBSERVATIONS_TYPE, + US_ASEC_REPORTED_INCOME_TYPE, + USAsecReportedIncomeKernel, + reported_income_declarations, +) +from .graph_context import US_FRAME_CONTEXT_TYPE +from .graph_housing_universe import ( + HOUSEHOLD_EVIDENCE_COLUMNS, + US_ASEC_HOUSING_UNIVERSE_TYPE, + bind_housing_universe, + verify_graph_housing_rows, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .graph_sources import frame_column_declarations + +ASEC_PREPARED_STAGE = "asec_prepared_v3" +ASEC_PREPARED_PHASE = "prepare_asec_current_money_slice" +ASEC_PREPARED_CODEC = "us-asec-prepared-current-money-v3" +ASEC_PREPARED_SOURCE_NAME = "asec_prepared" +ASEC_PREPARED_SOURCE = SourceRef( + ASEC_PREPARED_SOURCE_NAME, + ASEC_PREPARED_CODEC, + "Reviewed ASEC restoration inputs: P, H, T with its receipt, three housing " + "cohort HDFs and three official Census PERSON CSV members.", +) +ASEC_PREPARED_DEPENDENCIES = STAGE_DEPENDENCIES[ASEC_PREPARED_STAGE] + +PREFIX = "asec_prepared" +CREATE_NODE = f"{PREFIX}.create" +SELECT_NODE = f"{PREFIX}.select_households" +LEAVES_NODE = f"{PREFIX}.cps_carried_current" +ENGINE_NODE = f"{PREFIX}.engine_current_money" + +#: The one household column the selection and engine nodes declare. They need +#: the household table and its design weights; the executor exposes an entity's +#: weights only to a node that declares that entity. This is the first column +#: the reviewed housing attachment guarantees, so a changed attachment fails +#: closed here instead of silently reading something else. It is a carrier: it +#: never reaches the corrected leaves and never reaches the engine frame. +HOUSEHOLD_WEIGHT_CARRIER = housing_source.ATTACHED_COLUMNS[0] + +SELECTION_SCHEMA = "microcosm.us.asec_household_selection.v2" +SELECTION_SOURCE_LABEL = "US ASEC prepared current money" +SELECTION_STRATUM_COLUMN = "source_year" +LEAF_DTYPE = "float64" +_ENTITIES = US_SCHEMA.entities +_PERSON_ID = US_SCHEMA.entity_id_column("person") + + +class PreparedGraphError(ValueError): + """A declaration, binding or identity this slice refuses to execute on.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def _artifact(context: KernelContext, name: str, expected): + """Return one declared typed artifact, refusing an aliased or retyped edge.""" + value = context.artifacts.get(name) + edges = [edge for edge in context.node.artifact_inputs if edge.name == name] + _require( + value is not None + and len(edges) == 1 + and edges[0].type == expected + and value.type == expected, + f"ARTIFACT_EDGE:{name}", + ) + _require( + isinstance(value.producer_key, str) and len(value.producer_key) == 64, + f"ARTIFACT_PRODUCER:{name}", + ) + return value + + +def _same_producer(context: KernelContext, names: Sequence[str]) -> str: + """Require a set of edges to come from one actual producing node execution.""" + producers = {context.artifacts[name].producer_key for name in names} + edges = {edge.name: edge.producer for edge in context.node.artifact_inputs} + _require(len(producers) == 1, "ARTIFACT_PRODUCER_SPLIT") + _require(len({edges[name] for name in names}) == 1, "ARTIFACT_PRODUCER_SPLIT") + return next(iter(producers)) + + +def _phase(context: KernelContext, extra: tuple[str, ...] = ()) -> None: + _require(set(context.params) == {"phase", *extra}, "NODE_PARAMS") + _require(context.params["phase"] == ASEC_PREPARED_PHASE, "NODE_PHASE") + + +def _document(value) -> dict: + document = graph_context._decode(value.payload) + graph_context._mass_records(document["mass_log"]) + _require(canonical_json(document) == value.payload, "CONTEXT_CANONICAL") + _require(document["metadata"] == {}, "PREPARED_CONTEXT_METADATA") + _require(document["mass_log"] == [], "PREPARED_CONTEXT_MASS_LOG") + _require(document["weight_sources"] == {"household": "design"}, "WEIGHT_AUTHORITY") + return document + + +def _identity(ids: np.ndarray, entity: str) -> dict[str, object]: + """Row identity of an id vector, using the graph context's own digest.""" + column = US_SCHEMA.entity_id_column(entity) + return graph_context._row_identity( + pd.DataFrame({column: np.asarray(ids, dtype="int64")}), entity + ) + + +def _check_identity(document: dict, entity: str, table: pd.DataFrame) -> None: + declared = document["entities"][entity] + actual = graph_context._row_identity(table, entity) + _require( + all(declared[key] == value for key, value in actual.items()), + f"CONTEXT_IDENTITY:{entity}", + ) + _require( + not set(table.columns) - set(declared["columns"]), f"CONTEXT_COLUMNS:{entity}" + ) + + +def _group_ids(person: pd.DataFrame, group: str) -> np.ndarray: + """The exact id inventory a group table holds for these persons. + + ``Frame`` validates that a group table's ids are the sorted distinct values + of the person membership column, and ``Frame.select`` prunes to exactly + that set. Reconstructing the inventory from membership is therefore the + same operation, not an approximation; every reconstruction below is checked + against the producer's typed context before it is used. + """ + membership = person[US_SCHEMA.membership_column(group)].to_numpy(dtype="int64") + return np.unique(membership) + + +def _minimal_frame(person: pd.DataFrame, household: pd.DataFrame, weights, strata): + """A person/household view for the sampler; no group beyond household.""" + columns = [_PERSON_ID, US_SCHEMA.membership_column("household")] + extra = [name for name in (SELECTION_STRATUM_COLUMN,) if name in person] + return Frame( + { + "person": person.loc[:, columns + extra].copy(deep=True), + "household": household.loc[ + :, [US_SCHEMA.entity_id_column("household")] + ].copy(deep=True), + }, + EntitySchema(group_entities=("household",)), + { + "household": Weights( + np.asarray(weights.values, dtype="float64"), weights.kind + ) + }, + strata.copy(deep=True), + ) + + +def _household_strata(person: pd.DataFrame, household_ids: np.ndarray) -> np.ndarray: + """One declared source-year stratum per household row; mixed years refuse.""" + years = person[SELECTION_STRATUM_COLUMN] + _require(pd.api.types.is_integer_dtype(years.dtype), "STRATUM_DTYPE") + _require(not bool(years.isna().any()), "STRATUM_MISSING") + frame = pd.DataFrame( + { + "household": person[US_SCHEMA.membership_column("household")].to_numpy( + dtype="int64" + ), + "year": years.to_numpy(dtype="int64"), + } + ) + distinct = frame.drop_duplicates() + _require(not bool(distinct["household"].duplicated().any()), "MIXED_YEAR_HOUSEHOLD") + lookup = dict( + zip(distinct["household"].tolist(), distinct["year"].tolist(), strict=True) + ) + _require(set(lookup) == set(household_ids.tolist()), "STRATUM_COVERAGE") + return np.asarray( + [str(lookup[int(value)]) for value in household_ids], dtype=object + ) + + +class _PreparedKernel(KernelBase): + """Every kernel of this slice shares one reviewed implementation scope.""" + + def implementation_hash(self) -> str: + return implementation_hash(ASEC_PREPARED_STAGE) + + +class USAsecPreparedCreateKernel(_PreparedKernel): + """Prepare the whole authenticated ASEC population from one source directory.""" + + ref = "us.asec_prepared.create@3" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.CREATE, + dependencies=ASEC_PREPARED_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + _phase(context) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(tuple(node.sources) == (ASEC_PREPARED_SOURCE_NAME,), "NODE_SOURCES") + _require(not node.inputs and not node.artifact_inputs, "NODE_INPUTS") + _require( + node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ArtifactOutput("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ArtifactOutput("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ArtifactOutput("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ), + "NODE_ARTIFACT_OUTPUTS", + ) + prepared = prepare_asec_current_money_population( + context.sources[ASEC_PREPARED_SOURCE_NAME] + ) + _require( + frame_column_declarations(prepared.frame) == node.outputs, + "PREPARED_COLUMN_INVENTORY", + ) + receipt = prepared.receipt + consumers = load_graph_current_money_consumers() + _require(receipt["source_kind"] == PREPARED_SOURCE_KIND, "PREPARED_SOURCE_KIND") + _require(receipt["file_roster"] == list(PREPARED_SOURCE_FILES), "FILE_ROSTER") + return KernelResult( + frame=prepared.frame, + artifacts={ + "frame_context": graph_context.encode_us_frame_context(prepared.frame), + "current_money": prepared.money_payload, + "prepared_receipt": prepared.receipt_payload, + "housing_universe": prepared.housing_universe_payload, + "income_observations": prepared.income_observations_payload, + }, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "prepared": receipt, + "current_money_consumers": consumers, + "release_eligible": False, + }, + ) + + +class USAsecPreparedSelectionKernel(_PreparedKernel): + """Draw the seeded whole-household sample and slice the money body to it.""" + + ref = "us.asec_prepared.select_households@3" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.FILTER, + dependencies=ASEC_PREPARED_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + _phase(context, ("fraction", "seed")) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(not node.outputs and not node.sources, "NODE_OUTPUTS") + _require(node.structural is StructuralDelta.FILTER, "NODE_STRUCTURAL") + _require(node.mass == "declared", "NODE_MASS") + _require( + set(context.tables) == {"person", "household"} + and set(context.artifacts) + == { + "frame_context", + "current_money", + "prepared_receipt", + "housing_universe", + }, + "NODE_INPUTS", + ) + _require( + node.inputs + == ( + Slice("person", (SELECTION_STRATUM_COLUMN,)), + Slice( + "household", (HOUSEHOLD_WEIGHT_CARRIER, *HOUSEHOLD_EVIDENCE_COLUMNS) + ), + ), + "NODE_SLICES", + ) + _require( + node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ASEC_SELECTION_TYPE), + ArtifactOutput("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ), + "NODE_ARTIFACT_OUTPUTS", + ) + bound = _artifact(context, "frame_context", US_FRAME_CONTEXT_TYPE) + money = _artifact(context, "current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE) + receipt_value = _artifact( + context, "prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE + ) + producer_key = _same_producer( + context, + ("frame_context", "current_money", "prepared_receipt", "housing_universe"), + ) + document = _document(bound) + person, household = context.tables["person"], context.tables["household"] + for entity, table in (("person", person), ("household", household)): + _check_identity(document, entity, table) + _require( + document["weight_sources"] == {"household": "design"}, "WEIGHT_AUTHORITY" + ) + _require( + context.weights["household"].kind.value == "design", "WEIGHT_AUTHORITY" + ) + prepared = json.loads(receipt_value.payload) + _require( + canonical_json(prepared) == receipt_value.payload, "PREPARED_RECEIPT_JSON" + ) + _require( + prepared["source_kind"] == PREPARED_SOURCE_KIND + and prepared["release_eligible"] is False, + "PREPARED_RECEIPT_SCHEMA", + ) + _require( + prepared["entity_rows"]["person"] == document["entities"]["person"]["rows"] + and prepared["entity_rows"]["household"] + == document["entities"]["household"]["rows"], + "PREPARED_RECEIPT_ROWS", + ) + body = parse_current_money_body( + money.payload, + expected_header_sha256=prepared["money_header_sha256"], + expected_content_sha256=prepared["money_content_sha256"], + field_entities=tuple(tuple(item) for item in prepared["field_entities"]), + ) + _require( + body.person_rows == document["entities"]["person"]["rows"] + and body.household_rows == document["entities"]["household"]["rows"], + "BODY_ROW_ALIGNMENT", + ) + household_ids = household[US_SCHEMA.entity_id_column("household")].to_numpy( + dtype="int64" + ) + strata = _household_strata(person, household_ids) + universe_value = _artifact( + context, "housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE + ) + universe = bind_housing_universe( + universe_value.payload, prepared_receipt=prepared + ) + verify_graph_housing_rows( + household, + universe, + positions=np.arange(len(household), dtype="int64"), + income_years=strata.astype("int64"), + ) + view = _minimal_frame( + person, household, context.weights["household"], context.strata + ) + sampled, selection = sample_frame_households( + view, + fraction=context.params["fraction"], + seed=context.params["seed"], + source_name=SELECTION_SOURCE_LABEL, + unit_strata=strata, + unit_noun="household", + floor_context="the ASEC prepared current-money engineering sample", + ) + selected_household_ids = sampled.table("household")[ + US_SCHEMA.entity_id_column("household") + ].to_numpy(dtype="int64") + membership = person[US_SCHEMA.membership_column("household")].to_numpy( + dtype="int64" + ) + keep = np.isin(membership, selected_household_ids) + person_positions = np.flatnonzero(keep).astype("int64") + household_positions = np.flatnonzero( + np.isin(household_ids, selected_household_ids) + ).astype("int64") + _require(len(person_positions) > 0, "EMPTY_SELECTION") + _require( + np.array_equal(household_ids[household_positions], selected_household_ids), + "HOUSEHOLD_POSITIONS", + ) + # Whole households only: no selected household may leave a member behind. + _require( + np.array_equal( + person[_PERSON_ID].to_numpy(dtype="int64")[keep], + sampled.person[_PERSON_ID].to_numpy(dtype="int64"), + ), + "WHOLE_HOUSEHOLD", + ) + parent_weights = np.asarray( + context.weights["household"].values, dtype="float64" + ) + selected_weights = np.asarray( + sampled.weights_for("household").values, dtype="float64" + ) + _require( + selected_weights.tobytes() == parent_weights[household_positions].tobytes(), + "SELECTED_WEIGHT_BYTES", + ) + entities = {"person": person[_PERSON_ID].to_numpy(dtype="int64")[keep]} + for group in US_SCHEMA.group_entities: + entities[group] = _group_ids(person.loc[keep], group) + _require( + np.array_equal(entities["household"], selected_household_ids), + "SELECTED_HOUSEHOLD_INVENTORY", + ) + identities = { + entity: _identity(ids, entity) for entity, ids in entities.items() + } + before_mass = view.stratum_mass() + after_mass = sampled.stratum_mass() + payload = canonical_json( + { + "schema": SELECTION_SCHEMA, + "phase": ASEC_PREPARED_PHASE, + "release_eligible": False, + "representative": False, + "claim": "engineering_sample_only_no_district_or_calibration_claim", + "stratum_column": SELECTION_STRATUM_COLUMN, + "stratum_grain": "household_source_year", + "exact_count_rule": EXACT_COUNT_RULE, + "mass_normalization": "none", + "selection": _json(selection), + "entities": { + entity: { + "rows": identities[entity]["rows"], + "ordered_ids_sha256": identities[entity]["ordered_ids_sha256"], + } + for entity in _ENTITIES + }, + "person_positions_sha256": _sha(person_positions.tobytes()), + "household_positions_sha256": _sha(household_positions.tobytes()), + "parent": { + "producer_key": producer_key, + "frame_context_sha256": _sha(bound.payload), + "prepared_receipt_sha256": _sha(receipt_value.payload), + "money_header_sha256": prepared["money_header_sha256"], + "money_content_sha256": prepared["money_content_sha256"], + "housing_universe_payload_sha256": _sha(universe_value.payload), + }, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + }, + } + ) + selected = select_current_money( + body, + person_positions=person_positions, + household_positions=household_positions, + prepared_receipt_sha256=_sha(receipt_value.payload), + selection_sha256=_sha(payload), + person_identity_sha256=identities["person"]["ordered_ids_sha256"], + household_identity_sha256=identities["household"]["ordered_ids_sha256"], + ) + for entity in _ENTITIES: + document["entities"][entity].update(identities[entity]) + return KernelResult( + keep=pd.Series( + keep, + index=pd.Index(person[_PERSON_ID].to_numpy(), name=_PERSON_ID), + dtype=bool, + ), + artifacts={ + "frame_context": canonical_json(document), + "selection": payload, + "selected_current_money": encode_selected_current_money(selected), + }, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "selection_sha256": _sha(payload), + "selected_money_sha256": _sha(encode_selected_current_money(selected)), + "release_eligible": False, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + "stratum_before": { + str(key): float(value) for key, value in before_mass.items() + }, + "stratum_after": { + str(key): float(value) for key, value in after_mass.items() + }, + }, + }, + ) + + +def _json(value: object) -> object: + """Normalize a receipt for canonical JSON without inventing string casts.""" + return graph_context._json_data(value) + + +class USAsecPreparedLeavesKernel(_PreparedKernel): + """Derive the corrected monetary CPS leaves from the selected money alone.""" + + ref = "us.asec_prepared.cps_carried_current@3" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=ASEC_PREPARED_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + _phase(context) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(not node.sources, "NODE_SOURCES") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require( + node.inputs + == ( + Slice("person", CPS_CARRIED_CURRENT_ROUTING_COLUMNS), + Slice("household", HOUSEHOLD_EVIDENCE_COLUMNS), + ), + "NODE_SLICES", + ) + _require(node.outputs == owned_leaf_declarations(), "NODE_OUTPUTS") + _require( + set(context.artifacts) + == { + "frame_context", + "selection", + "selected_current_money", + "prepared_receipt", + "housing_universe", + }, + "NODE_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == (ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + "NODE_ARTIFACT_OUTPUTS", + ) + bound = _artifact(context, "frame_context", US_FRAME_CONTEXT_TYPE) + selection_value = _artifact(context, "selection", US_ASEC_SELECTION_TYPE) + money = _artifact( + context, "selected_current_money", US_ASEC_SELECTED_MONEY_TYPE + ) + receipt_value = _artifact( + context, "prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE + ) + _same_producer( + context, ("frame_context", "selection", "selected_current_money") + ) + universe_value = _artifact( + context, "housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE + ) + _same_producer(context, ("prepared_receipt", "housing_universe")) + document = _document(bound) + person, spm_unit = context.tables["person"], context.tables["spm_unit"] + for entity, table in (("person", person), ("spm_unit", spm_unit)): + _check_identity(document, entity, table) + selection = json.loads(selection_value.payload) + _require(canonical_json(selection) == selection_value.payload, "SELECTION_JSON") + _require(selection["schema"] == SELECTION_SCHEMA, "SELECTION_SCHEMA") + prepared = json.loads(receipt_value.payload) + _require( + selection["parent"]["prepared_receipt_sha256"] + == _sha(receipt_value.payload) + and selection["parent"]["producer_key"] == receipt_value.producer_key, + "SELECTION_PARENT_BINDING", + ) + _require( + selection["parent"]["housing_universe_payload_sha256"] + == _sha(universe_value.payload), + "SELECTION_HOUSING_UNIVERSE_BINDING", + ) + universe = bind_housing_universe( + universe_value.payload, prepared_receipt=prepared + ) + household = context.tables["household"] + _check_identity(document, "household", household) + positions = ( + pd.Index(universe.array("household_id")) + .get_indexer(household.household_id.to_numpy()) + .astype("int64") + ) + _require( + _sha(positions.tobytes()) == selection["household_positions_sha256"], + "SELECTED_HOUSING_UNIVERSE_POSITIONS", + ) + verify_graph_housing_rows(household, universe, positions=positions) + # The complete selected-coordinate check: the selection artifact, the + # producer's typed context and the actual rows must name one row set. + for entity in _ENTITIES: + declared = document["entities"][entity] + _require( + selection["entities"][entity]["rows"] == declared["rows"] + and selection["entities"][entity]["ordered_ids_sha256"] + == declared["ordered_ids_sha256"], + f"SELECTION_IDENTITY:{entity}", + ) + selected = decode_selected_current_money( + money.payload, + expected_parent_header_sha256=prepared["money_header_sha256"], + expected_parent_content_sha256=prepared["money_content_sha256"], + expected_prepared_receipt_sha256=_sha(receipt_value.payload), + expected_selection_sha256=_sha(selection_value.payload), + ) + header = selected.header_data + _require( + header["person_identity_sha256"] + == document["entities"]["person"]["ordered_ids_sha256"] + and header["household_identity_sha256"] + == document["entities"]["household"]["ordered_ids_sha256"], + "SELECTED_COORDINATES", + ) + _require( + selected.person_rows == len(person) + and selected.household_rows == document["entities"]["household"]["rows"], + "SELECTED_ROW_ALIGNMENT", + ) + incumbents = sorted( + f"{entity}.{column}" + for entity, column in _owned_coordinates() + if column in document["entities"][entity]["columns"] + ) + _require(not incumbents, "OWNED_LEAF_INCUMBENT") + routing = person.loc[:, list(CPS_CARRIED_CURRENT_ROUTING_COLUMNS)] + spm_ids = spm_unit[US_SCHEMA.entity_id_column("spm_unit")].to_numpy( + dtype="int64" + ) + leaves = derive_cps_carried_current_leaves( + selected, + routing=routing, + spm_membership=person[US_SCHEMA.membership_column("spm_unit")].to_numpy( + dtype="int64" + ), + spm_ids=spm_ids, + ) + person_index = pd.Index(person[_PERSON_ID].to_numpy(), name=_PERSON_ID) + spm_index = pd.Index(spm_ids, name=US_SCHEMA.entity_id_column("spm_unit")) + columns = { + ("person", name): pd.Series(values, index=person_index, dtype=LEAF_DTYPE) + for name, values in leaves.person.items() + } + columns.update( + { + ("spm_unit", name): pd.Series(values, index=spm_index, dtype=LEAF_DTYPE) + for name, values in leaves.spm_unit.items() + } + ) + for entity, column in _owned_coordinates(): + document["entities"][entity]["columns"].append(column) + return KernelResult( + columns=columns, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "contract": cps_carried_current_leaf_contract(), + "selected_money_sha256": _sha(money.payload), + "selection_sha256": _sha(selection_value.payload), + "person_rows": len(person), + "spm_unit_rows": len(spm_ids), + "release_eligible": False, + }, + ) + + +class USAsecPreparedEngineKernel(_PreparedKernel): + """Run the real PolicyEngine-US adapter for the admitted formula-owned roots.""" + + ref = "us.asec_prepared.engine_current_money@3" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=( + *ASEC_PREPARED_DEPENDENCIES, + "policyengine-us", + "policyengine-core", + ), + ) + + def implementation_hash(self) -> str: + # The executor checks this identity before cache lookup. Runtime or + # baseline parameter drift therefore refuses even when run() is unused. + return _sha( + canonical_json( + { + "implementation": super().implementation_hash(), + "runtime": engine_runtime_identity(), + } + ) + ) + + def __init__(self, engine=None) -> None: + # Only a real adapter instance is accepted; a substitute would make the + # evaluation artifact a claim about something other than the engine. + if engine is not None and type(engine) is not PolicyEngineUSEngine: + raise PreparedGraphError("ENGINE_ADAPTER_TYPE") + self._engine = engine + + def run(self, context: KernelContext) -> KernelResult: + _phase(context, ("period", "engine_version")) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(not node.sources, "NODE_SOURCES") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + # Engine outputs are formula-owned; this node owns no population cell. + _require(not node.outputs, "ENGINE_OWNS_NO_CELLS") + _require( + node.inputs + == ( + Slice("person", CPS_CARRIED_CURRENT_PERSON_LEAVES), + Slice("spm_unit", CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + Slice("household", (HOUSEHOLD_WEIGHT_CARRIER,)), + ), + "NODE_SLICES", + ) + _require(set(context.artifacts) == {"frame_context"}, "NODE_ARTIFACT_INPUTS") + _require( + node.artifact_outputs + == (ArtifactOutput("engine_evaluation", US_ASEC_ENGINE_EVALUATION_TYPE),), + "NODE_ARTIFACT_OUTPUTS", + ) + bound = _artifact(context, "frame_context", US_FRAME_CONTEXT_TYPE) + document = _document(bound) + for entity in ("person", "spm_unit", "household"): + _check_identity(document, entity, context.tables[entity]) + for entity, column in _owned_coordinates(): + _require( + column in document["entities"][entity]["columns"], + f"LEAF_NOT_IN_CONTEXT:{entity}.{column}", + ) + frame = self._engine_frame(context, document) + index = PolicyEngineUSVariableMetadataIndex() + contracts = admit_engine_outputs( + index, produced_leaves=CPS_CARRIED_CURRENT_PERSON_LEAVES + ) + _require( + all( + contract.engine_version == context.params["engine_version"] + for contract in contracts + ), + "DECLARED_ENGINE_VERSION", + ) + evaluation = materialize_engine_outputs( + frame, + PolicyEngineUSEngine() if self._engine is None else self._engine, + contracts, + period=context.params["period"], + context_bindings={ + "frame_context_sha256": _sha(bound.payload), + "frame_context_producer_key": bound.producer_key, + "produced_leaves": list(CPS_CARRIED_CURRENT_PERSON_LEAVES), + "spm_unit_leaves": list(CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + }, + ) + payload = encode_engine_evaluation(evaluation) + return KernelResult( + artifacts={"engine_evaluation": payload}, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "engine": { + "package": evaluation.header["engine_package"], + "version": evaluation.header["engine_version"], + "period": evaluation.header["period"], + "roots": list(ADMITTED_ROOTS), + "blocked_roots": list(BLOCKED_ENGINE_OUTPUTS), + "closures": evaluation.header["closures"], + "recorded_defaults": [], + "aggregates": evaluation.header["aggregates"], + }, + "evaluation_sha256": _sha(payload), + "cells_written": 0, + "release_eligible": False, + "claim": "engineering_evidence_only_no_tax_or_benefit_score", + }, + ) + + def _engine_frame(self, context: KernelContext, document: dict) -> Frame: + """Rebuild the selected frame carrying only leaves and id carriers. + + Every raw ASEC column, every routing code and the household carrier are + left behind: what the engine receives is exactly the corrected leaves + plus the structure they are indexed by. + """ + person = context.tables["person"] + keep = [ + _PERSON_ID, + *(US_SCHEMA.membership_column(group) for group in US_SCHEMA.group_entities), + *CPS_CARRIED_CURRENT_PERSON_LEAVES, + ] + _require(set(person.columns) == set(keep), "ENGINE_PERSON_COLUMNS") + tables = {"person": person.loc[:, keep].copy(deep=True)} + for group in US_SCHEMA.group_entities: + ids = _group_ids(person, group) + declared = document["entities"][group] + actual = _identity(ids, group) + _require( + all(declared[key] == value for key, value in actual.items()), + f"ENGINE_GROUP_IDENTITY:{group}", + ) + tables[group] = pd.DataFrame({US_SCHEMA.entity_id_column(group): ids}) + for group in ("household", "spm_unit"): + view = context.tables[group] + _require( + np.array_equal( + view[US_SCHEMA.entity_id_column(group)].to_numpy(dtype="int64"), + tables[group][US_SCHEMA.entity_id_column(group)].to_numpy(), + ), + f"ENGINE_GROUP_VIEW:{group}", + ) + spm = context.tables["spm_unit"] + for name in CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES: + tables["spm_unit"][name] = spm[name].to_numpy(dtype="float64", copy=True) + weights = context.weights["household"] + _require(weights.kind.value == "design", "ENGINE_WEIGHT_AUTHORITY") + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.asarray(weights.values, dtype="float64"), weights.kind + ) + }, + ) + + +def _owned_coordinates() -> tuple[tuple[str, str], ...]: + return ( + *(("person", name) for name in CPS_CARRIED_CURRENT_PERSON_LEAVES), + *(("spm_unit", name) for name in CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + ) + + +def owned_leaf_declarations() -> tuple[Owned, ...]: + """The cells the corrected-leaves node owns, in one canonical order.""" + return tuple( + Owned(entity, column, LEAF_DTYPE) for entity, column in _owned_coordinates() + ) + + +def us_asec_prepared_graph( + columns: Sequence[Owned], + *, + fraction: float, + seed: int, + period: int = 2024, + engine_version: str = "1.819.0", +) -> Graph: + """Declare the five-node slice over the caller's prepared column inventory.""" + inventory = {(owned.entity, owned.column) for owned in columns} + _require(len(inventory) == len(tuple(columns)), "COLUMN_INVENTORY_REPEATS") + _require( + ("household", HOUSEHOLD_WEIGHT_CARRIER) in inventory, "MISSING_WEIGHT_CARRIER" + ) + missing = sorted( + name + for name in (SELECTION_STRATUM_COLUMN, *CPS_CARRIED_CURRENT_ROUTING_COLUMNS) + if ("person", name) not in inventory + ) + _require(not missing, f"MISSING_DECLARED_INPUTS:{missing}") + _require( + all(("household", name) in inventory for name in HOUSEHOLD_EVIDENCE_COLUMNS), + "MISSING_HOUSING_UNIVERSE_INPUTS", + ) + # A nominal incumbent can be neither preserved nor silently rewritten. + incumbent = sorted( + f"{entity}.{column}" + for entity, column in _owned_coordinates() + if (entity, column) in inventory + ) + incumbent.extend( + f"person.{name}" for name in RESULT_COLUMNS if ("person", name) in inventory + ) + _require(not incumbent, f"OWNED_LEAF_INCUMBENT:{incumbent}") + parent_artifacts = ( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "current_money", + CREATE_NODE, + "current_money", + US_ASEC_CURRENT_MONEY_BODY_TYPE, + ), + ArtifactInput( + "prepared_receipt", + CREATE_NODE, + "prepared_receipt", + US_ASEC_PREPARED_RECEIPT_TYPE, + ), + ArtifactInput( + "housing_universe", + CREATE_NODE, + "housing_universe", + US_ASEC_HOUSING_UNIVERSE_TYPE, + ), + ) + create = Node( + id=CREATE_NODE, + kernel=USAsecPreparedCreateKernel.ref, + structural=StructuralDelta.CREATE, + sources=(ASEC_PREPARED_SOURCE_NAME,), + outputs=tuple(columns), + params={"phase": ASEC_PREPARED_PHASE}, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ArtifactOutput("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ArtifactOutput("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ArtifactOutput("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ), + ) + select = Node( + id=SELECT_NODE, + kernel=USAsecPreparedSelectionKernel.ref, + base=CREATE_NODE, + structural=StructuralDelta.FILTER, + mass="declared", + inputs=( + Slice("person", (SELECTION_STRATUM_COLUMN,)), + Slice("household", (HOUSEHOLD_WEIGHT_CARRIER, *HOUSEHOLD_EVIDENCE_COLUMNS)), + ), + params={"phase": ASEC_PREPARED_PHASE, "fraction": fraction, "seed": seed}, + artifact_inputs=parent_artifacts, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ASEC_SELECTION_TYPE), + ArtifactOutput("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ), + ) + leaves = Node( + id=LEAVES_NODE, + kernel=USAsecPreparedLeavesKernel.ref, + population=SELECT_NODE, + inputs=( + Slice("person", CPS_CARRIED_CURRENT_ROUTING_COLUMNS), + Slice("household", HOUSEHOLD_EVIDENCE_COLUMNS), + ), + outputs=owned_leaf_declarations(), + params={"phase": ASEC_PREPARED_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", SELECT_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "selection", SELECT_NODE, "selection", US_ASEC_SELECTION_TYPE + ), + ArtifactInput( + "selected_current_money", + SELECT_NODE, + "selected_current_money", + US_ASEC_SELECTED_MONEY_TYPE, + ), + parent_artifacts[2], + parent_artifacts[3], + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ) + engine = Node( + id=ENGINE_NODE, + kernel=USAsecPreparedEngineKernel.ref, + population=SELECT_NODE, + inputs=( + Slice("person", CPS_CARRIED_CURRENT_PERSON_LEAVES), + Slice("spm_unit", CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + Slice("household", (HOUSEHOLD_WEIGHT_CARRIER,)), + ), + params={ + "phase": ASEC_PREPARED_PHASE, + "period": period, + "engine_version": engine_version, + }, + artifact_inputs=( + ArtifactInput( + "frame_context", LEAVES_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ), + artifact_outputs=( + ArtifactOutput("engine_evaluation", US_ASEC_ENGINE_EVALUATION_TYPE), + ), + ) + reported = Node( + id=REPORTED_INCOME_NODE, + kernel=USAsecReportedIncomeKernel.ref, + population=SELECT_NODE, + inputs=(Slice("person", ("source_year",)),), + outputs=reported_income_declarations(), + params={"phase": ASEC_PREPARED_PHASE}, + artifact_inputs=( + *leaves.artifact_inputs[:3], + parent_artifacts[2], + ArtifactInput( + "income_observations", + CREATE_NODE, + "income_observations", + US_ASEC_INCOME_OBSERVATIONS_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + ) + return Graph( + country="us", + sources=(ASEC_PREPARED_SOURCE,), + nodes=(create, select, leaves, engine, reported), + ) + + +def us_asec_prepared_registry(engine=None) -> KernelRegistry: + """Register exactly this slice's five kernels.""" + registry = KernelRegistry() + registry.register(USAsecPreparedCreateKernel()) + registry.register(USAsecPreparedSelectionKernel()) + registry.register(USAsecPreparedLeavesKernel()) + registry.register(USAsecPreparedEngineKernel(engine)) + registry.register(USAsecReportedIncomeKernel()) + return registry + + +__all__ = [ + "ASEC_PREPARED_CODEC", + "ASEC_PREPARED_DEPENDENCIES", + "ASEC_PREPARED_PHASE", + "ASEC_PREPARED_SOURCE", + "ASEC_PREPARED_SOURCE_NAME", + "ASEC_PREPARED_STAGE", + "CREATE_NODE", + "ENGINE_NODE", + "HOUSEHOLD_WEIGHT_CARRIER", + "LEAVES_NODE", + "PreparedGraphError", + "SELECTION_SCHEMA", + "SELECT_NODE", + "USAsecPreparedCreateKernel", + "USAsecPreparedEngineKernel", + "USAsecPreparedLeavesKernel", + "USAsecPreparedSelectionKernel", + "owned_leaf_declarations", + "us_asec_prepared_graph", + "us_asec_prepared_registry", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py new file mode 100644 index 000000000..f65c71236 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py @@ -0,0 +1,56 @@ +"""Assign shared geography after the complete combined-survey support clone. + +This declaration has no country-specific geography kernel or source admission. +Callers provide normalized, qualified observed columns and declared support. +""" + +from microcosm.build.atomic_geography import validate_assignment_spec +from microcosm.build.graph_atomic_geography import atomic_geography_nodes + +from .graph_combined_clone import us_combined_survey_clone_nodes + +ASSIGNMENT_IDENTITY = ( + "survey_geography_origin_key", + "household_support_clone_index", +) + + +def atomic_survey_clone_nodes( + definition, + columns, + *, + base, + geography_prefix="geography", + clone_prefix="combined_survey_puf_support_clone", +): + """Complete initial clones, then draw on each stable source/role identity. + + The base carries qualified observed constraints, not assigned geography. + The clone-index Slice depends on its ownership claim; the population edge + depends on EXPAND. Numeric remapped household IDs are only row coordinates. + """ + columns = tuple(columns) + definition = validate_assignment_spec(definition) + if tuple(definition.get("identity", ())) != ASSIGNMENT_IDENTITY: + raise ValueError( + "US postclone geography requires its stable source/clone identity." + ) + clones = us_combined_survey_clone_nodes( + columns, + base=base, + source_channels=("acs", "asec"), + prefix=clone_prefix, + ) + inventory = {(o.entity, o.column): o for o in columns} + inventory.update({(o.entity, o.column): o for o in clones[1].outputs}) + origin = inventory.get(("household", ASSIGNMENT_IDENTITY[0])) + if origin is None or origin.dtype != "string": + raise ValueError("US postclone geography requires a string source identity.") + geography = atomic_geography_nodes( + definition, + tuple(inventory.values()), + base=clones[0].id, + prefix=geography_prefix, + emit_validation_artifact=True, + ) + return (*clones, *geography) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py new file mode 100644 index 000000000..358ead101 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py @@ -0,0 +1,1383 @@ +"""Checked atomic survey geography plus current financial development output. + +The base nineteen nodes retain the raw allocation and the pre-financial clone +separately. An explicit property-income option adds sixteen nodes and retains +the complete legacy financial population alongside its extended output. Support +bytes establish integrity, not publisher provenance or release eligibility. +""" + +from __future__ import annotations + +import json +import sys +import weakref +from dataclasses import dataclass, replace +from types import FunctionType, SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.fit import qrf_target +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.graph import ( + ArtifactInput, + ArtifactValue, + KernelResult, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph import population as population_ops +from microcosm.graph.artifact_edges import numeric_scope, typed_contracts +from microcosm.graph.executor import ( + _all_node_keys, + _apply_result, + _project_context, + _source_paths_and_keys, +) +from microcosm.graph.keys import _capabilities_projection, opaque_artifact_key +from microcosm.graph.population import Population +from microcosm.graph.serialize import graph_to_json + +from . import graph_atomic_survey_population as atomic +from . import graph_current_survey_predictors as financial + +values = financial.values +codec = financial.codec +survey = atomic.survey +reconstruction = atomic.reconstruction +require = values.require +RUN_PROTOCOL = "microcosm.us.atomic-survey-financial-run.v1" +_ISSUED_RUNS = {} + + +@dataclass(frozen=True) +class AtomicSurveyFinancialRunValues: + """Development output, retaining the independently admitted prefix values.""" + + prefix: atomic.AtomicSurveyPopulationRunValues + financial_population: Population + manifest: object + compiled: object + store: object + kernels: object + sources: dict + projection: bytes + matrix: bytes + + def checked_view(self): + """Recheck a completed actual run; public dataclass copies are unissued.""" + return check_atomic_survey_financial_run(self) + + +@dataclass(frozen=True) +class CheckedAtomicSurveyFinancialRun: + """Descriptive values; authority remains in the actual issued run handle.""" + + payload: bytes + digest: str + population: Population + + +@dataclass(frozen=True) +class _FinancialRunState: + prefix: object + prefix_objects: tuple + preparation_entry: tuple + financial_population: Population + populations: tuple + manifest: object + manifest_bytes: bytes + prefix_manifest_bytes: bytes + compiled: object + declaration: str + prefix_declaration: str + store: object + kernels: object + source_items: tuple + config_bytes: bytes + projection: bytes + matrix: bytes + pins: bytes + n_estimators: int + demographic_conditioning: bool + source_keys: tuple + keys: tuple + implementations: tuple + artifact_hashes: tuple + manifest_populations: tuple + prefix_manifest_populations: tuple + live: dict + property_income: object = None + property_income_bytes: bytes | None = None + legacy_financial_population: Population | None = None + legacy_financial_stamp: object = None + rebase_property_taxes: bool = False + property_population: Population | None = None + property_population_stamp: object = None + tax_verification: bytes | None = None + + +def _property_module(): + """Load the explicitly selected extension without changing the default path.""" + from . import graph_current_survey_property + + return graph_current_survey_property + + +def _tax_module(): + from . import graph_property_tax_leaves + + return graph_property_tax_leaves + + +def _tax_nodes(frame, population, options, *, anticipated_outputs=()): + property_graph = _property_module() + reconciliation = options.nodes((property_graph.PROPERTY_REPORTED_TOTAL,))[-1] + output = reconciliation.artifact_outputs[0] + return _tax_module().property_tax_leaf_nodes( + frame, + population=population, + projection=ArtifactInput( + "projection", + property_graph.PROJECTION_NODE, + "projection", + property_graph.PROJECTION_TYPE, + ), + reconciliation=ArtifactInput( + "reconciliation", + reconciliation.id, + output.name, + output.type, + ), + atol=float(options.atol), + rtol=float(options.rtol), + anticipated_outputs=anticipated_outputs, + ) + + +def _reconstruct_tax(population, *, compiled, kernels, keys, loaded, options): + """Replay three deterministic operations on the actual complete post35 Frame.""" + tax = _tax_module() + nodes = _tax_nodes(population.frame, population.version, options) + classes = ( + tax.PropertyTaxReceivingKernel, + tax.PropertyTaxLeavesKernel, + tax.PropertyTaxLeafGateKernel, + ) + populations, results = {}, {} + for node, cls in zip(nodes, classes, strict=True): + require( + node.normative() == compiled.graph.node(node.id).normative(), + "PROPERTY_TAX_DECLARATION", + ) + artifacts = { + edge.name: ArtifactValue( + loaded[edge.producer, edge.artifact], + edge.type, + opaque_artifact_key(keys[edge.producer], edge.artifact), + keys[edge.producer], + numeric_scope( + kernels.get(compiled.graph.node(edge.producer).kernel).capabilities + ), + ) + for edge in node.artifact_inputs + } + context = _project_context( + node, + population, + key=keys[node.id], + sources={}, + tolerances={}, + numerics={}, + artifacts=artifacts, + ) + result = cls().run(context) + require( + all( + loaded[node.id, name] == payload + for name, payload in result.artifacts.items() + ), + "PROPERTY_TAX_ARTIFACT", + ) + population = _apply_result( + node, + result, + population, + mass_partition=compiled.graph.mass_partition, + ) + populations[node.id], results[node.id] = population, result + return populations, results + + +def financial_tax_gate_edges(run): + """Declare the real numerical gate only for an issued opt-in run.""" + state = _run_entry(run)[2] + if not state.rebase_property_taxes: + return () + tax = _tax_module() + return ( + ArtifactInput( + "property_tax_verification", + tax.GATE_NODE, + "verification", + tax.VERIFICATION_TYPE, + ), + ) + + +def require_complete_property_taxes(run): + """Require the already checked numerical gate before PUF source qualification.""" + entry = _run_entry(run) + _pure_run(run, entry) + state = entry[2] + if state.rebase_property_taxes: + document = codec.decode_json(state.tax_verification) + require( + document["numeric_verified"] is True and document["complete"] is True, + "PROPERTY_TAX_INPUTS_INCOMPLETE", + ) + + +def _manifest_population_seals(manifest, compiled): + """Include transient attached Frames, which portable JSON deliberately omits.""" + return tuple( + ( + version, + reconstruction._population_stamp( + Population.from_frame( + manifest.population(version), + version, + mass_ledger=manifest.mass_ledger(version), + ) + ), + ) + for version in sorted(set(compiled.versions.values())) + ) + + +def _run_entry(run): + entry = _ISSUED_RUNS.get(id(run)) + require( + type(run) is AtomicSurveyFinancialRunValues + and entry is not None + and entry[0]() is run, + "UNISSUED_FINANCIAL_RUN", + ) + return entry + + +def financial_output_node(run): + """Name the final writer of an already issued financial run.""" + state = _run_entry(run)[2] + if state.rebase_property_taxes: + return _tax_module().GATE_NODE + return ( + financial.ATTACH_NODE + if state.property_income is None + else _property_module().ATTACH_NODE + ) + + +def _run_document(run, state): + """Portable ancestry omits timing/cache hits and private physical seals.""" + return codec.encode_json( + { + "protocol": RUN_PROTOCOL, + "graph_sha256": codec.sha(state.declaration.encode()), + "manifest_key": state.manifest.key, + "preparation_sha256": codec.sha(state.preparation_entry[1]), + "geography_config_sha256": codec.sha(state.config_bytes), + "projection_sha256": codec.sha(state.projection), + "matrix_sha256": codec.sha(state.matrix), + "host_edges": codec.decode_json(state.pins), + "node_keys": dict(state.keys), + "artifact_payload_sha256": [list(row) for row in state.artifact_hashes], + "financial_frame_sha256": values.source._frame_identity( + run.financial_population.frame + ), + "financial_version": run.financial_population.version, + "financial_owners": sorted( + (e, c, writer) + for (e, c), writer in run.financial_population.owners.items() + ), + "owned_columns": [ + *values.OUTPUTS, + *( + () + if state.property_income is None + else (owned.column for owned in _property_module().owned_columns()) + ), + ], + "demographic_conditioning": state.demographic_conditioning, + "n_estimators": state.n_estimators, + "release_eligible": False, + **( + {} + if state.property_income is None + else { + "property_income": codec.decode_json(state.property_income_bytes), + "property_node_count": 16, + "tax_split_rebased": state.rebase_property_taxes, + "capital_gains_conditioning": _property_module().CAP_LIMITATION, + "legacy_financial_frame_sha256": values.source._frame_identity( + state.legacy_financial_population.frame + ), + } + ), + **( + {} + if not state.rebase_property_taxes + else { + "tax_rebase_node_count": 3, + "tax_rebased_columns": _tax_module().TAX_LEAF_COLUMNS, + "tax_verification_sha256": codec.sha(state.tax_verification), + "property_frame_sha256": values.source._frame_identity( + state.property_population.frame + ), + "tax_leaf_complete": codec.decode_json(state.tax_verification)[ + "complete" + ], + } + ), + } + ) + + +def _pure_run(run, entry): + """Check retained graph/source/Population state after all external I/O.""" + require(_run_entry(run) is entry, "FINAL_FINANCIAL_RUN_ISSUANCE") + state = entry[2] + prefix = state.prefix + require( + run.prefix is prefix + and run.financial_population is state.financial_population + and run.manifest is state.manifest + and run.compiled is state.compiled + and run.store is state.store + and run.kernels is state.kernels + and run.projection == state.projection + and run.matrix == state.matrix + and tuple(sorted(run.sources.items())) == state.source_items + and tuple(sorted(prefix.sources.items())) == state.source_items + and all( + a is b + for a, b in zip( + ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + prefix.geography_config, + ), + state.prefix_objects, + strict=True, + ) + ) + and values.host.survey_budget._config_payload(prefix.geography_config) + == state.config_bytes + and run.manifest.to_json_bytes() == state.manifest_bytes + and prefix.manifest.to_json_bytes() == state.prefix_manifest_bytes + and graph_to_json(run.compiled.graph) == state.declaration + and graph_to_json(prefix.compiled.graph) == state.prefix_declaration + and run.compiled == compile_graph(run.compiled.graph) + and prefix.compiled == compile_graph(prefix.compiled.graph), + "FINANCIAL_RUN_BINDINGS_CHANGED", + ) + require( + _manifest_population_seals(run.manifest, run.compiled) + == state.manifest_populations + and _manifest_population_seals(prefix.manifest, prefix.compiled) + == state.prefix_manifest_populations, + "FINANCIAL_RUN_ATTACHED_POPULATION_CHANGED", + ) + require( + values.source._ISSUED.get(id(prefix.preparation)) is state.preparation_entry + and prefix.preparation.payload == state.preparation_entry[1], + "FINANCIAL_RUN_SOURCE_CHANGED", + ) + values.source._pure_final(state.preparation_entry[2]) + if state.property_income is not None: + require( + type(state.property_income) is _property_module().PropertyIncomeOptions + and state.property_income.to_bytes() == state.property_income_bytes + and reconstruction._population_stamp(state.legacy_financial_population) + == state.legacy_financial_stamp, + "PROPERTY_OPTIONS_OR_LEGACY_POPULATION_CHANGED", + ) + if state.rebase_property_taxes: + require( + state.property_income is not None + and reconstruction._population_stamp(state.property_population) + == state.property_population_stamp, + "PROPERTY_TAX_PARENT_POPULATION_CHANGED", + ) + for name, population, stamp in state.populations: + actual = ( + run.financial_population if name == "financial" else getattr(prefix, name) + ) + require( + actual is population and reconstruction._population_stamp(actual) == stamp, + "FINANCIAL_RUN_POPULATION_CHANGED", + ) + require( + _run_document(run, state) == entry[1] + and _live(state.property_income, state.rebase_property_taxes) == state.live, + "FINAL_FINANCIAL_RUN_SEAL", + ) + require(_run_entry(run) is entry, "FINAL_FINANCIAL_RUN_ISSUANCE") + + +def check_atomic_survey_financial_run(run): + """Requalify existing artifacts and live owners without fitting or execution.""" + entry = _run_entry(run) + _pure_run(run, entry) + state, prefix = entry[2], entry[2].prefix + _, source_keys = _source_paths_and_keys(run.compiled, run.sources, run.store) + keys, implementations = _all_node_keys(run.compiled, run.kernels, source_keys) + require( + tuple(sorted(source_keys.items())) == state.source_keys + and tuple(sorted(keys.items())) == state.keys + and tuple(sorted(implementations.items())) == state.implementations, + "FINANCIAL_RUN_IMPLEMENTATIONS_CHANGED", + ) + loaded = _artifacts( + run.manifest, run.compiled, run.store, run.kernels, keys, implementations + ) + require( + tuple( + sorted( + (node, name, codec.sha(payload)) + for (node, name), payload in loaded.items() + ) + ) + == state.artifact_hashes, + "FINANCIAL_RUN_ARTIFACT_CHANGED", + ) + if state.rebase_property_taxes: + tax_populations, _ = _reconstruct_tax( + state.property_population, + compiled=run.compiled, + kernels=run.kernels, + keys=keys, + loaded=loaded, + options=state.property_income, + ) + atomic.same_replayed_population( + tax_populations[_tax_module().GATE_NODE], + run.financial_population, + ) + # These exact fitted artifacts were checked at actual execution/required + # replay before issuance. Rechecking their identities needs no new pickle + # decode or fit; the materialized verifier freshly derives current values. + financial.verify_materialized_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + population=( + run.financial_population + if state.property_income is None + else state.legacy_financial_population + ), + projection=state.projection, + matrix=state.matrix, + matrix_producer_key=keys[financial.PROJECTION_NODE], + raw_draws=tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ), + apply_states=tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ), + host_pins=codec.decode_json(state.pins), + n_estimators=state.n_estimators, + demographic_conditioning=state.demographic_conditioning, + geography_config=prefix.geography_config, + ) + if state.property_income is not None: + _property_module().verify_materialized_property_income( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + legacy_population=state.legacy_financial_population, + population=( + state.property_population + if state.rebase_property_taxes + else run.financial_population + ), + host_pins=codec.decode_json(state.pins), + options=state.property_income, + artifacts=loaded, + legacy_matrix_producer_key=keys[financial.PROJECTION_NODE], + demographic_conditioning=state.demographic_conditioning, + geography_config=prefix.geography_config, + ) + result = CheckedAtomicSurveyFinancialRun( + entry[1], codec.sha(entry[1]), run.financial_population + ) + _pure_run(run, entry) + return result + + +def _issue_run( + result, + *, + preparation_entry, + pins, + n_estimators, + demographic_conditioning, + source_keys, + keys, + implementations, + loaded, + live, + property_income=None, + legacy_financial_population=None, + rebase_property_taxes=False, + property_population=None, +): + """Called only after this runner's complete materialization/replay checks.""" + prefix = result.prefix + populations = tuple( + (name, population, reconstruction._population_stamp(population)) + for name, population in ( + *( + (name, getattr(prefix, name)) + for name in ( + "allocated_population", + "observed_population", + "expanded_population", + "geography_population", + "clone_population", + ) + ), + ("financial", result.financial_population), + ) + ) + state = _FinancialRunState( + prefix, + ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + prefix.geography_config, + ), + preparation_entry, + result.financial_population, + populations, + result.manifest, + result.manifest.to_json_bytes(), + prefix.manifest.to_json_bytes(), + result.compiled, + graph_to_json(result.compiled.graph), + graph_to_json(prefix.compiled.graph), + result.store, + result.kernels, + tuple(sorted(result.sources.items())), + prefix.geography_config.to_bytes(), + result.projection, + result.matrix, + codec.encode_json(pins), + n_estimators, + demographic_conditioning, + tuple(sorted(source_keys.items())), + tuple(sorted(keys.items())), + tuple(sorted(implementations.items())), + tuple( + sorted( + (node, name, codec.sha(payload)) + for (node, name), payload in loaded.items() + ) + ), + _manifest_population_seals(result.manifest, result.compiled), + _manifest_population_seals(prefix.manifest, prefix.compiled), + live, + property_income, + None if property_income is None else property_income.to_bytes(), + legacy_financial_population, + None + if legacy_financial_population is None + else reconstruction._population_stamp(legacy_financial_population), + rebase_property_taxes, + property_population, + None + if property_population is None + else reconstruction._population_stamp(property_population), + None + if not rebase_property_taxes + else loaded[_tax_module().GATE_NODE, "verification"], + ) + identifier = id(result) + + def forget(reference): + entry = _ISSUED_RUNS.get(identifier) + if entry is not None and entry[0] is reference: + _ISSUED_RUNS.pop(identifier, None) + + reference = weakref.ref(result, forget) + _ISSUED_RUNS[identifier] = (reference, _run_document(result, state), state) + _pure_run(result, _run_entry(result)) + + +def _live(property_income=None, rebase_property_taxes=False): + """Pure final fence over this composition and its existing owner closure.""" + result = dict(values.host.survey_budget._live()) + modules = ( + sys.modules[__name__], + atomic, + financial, + values, + codec, + qrf_target, + financial.qrf, + financial.model_input, + sys.modules[LegacyQRFTrainKernel.__module__], + sys.modules[LegacyQRFApplyMatrixKernel.__module__], + ) + if property_income is not None: + from . import current_asec_property_basis, graph_property_income_receipts + + extension = _property_module() + modules = ( + *modules, + extension, + extension.completion, + extension.completion.provenance, + extension.sources, + current_asec_property_basis, + extension.sources.acs, + extension.sources.interest, + extension.sources.routing, + extension.sources.dividend, + extension.model, + extension.signed_graph, + graph_property_income_receipts, + sys.modules[extension.LegacyQRFApplyKernel.__module__], + ) + result["property_contract"] = values.source._runtime_marker( + ( + extension.PROTOCOL, + extension.completion.PROTOCOL, + extension.completion.PROPERTY_COMPLETION_TYPE, + extension.completion._COMPONENT_FIELDS, + extension.completion._ASEC_STATUSES, + extension.completion._ACS_STATUSES, + extension.completion._LITERAL_STATUSES, + extension.completion._PUBLIC_ROUTES, + extension.completion._PUBLIC_REASONS, + extension.completion._PUBLIC_MEASURES, + extension.PREFIX, + extension.CAP_LIMITATION, + extension.LEGACY_DIFFERENCE, + extension.BASIS_DIAGNOSTICS, + extension.PROPERTY_COMPONENTS, + extension.PROPERTY_DRAW_COLUMNS, + extension.PROPERTY_REPORTED_TOTAL, + extension.model.PROTOCOL, + extension.sources.PROTOCOL, + current_asec_property_basis.OTHER_PROPERTY_CATEGORIES, + current_asec_property_basis.OTHER_UNSPECIFIED_CATEGORY, + ) + ) + if rebase_property_taxes: + tax = _tax_module() + modules = (*modules, tax, tax.cps_carried) + result["property_tax_contract"] = values.source._runtime_marker( + ( + tax.PROTOCOL, + tax.RECEIVING_NODE, + tax.TAX_LEAVES_NODE, + tax.GATE_NODE, + tax.TAX_LEAF_COLUMNS, + tax.DIAGNOSTICS_TYPE, + tax.VERIFICATION_TYPE, + tax.PROPERTY_COMPONENTS, + tax._INPUTS, + tax.cps_carried.TAXABLE_INTEREST_FRACTION, + tax.cps_carried.QUALIFIED_DIVIDEND_FRACTION, + ) + ) + for module in modules: + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[module.__name__, name] = values.source._function_seal(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + result[module.__name__, name] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[module.__name__, name, method] = ( + values.source._function_seal(function) + ) + result["financial_contract"] = values.source._runtime_marker( + ( + RUN_PROTOCOL, + values.FEATURES, + values.DEMOGRAPHIC_FEATURES, + values.TARGETS, + values.OUTPUTS, + values.PROTOCOL, + values.PHASE, + values.SEED, + ) + ) + return result + + +def _artifacts(manifest, compiled, store, kernels, keys, implementations): + """Read only the declared outputs after binding their actual producer keys.""" + require(set(manifest.nodes) == set(compiled.order), "ATOMIC_NODE_ROSTER") + loaded = {} + for node_id in compiled.order: + node, record = compiled.graph.node(node_id), manifest.node(node_id) + kernel = kernels.get(node.kernel) + require( + record.key == keys[node_id] + and record.kernel_ref == node.kernel + and record.kernel_impl_hash == implementations[node_id] + and _capabilities_projection(record.capabilities) + == _capabilities_projection(kernel.capabilities) + and record.typed_artifacts + == typed_contracts(compiled, node, keys, kernels), + "ATOMIC_ARTIFACT_PRODUCER", + ) + require( + set(record.opaque_artifacts) == {o.name for o in node.artifact_outputs}, + "ATOMIC_ARTIFACT_ROSTER", + ) + for output in node.artifact_outputs: + payload = store.load_bytes(record.opaque_artifacts[output.name]) + survey._final_artifact( + manifest, + store, + node_id=node_id, + name=output.name, + type_=output.type, + payload=payload, + capabilities=kernel.capabilities, + ) + loaded[node_id, output.name] = payload + return loaded + + +def _model_receipts(nodes, donor, qualified, loaded, matrix_key): + """Bind fitted checkpoints to the exact source-derived design-weight donor. + + The start-chain operation only resolves inputs and initializes RNG state; + this verifier does not fit a second model. Pickles are decoded only after + the caller has checked the actual store and typed graph producer closure. + """ + fits = tuple(n for n in nodes if n.kernel == LegacyQRFTrainKernel.ref) + first = fits[0] + model_frame = codec.model_frame( + SimpleNamespace(weights={"person": donor.frame.resolve_weights("person")}), + first.inputs[0], + donor.frame.person, + ) + predictors = values.feature_columns(qualified.demographic_conditioning) + model = financial.qrf.RegimeGatedQRF( + seed=values.SEED, + n_estimators=first.params["n_estimators"], + zero_atol=0, + max_samples_leaf=None, + ) + before = qrf_target.LegacyQRFTrainingState.from_chain( + model.start_chain( + model_frame, list(predictors), list(values.TARGETS), weights="design" + ) + ) + receipts, history = {}, [] + for i, fit in enumerate(fits): + target = values.TARGETS[i] + payload = loaded[fit.id, "model"] + packet, after = codec.read_training(loaded[fit.id, "training_state"]) + require( + len(packet["models"]) == i + 1 + and packet["models"][:i] == history + and packet["models"][-1]["sha256"] == codec.sha(payload), + "ATOMIC_TRAINING_HISTORY", + ) + fitted = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=packet["models"][-1]["sha256"] + ) + require( + fitted.training_state == before + and fitted.next_training_state == after + and fitted.donor_sha256 + == qrf_target._consumed_values_sha256( + model_frame.person, (*predictors, *values.TARGETS[:i], target) + ), + "ATOMIC_TRAINING_DONOR", + ) + history = [ + *history, + { + "target": target, + "sha256": codec.sha(payload), + "training_id": fitted.training_id, + }, + ] + require(packet["models"] == history, "ATOMIC_TRAINING_HISTORY") + before = after + receipts[fit.id] = { + "phase": values.PHASE, + "target": target, + "training_id": fitted.training_id, + "model_sha256": codec.sha(payload), + "donor_rows": len(model_frame.person), + "entity": "person", + "weight_kind": "design", + "regime": fitted.regime, + } + apply_id = f"{financial.APPLY_PREFIX}.{i:03d}" + application = financial.decode_matrix_apply_state( + loaded[apply_id, "apply_state"] + ) + require( + application["application"]["models"] == history, + "ATOMIC_APPLY_MODEL_HISTORY", + ) + receipts[apply_id] = { + "phase": values.PHASE, + "target": target, + "recipient_rows": len( + financial.model_input.decode_recipient_matrix(qualified.matrix).features + ), + "entity": "person", + "model_sha256": codec.sha(payload), + "raw_sha256": codec.sha(loaded[apply_id, "raw_draw"]), + "regime": fitted.regime, + "matrix_sha256": codec.sha(qualified.matrix), + "matrix_producer_key": matrix_key, + } + return receipts + + +def run_atomic_survey_financial( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed, + geography_config, + demographic_conditioning=False, + n_estimators=100, + property_income=None, + rebase_property_taxes=False, + resume="auto", + return_values=False, +): + """Verify the base financial graph and its explicitly selected extension.""" + require(type(return_values) is bool, "RETURN_VALUES_FLAG") + require(type(rebase_property_taxes) is bool, "PROPERTY_TAX_FLAG") + require( + not rebase_property_taxes or property_income is not None, + "PROPERTY_TAX_REQUIRES_PROPERTY", + ) + values.feature_columns(demographic_conditioning) + property_graph = None if property_income is None else _property_module() + if property_graph is not None: + require( + type(property_income) is property_graph.PropertyIncomeOptions, + "PROPERTY_OPTIONS_TYPE", + ) + property_income_bytes = property_income.to_bytes() + else: + property_income_bytes = None + live = _live(property_income, rebase_property_taxes) + config_bytes = values.host.survey_budget._config_payload(geography_config) + require(config_bytes is not None, "ATOMIC_GEOGRAPHY_REQUIRED") + prefix = atomic.run_atomic_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed, + geography_config=geography_config, + resume=resume, + return_values=True, + ) + entry = prefix.preparation._checked() + prefix_objects = ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + ) + prefix_manifest_bytes = prefix.manifest.to_json_bytes() + prefix_declaration = graph_to_json(prefix.compiled.graph) + retained = { + name: ( + getattr(prefix, name), + reconstruction._population_stamp(getattr(prefix, name)), + ) + for name in ( + "allocated_population", + "observed_population", + "expanded_population", + "geography_population", + "clone_population", + ) + } + qualified = values.qualify_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + projection_bytes, matrix_bytes = qualified.projection, qualified.matrix + property_qualified = ( + None + if property_graph is None + else property_graph.sources.qualify_current_property_income_sources( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + ) + geography = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + pins, prefix_artifacts = {}, {} + for node_id, record in prefix.manifest.nodes.items(): + for name, key in record.opaque_artifacts.items(): + prefix_artifacts[node_id, name] = prefix.store.load_bytes(key) + for edge in ( + *financial.host.current_survey_host_edges(), + financial._geography_edge(), + ): + record = prefix.manifest.node(edge.producer) + pins[edge.name] = { + "producer_key": record.key, + "artifact_key": record.opaque_artifacts[edge.artifact], + "payload_sha256": codec.sha(prefix_artifacts[edge.producer, edge.artifact]), + } + nodes = financial.current_survey_predictor_nodes( + qualified, + prefix.clone_population.frame, + host_pins=pins, + n_estimators=n_estimators, + ) + property_nodes = ( + () + if property_graph is None + else property_graph.current_survey_property_nodes( + property_qualified, + prefix.clone_population.frame, + host_pins=pins, + options=property_income, + ) + ) + tax_nodes = ( + () + if not rebase_property_taxes + else _tax_nodes( + prefix.clone_population.frame, + prefix.clone_population.version, + property_income, + anticipated_outputs=( + *next(n.outputs for n in nodes if n.id == financial.ATTACH_NODE), + *property_graph.owned_columns(), + ), + ) + ) + compiled = compile_graph( + replace( + prefix.compiled.graph, + nodes=(*prefix.compiled.graph.nodes, *nodes, *property_nodes, *tax_nodes), + ) + ) + require( + len(prefix.compiled.order) == 9 + and len(property_nodes) == (0 if property_graph is None else 16) + and len(tax_nodes) == (3 if rebase_property_taxes else 0) + and len(compiled.order) == 19 + len(property_nodes) + len(tax_nodes), + "ATOMIC_COMPILER_ROSTER", + ) + gate_edge = financial._geography_edge() + require( + gate_edge.producer in compiled.predecessors[financial.DONOR_NODE] + and prefix_artifacts[gate_edge.producer, gate_edge.artifact] + == qualified.geography_validation, + "ATOMIC_GEOGRAPHY_GATE_EDGE", + ) + declaration = graph_to_json(compiled.graph) + kernels, store, sources = prefix.kernels, prefix.store, dict(prefix.sources) + source_items = tuple(sorted(sources.items())) + for cls in ( + financial.CurrentSurveyPredictorProjectionKernel, + financial.CurrentSurveyPredictorDonorFilterKernel, + financial.CurrentSurveyPredictorDonorColumnsKernel, + financial.CurrentSurveyPredictorAttachKernel, + ): + kernels.register( + cls( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + host_pins=pins, + n_estimators=n_estimators, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + ) + kernels.register(LegacyQRFTrainKernel()) + kernels.register(LegacyQRFApplyMatrixKernel()) + if property_graph is not None: + property_graph.register_property_kernels( + kernels, + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + host_pins=pins, + options=property_income, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + if rebase_property_taxes: + tax = _tax_module() + for cls in ( + tax.PropertyTaxReceivingKernel, + tax.PropertyTaxLeavesKernel, + tax.PropertyTaxLeafGateKernel, + ): + kernels.register(cls()) + _, source_keys = _source_paths_and_keys(compiled, sources, store) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + base_expected = { + survey.CREATE_NODE: Population.from_frame(entry[2].frame, survey.CREATE_NODE), + survey.ALLOCATION_NODE: prefix.allocated_population, + **{s.node.id: s.population for s in geography.stages}, + } + receipts = { + **{n: r.receipt for n, r in prefix.manifest.nodes.items()}, + **{s.node.id: json.loads(s.receipt) for s in geography.stages}, + } + donor_node = compiled.graph.node(financial.DONOR_NODE) + donor = population_ops.patch( + base_expected[survey.CREATE_NODE], + donor_node, + KernelResult(frame=qualified.donor_frame), + ) + receipts[donor_node.id] = { + "selection": "ASEC_native_whole_households", + "fit_weight_kind": "design", + "release_eligible": False, + } + columns_node = compiled.graph.node(financial.DONOR_COLUMNS_NODE) + donor_columns = population_ops.patch( + donor, + columns_node, + KernelResult( + columns={ + ("person", c): qualified.donor_columns[c] + for c in ( + *values.feature_columns(demographic_conditioning), + *values.TARGETS, + ) + } + ), + ) + receipts[financial.PROJECTION_NODE] = qualified.evidence + receipts[columns_node.id] = qualified.evidence + observed, observed_stamps = {}, {} + + def observe(node_id, population): + require(node_id not in observed, "ATOMIC_OBSERVER_DUPLICATE") + observed[node_id] = population + observed_stamps[node_id] = reconstruction._population_stamp(population) + + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + require(tuple(observed) == compiled.order, "ATOMIC_OBSERVER_ROSTER") + loaded = _artifacts(manifest, compiled, store, kernels, keys, implementations) + require( + all(loaded[k] == v for k, v in prefix_artifacts.items()), + "ATOMIC_PREFIX_ARTIFACTS", + ) + require( + loaded[financial.PROJECTION_NODE, "projection"] == projection_bytes + and loaded[financial.PROJECTION_NODE, "matrix"] == matrix_bytes, + "ATOMIC_SOURCE_ARTIFACTS", + ) + raw = tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ) + applications = tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ) + matrix_key = keys[financial.PROJECTION_NODE] + drawn = financial.read_current_survey_draws( + matrix_bytes, + matrix_key, + raw, + applications, + demographic_conditioning=demographic_conditioning, + ) + receipts.update( + _model_receipts(nodes, donor_columns, qualified, loaded, matrix_key) + ) + receipts[financial.ATTACH_NODE] = { + **qualified.evidence, + "raw_sha256": [codec.sha(r) for r in raw], + "all_output_cells_available": True, + "ACS_financial_origin": "modeled", + "paired_draws": "source_origin_join", + "host_weights_changed": False, + } + property_results = ( + {} + if property_graph is None + else property_graph.reconstruct_property_results( + property_qualified, + prefix.clone_population.frame, + host_pins=pins, + options=property_income, + artifacts=loaded, + legacy_matrix_producer_key=matrix_key, + ) + ) + receipts.update({name: result.receipt for name, result in property_results.items()}) + expected, current, tax_populations = {}, {}, {} + for node_id in compiled.order: + node, version = compiled.graph.node(node_id), compiled.versions[node_id] + if rebase_property_taxes and node_id == _tax_module().RECEIVING_NODE: + tax_populations, tax_results = _reconstruct_tax( + current[node.base], + compiled=compiled, + kernels=kernels, + keys=keys, + loaded=loaded, + options=property_income, + ) + receipts.update( + {name: result.receipt for name, result in tax_results.items()} + ) + population = tax_populations[node_id] + elif node_id in tax_populations: + population = tax_populations[node_id] + elif node_id in property_results: + incumbent = ( + version if node.structural is StructuralDelta.NONE else node.base + ) + property_result = property_results[node_id] + if node.structural is StructuralDelta.FILTER: + frame = current[incumbent].frame + entity = frame.schema.person_entity + id_column = frame.schema.entity_id_column(entity) + ids = pd.Index(frame.table(entity)[id_column], name=id_column) + mask = property_result.keep.reindex(ids).to_numpy( + dtype=np.bool_, copy=True + ) + property_result = replace( + property_result, frame=frame.select(mask), keep=None + ) + population = population_ops.patch(current[incumbent], node, property_result) + elif node_id == donor_node.id: + population = donor + elif node_id == columns_node.id: + population = donor_columns + elif node_id == financial.ATTACH_NODE: + population = population_ops.patch( + current[version], + node, + KernelResult( + columns=values.complete_predictor_columns( + qualified, prefix.clone_population.frame, drawn + ) + ), + ) + elif node_id in base_expected: + population = base_expected[node_id] + else: + population = current[version] + expected[node_id] = current[version] = population + atomic.same_replayed_population(population, observed[node_id]) + if property_graph is not None: + from .graph_property_income_receipts import verify_property_model_receipts + + receipts.update( + verify_property_model_receipts( + property_nodes, + expected[property_graph.DONOR_COLUMNS_NODE], + expected[property_graph.RECIPIENT_COLUMNS_NODE], + loaded, + ) + ) + expected_stamps = { + n: reconstruction._population_stamp(p) for n, p in expected.items() + } + states = atomic._states(compiled, kernels, source_keys, expected, receipts) + survey._check_node_states(manifest, states) + final_node = ( + financial.ATTACH_NODE if property_graph is None else property_graph.ATTACH_NODE + ) + property_population = observed[final_node] if rebase_property_taxes else None + if rebase_property_taxes: + final_node = _tax_module().GATE_NODE + legacy_population = observed[financial.ATTACH_NODE] + result = AtomicSurveyFinancialRunValues( + prefix, + observed[final_node], + manifest, + compiled, + store, + kernels, + sources, + projection_bytes, + matrix_bytes, + ) + # Implementation/source/store I/O precedes the last owner/support borrow. + current_keys, current_implementations = _all_node_keys( + compiled, kernels, source_keys + ) + require( + current_keys == keys and current_implementations == implementations, + "ATOMIC_FINAL_IMPLEMENTATIONS", + ) + financial.verify_materialized_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + population=legacy_population, + projection=projection_bytes, + matrix=matrix_bytes, + matrix_producer_key=matrix_key, + raw_draws=raw, + apply_states=applications, + host_pins=pins, + n_estimators=n_estimators, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + if property_graph is not None: + property_graph.verify_materialized_property_income( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + legacy_population=legacy_population, + population=( + property_population + if rebase_property_taxes + else result.financial_population + ), + host_pins=pins, + options=property_income, + artifacts=loaded, + legacy_matrix_producer_key=matrix_key, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + require( + property_income.to_bytes() == property_income_bytes, + "PROPERTY_OPTIONS_CHANGED", + ) + values.source._pure_final(entry[2]) + require( + values.source._ISSUED.get(id(prefix.preparation)) is entry + and prefix.preparation.payload == entry[1] + and prefix.geography_config is geography_config + and values.host.survey_budget._config_payload(geography_config) == config_bytes + and result.prefix is prefix + and result.manifest is manifest + and result.compiled is compiled + and result.store is store + and result.kernels is kernels + and result.financial_population is observed[final_node] + and result.projection == projection_bytes + and result.matrix == matrix_bytes + and all( + a is b + for a, b in zip( + ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + ), + prefix_objects, + strict=True, + ) + ) + and prefix.manifest.to_json_bytes() == prefix_manifest_bytes + and graph_to_json(prefix.compiled.graph) == prefix_declaration + and prefix.compiled == compile_graph(prefix.compiled.graph) + and tuple(sorted(result.sources.items())) == source_items + and tuple(sorted(prefix.sources.items())) == source_items + and graph_to_json(compiled.graph) == declaration + and compiled == compile_graph(compiled.graph) + and _live(property_income, rebase_property_taxes) == live, + "ATOMIC_FINAL_BINDINGS", + ) + for name, (population, stamp) in retained.items(): + require( + getattr(prefix, name) is population + and reconstruction._population_stamp(population) == stamp, + "ATOMIC_FINAL_PREFIX_MUTATION", + ) + for version, population in ( + (survey.CREATE_NODE, base_expected[survey.CREATE_NODE]), + (survey.ALLOCATION_NODE, prefix.observed_population), + (atomic.clone.COMBINED_CLONE_NODE, prefix.clone_population), + ): + survey._same_frame(population.frame, prefix.manifest.population(version)) + require( + population.mass_ledger == prefix.manifest.mass_ledger(version), + "ATOMIC_FINAL_PREFIX_LEDGER", + ) + for node_id, population in expected.items(): + require( + reconstruction._population_stamp(population) == expected_stamps[node_id] + and reconstruction._population_stamp(observed[node_id]) + == observed_stamps[node_id], + "ATOMIC_FINAL_POPULATION_MUTATION", + ) + atomic.same_replayed_population(population, observed[node_id]) + for version, population in current.items(): + survey._same_frame(population.frame, manifest.population(version)) + require( + population.mass_ledger == manifest.mass_ledger(version), + "ATOMIC_FINAL_LEDGER", + ) + survey._check_node_states(manifest, states) + _issue_run( + result, + preparation_entry=entry, + pins=pins, + n_estimators=n_estimators, + demographic_conditioning=demographic_conditioning, + source_keys=source_keys, + keys=keys, + implementations=implementations, + loaded=loaded, + live=live, + property_income=property_income, + legacy_financial_population=None + if property_graph is None + else legacy_population, + rebase_property_taxes=rebase_property_taxes, + property_population=property_population, + ) + return result if return_values else manifest diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py new file mode 100644 index 000000000..5bd732768 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py @@ -0,0 +1,334 @@ +"""Complete initial support clones before assigning qualified survey geography. + +The existing runner independently admits the raw allocation. This extension +reconstructs every added column from the live preparation and pinned normalized +support, then checks the complete graph populations on cold and warm execution. +Normalized support integrity does not establish its publisher provenance. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from types import SimpleNamespace + +from microcosm.build.graph_atomic_geography import ( + ATOMIC_SUPPORT_TYPE as ATOMIC_SUPPORT_TYPE, +) +from microcosm.build.graph_atomic_geography import ( + AtomicSupportImportKernel as AtomicSupportImportKernel, +) +from microcosm.build.graph_atomic_geography import ( + register_atomic_geography_kernels, +) +from microcosm.graph import ( + Graph, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.artifact_edges import typed_contracts +from microcosm.graph.codecs import load_raw_bytes +from microcosm.graph.executor import ( + _all_node_keys, + _input_writers, + _source_paths_and_keys, + _tolerance_writer_payload, +) +from microcosm.graph.keys import ( + _capabilities_projection, + artifact_key, + frame_key, + opaque_artifact_key, + weights_key, +) +from microcosm.graph.keys import seed as node_seed +from microcosm.graph.manifest import _freeze_json +from microcosm.graph.population import ( + Population, + mass_record_receipt, + weight_cap_receipt, +) + +from . import graph_combined_clone as clone +from . import graph_current_survey_geography as projection +from . import graph_survey_population as survey +from . import survey_atomic_geography as reconstruction +from .survey_population_replay import same_replayed_population + + +@dataclass(frozen=True) +class AtomicSurveyPopulationRunValues: + """Actual run values, without a new source or release certificate.""" + + manifest: object + preparation: object + allocated_population: Population + observed_population: Population + expanded_population: Population + geography_population: Population + clone_population: Population + geography_config: reconstruction.AtomicSurveyReconstruction + compiled: object + store: object + kernels: object + sources: dict + + +def _states(compiled, kernels, source_keys, expected_populations, raw_receipts): + """Bind independent domain receipts to current implementation and graph keys.""" + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + states, writer_receipts = {}, {} + for node_id in compiled.order: + node = compiled.graph.node(node_id) + population = expected_populations[node_id] + key = keys[node_id] + structural = node.structural is not StructuralDelta.NONE + cells = ( + tuple( + (e, str(c)) + for e in population.frame.entities + for c in population.frame.table(e) + ) + if structural + else tuple((o.entity, o.column) for o in node.outputs) + ) + weight_entity = ( + node.weights.entity + if node.weights is not None + else node.params.get("expand_weight_entity") + if node.structural is StructuralDelta.EXPAND + else None + ) + capabilities = _capabilities_projection(kernels.get(node.kernel).capabilities) + receipt = dict(raw_receipts[node_id]) + receipt["capabilities"] = dict(capabilities) + writers = _tolerance_writer_payload( + _input_writers(compiled, node_id, receipts=writer_receipts) + ) + if writers: + receipt["capabilities"]["tolerance_writers"] = writers + if structural and node.structural is not StructuralDelta.CREATE: + receipt["mass"] = { + **receipt.get("mass", {}), + **mass_record_receipt(population.mass_ledger[-1]), + } + receipt.update(weight_cap_receipt(population, node)) + states[node_id] = { + "key": key, + "ref": node.kernel, + "capabilities": capabilities, + "implementation": implementations[node_id], + "typed_artifacts": typed_contracts(compiled, node, keys, kernels), + "seed": node_seed(key), + "frame_key": frame_key(key) if structural else None, + "weight_key": weights_key(key, weight_entity) if weight_entity else None, + "artifacts": {(e, c): artifact_key(key, e, c) for e, c in cells}, + "opaque_artifacts": { + o.name: opaque_artifact_key(key, o.name) for o in node.artifact_outputs + }, + "receipt": _freeze_json(receipt), + } + writer_receipts[node_id] = SimpleNamespace(receipt=states[node_id]["receipt"]) + return states + + +def run_atomic_survey_population( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed, + geography_config, + resume="auto", + return_values=False, +): + """Assign each completed source/clone household and independently verify replay.""" + survey._require(type(return_values) is bool, "RETURN_VALUES_FLAG") + prefix = survey.run_authenticated_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed, + resume=resume, + clones=False, + return_values=True, + ) + source_owner = survey._source_owner() + preparation_entry = prefix.preparation._checked() + geography = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + _, view = survey._checked_preparation(prefix.preparation) + instructions = survey.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _, allocated_context, allocation_payload, _, _ = survey._allocation_output( + view.frame, view.context, instructions, survey._sha(view.payload) + ) + prefix_artifacts = { + (survey.CREATE_NODE, "preparation"): view.payload, + (survey.CREATE_NODE, "frame_context"): view.context, + (survey.ALLOCATION_NODE, "allocation"): allocation_payload, + (survey.ALLOCATION_NODE, "frame_context"): allocated_context, + } + additions = geography.nodes + compiled = compile_graph( + Graph( + "us", + ( + *prefix.compiled.graph.sources, + *(SourceRef(name, "raw-bytes-v1") for name, _ in geography.sources), + ), + (*prefix.compiled.graph.nodes, *additions), + ) + ) + survey._require(len(compiled.order) == 9, "ATOMIC_COMPILER_ROSTER") + sources = {**prefix.sources, **dict(geography.sources)} + kernels, store = prefix.kernels, prefix.store + store.codecs.register_bytes("raw-bytes-v1", load_raw_bytes) + kernels.register(projection.CurrentSurveyGeographyKernel(prefix.preparation)) + register_atomic_geography_kernels(kernels) + clone.register_us_combined_survey_clone_kernels(kernels) + expected = { + survey.CREATE_NODE: Population.from_frame( + prefix.manifest.population(survey.CREATE_NODE), survey.CREATE_NODE + ), + survey.ALLOCATION_NODE: prefix.allocated_population, + **{stage.node.id: stage.population for stage in geography.stages}, + } + receipts = { + **{name: record.receipt for name, record in prefix.manifest.nodes.items()}, + **{stage.node.id: json.loads(stage.receipt) for stage in geography.stages}, + } + expected_stamps = { + name: reconstruction._population_stamp(value) + for name, value in expected.items() + } + raw_stamp = reconstruction._population_stamp(prefix.allocated_population) + source_items = tuple(sorted(sources.items())) + config_bytes = geography_config.to_bytes() + _, source_keys = _source_paths_and_keys(compiled, sources, store) + states = _states(compiled, kernels, source_keys, expected, receipts) + observed, observed_stamps = {}, {} + + def observe(node_id, population): + survey._require(node_id not in observed, "DUPLICATE_POPULATION_OBSERVATION") + same_replayed_population(expected[node_id], population) + observed[node_id] = population + observed_stamps[node_id] = reconstruction._population_stamp(population) + + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + survey._require(tuple(observed) == compiled.order, "POPULATION_OBSERVER_COVERAGE") + survey._check_node_states(manifest, states) + for stage in geography.stages: + for name, payload in stage.artifacts: + output = next(o for o in stage.node.artifact_outputs if o.name == name) + survey._final_artifact( + manifest, + store, + node_id=stage.node.id, + name=name, + type_=output.type, + payload=payload, + capabilities=kernels.get(stage.node.kernel).capabilities, + ) + # Validate prefix artifacts separately from column materialization. + for (node_id, name), payload in prefix_artifacts.items(): + key = prefix.manifest.node(node_id).opaque_artifacts[name] + survey._require( + manifest.node(node_id).opaque_artifacts.get(name) == key, + "ATOMIC_PREFIX_ARTIFACT_KEY", + ) + survey._require( + store.load_bytes(key) == payload, "ATOMIC_PREFIX_ARTIFACT_BYTES" + ) + geography_terminal = geography.nodes[-1].id + result = AtomicSurveyPopulationRunValues( + manifest=manifest, + preparation=prefix.preparation, + allocated_population=prefix.allocated_population, + observed_population=observed[projection.NODE], + expanded_population=observed[clone.COMBINED_CLONE_CLAIM_NODE], + geography_population=observed[geography_terminal], + # This historical field is the terminal receiving population. The + # explicit expanded_population is the preassignment clone snapshot. + clone_population=observed[geography_terminal], + geography_config=geography_config, + compiled=compiled, + store=store, + kernels=kernels, + sources=sources, + ) + current_keys, current_implementations = _all_node_keys( + compiled, kernels, source_keys + ) + survey._require( + current_keys == {name: value["key"] for name, value in states.items()} + and current_implementations + == {name: value["implementation"] for name, value in states.items()}, + "ATOMIC_FINAL_IMPLEMENTATIONS", + ) + # Finish source/support I/O before final comparisons of returned objects. + fresh = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + survey._require( + source_owner._ISSUED.get(id(prefix.preparation)) is preparation_entry + and prefix.preparation.payload == preparation_entry[1], + "ATOMIC_FINAL_PREPARATION", + ) + source_owner._pure_final(preparation_entry[2]) + survey._require( + fresh.config_sha256 == geography.config_sha256 + and fresh.projection_receipt == geography.projection_receipt + and fresh.support_payload == geography.support_payload + and fresh.definition == geography.definition + and geography_config.to_bytes() == config_bytes + and result.geography_config is geography_config + and result.preparation is prefix.preparation + and result.allocated_population is prefix.allocated_population + and result.observed_population is observed[projection.NODE] + and result.expanded_population is observed[clone.COMBINED_CLONE_CLAIM_NODE] + and result.geography_population is observed[geography_terminal] + and result.clone_population is result.geography_population + and result.manifest is manifest + and result.compiled is compiled + and result.store is store + and result.kernels is kernels + and tuple(sorted(result.sources.items())) == source_items + and reconstruction._population_stamp(prefix.allocated_population) == raw_stamp, + "ATOMIC_FINAL_RECONSTRUCTION", + ) + same_replayed_population(fresh.observed_population, result.observed_population) + same_replayed_population(fresh.expanded_population, result.expanded_population) + same_replayed_population(fresh.population, result.geography_population) + for node_id, population in expected.items(): + survey._require( + reconstruction._population_stamp(population) == expected_stamps[node_id] + and reconstruction._population_stamp(observed[node_id]) + == observed_stamps[node_id], + "ATOMIC_FINAL_POPULATION_MUTATION", + ) + same_replayed_population(population, observed[node_id]) + latest = {} + for node_id in compiled.order: + latest[compiled.versions[node_id]] = node_id + for version, node_id in latest.items(): + survey._same_frame(expected[node_id].frame, manifest.population(version)) + survey._require( + manifest.mass_ledger(version) == expected[node_id].mass_ledger, + "ATOMIC_FINAL_MASS_LEDGER", + ) + survey._check_node_states(manifest, states) + return result if return_values else manifest diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py new file mode 100644 index 000000000..c9b1276fe --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py @@ -0,0 +1,830 @@ +"""The combined-survey PUF-support clone, as a real graph ``EXPAND`` stage. + +What this stage is +------------------ + +It takes an already combined ASEC + ACS population — every entity native, every +``*_support_clone_index`` zero, household weights already IMPORTANCE — and +returns the two-role host the PUF detail pass needs: the untouched native role +plus exactly one full-attachment detail clone of every household, with each +incoming household weight split evenly across the pair. + +It is only that. No PUF donor is read, no quantile forest is fitted, no draw is +taken, no amount is placed, no tax or benefit value is computed, and no prior +wage stage runs. The donor sampling weight ``S006`` never touches a host weight: +this stage never sees a donor at all. The profile is development; nothing here +is release eligible. + +Why it is an ``EXPAND`` and not a Frame handed to the executor +------------------------------------------------------------- + +:func:`~.puf_support.clone_us_frame_for_puf_support` already performs this +operation directly, and it stays the behavioral authority: the kernel calls it +and validates its output with +:func:`~.puf_support.validate_puf_clone_attachment`, exactly as the direct path +does. What the kernel returns to the executor, though, is not that Frame. It +returns what :class:`~microcosm.graph.KernelResult` declares for a structural +node — per-entity clone lineage, the declared cell overlays, and explicit +weights — and ``microcosm.graph.population._patch_expand`` does the structural +work: it carries every column from each copied row, remaps every copied +person's five memberships onto the copied groups, appends the strata, installs +the weights and records the mass ledger. A Frame smuggled through as an opaque +artifact would make the executor's lineage, membership, storage, weight and +mass checks vacuous, so this stage does not do that. + +The ID relationship between the two paths +----------------------------------------- + +The executor's contract is that every base row survives with its own id and new +rows get new ids. The direct operator honours the same rule on this input: for a +preassembled frame it copies the native block unchanged and shifts only the +detail block by a decimal multiplier. The two paths therefore agree on raw ids +here — but the stage does not depend on that. Lineage is derived from the +operator's own output through the ``(entity_source_id, clone_index)`` bijection, +with the positional pairing checked independently, and the receipt publishes a +digest of that bijection. A comparison against the direct operator is a +comparison by declared source and role, not an assumption about integers. + +What the node key binds +----------------------- + +The kernel's implementation hash covers this module, the clone operator's module +and the provenance owner's module, plus the pinned ``numpy``/``pandas`` +versions, so an edit to the operator moves every cached identity. The node's own +key additionally binds its declared parameters, the artifact keys of the +provenance columns it reads and — through ``population_input['base']`` — the +whole base population version: its amounts, its ids, its row order, its +provenance and its incoming weights. Changing any of those changes this node's +key, so warm reuse can never serve a clone of a different population. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping, Sequence + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import token_for_dtype + +from . import puf_support as _puf_support_module +from . import support_provenance as _support_provenance_module +from .puf_support import ( + clone_us_frame_for_puf_support, + validate_puf_clone_attachment, +) +from .support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + PUF_TAX_DETAIL_CLONE_INDEX, + PUF_TAX_DETAIL_SUPPORT_CHANNEL, + spine_assembly_manifest, + spine_provenance_counts, + spine_source_id_column, + support_channel_column, + support_clone_index_column, + support_source_id_column, +) + +__all__ = [ + "COMBINED_CLONE_ATTACHMENT_FRACTION", + "COMBINED_CLONE_ATTACHMENT_SEED", + "COMBINED_CLONE_CELL_DTYPE", + "COMBINED_CLONE_CLAIM_NODE", + "COMBINED_CLONE_NODE", + "COMBINED_CLONE_PHASE", + "COMBINED_CLONE_PREFIX", + "COMBINED_CLONE_PROFILE", + "COMBINED_CLONE_ROLES", + "COMBINED_CLONE_WEIGHT_ENTITY", + "COMBINED_CLONE_WEIGHT_KIND", + "COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR", + "CombinedCloneError", + "USCombinedSurveyCloneClaimKernel", + "USCombinedSurveyCloneExpandKernel", + "combined_clone_provenance_columns", + "register_us_combined_survey_clone_kernels", + "us_combined_survey_clone_nodes", +] + + +class CombinedCloneError(ValueError): + """A combined-survey clone declaration or input violates the contract.""" + + +#: Descriptive phase label carried in the node params and every receipt. +COMBINED_CLONE_PHASE = "us_combined_survey_puf_support_clone" +COMBINED_CLONE_PREFIX = "combined_survey_puf_support_clone" +COMBINED_CLONE_NODE = COMBINED_CLONE_PREFIX +COMBINED_CLONE_CLAIM_NODE = f"{COMBINED_CLONE_PREFIX}.owned" + +#: This stage is an engineering intermediate, not a candidate for release. +COMBINED_CLONE_PROFILE = "development" + +#: The two operator roles the clone writes: the incoming native role keeps +#: clone index 0, the single detail copy takes ``PUF_TAX_DETAIL_CLONE_INDEX``. +#: These are operator roles, never source channels: the ASEC and ACS source +#: channels are carried unchanged into *both* roles. +COMBINED_CLONE_ROLES = (BASE_ASEC_SUPPORT_CHANNEL, PUF_TAX_DETAIL_SUPPORT_CHANNEL) + +#: Full one-copy attachment only. A seeded partial arm is a separate, explicitly +#: declared control and is deliberately not reachable from this stage. +COMBINED_CLONE_ATTACHMENT_FRACTION = 1.0 +COMBINED_CLONE_ATTACHMENT_SEED = 0 + +#: Each incoming household weight is split across exactly two roles. +COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR = 2 +COMBINED_CLONE_WEIGHT_ENTITY = "household" +COMBINED_CLONE_WEIGHT_KIND = WeightKind.IMPORTANCE + +#: The clone-index columns are the only cells this stage writes. +COMBINED_CLONE_CELL_DTYPE = "int64" + +_PARAM_KEYS = ( + "clone_attachment_fraction", + "clone_attachment_seed", + "clone_roles", + "expand_cells", + "expand_weight_entity", + "expand_weight_kind", + "phase", + "profile", + "release_eligible", + "source_channels", + "weight_split_denominator", +) + +_LINEAGE_DIGEST_DOMAIN = b"microcosm.us.combined-survey-clone-lineage.v1\0" + + +def combined_clone_provenance_columns(entity: str) -> tuple[str, ...]: + """The four provenance columns this stage reads on ``entity``. + + Order is fixed so a node declaration is a stable, reviewable literal. + """ + + return ( + support_source_id_column(entity), + spine_source_id_column(entity), + support_channel_column(entity), + support_clone_index_column(entity), + ) + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise CombinedCloneError(f"{COMBINED_CLONE_PHASE}: {reason}") + + +def _expected_expand_cells() -> tuple[tuple[str, str, str], ...]: + return tuple( + (entity, support_clone_index_column(entity), COMBINED_CLONE_CELL_DTYPE) + for entity in US_SCHEMA.entities + ) + + +def _validated_source_channels(value: object) -> tuple[str, ...]: + _require(isinstance(value, tuple), "SOURCE_CHANNELS_TYPE") + channels = tuple(value) # type: ignore[arg-type] + _require( + len(channels) >= 2 + and len(set(channels)) == len(channels) + and all(isinstance(name, str) and bool(name) for name in channels), + "SOURCE_CHANNELS_VALUE", + ) + _require(tuple(sorted(channels)) == channels, "SOURCE_CHANNELS_ORDER") + return channels + + +def _digest(payload: object) -> str: + return hashlib.sha256(_LINEAGE_DIGEST_DOMAIN + canonical_json(payload)).hexdigest() + + +def _int64_ids(table: pd.DataFrame, column: str, *, reason: str) -> np.ndarray: + values = table[column] + _require(str(values.dtype) == "int64", reason) + return values.to_numpy(dtype=np.int64, copy=True) + + +def _entity_lineage( + before: Frame, + after: Frame, + entity: str, +) -> tuple[pd.Series, dict[str, object]]: + """Derive one entity's new-id -> copied-id lineage and its receipt facts. + + The pairing is by ``(entity_source_id, clone_index)``: the detail row whose + assembly-unique source id is *s* copies the native row whose source id is + *s*. Nothing here reads or reconstructs an ID-offset rule. The positional + pairing the operator happens to produce is checked against that bijection + rather than trusted in its place. + """ + + id_column = US_SCHEMA.entity_id_column(entity) + source_column = support_source_id_column(entity) + clone_column = support_clone_index_column(entity) + + before_table = before.table(entity) + after_table = after.table(entity) + before_ids = _int64_ids(before_table, id_column, reason=f"BASE_ID_DTYPE:{entity}") + after_ids = _int64_ids(after_table, id_column, reason=f"CLONE_ID_DTYPE:{entity}") + clone_index = after_table[clone_column].to_numpy(dtype=np.int64, copy=True) + source_ids = _int64_ids( + after_table, source_column, reason=f"CLONE_SOURCE_ID_DTYPE:{entity}" + ) + + unexpected = sorted( + set(int(value) for value in np.unique(clone_index)) + - {0, PUF_TAX_DETAIL_CLONE_INDEX} + ) + _require(not unexpected, f"CLONE_ROLE_UNEXPECTED:{entity}:{unexpected}") + + native = clone_index == 0 + detail = clone_index == PUF_TAX_DETAIL_CLONE_INDEX + _require(int(native.sum()) == len(before_ids), f"NATIVE_ROW_COUNT:{entity}") + _require(int(detail.sum()) == len(before_ids), f"DETAIL_ROW_COUNT:{entity}") + _require( + np.array_equal(after_ids[native], before_ids), f"NATIVE_IDS_PRESERVED:{entity}" + ) + + native_sources = source_ids[native] + detail_sources = source_ids[detail] + _require( + len(np.unique(native_sources)) == len(native_sources), + f"NATIVE_SOURCE_IDS_UNIQUE:{entity}", + ) + _require( + len(np.unique(detail_sources)) == len(detail_sources), + f"DETAIL_SOURCE_IDS_UNIQUE:{entity}", + ) + native_by_source = pd.Index(native_sources) + positions = native_by_source.get_indexer(detail_sources) + _require(bool((positions >= 0).all()), f"DETAIL_SOURCE_ID_UNMATCHED:{entity}") + + target_ids = after_ids[detail] + copied_ids = before_ids[positions] + _require( + not len(np.intersect1d(target_ids, before_ids)), + f"TARGET_ID_COLLIDES_WITH_BASE:{entity}", + ) + _require( + len(np.unique(target_ids)) == len(target_ids), f"TARGET_IDS_UNIQUE:{entity}" + ) + # Independent cross-check: the operator emits the detail block row-aligned + # to the native block, so position i of the detail block must resolve to + # position i of the native block. A disagreement means the bijection and + # the physical layout describe different pairings; refuse rather than pick. + _require( + np.array_equal(positions, np.arange(len(positions), dtype=np.int64)), + f"LINEAGE_POSITION_DISAGREES:{entity}", + ) + + lineage = pd.Series( + copied_ids, + index=pd.Index(target_ids, name=id_column, dtype=before_table[id_column].dtype), + dtype=before_table[id_column].dtype, + name=id_column, + ) + facts = { + "base_rows": int(len(before_ids)), + "expanded_rows": int(len(after_ids)), + "native_rows": int(native.sum()), + "detail_rows": int(detail.sum()), + "lineage_sha256": _digest( + [ + [int(source), int(PUF_TAX_DETAIL_CLONE_INDEX), int(target)] + for source, target in zip(detail_sources, target_ids, strict=True) + ] + ), + "native_source_ids_sha256": _digest( + [int(value) for value in native_sources.tolist()] + ), + } + return lineage, facts + + +def _channel_mass(frame: Frame, channels: Sequence[str]) -> dict[str, float]: + """Weighted household mass per source channel, for the receipt only.""" + + table = frame.table(COMBINED_CLONE_WEIGHT_ENTITY) + values = np.asarray( + frame.weights_for(COMBINED_CLONE_WEIGHT_ENTITY).values, dtype=np.float64 + ) + labels = table[support_channel_column(COMBINED_CLONE_WEIGHT_ENTITY)].astype(str) + return { + channel: float(values[labels.eq(channel).to_numpy()].sum()) + for channel in channels + } + + +class USCombinedSurveyCloneExpandKernel(KernelBase): + """Run the registered clone operator and hand the executor its lineage.""" + + ref = "us.combined_survey.puf_support_clone.expand@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.EXPAND, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self) -> str: + # The behavior of this node is this module plus the operator it calls + # and the provenance owner whose columns define both roles. Hashing + # only this file would let an operator edit reuse a stale clone. + return source_hash( + type(self), + _puf_support_module, + _support_provenance_module, + dependencies=self.capabilities.dependencies, + ) + + # ------------------------------------------------------------------ + + def _validated_params(self, context: KernelContext) -> tuple[str, ...]: + params = context.params + _require(set(params) == set(_PARAM_KEYS), f"NODE_PARAMS:{sorted(params)}") + _require(params["phase"] == COMBINED_CLONE_PHASE, "NODE_PHASE") + _require(params["profile"] == COMBINED_CLONE_PROFILE, "NODE_PROFILE") + _require(params["release_eligible"] is False, "NODE_RELEASE_ELIGIBLE") + _require( + params["clone_attachment_fraction"] == COMBINED_CLONE_ATTACHMENT_FRACTION, + "NODE_ATTACHMENT_FRACTION", + ) + _require( + params["clone_attachment_seed"] == COMBINED_CLONE_ATTACHMENT_SEED, + "NODE_ATTACHMENT_SEED", + ) + _require(tuple(params["clone_roles"]) == COMBINED_CLONE_ROLES, "NODE_ROLES") + _require( + params["weight_split_denominator"] + == COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR, + "NODE_WEIGHT_SPLIT", + ) + _require( + params["expand_weight_entity"] == COMBINED_CLONE_WEIGHT_ENTITY, + "NODE_WEIGHT_ENTITY", + ) + _require( + params["expand_weight_kind"] == COMBINED_CLONE_WEIGHT_KIND.value, + "NODE_WEIGHT_KIND", + ) + _require( + tuple(params["expand_cells"]) == _expected_expand_cells(), + "NODE_EXPAND_CELLS", + ) + return _validated_source_channels(params["source_channels"]) + + def _validated_declaration(self, context: KernelContext) -> None: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(node.structural is StructuralDelta.EXPAND, "NODE_STRUCTURAL") + _require(node.entrants is False, "NODE_ENTRANTS") + _require(node.mass == "conserve", "NODE_MASS") + _require(not node.sources, "NODE_SOURCES") + _require(not node.artifact_inputs, "NODE_ARTIFACT_INPUTS") + _require(not node.artifact_outputs, "NODE_ARTIFACT_OUTPUTS") + declared = { + (slice_.entity, column) + for slice_ in node.inputs + for column in slice_.columns + } + expected = { + (entity, column) + for entity in US_SCHEMA.entities + for column in combined_clone_provenance_columns(entity) + } + _require(declared == expected, f"NODE_INPUTS:{sorted(declared ^ expected)}") + _require( + all(slice_.rows == "all" for slice_ in node.inputs), "NODE_INPUT_ROW_MASK" + ) + + def _minimal_frame( + self, context: KernelContext, channels: tuple[str, ...] + ) -> Frame: + """Rebuild the combined host from the declared slices only. + + The frame carries the four provenance columns, the structural id and + membership columns the executor always projects, the household weights + and the strata — nothing else. Every amount the population holds is + carried by the executor from the copied rows, so the operator does not + need to see one, and this kernel deliberately cannot read one. + """ + + missing = [ + entity for entity in US_SCHEMA.entities if entity not in context.tables + ] + _require(not missing, f"MISSING_SLICES:{missing}") + tables = { + entity: context.tables[entity].copy(deep=True) + for entity in US_SCHEMA.entities + } + _require( + COMBINED_CLONE_WEIGHT_ENTITY in context.weights, "MISSING_HOUSEHOLD_WEIGHTS" + ) + household_weights = context.weights[COMBINED_CLONE_WEIGHT_ENTITY] + _require( + household_weights.kind is COMBINED_CLONE_WEIGHT_KIND, + f"INCOMING_WEIGHT_KIND:{household_weights.kind.value}", + ) + self._require_inherited_weights(context, tables, household_weights) + metadata = spine_assembly_manifest(tables, channels=channels) + frame = Frame( + tables, + US_SCHEMA, + {COMBINED_CLONE_WEIGHT_ENTITY: household_weights}, + context.strata.copy(deep=True), + metadata=metadata, + ) + observed = { + str(value) + for entity in US_SCHEMA.entities + for value in frame.table(entity)[support_channel_column(entity)].unique() + } + _require(observed == set(channels), f"SOURCE_CHANNELS_LIVE:{sorted(observed)}") + return frame + + @staticmethod + def _require_inherited_weights( + context: KernelContext, + tables: Mapping[str, pd.DataFrame], + household_weights: Weights, + ) -> None: + """Refuse a base that weights anything but the household explicitly. + + ``KernelContext.weights`` resolves inheritance, so it cannot by itself + say which entity stores a vector. What it can say is whether every + other entity's effective weights are exactly the household weights + broadcast through membership; anything else is a base this stage would + silently mis-split, because the executor carries a non-weight-entity's + weights unhalved while the operator halves them. The residual case — a + stored vector numerically equal to the inherited one — cannot pass the + executor's ``conserve`` person-mass ledger either. + """ + + household = tables[COMBINED_CLONE_WEIGHT_ENTITY] + by_id = pd.Series( + np.asarray(household_weights.values, dtype=np.float64), + index=pd.Index(household["household_id"].to_numpy()), + ) + person = tables[US_SCHEMA.person_entity] + person_weights = by_id.reindex( + person[US_SCHEMA.membership_column(COMBINED_CLONE_WEIGHT_ENTITY)].to_numpy() + ).to_numpy(dtype=np.float64) + for entity, weights in context.weights.items(): + if entity == COMBINED_CLONE_WEIGHT_ENTITY: + continue + values = np.asarray(weights.values, dtype=np.float64) + if entity == US_SCHEMA.person_entity: + expected = person_weights + else: + member = pd.DataFrame( + { + "group": person[US_SCHEMA.membership_column(entity)].to_numpy(), + "weight": person_weights, + } + ) + grouped = member.groupby("group")["weight"] + _require( + bool((grouped.nunique() == 1).all()), + f"AMBIGUOUS_INHERITED_WEIGHTS:{entity}", + ) + expected = ( + grouped.first() + .reindex(tables[entity][US_SCHEMA.entity_id_column(entity)]) + .to_numpy(dtype=np.float64) + ) + _require( + values.shape == expected.shape and np.array_equal(values, expected), + f"NON_INHERITED_WEIGHTS:{entity}", + ) + + # ------------------------------------------------------------------ + + def run(self, context: KernelContext) -> KernelResult: + self._validated_declaration(context) + channels = self._validated_params(context) + before = self._minimal_frame(context, channels) + + after = clone_us_frame_for_puf_support( + before, + channels=COMBINED_CLONE_ROLES, + clone_attachment_fraction=COMBINED_CLONE_ATTACHMENT_FRACTION, + clone_attachment_seed=COMBINED_CLONE_ATTACHMENT_SEED, + ) + authority = validate_puf_clone_attachment( + after, + boundary=f"{COMBINED_CLONE_PHASE} clone output", + expected_fraction=COMBINED_CLONE_ATTACHMENT_FRACTION, + expected_seed=COMBINED_CLONE_ATTACHMENT_SEED, + ) + _require( + authority["authority_form"] == "full_clone_identity_no_manifest", + "ATTACHMENT_AUTHORITY_FORM", + ) + + lineage: dict[str, pd.Series] = {} + entity_facts: dict[str, object] = {} + for entity in US_SCHEMA.entities: + lineage[entity], entity_facts[entity] = _entity_lineage( + before, after, entity + ) + + columns: dict[tuple[str, str], pd.Series] = {} + for entity, column, dtype in _expected_expand_cells(): + incumbent = before.table(entity)[column] + _require( + token_for_dtype(incumbent.dtype) == dtype, + f"CELL_DTYPE:{entity}.{column}", + ) + table = after.table(entity) + values = table[column] + _require(str(values.dtype) == dtype, f"CLONE_CELL_DTYPE:{entity}.{column}") + columns[(entity, column)] = pd.Series( + values.to_numpy(dtype=np.int64, copy=True), + index=pd.Index( + table[US_SCHEMA.entity_id_column(entity)].to_numpy(copy=True), + name=US_SCHEMA.entity_id_column(entity), + ), + name=column, + dtype=dtype, + ) + + weights = self._split_weights(before, after, lineage) + receipt = self._receipt(before, after, channels, authority, entity_facts) + return KernelResult( + columns=columns, expand=lineage, weights=weights, receipt=receipt + ) + + @staticmethod + def _split_weights( + before: Frame, after: Frame, lineage: Mapping[str, pd.Series] + ) -> Weights: + """Household weights in the executor's target order, pair-sum checked.""" + + entity = COMBINED_CLONE_WEIGHT_ENTITY + id_column = US_SCHEMA.entity_id_column(entity) + after_weights = after.weights_for(entity) + _require(after_weights.kind is COMBINED_CLONE_WEIGHT_KIND, "CLONE_WEIGHT_KIND") + by_id = pd.Series( + np.asarray(after_weights.values, dtype=np.float64), + index=pd.Index(after.table(entity)[id_column].to_numpy(copy=True)), + ) + base_ids = before.table(entity)[id_column].to_numpy(copy=True) + target_ids = np.concatenate( + [base_ids, lineage[entity].index.to_numpy(copy=True)] + ) + ordered = by_id.reindex(target_ids) + _require(not bool(ordered.isna().any()), "WEIGHT_TARGET_ALIGNMENT") + values = ordered.to_numpy(dtype=np.float64) + + incoming = np.asarray( + before.weights_for(entity).values, dtype=np.float64, copy=True + ) + native = values[: len(base_ids)] + detail = values[len(base_ids) :] + # Require an exact pair sum rather than a tolerance that could hide + # a rescale. Halving can lose the low bit of a subnormal double; + # such a weight refuses here instead of silently losing mass. + _require(np.array_equal(native + detail, incoming), "PAIR_WEIGHT_SUM") + _require(np.array_equal(native, detail), "PAIR_WEIGHT_SYMMETRY") + return Weights(values, after_weights.kind) + + @staticmethod + def _receipt( + before: Frame, + after: Frame, + channels: tuple[str, ...], + authority: Mapping[str, object], + entity_facts: Mapping[str, object], + ) -> dict[str, object]: + """Counts, digests, labels and platform-sensitive mass summaries. + + Floating reductions here and in ``_channel_mass`` are receipt evidence, + outside the BITWISE population-output promise. Their last bits can + differ across platforms, so receipt/manifest hashes are scoped too. + """ + + return { + "phase": COMBINED_CLONE_PHASE, + "profile": COMBINED_CLONE_PROFILE, + "release_eligible": False, + "operator": { + "module": _puf_support_module.__name__, + "clone": clone_us_frame_for_puf_support.__name__, + "validator": validate_puf_clone_attachment.__name__, + }, + "source_channels": list(channels), + "clone_roles": list(COMBINED_CLONE_ROLES), + "clone_attachment": { + "fraction": COMBINED_CLONE_ATTACHMENT_FRACTION, + "seed": COMBINED_CLONE_ATTACHMENT_SEED, + "authority_form": authority["authority_form"], + "eligible_household_count": int(authority["eligible_household_count"]), + "realized_household_count": int(authority["realized_household_count"]), + "exact_count_rule": authority["exact_count_rule"], + "selected_household_source_ids_sha256": authority[ + "selected_household_source_ids_sha256" + ], + }, + "entities": {entity: entity_facts[entity] for entity in US_SCHEMA.entities}, + "provenance_counts": { + "base": spine_provenance_counts( + before, boundary=f"{COMBINED_CLONE_PHASE} base" + ), + "expanded": spine_provenance_counts( + after, boundary=f"{COMBINED_CLONE_PHASE} expanded" + ), + }, + "household_weights": { + "kind": COMBINED_CLONE_WEIGHT_KIND.value, + # The operator divides by its role count. This literal is + # the re-validated declaration of that two-role contract, + # not a separate arithmetic control for the operator. + "split_denominator": COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR, + "base_total": float( + before.weights_for(COMBINED_CLONE_WEIGHT_ENTITY).total + ), + "expanded_total": float( + after.weights_for(COMBINED_CLONE_WEIGHT_ENTITY).total + ), + "base_by_source_channel": _channel_mass(before, channels), + "expanded_by_source_channel": _channel_mass(after, channels), + }, + "donor_inputs": { + "puf_donor_read": False, + "donor_sampling_weight_in_host_weights": False, + "draws_or_tax_benefit_values_written": False, + "prior_wage_stage": False, + }, + } + + +class USCombinedSurveyCloneClaimKernel(KernelBase): + """Own the clone-index cells the ``EXPAND`` materialized, unchanged. + + A copied row's clone index differs from its source's, which the executor + only accepts when a same-version claimant declares that coordinate a + full-cell rewrite. This kernel is that claimant: it re-emits exactly the + values the ``EXPAND`` installed, so ownership becomes explicit without any + second opinion about what the clone index is. + """ + + ref = "us.combined_survey.puf_support_clone.claim@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + ) + + def implementation_hash(self) -> str: + return source_hash( + type(self), + _support_provenance_module, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + expected = { + (entity, support_clone_index_column(entity)) + for entity in US_SCHEMA.entities + } + owned = {(output.entity, output.column) for output in node.outputs} + _require(owned == expected, f"CLAIM_OUTPUTS:{sorted(owned ^ expected)}") + _require( + all(output.rewrite and output.rows == "all" for output in node.outputs), + "CLAIM_REWRITE", + ) + columns: dict[tuple[str, str], pd.Series] = {} + for output in node.outputs: + table = context.tables[output.entity] + id_column = US_SCHEMA.entity_id_column(output.entity) + _require( + str(table[output.column].dtype) == output.dtype, + f"CLAIM_DTYPE:{output.entity}.{output.column}", + ) + columns[(output.entity, output.column)] = pd.Series( + table[output.column].to_numpy(dtype=np.int64, copy=True), + index=pd.Index(table[id_column].to_numpy(copy=True), name=id_column), + name=output.column, + dtype=output.dtype, + ) + return KernelResult( + columns=columns, + receipt={ + "phase": COMBINED_CLONE_PHASE, + "claimed_cells": sorted( + f"{entity}.{column}" for entity, column in owned + ), + }, + ) + + +def us_combined_survey_clone_nodes( + columns: Sequence[Owned], + *, + base: str, + source_channels: Sequence[str], + prefix: str = COMBINED_CLONE_PREFIX, +) -> tuple[Node, ...]: + """Declare the clone ``EXPAND`` and the claim node that owns its cells. + + Args: + columns: The base population version's full column inventory. Only the + four provenance columns of each entity are read; the inventory is + required so a missing or wrongly typed provenance column is a + declaration-time refusal rather than a run-time surprise. + base: The population version holding the combined native host. + source_channels: The exact source channels the host carries, sorted. + Both survey arms must be named; this is what the kernel then + requires the live rows to match. + prefix: Node-id prefix, so a caller may declare more than one profile. + + Returns: + ``(expand_node, claim_node)``. The claim node's population is the + expand node, so the claimed cells are owned inside the cloned version. + """ + + _require(bool(base), "BASE_REQUIRED") + channels = _validated_source_channels(tuple(source_channels)) + inventory = {(owned.entity, owned.column): owned for owned in columns} + _require(len(inventory) == len(tuple(columns)), "COLUMN_INVENTORY_REPEATS") + for entity in US_SCHEMA.entities: + for column in combined_clone_provenance_columns(entity): + owned = inventory.get((entity, column)) + _require(owned is not None, f"MISSING_PROVENANCE_COLUMN:{entity}.{column}") + cells = _expected_expand_cells() + for entity, column, dtype in cells: + _require( + inventory[(entity, column)].dtype == dtype, + f"CLONE_INDEX_DTYPE:{entity}.{column}", + ) + + expand_node = f"{prefix}" + claim_node = f"{prefix}.owned" + expand = Node( + id=expand_node, + kernel=USCombinedSurveyCloneExpandKernel.ref, + inputs=tuple( + Slice(entity, combined_clone_provenance_columns(entity)) + for entity in US_SCHEMA.entities + ), + params={ + "clone_attachment_fraction": COMBINED_CLONE_ATTACHMENT_FRACTION, + "clone_attachment_seed": COMBINED_CLONE_ATTACHMENT_SEED, + "clone_roles": COMBINED_CLONE_ROLES, + "expand_cells": cells, + "expand_weight_entity": COMBINED_CLONE_WEIGHT_ENTITY, + "expand_weight_kind": COMBINED_CLONE_WEIGHT_KIND.value, + "phase": COMBINED_CLONE_PHASE, + "profile": COMBINED_CLONE_PROFILE, + "release_eligible": False, + "source_channels": channels, + "weight_split_denominator": COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR, + }, + structural=StructuralDelta.EXPAND, + base=base, + mass="conserve", + description=( + "Attach one whole-household PUF-detail clone to the combined " + "ASEC/ACS host and split every household weight across the pair." + ), + ) + claim = Node( + id=claim_node, + kernel=USCombinedSurveyCloneClaimKernel.ref, + outputs=tuple( + Owned(entity, column, dtype, rewrite=True) + for entity, column, dtype in cells + ), + population=expand_node, + params={"phase": COMBINED_CLONE_PHASE}, + description="Own the clone-index cells the clone stage materialized.", + ) + return (expand, claim) + + +def register_us_combined_survey_clone_kernels(registry: KernelRegistry) -> None: + """Register both kernels of the bounded combined-survey clone stage.""" + + registry.register(USCombinedSurveyCloneExpandKernel()) + registry.register(USCombinedSurveyCloneClaimKernel()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py new file mode 100644 index 000000000..7ce6a9b24 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py @@ -0,0 +1,1404 @@ +"""Bind the carried prepared-ASEC evidence to the ASEC rows of a composed population. + +:mod:`.graph_composed_population` reads both authenticated sources whole and +carries the prepared arm's four typed evidence artifacts plus its own +``asec_frame_context`` unchanged. Those artifacts are **positional over the whole +prepared ASEC population**, not over the composed rows, so the accepted +prepared-slice kernels cannot attach: their contracts require an empty context +metadata, an empty mass history, a design household weight and prepared-receipt +rows equal to the population's rows. A harmonized composed population satisfies +none of them. Those refusals are real, are reproduced in the test module, and +nothing here weakens them. + +What this module adds is the missing edge and nothing else: one node that +resolves which composed rows came from the ASEC arm and which source row each +one is, and publishes that resolution as a typed artifact together with a +:class:`~.asec_current_money_selection.SelectedCurrentMoney` sliced to exactly +those source coordinates by the same reviewed selector the prepared slice uses. + +How rows are resolved +--------------------- +By source arm and the arm's own pre-remap identifier, never by composed row +position and never by the post-offset ``*_source_id``. For each entity the node +reads ``*_support_channel`` to find the arm and ``*_spine_source_id`` for the +arm's original ID, then locates each original ID in the arm-ordered identity the +carried evidence itself declares: ``income_observations``' ``person_id`` for +persons and ``housing_universe``' ``household_id`` for households. Both buffers +are bound to the prepared receipt before they are used, so that ordering is +receipt-authenticated rather than assumed. The resolution refuses an empty, +duplicated, unresolved or non-monotone mapping, and refuses a cohort +disagreement: the composed ``source_year`` must equal the source ``income_year`` +at the resolved positions, and the income artifact's own three declared +crosschecks must agree cell by cell. + +Nothing is assumed about where the arm's rows sit. They need not be a prefix, +need not be contiguous, and group tables may be sorted or interleaved with the +other arm's; the mask is read, never constructed from a row range. Every +reconstruction is then checked back against ``source_origin``: the origin +document's per-entity provenance digests are recomputed against the actual +composed tables, so a document that describes a different frame refuses. + +Why this module may read source provenance +------------------------------------------ +Resolving "which arm did this row come from" *is* this node's operation, so it +is a reviewed source-spine provenance owner. It applies no population treatment: +it writes one boolean arm-membership cell per bound entity and no measurement. +The measurements live in :mod:`.graph_composed_asec_measures`, consume this +node's typed artifact and its declared row mask, and stay source-blind. +""" + +from __future__ import annotations + +import json +import struct +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import InitVar, dataclass + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from .asec_current_money import ( + HEADER_MAX_BYTES, + MAX_HOUSEHOLDS, + MAX_PERSONS, + _json, + _parse, + _sha, +) +from .asec_current_money_selection import ( + US_ASEC_CURRENT_MONEY_BODY_TYPE, + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + encode_selected_current_money, + parse_current_money_body, + select_current_money, +) +from .asec_prepared_source import PREPARED_SOURCE_KIND +from .graph_asec_income import ( + US_ASEC_INCOME_OBSERVATIONS_TYPE, + bind_income_observations, +) +from .graph_asec_prepared import PreparedGraphError, _artifact, _same_producer +from .graph_composed_contracts import ( + BIND_NODE as BIND_NODE, +) +from .graph_composed_contracts import ( + US_COMPOSED_ASEC_ARM_ROWS_TYPE as US_COMPOSED_ASEC_ARM_ROWS_TYPE, +) +from .graph_composed_contracts import ( + US_COMPOSED_ASEC_BINDING_TYPE as US_COMPOSED_ASEC_BINDING_TYPE, +) +from .graph_composed_population import ( + CREATE_NODE, + US_COMPOSED_SOURCE_ORIGIN_TYPE, + bind_composed_source_origin, +) +from .graph_context import US_FRAME_CONTEXT_TYPE, _decode, _mass_records, _row_identity +from .graph_housing_universe import ( + HOUSEHOLD_EVIDENCE_COLUMNS, + US_ASEC_HOUSING_UNIVERSE_TYPE, + bind_housing_universe, + verify_graph_housing_rows, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .stacked_spine import ACS_STACKED_SUPPORT_CHANNEL +from .support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + spine_source_id_column, + support_channel_column, + support_source_id_column, +) + +COMPOSED_ASEC_STAGE = "composed_asec_binding_v1" +COMPOSED_ASEC_PHASE = "bind_composed_asec_evidence" +COMPOSED_ASEC_DEPENDENCIES = STAGE_DEPENDENCIES[COMPOSED_ASEC_STAGE] + + +#: The arm this binding resolves. The other arm is named only so an unexpected +#: third channel refuses; no ACS row is read, written or measured here. +ARM_CHANNEL = BASE_ASEC_SUPPORT_CHANNEL +_OTHER_CHANNEL = ACS_STACKED_SUPPORT_CHANNEL + +#: The declared boolean arm-membership cell, entity-prefixed because ``Frame`` +#: requires column names to be globally unique across entity tables. Every +#: measurement node writes under this mask, so the executor — not a convention — +#: is what keeps the opposite arm's storage byte-identical: +#: ``population._patch_columns`` refuses any change outside an ``Owned`` mask, +#: missingness included. +ARM_ROW_SUFFIX = "composed_asec_row" +ARM_ROW_DTYPE = "bool" + + +def arm_row_column(entity: str) -> str: + """Return the entity-prefixed arm-membership cell name.""" + _require(entity in US_SCHEMA.entities, f"ARM_ROW_ENTITY:{entity}") + return f"{entity}_{ARM_ROW_SUFFIX}" + + +#: The entities a measurement of this arm addresses: persons, the households +#: whose carried housing evidence is verified against the source, and the SPM +#: units the corrected childcare leaf is grained on. +BOUND_ENTITIES = ("person", "household", "spm_unit") +#: The two entities the carried evidence declares an ordered source identity for. +_ID_ENTITIES = ("person", "household") + +BINDING_SCHEMA = "microcosm.us.composed-asec-binding.v1" + +#: The income artifact's own declared crosschecks, verified row by row between +#: the composed cells and the source buffers at the resolved positions. +PERSON_COORDINATE_CROSSCHECKS = ("source_household_id", "A_LINENO", "A_AGE") +COHORT_COLUMN = "source_year" + +#: Demographic columns a later calibration wants on both arms and which this +#: stage deliberately does not bind: each names the exact source column the +#: reviewed ASEC mapping needs, the mapping itself, and what the native ACS arm +#: uses instead. Nothing is imputed, aged or defaulted. +#: +#: Whether the prepared arm carries the source column is an **observed property +#: of the population in hand**, recorded per run rather than asserted: the +#: invented fixture parent has neither column, and the genuine prepared arm has +#: both. Presence is not a licence — binding either column still needs a +#: reviewed cross-arm mapping decision that this stage may not make — so what +#: fails closed here is an attempted **write** of one of these columns +#: (``ASEC_DEMOGRAPHIC_COLUMN_BOUND``), not the source column's presence. Keying +#: the refusal on presence would have refused every genuine run while proving +#: nothing about what this stage binds. +#: Each entry is (column, required source columns, diagnostic-only source +#: columns, the explicit source contract that would supply it, the mappings +#: deliberately not adopted as proof, the caveats on reading presence as +#: observation, the ACS arm's own observed mapping). +#: The required columns are the ones an *explicit* binding needs, which is not +#: the same as the columns some existing mapping happens to read: the root +#: adjudication of 2026-09-06 refused ``P_SEQ == 1`` as semantic proof of +#: headship and required sex to carry its allocation provenance, so those +#: mappings are recorded here as not adopted rather than as the requirement. +_UNBOUND_DEMOGRAPHICS = ( + ( + "is_female", + ("A_SEX", "AXSEX"), + (), + "asec_demographic_source: A_SEX printed codes 1 = Male and 2 = Female, " + "admitted only with AXSEX printed codes 0 = No change or 4 = Allocated; " + "any other token leaves the person unbound", + ( + ( + "cps_carried.derive_us_cps_carried_inputs (is_female = A_SEX == 2)", + "maps every token other than 2 onto male, so an unprinted code " + "would be admitted as male, and it reads no allocation flag", + ), + ), + ( + "A_SEX reaches the prepared person roster as a carried source " + "column, but its presence attests the column, not that every " + "delivered token is one of the two printed codes", + ), + "SEX == 2 on the native ACS arm, whose pinned dictionary prints the " + "same 1 = Male / 2 = Female convention", + ), + ( + "is_household_head", + ("A_EXPRRP",), + ("P_SEQ",), + "asec_demographic_source: A_EXPRRP printed codes 1 = Reference person " + "with relatives and 2 = Reference person without relatives, with " + "exactly one such person per household and every household relationship " + "token inside the printed named codes", + ( + ( + "relationship_inputs (is_household_head = P_SEQ == 1)", + "P_SEQ's dictionary entry labels no code as reference person, " + "and the ordering statement behind it is expressly limited to " + "the ASCII file while this arm is restored from the CSV member", + ), + ( + "A_FAMREL == 1", + "a family relationship scoped to primary-family membership, " + "which a subfamily reference person does not carry; household " + "and family roles stay separate", + ), + ( + "asec_pool relationship recode (A_LINENO == 1)", + "a separately assigned Basic-CPS roster line number with no " + "source guarantee of agreement with any ASEC sequence", + ), + ), + ( + "asec_pool._with_relationship_recode derives A_EXPRRP from " + "A_LINENO == 1 whenever the locked input omits it, so the column's " + "presence on a prepared arm does not certify an observed source " + "value for that cohort", + "the native ACS arm's A_EXPRRP is likewise derived by acs_pums " + "from RELSHIPP, and its group-quarters households carry the " + "nonrelative code 14 rather than any reference-person code", + ), + "RELSHIPP == 20 observed on the native ACS arm's housing units only; " + "RELSHIPP 37 and 38 are the group-quarters populations and carry no " + "observed reference person", + ), +) +#: The population columns this stage records as unbound, and therefore may not +#: own on any node. Public because the measurement module's own declarations are +#: checked against it. +UNBOUND_DEMOGRAPHIC_COLUMNS = tuple(column for column, *_rest in _UNBOUND_DEMOGRAPHICS) + +_ARTIFACT_TYPES = ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("source_origin", US_COMPOSED_SOURCE_ORIGIN_TYPE), + ("asec_frame_context", US_FRAME_CONTEXT_TYPE), + ("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), +) +#: The prepared context and the four evidence artifacts are positional over one +#: prepared population, so they must come from one producing node execution. +_SOURCE_EDGES = ( + "asec_frame_context", + "current_money", + "housing_universe", + "income_observations", + "prepared_receipt", +) +#: Every edge the composed CREATE produces, the origin document included. The +#: origin is positional over that same CREATE execution, so it is held to the +#: same producer identity as the evidence. Its content is independently +#: recomputed against the actual tables (``ORIGIN_DOES_NOT_DESCRIBE_THIS_FRAME``) +#: and that check is unchanged — but recomputation cannot tell two CREATE +#: executions describing identical tables apart, so the producer key the +#: document records would otherwise be carried unverified. +_CREATE_EDGES = (*_SOURCE_EDGES, "source_origin") + +_BINDING_KEYS = frozenset( + { + "arm", + "arm_channel", + "arm_row_columns", + "certified", + "claim", + "cohorts", + "composed", + "composed_is_whole_arm", + "demographics", + "inputs", + "origin", + "phase", + "prepared_arm_rows", + "release_eligible", + "resolution", + "schema", + } +) +_ARM_KEYS = frozenset({"rows", "composed_ordered_ids_sha256", "mask_sha256"}) +_ARM_ID_KEYS = _ARM_KEYS | { + "original_ordered_ids_sha256", + "source_positions_sha256", +} + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def refuse_unbound_demographic_writes(columns: Iterable[str]) -> None: + """Refuse if this stage would own a demographic its binding records as open. + + This is the fail-closed half of the recorded gap. The binding document says + ``is_female`` and ``is_household_head`` are not bound on this population; + that statement is only honest while no node of the stage writes them, so + every owned-column roster of the stage passes through here. A reviewed + mapping that started producing one of them would refuse at declaration time + rather than silently turning a recorded gap into an unreviewed binding. + """ + for column in columns: + _require( + column not in UNBOUND_DEMOGRAPHIC_COLUMNS, + f"ASEC_DEMOGRAPHIC_COLUMN_BOUND:{column}", + ) + + +def _is_digest(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and not set(value) - set("0123456789abcdef") + ) + + +@dataclass(frozen=True) +class ComposedAsecBinding: + """One resolved arm binding: the row masks, the source rows and its document.""" + + document: Mapping[str, object] + context_document: Mapping[str, object] + masks: Mapping[str, np.ndarray] + positions: Mapping[str, np.ndarray] + original_ids: Mapping[str, np.ndarray] + income_years: np.ndarray + selected_money: bytes + + @property + def payload(self) -> bytes: + return canonical_json(self.document) + + +def arm_mask(table: pd.DataFrame, entity: str) -> np.ndarray: + """Which rows of a composed entity table came from the ASEC arm.""" + column = support_channel_column(entity) + _require(column in table, f"ARM_CHANNEL_COLUMN:{entity}") + channels = table[column] + _require(not bool(channels.isna().any()), f"ARM_CHANNEL_MISSING:{entity}") + values = channels.astype(str).to_numpy() + _require( + set(values.tolist()) <= {ARM_CHANNEL, _OTHER_CHANNEL}, + f"ARM_CHANNEL_UNKNOWN:{entity}", + ) + mask = values == ARM_CHANNEL + _require(bool(mask.any()), f"ARM_EMPTY:{entity}") + return mask + + +def _original_ids(table: pd.DataFrame, entity: str, mask: np.ndarray) -> np.ndarray: + """The arm's own pre-remap identifiers, never the post-offset source IDs.""" + column = spine_source_id_column(entity) + _require(column in table, f"ARM_SPINE_SOURCE_ID:{entity}") + return _int64_view(table.loc[mask, column], reason=f"ARM_SPINE_SOURCE_ID:{entity}") + + +def _resolved_positions( + original_ids: np.ndarray, arm_ids: np.ndarray, *, entity: str +) -> np.ndarray: + """Locate each original ID in the source's own ordered identity. + + Every failure mode refuses instead of dropping a row: a source identity that + repeats an ID is ambiguous, an original ID the source never declares is + unresolved, and a non-increasing mapping cannot be a whole-household slice of + the source buffers, which the reviewed selector requires to be strictly + ordered. The order is a property of the actual mapping, checked here; it is + never imposed by sorting the rows. + """ + _require(len(original_ids) > 0, f"BINDING_EMPTY:{entity}") + _require( + len(np.unique(arm_ids)) == len(arm_ids), f"BINDING_AMBIGUOUS_SOURCE:{entity}" + ) + _require( + len(np.unique(original_ids)) == len(original_ids), + f"BINDING_AMBIGUOUS_ROWS:{entity}", + ) + positions = pd.Index(arm_ids).get_indexer(original_ids).astype("int64") + _require(bool((positions >= 0).all()), f"BINDING_UNRESOLVED:{entity}") + _require( + len(positions) < 2 or bool((np.diff(positions) > 0).all()), + f"BINDING_NON_MONOTONE:{entity}", + ) + return positions + + +def _int64_view(series: pd.Series, *, reason: str) -> np.ndarray: + """Narrow the stack's nullable storage back to int64 on this arm's own rows. + + Stacking widens an arm-specific integer column to ``Int64`` so the opposite + arm's absence stays a declared null. On the arm's own rows nothing may be + missing and the reviewed verifiers compare native ``int64`` buffers, so the + narrowing is explicit and refuses rather than filling. + """ + _require(pd.api.types.is_integer_dtype(series.dtype), f"{reason}_DTYPE") + _require(not bool(series.isna().any()), f"{reason}_MISSING") + return series.to_numpy(dtype="int64") + + +def _arm_household_evidence( + household: pd.DataFrame, mask: np.ndarray, original_ids: np.ndarray +) -> pd.DataFrame: + """The arm's carried housing evidence restated under its own source identity. + + ``verify_graph_housing_rows`` compares native source buffers against a + household view keyed by ``household_id``. On a composed population that + column is the assembly-remapped identity, so this view restates the arm's own + pre-remap identity instead. The evidence columns themselves are carried + unchanged; only the stack's nullable storage is narrowed. + """ + view = pd.DataFrame(index=pd.RangeIndex(int(mask.sum()))) + view["household_id"] = original_ids + for column in HOUSEHOLD_EVIDENCE_COLUMNS: + _require(column in household, f"ARM_HOUSING_EVIDENCE:{column}") + view[column] = _int64_view( + household.loc[mask, column], reason=f"ARM_HOUSING_EVIDENCE:{column}" + ) + return view + + +def _arm_household_cohorts( + person: pd.DataFrame, + person_mask: np.ndarray, + income_years: np.ndarray, + composed_household_ids: np.ndarray, +) -> np.ndarray: + """One source income year per ASEC household; a mixed household refuses. + + The household evidence is cohort-scoped, so the housing verifier is given the + year the arm's own persons declare rather than the year the source buffer + would have supplied — a household whose members disagree is a real source + contract failure and must not be reduced away. + """ + membership = person.loc[ + person_mask, US_SCHEMA.membership_column("household") + ].to_numpy(dtype="int64") + pairs = pd.DataFrame({"household": membership, "year": income_years}) + distinct = pairs.drop_duplicates() + _require( + not bool(distinct["household"].duplicated().any()), "ARM_MIXED_YEAR_HOUSEHOLD" + ) + lookup = dict( + zip(distinct["household"].tolist(), distinct["year"].tolist(), strict=True) + ) + _require( + set(lookup) == set(composed_household_ids.tolist()), + "ARM_HOUSEHOLD_COHORT_COVERAGE", + ) + return np.asarray( + [lookup[int(value)] for value in composed_household_ids], dtype="int64" + ) + + +def _composed_document(payload: bytes) -> dict: + """Decode the composed population's typed context without relaxing anything. + + The prepared slice's ``_document`` refuses exactly the three things a + composed population always has — assembly metadata, a mass history and an + importance household weight — so it cannot be reused. This decoder keeps the + canonical-bytes, normative-metadata and mass-record validation that + ``graph_context`` owns and asserts the composed shape positively. + """ + document = _decode(payload) + _mass_records(document["mass_log"]) + _require(canonical_json(document) == payload, "COMPOSED_CONTEXT_CANONICAL") + _require(set(document["weight_sources"]) == {"household"}, "COMPOSED_WEIGHT_ENTITY") + return document + + +def _prepared_document(payload: bytes) -> dict: + """The prepared arm's own pre-stack context, which the evidence is ordered in.""" + document = _decode(payload) + _mass_records(document["mass_log"]) + _require(canonical_json(document) == payload, "PREPARED_CONTEXT_CANONICAL") + _require(document["metadata"] == {}, "PREPARED_CONTEXT_METADATA") + _require(document["mass_log"] == [], "PREPARED_CONTEXT_MASS_LOG") + _require( + document["weight_sources"] == {"household": "design"}, + "PREPARED_WEIGHT_AUTHORITY", + ) + return document + + +def _origin_entity_digests(tables: Mapping[str, pd.DataFrame]) -> dict: + """Recompute the origin document's per-entity mapping from the actual rows. + + ``bind_composed_source_origin`` deliberately verifies no digest against any + population, because it receives only bytes. A consumer that relies on the + mapping has to recompute it against the frame in hand; this is that + recomputation, in the producer's own order and encoding. + """ + digests = {} + for entity in US_SCHEMA.entities: + table = tables[entity] + channel_column = support_channel_column(entity) + spine_column = spine_source_id_column(entity) + source_column = support_source_id_column(entity) + for column in (channel_column, spine_column, source_column): + _require(column in table, f"ORIGIN_RECOMPUTE_COLUMN:{entity}.{column}") + channels = table[channel_column].astype(str).tolist() + spine_ids = table[spine_column].to_numpy(dtype="int64").tolist() + ids = table[source_column].to_numpy(dtype="int64").tolist() + outputs = ( + table[US_SCHEMA.entity_id_column(entity)].to_numpy(dtype="int64").tolist() + ) + digests[entity] = { + "rows_by_channel": { + channel: int(channels.count(channel)) + for channel in sorted(set(channels)) + }, + "ordered_channels_sha256": _sha(canonical_json(channels)), + "ordered_spine_source_ids_sha256": _sha(canonical_json(spine_ids)), + "ordered_source_to_output_sha256": _sha( + canonical_json( + [ + [channel, spine_id, source_id, output_id] + for channel, spine_id, source_id, output_id in zip( + channels, spine_ids, ids, outputs, strict=True + ) + ] + ) + ), + } + return digests + + +def _demographics(prepared_columns: Sequence[str], cohorts: Mapping[str, int]) -> dict: + """What a cross-arm demographic can honestly claim on this population today. + + ``age`` is bound through the reviewed corrected-leaf mapping whose only input + is the routing column ``A_AGE``. Its ASEC observation is the interview + household one year after the income year — the reference period the income + artifact declares and this node verifies cell by cell — while the native ACS + arm's ``age`` is observed in its own vintage. The two are **not** harmonized + here and no person is aged from any date; the periods are recorded per cohort + so a calibration decides rather than inherits an assumption. + + ``is_female`` and ``is_household_head`` stay unbound. What their entries say + about the source is **observed on the prepared arm actually in hand**, and + is stated both ways: which required source columns this population carries + and which it does not. That observation is evidence, not authority — + ``bound_by_this_stage`` is ``False`` either way, and + :func:`refuse_unbound_demographic_writes` is what keeps it true. + + Each entry also records the explicit source contract that would supply the + column, the mappings deliberately **not** adopted as proof of it, and the + diagnostic-only columns that are crosschecked against that contract and + never substituted for it. + """ + present = set(prepared_columns) + unbound = [] + for ( + column, + sources, + diagnostics, + contract, + not_adopted, + caveats, + acs_mapping, + ) in _UNBOUND_DEMOGRAPHICS: + found = sorted(name for name in sources if name in present) + absent = sorted(name for name in sources if name not in present) + unbound.append( + { + "column": column, + "required_asec_source_columns": list(sources), + "present_in_prepared_arm": found, + "absent_from_prepared_arm": absent, + "diagnostic_only_source_columns": list(diagnostics), + "diagnostic_only_present_in_prepared_arm": sorted( + name for name in diagnostics if name in present + ), + "diagnostic_only_is_semantic_proof": False, + "bound_by_this_stage": False, + "explicit_asec_source_contract": contract, + "asec_mappings_not_adopted": [ + {"mapping": mapping, "refusal": refusal} + for mapping, refusal in not_adopted + ], + "presence_certifies_observation": False, + "presence_caveats": list(caveats), + "acs_arm_mapping": acs_mapping, + "reason": ( + "required_asec_source_column_absent_from_this_prepared_population" + if absent + else "no_reviewed_cross_arm_mapping_decision_for_this_column" + ), + "resolution": "independent_source_decision_required", + } + ) + bound = [ + { + "column": "age", + "dtype": "float64", + "asec_source_column": "A_AGE", + "mapping": "derive_cps_carried_current_leaves", + "observed_period_kind": "interview_household_one_year_after_income_year", + "observed_period_by_cohort": { + year: {"income_year": int(year), "asec_survey_year": int(year) + 1} + for year in sorted(cohorts) + }, + "aged_from_a_date": False, + "cross_arm_period_harmonized": False, + "acs_arm_note": ( + "the native ACS arm's age is carried unchanged in its own " + "vintage and is never rewritten by this binding" + ), + } + ] + refuse_unbound_demographic_writes(item["column"] for item in bound) + return {"bound": bound, "unbound": unbound} + + +def resolve_composed_asec_binding( + tables: Mapping[str, pd.DataFrame], + *, + context_payload: bytes, + origin_payload: bytes, + prepared_context_payload: bytes, + money_payload: bytes, + receipt_payload: bytes, + housing_payload: bytes, + income_payload: bytes, + producers: Mapping[str, str], +) -> ComposedAsecBinding: + """Resolve the ASEC arm's composed rows against the carried source evidence.""" + context = _composed_document(context_payload) + origin = bind_composed_source_origin(origin_payload) + prepared_context = _prepared_document(prepared_context_payload) + receipt = json.loads(receipt_payload) + _require(canonical_json(receipt) == receipt_payload, "PREPARED_RECEIPT_JSON") + _require( + receipt["source_kind"] == PREPARED_SOURCE_KIND + and receipt["release_eligible"] is False, + "PREPARED_RECEIPT_SCHEMA", + ) + for entity in _ID_ENTITIES: + _require( + int(receipt["entity_rows"][entity]) + == int(prepared_context["entities"][entity]["rows"]), + f"PREPARED_EVIDENCE_ROWS:{entity}", + ) + income = bind_income_observations(income_payload, prepared_receipt=receipt) + universe = bind_housing_universe(housing_payload, prepared_receipt=receipt) + body = parse_current_money_body( + money_payload, + expected_header_sha256=receipt["money_header_sha256"], + expected_content_sha256=receipt["money_content_sha256"], + field_entities=tuple(tuple(item) for item in receipt["field_entities"]), + ) + _require( + body.person_rows == int(receipt["entity_rows"]["person"]) + and body.household_rows == int(receipt["entity_rows"]["household"]), + "BODY_ROW_ALIGNMENT", + ) + + for entity in US_SCHEMA.entities: + _require(entity in tables, f"COMPOSED_TABLE_MISSING:{entity}") + declared = context["entities"][entity] + actual = _row_identity(tables[entity], entity) + _require( + all(declared[key] == value for key, value in actual.items()), + f"COMPOSED_CONTEXT_IDENTITY:{entity}", + ) + _require( + _origin_entity_digests(tables) == origin["assembled"], + "ORIGIN_DOES_NOT_DESCRIBE_THIS_FRAME", + ) + + masks = {entity: arm_mask(tables[entity], entity) for entity in US_SCHEMA.entities} + alignment = origin["asec_evidence_alignment"] + for entity in US_SCHEMA.entities: + _require( + int(alignment["composed_rows"][entity]) == int(masks[entity].sum()), + f"ORIGIN_COMPOSED_ROWS:{entity}", + ) + whole = bool(alignment["composed_is_whole_arm"]) + _require( + whole == (alignment["composed_rows"] == alignment["arm_rows"]), + "ORIGIN_WHOLE_ARM_FLAG", + ) + for entity in _ID_ENTITIES: + _require( + int(alignment["evidence_rows"][entity]) + == int(receipt["entity_rows"][entity]), + f"ORIGIN_EVIDENCE_ROWS:{entity}", + ) + _require( + int(masks[entity].sum()) <= int(receipt["entity_rows"][entity]), + f"ARM_OVERFLOW:{entity}", + ) + + original = { + entity: _original_ids(tables[entity], entity, masks[entity]) + for entity in _ID_ENTITIES + } + arm_identity = { + "person": income.array("person_id").astype("int64"), + "household": universe.array("household_id").astype("int64"), + } + for entity in _ID_ENTITIES: + _require( + len(arm_identity[entity]) == int(receipt["entity_rows"][entity]), + f"ARM_IDENTITY_ROWS:{entity}", + ) + positions = { + entity: _resolved_positions( + original[entity], arm_identity[entity], entity=entity + ) + for entity in _ID_ENTITIES + } + + person = tables["person"] + income_years = _int64_view( + person.loc[masks["person"], COHORT_COLUMN], reason="ARM_COHORT" + ) + _require( + np.array_equal(income_years, income.array("income_year")[positions["person"]]), + "BINDING_COHORT_MISMATCH", + ) + for column in PERSON_COORDINATE_CROSSCHECKS: + _require(column in person, f"ARM_CROSSCHECK_COLUMN:{column}") + _require( + np.array_equal( + _int64_view( + person.loc[masks["person"], column], + reason=f"ARM_CROSSCHECK:{column}", + ), + income.array(column)[positions["person"]], + ), + f"BINDING_CROSSCHECK:{column}", + ) + composed_household_ids = ( + tables["household"] + .loc[masks["household"], US_SCHEMA.entity_id_column("household")] + .to_numpy(dtype="int64") + ) + verify_graph_housing_rows( + _arm_household_evidence( + tables["household"], masks["household"], original["household"] + ), + universe, + positions=positions["household"], + income_years=_arm_household_cohorts( + person, masks["person"], income_years, composed_household_ids + ), + ) + + identities = { + entity: _row_identity( + pd.DataFrame({US_SCHEMA.entity_id_column(entity): original[entity]}), entity + ) + for entity in _ID_ENTITIES + } + cohorts = { + str(int(year)): int((income_years == year).sum()) + for year in np.unique(income_years).tolist() + } + arm = {} + for entity in BOUND_ENTITIES: + table = tables[entity] + id_column = US_SCHEMA.entity_id_column(entity) + composed_ids = table.loc[masks[entity], id_column].to_numpy(dtype="int64") + record = { + "rows": int(masks[entity].sum()), + "composed_ordered_ids_sha256": _row_identity( + pd.DataFrame({id_column: composed_ids}), entity + )["ordered_ids_sha256"], + "mask_sha256": _sha(masks[entity].tobytes()), + } + if entity in _ID_ENTITIES: + record["original_ordered_ids_sha256"] = identities[entity][ + "ordered_ids_sha256" + ] + record["source_positions_sha256"] = _sha(positions[entity].tobytes()) + arm[entity] = record + + document = { + "schema": BINDING_SCHEMA, + "phase": COMPOSED_ASEC_PHASE, + "release_eligible": False, + "certified": False, + "claim": "engineering_binding_only_no_calibration_or_district_claim", + "arm_channel": ARM_CHANNEL, + "arm_row_columns": { + entity: arm_row_column(entity) for entity in BOUND_ENTITIES + }, + "resolution": { + "keys": ["support_channel", "spine_source_id", COHORT_COLUMN], + "source_identity": { + "household": "housing_universe.household_id", + "person": "income_observations.person_id", + }, + "crosschecks": list(PERSON_COORDINATE_CROSSCHECKS), + "uses_composed_row_position": False, + "uses_post_offset_source_id": False, + }, + "composed": { + entity: { + "rows": int(context["entities"][entity]["rows"]), + "ordered_ids_sha256": context["entities"][entity]["ordered_ids_sha256"], + } + for entity in US_SCHEMA.entities + }, + "arm": arm, + "prepared_arm_rows": { + entity: int(receipt["entity_rows"][entity]) for entity in _ID_ENTITIES + }, + "composed_is_whole_arm": whole, + "cohorts": cohorts, + "demographics": _demographics( + prepared_context["entities"]["person"]["columns"], cohorts + ), + "inputs": { + "asec_frame_context_sha256": _sha(prepared_context_payload), + "current_money_sha256": _sha(money_payload), + "frame_context_sha256": _sha(context_payload), + "housing_universe_sha256": _sha(housing_payload), + "income_observations_sha256": _sha(income_payload), + "money_content_sha256": receipt["money_content_sha256"], + "money_header_sha256": receipt["money_header_sha256"], + "prepared_receipt_sha256": _sha(receipt_payload), + "producers": {name: producers[name] for name in sorted(producers)}, + "source_origin_sha256": _sha(origin_payload), + }, + "origin": { + "preparation_sha256": origin["preparation_sha256"], + "sample_fraction": float(origin["sampling"]["sample_fraction"]), + "sample_seed": int(origin["sampling"]["sample_seed"]), + }, + } + payload = canonical_json(document) + selected = select_current_money( + body, + person_positions=positions["person"], + household_positions=positions["household"], + prepared_receipt_sha256=_sha(receipt_payload), + selection_sha256=_sha(payload), + person_identity_sha256=identities["person"]["ordered_ids_sha256"], + household_identity_sha256=identities["household"]["ordered_ids_sha256"], + ) + for entity in BOUND_ENTITIES: + context["entities"][entity]["columns"].append(arm_row_column(entity)) + return ComposedAsecBinding( + document=document, + context_document=context, + masks={entity: masks[entity] for entity in BOUND_ENTITIES}, + positions=positions, + original_ids=original, + income_years=income_years, + selected_money=encode_selected_current_money(selected), + ) + + +def bind_composed_asec_document(payload: bytes) -> dict: + """Decode the binding artifact as a typed shape, refusing a malformed one. + + Like the origin binder this checks canonical bytes and declared shape only. + It verifies no digest against any population; a consumer holding the frame + recomputes what it needs. + """ + document = json.loads(payload) + _require(canonical_json(document) == payload, "BINDING_CANONICAL") + _require(isinstance(document, dict), "BINDING_DOCUMENT") + _require(set(document) == _BINDING_KEYS, "BINDING_KEYS") + _require(document["schema"] == BINDING_SCHEMA, "BINDING_SCHEMA") + _require(document["phase"] == COMPOSED_ASEC_PHASE, "BINDING_PHASE") + _require(document["arm_channel"] == ARM_CHANNEL, "BINDING_ARM_CHANNEL") + _require( + document["arm_row_columns"] + == {entity: arm_row_column(entity) for entity in BOUND_ENTITIES}, + "BINDING_ARM_ROW_COLUMNS", + ) + _require( + document["release_eligible"] is False and document["certified"] is False, + "BINDING_RELEASE_CLAIM", + ) + _require( + set(document["composed"]) == set(US_SCHEMA.entities), + "BINDING_COMPOSED_ENTITIES", + ) + _require(set(document["arm"]) == set(BOUND_ENTITIES), "BINDING_ARM_ENTITIES") + for entity, record in document["arm"].items(): + expected = _ARM_ID_KEYS if entity in _ID_ENTITIES else _ARM_KEYS + _require(set(record) == expected, f"BINDING_ARM_KEYS:{entity}") + _require( + type(record["rows"]) is int and record["rows"] > 0, + f"BINDING_ARM_ROWS:{entity}", + ) + _require( + all( + _is_digest(value) + for key, value in record.items() + if key.endswith("_sha256") + ), + f"BINDING_ARM_DIGEST:{entity}", + ) + resolution = document["resolution"] + _require( + resolution["uses_composed_row_position"] is False + and resolution["uses_post_offset_source_id"] is False + and resolution["keys"] == ["support_channel", "spine_source_id", COHORT_COLUMN] + and resolution["crosschecks"] == list(PERSON_COORDINATE_CROSSCHECKS), + "BINDING_RESOLUTION", + ) + demographics = document["demographics"] + _require( + set(demographics) == {"bound", "unbound"} + and bool(demographics["bound"]) + and all(item["aged_from_a_date"] is False for item in demographics["bound"]), + "BINDING_DEMOGRAPHICS", + ) + # What the unbound entries must satisfy is that they are unbound and that + # they say the whole truth about the source, not that the source lacked the + # column: a genuine prepared arm does carry some of these, and a document + # recording that is correct. Every required column is accounted for exactly + # once, on one side or the other. A diagnostic-only column is held apart + # from the requirement and is never allowed to claim it proves anything. + _require( + [item["column"] for item in demographics["unbound"]] + == list(UNBOUND_DEMOGRAPHIC_COLUMNS) + and all( + item["bound_by_this_stage"] is False + and sorted( + [*item["present_in_prepared_arm"], *item["absent_from_prepared_arm"]] + ) + == sorted(item["required_asec_source_columns"]) + and not set(item["present_in_prepared_arm"]) + & set(item["absent_from_prepared_arm"]) + and item["diagnostic_only_is_semantic_proof"] is False + and not set(item["diagnostic_only_source_columns"]) + & set(item["required_asec_source_columns"]) + and set(item["diagnostic_only_present_in_prepared_arm"]) + <= set(item["diagnostic_only_source_columns"]) + and bool(item["explicit_asec_source_contract"]) + and bool(item["asec_mappings_not_adopted"]) + and item["presence_certifies_observation"] is False + and bool(item["presence_caveats"]) + and all( + bool(entry["mapping"]) and bool(entry["refusal"]) + for entry in item["asec_mappings_not_adopted"] + ) + for item in demographics["unbound"] + ), + "BINDING_UNBOUND_DEMOGRAPHICS", + ) + # Presence varies with the prepared roster. The required columns and their + # semantics do not: a transported document cannot redefine this registry. + declared_presence = [ + name + for item in demographics["unbound"] + for name in ( + *item["present_in_prepared_arm"], + *item["diagnostic_only_present_in_prepared_arm"], + ) + ] + _require( + demographics["unbound"] == _demographics(declared_presence, {})["unbound"], + "BINDING_UNBOUND_DEMOGRAPHICS", + ) + refuse_unbound_demographic_writes(item["column"] for item in demographics["bound"]) + _require( + all( + _is_digest(value) + for key, value in document["inputs"].items() + if key.endswith("_sha256") + ) + and all( + _is_digest(value) for value in document["inputs"]["producers"].values() + ), + "BINDING_INPUT_DIGESTS", + ) + _require( + type(document["composed_is_whole_arm"]) is bool + and bool(document["cohorts"]) + and all( + type(count) is int and count > 0 for count in document["cohorts"].values() + ), + "BINDING_COHORTS", + ) + return document + + +#: The resolved source rows, as a framed binary artifact rather than a JSON list. +#: The measurement nodes need the arm's original IDs and its source positions to +#: reconstruct the reviewed accounting, and they must get them without reading a +#: provenance column. This carries exactly those four int64 buffers, bound to the +#: binding document that already digests every one of them. +ARM_ROWS_MAGIC = b"MCCASECR\x01" +ARM_ROWS_KIND = "microcosm.us.composed_asec_arm_rows.v1" +_ARM_ROWS_BUFFERS = ( + ("person", "source_positions"), + ("person", "original_ids"), + ("household", "source_positions"), + ("household", "original_ids"), +) +_ARM_ROWS_HEADER_KEYS = frozenset( + { + "schema_version", + "artifact_kind", + "release_eligible", + "binding_sha256", + "buffers", + "rows", + "dtype", + } +) +ARM_ROWS_PAYLOAD_MAX_BYTES = ( + len(ARM_ROWS_MAGIC) + + 4 + + HEADER_MAX_BYTES + + 16 * (MAX_PERSONS + MAX_HOUSEHOLDS) + + 32 +) +_ARM_ROWS_TOKEN = object() + + +@dataclass(frozen=True) +class BoundComposedAsecArmRows: + """The arm's resolved source rows, admitted only against its binding document.""" + + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _ARM_ROWS_TOKEN, "ARM_ROWS_CONSTRUCTOR") + + def array(self, entity: str, name: str) -> np.ndarray: + _require((entity, name) in _ARM_ROWS_BUFFERS, "ARM_ROWS_BUFFER") + return np.frombuffer( + self.buffers[_ARM_ROWS_BUFFERS.index((entity, name))], dtype=" tuple[bytes, tuple[bytes, ...]]: + values = {"source_positions": positions, "original_ids": original_ids} + buffers = tuple( + np.ascontiguousarray(values[name][entity], dtype=" bytes: + """Frame the resolved source rows with a transport checksum over their bytes.""" + header, buffers = _arm_rows_parts( + positions, original_ids, binding_sha256=binding_sha256 + ) + payload = ( + ARM_ROWS_MAGIC + struct.pack(" BoundComposedAsecArmRows: + """Admit the resolved rows only against the binding document that digests them. + + Every buffer is re-digested here in the binding document's own encodings: the + original IDs through the graph context's ordered-identity digest and the + source positions through their raw bytes. A payload that does not reproduce + both, for the exact row counts the binding declares, is refused. + """ + _require( + type(payload) is bytes + and len(ARM_ROWS_MAGIC) + 4 + 32 < len(payload) <= ARM_ROWS_PAYLOAD_MAX_BYTES, + "ARM_ROWS_SIZE", + ) + _require(payload.startswith(ARM_ROWS_MAGIC), "ARM_ROWS_MAGIC") + _require(_sha(payload[:-32]) == payload[-32:].hex(), "ARM_ROWS_CHECKSUM") + size = struct.unpack_from("= 0).all()) + and (len(places) < 2 or bool((np.diff(places) > 0).all())), + f"ARM_ROWS_ORDER:{entity}", + ) + declared = binding_document["arm"][entity] + _require( + _row_identity( + pd.DataFrame({US_SCHEMA.entity_id_column(entity): ids}), entity + )["ordered_ids_sha256"] + == declared["original_ordered_ids_sha256"], + f"ARM_ROWS_IDENTITY:{entity}", + ) + _require( + _sha(places.tobytes()) == declared["source_positions_sha256"], + f"ARM_ROWS_POSITIONS:{entity}", + ) + return bound + + +def arm_row_declarations() -> tuple[Owned, ...]: + """The boolean arm-membership cells this node owns, in one canonical order.""" + return tuple( + Owned(entity, arm_row_column(entity), ARM_ROW_DTYPE) + for entity in BOUND_ENTITIES + ) + + +def bind_node_inputs() -> tuple[Slice, ...]: + """The declared views the resolution reads, and nothing else.""" + provenance = tuple( + ( + support_channel_column(entity), + spine_source_id_column(entity), + support_source_id_column(entity), + ) + for entity in US_SCHEMA.entities + ) + extra = { + "person": (COHORT_COLUMN, *PERSON_COORDINATE_CROSSCHECKS), + "household": HOUSEHOLD_EVIDENCE_COLUMNS, + } + return tuple( + Slice(entity, (*columns, *extra.get(entity, ()))) + for entity, columns in zip(US_SCHEMA.entities, provenance, strict=True) + ) + + +def bind_node_artifact_inputs(*, population_context: str) -> tuple[ArtifactInput, ...]: + """The population's own context plus the composed CREATE's carried evidence.""" + return ( + ArtifactInput( + "frame_context", population_context, "frame_context", US_FRAME_CONTEXT_TYPE + ), + *( + ArtifactInput(name, CREATE_NODE, name, kind) + for name, kind in _ARTIFACT_TYPES + if name != "frame_context" + ), + ) + + +_BIND_ARTIFACTS = ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("asec_binding", US_COMPOSED_ASEC_BINDING_TYPE), + ArtifactOutput("arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE), + ArtifactOutput("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), +) + + +def composed_asec_bind_node(*, population: str, population_context: str) -> Node: + """Declare the one binding node over an already composed population version.""" + return Node( + id=BIND_NODE, + kernel=USComposedAsecBindKernel.ref, + population=population, + inputs=bind_node_inputs(), + outputs=arm_row_declarations(), + params={"phase": COMPOSED_ASEC_PHASE}, + artifact_inputs=bind_node_artifact_inputs( + population_context=population_context + ), + artifact_outputs=_BIND_ARTIFACTS, + ) + + +class USComposedAsecBindKernel(KernelBase): + """Resolve the ASEC arm's composed rows and slice its evidence to them.""" + + ref = "us.composed_population.asec_bind@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=COMPOSED_ASEC_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return implementation_hash(COMPOSED_ASEC_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(set(context.params) == {"phase"}, "NODE_PARAMS") + _require(context.params["phase"] == COMPOSED_ASEC_PHASE, "NODE_PHASE") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require(not node.sources, "NODE_SOURCES") + _require(node.inputs == bind_node_inputs(), "NODE_SLICES") + _require(node.outputs == arm_row_declarations(), "NODE_OUTPUTS") + _require( + set(context.artifacts) == {name for name, _kind in _ARTIFACT_TYPES}, + "NODE_ARTIFACT_INPUTS", + ) + _require(node.artifact_outputs == _BIND_ARTIFACTS, "NODE_ARTIFACT_OUTPUTS") + _require( + { + edge.producer + for edge in node.artifact_inputs + if edge.name in _CREATE_EDGES + } + == {CREATE_NODE}, + "NODE_EVIDENCE_PRODUCER", + ) + values = { + name: _artifact(context, name, kind) for name, kind in _ARTIFACT_TYPES + } + create_producer = _same_producer(context, _CREATE_EDGES) + binding = resolve_composed_asec_binding( + {entity: context.tables[entity] for entity in US_SCHEMA.entities}, + context_payload=values["frame_context"].payload, + origin_payload=values["source_origin"].payload, + prepared_context_payload=values["asec_frame_context"].payload, + money_payload=values["current_money"].payload, + receipt_payload=values["prepared_receipt"].payload, + housing_payload=values["housing_universe"].payload, + income_payload=values["income_observations"].payload, + producers={ + "population": values["frame_context"].producer_key, + "prepared_source": create_producer, + "source_origin": values["source_origin"].producer_key, + }, + ) + columns = {} + for entity in BOUND_ENTITIES: + id_column = US_SCHEMA.entity_id_column(entity) + index = pd.Index( + context.tables[entity][id_column].to_numpy(), name=id_column + ) + columns[(entity, arm_row_column(entity))] = pd.Series( + binding.masks[entity], index=index, dtype=ARM_ROW_DTYPE + ) + payload = binding.payload + return KernelResult( + columns=columns, + artifacts={ + "frame_context": canonical_json(binding.context_document), + "asec_binding": payload, + "arm_rows": encode_composed_asec_arm_rows( + binding.positions, + binding.original_ids, + binding_sha256=_sha(payload), + ), + "selected_current_money": binding.selected_money, + }, + receipt={ + "phase": COMPOSED_ASEC_PHASE, + "implementation": implementation_manifest(COMPOSED_ASEC_STAGE), + "binding_sha256": _sha(payload), + "selected_money_sha256": _sha(binding.selected_money), + "arm_rows": { + entity: int(binding.masks[entity].sum()) + for entity in BOUND_ENTITIES + }, + "cohorts": binding.document["cohorts"], + "demographics": binding.document["demographics"], + "release_eligible": False, + "certified": False, + }, + ) + + +__all__ = [ + "ARM_CHANNEL", + "ARM_ROW_DTYPE", + "ARM_ROW_SUFFIX", + "BINDING_SCHEMA", + "BIND_NODE", + "BOUND_ENTITIES", + "COHORT_COLUMN", + "COMPOSED_ASEC_DEPENDENCIES", + "COMPOSED_ASEC_PHASE", + "COMPOSED_ASEC_STAGE", + "PERSON_COORDINATE_CROSSCHECKS", + "UNBOUND_DEMOGRAPHIC_COLUMNS", + "US_COMPOSED_ASEC_ARM_ROWS_TYPE", + "US_COMPOSED_ASEC_BINDING_TYPE", + "BoundComposedAsecArmRows", + "ComposedAsecBinding", + "USComposedAsecBindKernel", + "arm_mask", + "arm_row_column", + "arm_row_declarations", + "bind_composed_asec_arm_rows", + "bind_composed_asec_document", + "bind_node_artifact_inputs", + "bind_node_inputs", + "composed_asec_bind_node", + "encode_composed_asec_arm_rows", + "refuse_unbound_demographic_writes", + "resolve_composed_asec_binding", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py new file mode 100644 index 000000000..a945506b4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py @@ -0,0 +1,652 @@ +"""Corrected ASEC monetary and demographic measurements over composed ASEC rows. + +Two pure nodes hang off :mod:`.graph_composed_asec_binding`. Each writes only +under the binding's declared boolean arm mask, so the executor holds every +native ACS cell — value and missingness alike — byte-identical: a masked +``Owned`` is checked by ``population._patch_columns``, which refuses any change +to non-owned storage. Where a column is new the opposite arm keeps a declared +null; nothing is inferred as zero. + +Neither kernel reads a source-channel or spine-source-ID column. They receive +the arm's rows through the declared mask and the arm's source coordinates +through the typed ``arm_rows`` artifact, both already resolved and digested by +the binding node. The arithmetic is the accepted one, unchanged and imported: +:func:`~.cps_carried_current.derive_cps_carried_current_leaves` for the +corrected leaves (which is also where ``age`` comes from, mapped from ``A_AGE``) +and :func:`~.graph_asec_income.derive_reported_income` for the six reported +observations. Raw source observations and their quality/zero/allocation axes +stay inside those artifacts; only the computed quantities become cells. + +Nothing here is a release, a calibration, a transfer or a score. +""" + +from __future__ import annotations + +import json + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from .asec_current_money import _sha +from .asec_current_money_selection import ( + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + decode_selected_current_money, +) +from .cps_carried_current import ( + CPS_CARRIED_CURRENT_PERSON_LEAVES, + CPS_CARRIED_CURRENT_ROUTING_COLUMNS, + CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES, + cps_carried_current_leaf_contract, + derive_cps_carried_current_leaves, +) +from .graph_asec_income import ( + RESULT_COLUMNS, + US_ASEC_INCOME_OBSERVATIONS_TYPE, + US_ASEC_REPORTED_INCOME_TYPE, + bind_income_observations, + derive_reported_income, + encode_reported_income, +) +from .graph_asec_prepared import PreparedGraphError, _artifact, _same_producer +from .graph_composed_asec_binding import ( + BIND_NODE, + COHORT_COLUMN, + COMPOSED_ASEC_DEPENDENCIES, + COMPOSED_ASEC_PHASE, + COMPOSED_ASEC_STAGE, + US_COMPOSED_ASEC_ARM_ROWS_TYPE, + US_COMPOSED_ASEC_BINDING_TYPE, + USComposedAsecBindKernel, + arm_row_column, + bind_composed_asec_arm_rows, + bind_composed_asec_document, + composed_asec_bind_node, + refuse_unbound_demographic_writes, +) +from .graph_composed_contracts import ( + LEAVES_NODE as LEAVES_NODE, +) +from .graph_composed_contracts import ( + REPORTED_INCOME_NODE as REPORTED_INCOME_NODE, +) +from .graph_composed_population import ( + COMPOSED_SOURCES, + CREATE_NODE, + GEOGRAPHY_PHASE, + HARMONIZE_NODE, + composed_population_nodes, + composed_population_registry, +) +from .graph_context import US_FRAME_CONTEXT_TYPE, _decode, _mass_records, _row_identity +from .graph_geography import LOOKUP_SOURCES, us_geography_nodes +from .graph_implementation import implementation_hash, implementation_manifest + +LEAF_DTYPE = "float64" + +#: Corrected leaves the native ACS arm already carries under the same name. +#: On those the masked write fills the ASEC rows and the executor holds every +#: ACS cell byte-identical; on every other leaf the column is new and the ACS +#: rows stay declared-null. Any other incumbent is unreviewed and refuses. +ACS_SHARED_LEAVES = ( + "age", + "employment_income_before_lsr", + "self_employment_income_before_lsr", +) + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def _leaf_coordinates() -> tuple[tuple[str, str], ...]: + return ( + *(("person", name) for name in CPS_CARRIED_CURRENT_PERSON_LEAVES), + *(("spm_unit", name) for name in CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + ) + + +def composed_leaf_declarations() -> tuple[Owned, ...]: + """The corrected-leaf cells, owned only at the bound arm's rows. + + The binding document records ``is_female`` and ``is_household_head`` as + unbound on this population. That record stays honest only while no node of + the stage writes them, so the declaration itself refuses a leaf roster that + would (``ASEC_DEMOGRAPHIC_COLUMN_BOUND``) rather than letting a reviewed + mapping quietly turn a recorded gap into an unreviewed binding. + """ + coordinates = _leaf_coordinates() + refuse_unbound_demographic_writes(column for _entity, column in coordinates) + return tuple( + Owned(entity, column, LEAF_DTYPE, rows=arm_row_column(entity)) + for entity, column in coordinates + ) + + +def composed_reported_income_declarations() -> tuple[Owned, ...]: + """The six reported-income cells, owned only at the bound arm's rows.""" + refuse_unbound_demographic_writes(RESULT_COLUMNS) + return tuple( + Owned("person", name, LEAF_DTYPE, rows=arm_row_column("person")) + for name in RESULT_COLUMNS + ) + + +def _document(value) -> dict: + """Decode the producing node's context, keeping every graph-context check.""" + document = _decode(value.payload) + _mass_records(document["mass_log"]) + _require(canonical_json(document) == value.payload, "CONTEXT_CANONICAL") + return document + + +def _bound_population(document: dict, binding: dict) -> None: + """The context and the binding must describe one composed population.""" + for entity in US_SCHEMA.entities: + declared = document["entities"][entity] + recorded = binding["composed"][entity] + _require( + declared["rows"] == recorded["rows"] + and declared["ordered_ids_sha256"] == recorded["ordered_ids_sha256"], + f"BINDING_POPULATION_IDENTITY:{entity}", + ) + for entity, column in binding["arm_row_columns"].items(): + _require( + column == arm_row_column(entity) + and column in document["entities"][entity]["columns"], + f"BINDING_ARM_COLUMN:{entity}", + ) + + +def _arm_rows(table: pd.DataFrame, entity: str, binding: dict) -> np.ndarray: + """The masked view's own ids, checked against the binding's recorded arm.""" + id_column = US_SCHEMA.entity_id_column(entity) + ids = table[id_column].to_numpy(dtype="int64") + recorded = binding["arm"][entity] + _require(len(ids) == recorded["rows"], f"ARM_VIEW_ROWS:{entity}") + _require( + _row_identity(pd.DataFrame({id_column: ids}), entity)["ordered_ids_sha256"] + == recorded["composed_ordered_ids_sha256"], + f"ARM_VIEW_IDENTITY:{entity}", + ) + column = arm_row_column(entity) + _require( + column in table and bool(table[column].to_numpy().all()), + f"ARM_VIEW_MASK:{entity}", + ) + return ids + + +def _selected_money(binding: dict, prepared: dict, values): + """Decode the arm's money slice against the binding it was cut for.""" + selected = decode_selected_current_money( + values["selected_current_money"].payload, + expected_parent_header_sha256=prepared["money_header_sha256"], + expected_parent_content_sha256=prepared["money_content_sha256"], + expected_prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + expected_selection_sha256=_sha(values["asec_binding"].payload), + ) + header = selected.header_data + _require( + header["person_identity_sha256"] + == binding["arm"]["person"]["original_ordered_ids_sha256"] + and header["household_identity_sha256"] + == binding["arm"]["household"]["original_ordered_ids_sha256"], + "SELECTED_ARM_COORDINATES", + ) + _require( + selected.person_rows == binding["arm"]["person"]["rows"] + and selected.household_rows == binding["arm"]["household"]["rows"], + "SELECTED_ARM_ROWS", + ) + return selected + + +def _binding(values) -> dict: + binding = bind_composed_asec_document(values["asec_binding"].payload) + _require( + binding["inputs"]["prepared_receipt_sha256"] + == _sha(values["prepared_receipt"].payload), + "BINDING_PREPARED_RECEIPT", + ) + _require( + binding["inputs"]["producers"]["prepared_source"] + == values["prepared_receipt"].producer_key, + "BINDING_SOURCE_PRODUCER", + ) + return binding + + +def _prepared(value) -> dict: + """The producing CREATE's own receipt, re-read as canonical bytes.""" + document = json.loads(value.payload) + _require(canonical_json(document) == value.payload, "PREPARED_RECEIPT_JSON") + _require(document["release_eligible"] is False, "PREPARED_RECEIPT_SCHEMA") + return document + + +def _new_columns(document: dict, coordinates) -> None: + """Record only the coordinates this stage actually adds to the population.""" + for entity, column in coordinates: + columns = document["entities"][entity]["columns"] + if column not in columns: + columns.append(column) + + +class USComposedAsecLeavesKernel(KernelBase): + """Derive the corrected monetary and age leaves over the bound arm's rows.""" + + ref = "us.composed_population.asec_cps_carried_current@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=COMPOSED_ASEC_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return implementation_hash(COMPOSED_ASEC_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(set(context.params) == {"phase"}, "NODE_PARAMS") + _require(context.params["phase"] == COMPOSED_ASEC_PHASE, "NODE_PHASE") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require(not node.sources, "NODE_SOURCES") + _require(node.inputs == _leaves_inputs(), "NODE_SLICES") + _require(node.outputs == composed_leaf_declarations(), "NODE_OUTPUTS") + _require( + set(context.artifacts) + == { + "frame_context", + "asec_binding", + "arm_rows", + "selected_current_money", + "prepared_receipt", + }, + "NODE_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == (ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + "NODE_ARTIFACT_OUTPUTS", + ) + values = { + name: _artifact(context, name, kind) + for name, kind in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("asec_binding", US_COMPOSED_ASEC_BINDING_TYPE), + ("arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE), + ("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ) + } + _same_producer( + context, + ("frame_context", "asec_binding", "arm_rows", "selected_current_money"), + ) + binding = _binding(values) + prepared = _prepared(values["prepared_receipt"]) + document = _document(values["frame_context"]) + _bound_population(document, binding) + incumbent = sorted( + f"{entity}.{column}" + for entity, column in _leaf_coordinates() + if column in document["entities"][entity]["columns"] + ) + _require( + set(incumbent) <= {f"person.{name}" for name in ACS_SHARED_LEAVES}, + f"UNREVIEWED_LEAF_INCUMBENT:{incumbent}", + ) + person, spm_unit = context.tables["person"], context.tables["spm_unit"] + person_ids = _arm_rows(person, "person", binding) + spm_ids = _arm_rows(spm_unit, "spm_unit", binding) + selected = _selected_money(binding, prepared, values) + rows = bind_composed_asec_arm_rows( + values["arm_rows"].payload, binding_document=binding + ) + _require( + len(rows.array("person", "original_ids")) == len(person_ids), + "ARM_ROWS_PERSON_ALIGNMENT", + ) + leaves = derive_cps_carried_current_leaves( + selected, + routing=person.loc[:, list(CPS_CARRIED_CURRENT_ROUTING_COLUMNS)], + spm_membership=person[US_SCHEMA.membership_column("spm_unit")].to_numpy( + dtype="int64" + ), + spm_ids=spm_ids, + ) + person_index = pd.Index(person_ids, name=US_SCHEMA.entity_id_column("person")) + spm_index = pd.Index(spm_ids, name=US_SCHEMA.entity_id_column("spm_unit")) + columns = { + ("person", name): pd.Series(values_, index=person_index, dtype=LEAF_DTYPE) + for name, values_ in leaves.person.items() + } + columns.update( + { + ("spm_unit", name): pd.Series( + values_, index=spm_index, dtype=LEAF_DTYPE + ) + for name, values_ in leaves.spm_unit.items() + } + ) + _new_columns(document, _leaf_coordinates()) + return KernelResult( + columns=columns, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": COMPOSED_ASEC_PHASE, + "implementation": implementation_manifest(COMPOSED_ASEC_STAGE), + "contract": cps_carried_current_leaf_contract(), + "binding_sha256": _sha(values["asec_binding"].payload), + "selected_money_sha256": _sha(values["selected_current_money"].payload), + "arm_person_rows": int(len(person_ids)), + "arm_spm_unit_rows": int(len(spm_ids)), + "acs_shared_incumbents": incumbent, + "demographics": binding["demographics"], + "release_eligible": False, + "certified": False, + }, + ) + + +class USComposedAsecReportedIncomeKernel(KernelBase): + """Reconstruct the six reported observations over the bound arm's rows.""" + + ref = "us.composed_population.asec_reported_income@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=COMPOSED_ASEC_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return implementation_hash(COMPOSED_ASEC_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(set(context.params) == {"phase"}, "NODE_PARAMS") + _require(context.params["phase"] == COMPOSED_ASEC_PHASE, "NODE_PHASE") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require(not node.sources, "NODE_SOURCES") + _require(node.inputs == _reported_income_inputs(), "NODE_SLICES") + _require( + node.outputs == composed_reported_income_declarations(), "NODE_OUTPUTS" + ) + _require( + set(context.artifacts) + == { + "frame_context", + "asec_binding", + "arm_rows", + "selected_current_money", + "prepared_receipt", + "income_observations", + }, + "NODE_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == ( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + "NODE_ARTIFACT_OUTPUTS", + ) + values = { + name: _artifact(context, name, kind) + for name, kind in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("asec_binding", US_COMPOSED_ASEC_BINDING_TYPE), + ("arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE), + ("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ) + } + binding_producer = _same_producer( + context, ("asec_binding", "arm_rows", "selected_current_money") + ) + source_producer = _same_producer( + context, ("prepared_receipt", "income_observations") + ) + binding = _binding(values) + prepared = _prepared(values["prepared_receipt"]) + document = _document(values["frame_context"]) + _bound_population(document, binding) + _require( + not set(RESULT_COLUMNS) & set(document["entities"]["person"]["columns"]), + "OWNED_LEAF_INCUMBENT", + ) + person = context.tables["person"] + person_ids = _arm_rows(person, "person", binding) + selected = _selected_money(binding, prepared, values) + rows = bind_composed_asec_arm_rows( + values["arm_rows"].payload, binding_document=binding + ) + income = bind_income_observations( + values["income_observations"].payload, prepared_receipt=prepared + ) + _require( + person[COHORT_COLUMN].dtype == np.dtype("int64"), "ARM_SOURCE_YEAR_DTYPE" + ) + result = derive_reported_income( + selected, + income, + # The arm's own original person identity, resolved and digested by + # the binding node; the composed row ids never enter the accounting. + person_ids=rows.array("person", "original_ids"), + income_years=person[COHORT_COLUMN].to_numpy(dtype="int64"), + person_positions_sha256=binding["arm"]["person"]["source_positions_sha256"], + prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + selection_sha256=_sha(values["asec_binding"].payload), + selected_money_sha256=_sha(values["selected_current_money"].payload), + income_payload_sha256=_sha(values["income_observations"].payload), + source_producer_key=source_producer, + selection_producer_key=binding_producer, + frame_context_sha256=_sha(values["frame_context"].payload), + ) + index = pd.Index(person_ids, name=US_SCHEMA.entity_id_column("person")) + columns = { + ("person", name): pd.Series( + result.array(name), index=index, dtype=LEAF_DTYPE + ) + for name in RESULT_COLUMNS + } + _new_columns(document, tuple(("person", name) for name in RESULT_COLUMNS)) + payload = encode_reported_income(result) + return KernelResult( + columns=columns, + artifacts={ + "reported_income": payload, + "frame_context": canonical_json(document), + }, + receipt={ + "phase": COMPOSED_ASEC_PHASE, + "implementation": implementation_manifest(COMPOSED_ASEC_STAGE), + "accounting_sha256": _sha(payload), + "binding_sha256": _sha(values["asec_binding"].payload), + "arm_person_rows": int(len(person_ids)), + "release_eligible": False, + "certified": False, + }, + ) + + +def _leaves_inputs() -> tuple[Slice, ...]: + return ( + Slice( + "person", + (*CPS_CARRIED_CURRENT_ROUTING_COLUMNS, arm_row_column("person")), + rows=arm_row_column("person"), + ), + Slice( + "spm_unit", + (arm_row_column("spm_unit"),), + rows=arm_row_column("spm_unit"), + ), + ) + + +def _reported_income_inputs() -> tuple[Slice, ...]: + return ( + Slice( + "person", + (COHORT_COLUMN, arm_row_column("person")), + rows=arm_row_column("person"), + ), + ) + + +def composed_asec_measure_nodes(*, population: str, population_context: str): + """Bind the arm, then the corrected leaves, then the reported observations.""" + bind = composed_asec_bind_node( + population=population, population_context=population_context + ) + binding_artifacts = ( + ArtifactInput( + "asec_binding", BIND_NODE, "asec_binding", US_COMPOSED_ASEC_BINDING_TYPE + ), + ArtifactInput( + "arm_rows", BIND_NODE, "arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE + ), + ArtifactInput( + "selected_current_money", + BIND_NODE, + "selected_current_money", + US_ASEC_SELECTED_MONEY_TYPE, + ), + ) + prepared_receipt = ArtifactInput( + "prepared_receipt", + CREATE_NODE, + "prepared_receipt", + US_ASEC_PREPARED_RECEIPT_TYPE, + ) + leaves = Node( + id=LEAVES_NODE, + kernel=USComposedAsecLeavesKernel.ref, + population=population, + inputs=_leaves_inputs(), + outputs=composed_leaf_declarations(), + params={"phase": COMPOSED_ASEC_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", BIND_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + *binding_artifacts, + prepared_receipt, + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ) + reported = Node( + id=REPORTED_INCOME_NODE, + kernel=USComposedAsecReportedIncomeKernel.ref, + population=population, + inputs=_reported_income_inputs(), + outputs=composed_reported_income_declarations(), + params={"phase": COMPOSED_ASEC_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", LEAVES_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + *binding_artifacts, + prepared_receipt, + ArtifactInput( + "income_observations", + CREATE_NODE, + "income_observations", + US_ASEC_INCOME_OBSERVATIONS_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + ) + return (bind, leaves, reported) + + +def composed_asec_population_graph( + columns, + *, + sample_fraction: float, + sample_seed: int, + geography: bool = True, +) -> Graph: + """The composed development population plus this arm's bound measurements.""" + composed = composed_population_nodes( + columns, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + if not geography: + return Graph( + country="us", + sources=COMPOSED_SOURCES, + nodes=( + *composed, + *composed_asec_measure_nodes( + population=HARMONIZE_NODE, population_context=HARMONIZE_NODE + ), + ), + ) + geography_nodes = us_geography_nodes( + columns, base=HARMONIZE_NODE, context_producer=HARMONIZE_NODE + ) + return Graph( + country="us", + sources=(*COMPOSED_SOURCES, *LOOKUP_SOURCES), + nodes=( + *composed, + *geography_nodes, + *composed_asec_measure_nodes( + population=f"{GEOGRAPHY_PHASE}.boundary", + population_context=GEOGRAPHY_PHASE, + ), + ), + ) + + +def composed_asec_population_registry(*, geography: bool = True) -> KernelRegistry: + """The composed registry plus exactly this stage's three kernels.""" + registry = composed_population_registry(geography=geography) + registry.register(USComposedAsecBindKernel()) + registry.register(USComposedAsecLeavesKernel()) + registry.register(USComposedAsecReportedIncomeKernel()) + return registry + + +__all__ = [ + "ACS_SHARED_LEAVES", + "LEAF_DTYPE", + "LEAVES_NODE", + "REPORTED_INCOME_NODE", + "USComposedAsecLeavesKernel", + "USComposedAsecReportedIncomeKernel", + "composed_asec_measure_nodes", + "composed_asec_population_graph", + "composed_asec_population_registry", + "composed_leaf_declarations", + "composed_reported_income_declarations", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_contracts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_contracts.py new file mode 100644 index 000000000..2f49f6e89 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_contracts.py @@ -0,0 +1,16 @@ +"""Source-free declarations shared by composed producers and their consumers. + +These are the existing legacy producer IDs and artifact types. Importing their +names must not select legacy source assembly, evaluation, or country resources. +The defining producers re-export the same objects for compatibility; operations +and source authority remain in those producers. +""" + +from microcosm.graph import ArtifactType + +CREATE_NODE = "composed_population.prepare" +BIND_NODE = "composed_population.asec_bind" +LEAVES_NODE = "composed_population.asec_cps_carried_current" +REPORTED_INCOME_NODE = "composed_population.asec_reported_income" +US_COMPOSED_ASEC_BINDING_TYPE = ArtifactType("microcosm.us.composed_asec_binding", 1) +US_COMPOSED_ASEC_ARM_ROWS_TYPE = ArtifactType("microcosm.us.composed_asec_arm_rows", 1) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py new file mode 100644 index 000000000..b07496f8e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py @@ -0,0 +1,751 @@ +"""Compose the prepared ASEC population with the native ACS population. + +This is a separately versioned seam. The reviewed ``asec_raw_stage`` assembly +kernel in :mod:`.graph_sources` keeps its kernel ref, contract, params, sources +and node ids unchanged; nothing here rewrites them. What changes is only which +authenticated ASEC source the ASEC arm reads: the reviewed prepared +current-money directory (``us-asec-prepared-current-money-v3``) instead of the +raw-stage checkpoint. + +Node *keys* are a different matter and do move. Every stage manifest binds the +inventory and the bytes of ``graph_implementation.py``, which is in every +stage's module list, and declaring a new stage necessarily edits both. So every +existing US node key changes and every existing US cache is invalidated — +exactly as when any other attested module changes. + +Why this is a two-source ``CREATE`` and not two parent populations +------------------------------------------------------------------ +``Node.base`` is a single population version. A ``CREATE`` node may declare no +base at all, a structural node may not declare a second ``population``, and a +tuple ``base`` is refused as "not a structural node". So the executor cannot +structurally combine two parent populations; an ``EXPAND`` over one arm can +only invent entrant rows, never bind the other arm's rows or lineage. The +composed population is therefore one ``CREATE`` over two declared sources — +exactly the shape :class:`~.graph_sources.USAssemblyCreateKernel` already uses. +The refusals are recorded in the review root's two-parent probe. + +The stack itself is not reimplemented. Sampling, stacking, provenance and +allocation stay in :mod:`.stacked_spine` / :mod:`.spine_assembly`, and the +harmonization node reuses :class:`~.graph_sources.USSpineHarmonizeKernel` +unchanged by emitting the same two typed producer artifacts it declares. + +Nothing here is a release, a calibration, a certified population, a transfer or +a score. It is an engineering intermediate whose receipts say so. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import ( + CANONICAL_STRING_DTYPE, + canonicalize_frame_string_dtypes, +) +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, +) +from microcosm.graph.canonical import canonical_json + +from .asec_current_money import _sha +from .asec_current_money_selection import ( + US_ASEC_CURRENT_MONEY_BODY_TYPE, + US_ASEC_PREPARED_RECEIPT_TYPE, +) +from .asec_prepared_source import ( + PREPARED_SOURCE_FILES, + PREPARED_SOURCE_KIND, + prepare_asec_current_money_population, +) +from .graph_asec_income import US_ASEC_INCOME_OBSERVATIONS_TYPE +from .graph_asec_prepared import ( + ASEC_PREPARED_CODEC, + ASEC_PREPARED_SOURCE_NAME, + PreparedGraphError, +) +from .graph_composed_contracts import CREATE_NODE as CREATE_NODE +from .graph_context import US_FRAME_CONTEXT_TYPE, _json_data, encode_us_frame_context +from .graph_geography import ( + GEOGRAPHY_PHASE, + LOOKUP_SOURCES, + register_us_geography_kernels, + us_geography_nodes, +) +from .graph_housing_universe import US_ASEC_HOUSING_UNIVERSE_TYPE +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .graph_sources import ( + ACS_CODEC, + ASSEMBLY_PHASE, + US_STACK_PREPARATION_TYPE, + USSpineHarmonizeKernel, + _canonical_assembly, + frame_column_declarations, + load_graph_acs, + us_source_codecs, +) +from .stacked_spine import ACS_STACKED_SUPPORT_CHANNEL, prepare_stacked_spine +from .support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + spine_source_id_column, + support_channel_column, + support_source_id_column, +) + +COMPOSED_STAGE = "composed_population_v1" +COMPOSED_PHASE = "compose_prepared_asec_native_acs" +ACS_NATIVE_SOURCE_NAME = "acs_native" +COMPOSED_DEPENDENCIES = STAGE_DEPENDENCIES[COMPOSED_STAGE] + +PREFIX = "composed_population" +HARMONIZE_NODE = PREFIX + +US_COMPOSED_SOURCE_ORIGIN_TYPE = ArtifactType( + "microcosm.us.composed_population_source_origin", 1 +) + +COMPOSED_SOURCES = ( + SourceRef( + ASEC_PREPARED_SOURCE_NAME, + ASEC_PREPARED_CODEC, + "Reviewed ASEC restoration inputs: P, H, T with its receipt, three " + "housing cohort HDFs and three official Census PERSON CSV members.", + ), + SourceRef( + ACS_NATIVE_SOURCE_NAME, + ACS_CODEC, + "2024 ACS one-year national PUMS archives.", + ), +) + +#: Native source-record identity carried by each arm, on both entities. These +#: are the columns a later node must be able to bind back to an actual archive +#: member, so the origin artifact digests them in addition to — never instead of +#: — the arm's own pre-remap IDs. Every declared column must be present: a +#: silently skipped one would leave an arm unbindable while still receipting. +NATIVE_IDENTITY_COLUMNS = { + BASE_ASEC_SUPPORT_CHANNEL: ( + ("household", "asec_H_SEQ"), + ("person", "PERIDNUM"), + ("person", "PH_SEQ"), + ), + ACS_STACKED_SUPPORT_CHANNEL: ( + ("household", "SERIALNO"), + ("person", "source_household_id"), + ("person", "source_person_id"), + ), +} +#: The per-arm cohort/vintage axis. ASEC restores three income years into one +#: prepared population, so a single scalar vintage would misdescribe it. +COHORT_COLUMN = "source_year" + +_SOURCE_ORIGIN_SCHEMA = "microcosm.us.composed-source-origin.v1" + + +@dataclass(frozen=True) +class ComposedPopulationResult: + """A direct-call composition and everything its CREATE kernel emits.""" + + frame: Frame + preparation: object + dtype_transitions: tuple[dict[str, str], ...] + storage_bridge: tuple[dict[str, str], ...] + source_origin: dict[str, object] + asec_frame_context: bytes + money_payload: bytes + receipt_payload: bytes + housing_universe_payload: bytes + income_observations_payload: bytes + prepared_receipt: Mapping[str, object] + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def bridge_prepared_string_storage(frame: Frame) -> tuple[dict[str, str], ...]: + """Put the prepared arm on the pre-assembly string storage, in place. + + The one explicit schema difference between the two arms. The prepared + source hands back graph column storage (``string[python, pd.NA]``) because + its own CREATE writes graph cells directly; every pre-assembly build + boundary, and therefore the native ACS arm, uses ``CANONICAL_STRING_DTYPE`` + (``string[python, nan]``). ``_stack_spine_tables`` requires shared columns + to have identical dtypes, so one arm has to move. Moving the prepared arm + down leaves the ACS arm byte-identical to today's production assembly, and + :func:`~.graph_sources._canonical_assembly` promotes the assembled frame + back to graph storage afterwards under its own null/value equality checks. + + This is a physical storage bridge only: no value, no missingness and no + column is added, dropped or filled. + """ + transitions = [] + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table.columns: + dtype = table[column].dtype + if isinstance(dtype, pd.StringDtype) and dtype != CANONICAL_STRING_DTYPE: + transitions.append( + { + "entity": entity, + "column": column, + "from": str(dtype), + "to": str(CANONICAL_STRING_DTYPE), + } + ) + before = { + (item["entity"], item["column"]): frame.table(item["entity"])[item["column"]] + .isna() + .to_numpy(copy=True) + for item in transitions + } + canonicalize_frame_string_dtypes( + frame, boundary="US composed population source bridge", in_place=True + ) + for (entity, column), missing in before.items(): + actual = frame.table(entity)[column] + _require( + actual.dtype == CANONICAL_STRING_DTYPE + and np.array_equal(actual.isna().to_numpy(), missing), + f"STRING_BRIDGE_MISSINGNESS:{entity}.{column}", + ) + return tuple(transitions) + + +def _arm_origin(source: Frame, channel: str) -> dict[str, object]: + """Digest one arm's native identity and cohort axis before assembly.""" + identity = {} + for entity, column in NATIVE_IDENTITY_COLUMNS[channel]: + table = source.table(entity) + _require(column in table, f"ARM_NATIVE_IDENTITY:{channel}.{entity}.{column}") + series = table[column] + identity[f"{entity}.{column}"] = { + "dtype": str(series.dtype), + "rows": int(len(series)), + "missing": int(series.isna().sum()), + "ordered_sha256": _sha( + canonical_json( + [ + None if value is None or value is pd.NA else str(value) + for value in series.tolist() + ] + ) + ), + } + person = source.table("person") + _require(COHORT_COLUMN in person, f"ARM_COHORT_COLUMN:{channel}") + years = person[COHORT_COLUMN] + counts = years.value_counts(dropna=False).to_dict() + cohorts = { + ("missing" if pd.isna(year) else str(int(year))): int(count) + for year, count in counts.items() + } + cohorts = {key: cohorts[key] for key in sorted(cohorts)} + return { + "channel": channel, + "rows": {entity: int(source.n(entity)) for entity in source.entities}, + "household_mass": float(source.weights_for("household").total), + "weight_kind": source.weights_for("household").kind.value, + "native_identity": identity, + "person_cohorts": cohorts, + "columns": { + entity: sorted(source.table(entity).columns) for entity in source.entities + }, + } + + +def _source_origin( + frame: Frame, + sources: Mapping[str, Frame], + *, + sampling: Mapping[str, object], + preparation_sha256: str, + prepared_receipt: Mapping[str, object], +) -> dict[str, object]: + """Bind each output row to the arm and the arm's own pre-remap source ID. + + The mapping is read back off the assembled frame's own receipt-validated + support-provenance columns, so it records what assembly actually did rather + than restating what the caller intended. Both provenance IDs are digested: + ``*_spine_source_id`` is the arm's raw pre-remap identifier and is what + binds an output row back to its source row, while ``*_source_id`` is the + assembly-unique pre-clone identifier the clone stage will later offset. At + CREATE the latter equals the entity ID, so digesting it alone would bind + nothing. + """ + mapping: dict[str, object] = {} + for entity in US_SCHEMA.entities: + table = frame.table(entity) + channel_column = support_channel_column(entity) + spine_column = spine_source_id_column(entity) + source_column = support_source_id_column(entity) + _require(channel_column in table, f"ORIGIN_CHANNEL_MISSING:{entity}") + _require(spine_column in table, f"ORIGIN_SPINE_SOURCE_ID_MISSING:{entity}") + _require(source_column in table, f"ORIGIN_SOURCE_ID_MISSING:{entity}") + channels = table[channel_column].astype(str).tolist() + spine_ids = table[spine_column].to_numpy(dtype="int64").tolist() + ids = table[source_column].to_numpy(dtype="int64").tolist() + outputs = ( + table[US_SCHEMA.entity_id_column(entity)].to_numpy(dtype="int64").tolist() + ) + mapping[entity] = { + "rows_by_channel": { + channel: int(channels.count(channel)) + for channel in sorted(set(channels)) + }, + "ordered_channels_sha256": _sha(canonical_json(channels)), + "ordered_spine_source_ids_sha256": _sha(canonical_json(spine_ids)), + "ordered_source_to_output_sha256": _sha( + canonical_json( + [ + [channel, spine_id, source_id, output_id] + for channel, spine_id, source_id, output_id in zip( + channels, spine_ids, ids, outputs, strict=True + ) + ] + ) + ), + } + asec = sources[BASE_ASEC_SUPPORT_CHANNEL] + composed_asec = { + entity: int( + ( + frame.table(entity)[support_channel_column(entity)] + == BASE_ASEC_SUPPORT_CHANNEL + ).sum() + ) + for entity in US_SCHEMA.entities + } + return { + "schema": _SOURCE_ORIGIN_SCHEMA, + "phase": COMPOSED_PHASE, + "preparation_sha256": preparation_sha256, + "sampling": { + "sample_fraction": float(sampling["sample_fraction"]), + "sample_seed": int(sampling["sample_seed"]), + "survey_samples": { + channel: { + key: sample[key] + for key in ( + "realized_household_count", + "selected_household_ids_sha256", + "incoming_household_mass", + "sampled_household_mass", + ) + } + for channel, sample in sampling["survey_samples"].items() + }, + }, + # The four carried prepared artifacts describe the whole prepared ASEC + # population, not the composed rows. They coincide only when the draw + # keeps the arm entire. Recording both counts makes the difference + # readable instead of leaving a later consumer to assume alignment. + "asec_evidence_alignment": { + "basis": "full_prepared_population", + "evidence_rows": { + entity: int(count) + for entity, count in sorted(prepared_receipt["entity_rows"].items()) + }, + "arm_rows": {entity: int(asec.n(entity)) for entity in asec.entities}, + "composed_rows": composed_asec, + "composed_is_whole_arm": all( + composed_asec[entity] == int(asec.n(entity)) + for entity in US_SCHEMA.entities + ), + }, + "arms": { + channel: _arm_origin(source, channel) + for channel, source in sorted(sources.items()) + }, + "assembled": mapping, + "release_eligible": False, + } + + +def compose_from_sources( + asec_prepared_path, + acs_path, + *, + sample_fraction: float, + sample_seed: int, +) -> ComposedPopulationResult: + """The direct-call parity oracle the CREATE kernel shares. + + Source preparation is deliberately separate from sampling: the prepared + ASEC directory and the ACS archives are read whole, and only then does + :func:`prepare_stacked_spine` draw its declared seeded sample. Reading a + fraction of a source is not what this does and would not cost a fraction + of a whole-source read. + """ + prepared = prepare_asec_current_money_population(asec_prepared_path) + asec = prepared.frame + # The carried evidence is positional over the whole prepared population. + # If the receipt and the arm ever disagreed, everything downstream that + # reads the evidence by position would be silently misaligned. + _require( + { + entity: int(count) + for entity, count in prepared.receipt["entity_rows"].items() + } + == {entity: int(asec.n(entity)) for entity in prepared.receipt["entity_rows"]}, + "PREPARED_EVIDENCE_ROWS", + ) + acs = load_graph_acs(acs_path) + storage_bridge = bridge_prepared_string_storage(asec) + # Encoded before stacking, so the identity it declares is the prepared + # arm's own ordered identity, which is what the evidence is positional in. + asec_context = encode_us_frame_context(asec) + arms = { + BASE_ASEC_SUPPORT_CHANNEL: asec, + ACS_STACKED_SUPPORT_CHANNEL: acs, + } + stacked = prepare_stacked_spine( + asec, acs, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + transitions = _canonical_assembly(stacked.frame, (asec, acs)) + preparation_payload = canonical_json(_json_data(stacked.receipt)) + return ComposedPopulationResult( + frame=stacked.frame, + preparation=stacked.receipt, + dtype_transitions=transitions, + storage_bridge=storage_bridge, + source_origin=_source_origin( + stacked.frame, + arms, + sampling=stacked.receipt["sampling"], + preparation_sha256=_sha(preparation_payload), + prepared_receipt=prepared.receipt, + ), + asec_frame_context=asec_context, + money_payload=prepared.money_payload, + receipt_payload=prepared.receipt_payload, + housing_universe_payload=prepared.housing_universe_payload, + income_observations_payload=prepared.income_observations_payload, + prepared_receipt=prepared.receipt, + ) + + +class USComposedPopulationCreateKernel(KernelBase): + """Read both authenticated sources whole and stack them into one spine.""" + + ref = "us.composed_population.prepare@1" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.CREATE, + dependencies=COMPOSED_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + # Runs before cache lookup, including resume=require, exactly as the + # raw-stage assembly kernel does: a renamed or substituted public + # loader cannot claim the declared codec implementation. + us_source_codecs() + return implementation_hash(COMPOSED_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require( + set(context.params) == {"phase", "sample_fraction", "sample_seed"}, + "NODE_PARAMS", + ) + _require(context.params["phase"] == COMPOSED_PHASE, "NODE_PHASE") + _require( + tuple(node.sources) == (ASEC_PREPARED_SOURCE_NAME, ACS_NATIVE_SOURCE_NAME), + "NODE_SOURCES", + ) + _require(not node.inputs and not node.artifact_inputs, "NODE_INPUTS") + _require(node.artifact_outputs == _CREATE_ARTIFACTS, "NODE_ARTIFACT_OUTPUTS") + result = compose_from_sources( + context.sources[ASEC_PREPARED_SOURCE_NAME], + context.sources[ACS_NATIVE_SOURCE_NAME], + sample_fraction=context.params["sample_fraction"], + sample_seed=context.params["sample_seed"], + ) + receipt = result.prepared_receipt + _require(receipt["source_kind"] == PREPARED_SOURCE_KIND, "PREPARED_SOURCE_KIND") + _require( + receipt["file_roster"] == list(PREPARED_SOURCE_FILES), + "PREPARED_FILE_ROSTER", + ) + _require( + frame_column_declarations(result.frame) == node.outputs, + "COMPOSED_COLUMN_INVENTORY", + ) + return KernelResult( + frame=result.frame, + artifacts={ + "frame_context": encode_us_frame_context(result.frame), + "preparation": canonical_json(_json_data(result.preparation)), + "source_origin": canonical_json(result.source_origin), + "asec_frame_context": result.asec_frame_context, + "current_money": result.money_payload, + "prepared_receipt": result.receipt_payload, + "housing_universe": result.housing_universe_payload, + "income_observations": result.income_observations_payload, + }, + receipt={ + "phase": COMPOSED_PHASE, + "implementation": implementation_manifest(COMPOSED_STAGE), + "preparation": result.preparation, + "prepared": receipt, + "dtype_transitions": result.dtype_transitions, + "storage_bridge": result.storage_bridge, + "release_eligible": False, + "certified": False, + }, + ) + + +_CREATE_ARTIFACTS = ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("preparation", US_STACK_PREPARATION_TYPE), + ArtifactOutput("source_origin", US_COMPOSED_SOURCE_ORIGIN_TYPE), + # The prepared arm's own typed context, encoded before stacking. The four + # evidence artifacts below are positional over that population, so a later + # binding node needs its ordered identity to align them to composed rows. + ArtifactOutput("asec_frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ArtifactOutput("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ArtifactOutput("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ArtifactOutput("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), +) + + +#: Top-level keys the origin document must carry, exactly. +_ORIGIN_KEYS = frozenset( + { + "schema", + "phase", + "preparation_sha256", + "sampling", + "asec_evidence_alignment", + "arms", + "assembled", + "release_eligible", + } +) +_ORIGIN_ENTITY_KEYS = frozenset( + { + "rows_by_channel", + "ordered_channels_sha256", + "ordered_spine_source_ids_sha256", + "ordered_source_to_output_sha256", + } +) + + +def bind_composed_source_origin(payload: bytes) -> dict[str, object]: + """Decode the origin artifact as a typed shape, refusing a malformed one. + + This checks canonical bytes and the document's declared shape: its exact + key set, its schema/phase, that both arms and every US entity are present + with well-formed digests and counts, and that it makes no release claim. It + verifies **no** digest against any population — it cannot, since it receives + only bytes. A consumer that needs the mapping to be true of a frame in hand + must recompute the digests against that frame. + """ + document = json.loads(payload) + _require(canonical_json(document) == payload, "ORIGIN_CANONICAL") + _require(isinstance(document, dict), "ORIGIN_DOCUMENT") + _require(set(document) == _ORIGIN_KEYS, "ORIGIN_KEYS") + _require(document["schema"] == _SOURCE_ORIGIN_SCHEMA, "ORIGIN_SCHEMA") + _require(document["phase"] == COMPOSED_PHASE, "ORIGIN_PHASE") + _require(_is_digest(document["preparation_sha256"]), "ORIGIN_PREPARATION_DIGEST") + _require(document["release_eligible"] is False, "ORIGIN_RELEASE_CLAIM") + channels = {BASE_ASEC_SUPPORT_CHANNEL, ACS_STACKED_SUPPORT_CHANNEL} + _require(set(document["arms"]) == channels, "ORIGIN_ARMS") + sampling = document["sampling"] + _require( + set(sampling) == {"sample_fraction", "sample_seed", "survey_samples"} + and set(sampling["survey_samples"]) == channels, + "ORIGIN_SAMPLING", + ) + alignment = document["asec_evidence_alignment"] + _require( + set(alignment) + == { + "basis", + "evidence_rows", + "arm_rows", + "composed_rows", + "composed_is_whole_arm", + } + and alignment["basis"] == "full_prepared_population" + and isinstance(alignment["composed_is_whole_arm"], bool), + "ORIGIN_ALIGNMENT", + ) + _require(set(document["assembled"]) == set(US_SCHEMA.entities), "ORIGIN_ENTITIES") + for entity, record in document["assembled"].items(): + _require(set(record) == _ORIGIN_ENTITY_KEYS, f"ORIGIN_ENTITY_KEYS:{entity}") + _require( + all( + _is_digest(record[key]) + for key in _ORIGIN_ENTITY_KEYS + if key.endswith("_sha256") + ), + f"ORIGIN_ENTITY_DIGEST:{entity}", + ) + _require( + set(record["rows_by_channel"]) <= channels + and all( + isinstance(count, int) and count >= 0 + for count in record["rows_by_channel"].values() + ), + f"ORIGIN_ENTITY_ROWS:{entity}", + ) + return document + + +def _is_digest(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and not set(value) - set("0123456789abcdef") + ) + + +def composed_population_nodes( + columns: Sequence[Owned], *, sample_fraction: float, sample_seed: int +) -> tuple[Node, ...]: + """The two composition nodes: whole-source CREATE, then reused harmonize.""" + inventory = {(owned.entity, owned.column) for owned in columns} + _require(len(inventory) == len(tuple(columns)), "COLUMN_INVENTORY_REPEATS") + for entity in US_SCHEMA.entities: + _require( + (entity, support_channel_column(entity)) in inventory, + f"MISSING_SUPPORT_CHANNEL:{entity}", + ) + _require( + (entity, support_source_id_column(entity)) in inventory, + f"MISSING_SUPPORT_SOURCE_ID:{entity}", + ) + return ( + Node( + id=CREATE_NODE, + kernel=USComposedPopulationCreateKernel.ref, + structural=StructuralDelta.CREATE, + sources=(ASEC_PREPARED_SOURCE_NAME, ACS_NATIVE_SOURCE_NAME), + outputs=tuple(columns), + params={ + "phase": COMPOSED_PHASE, + "sample_fraction": sample_fraction, + "sample_seed": sample_seed, + }, + artifact_outputs=_CREATE_ARTIFACTS, + ), + Node( + id=HARMONIZE_NODE, + kernel=USSpineHarmonizeKernel.ref, + base=CREATE_NODE, + structural=StructuralDelta.REWEIGHT, + inputs=( + Slice("household", (support_channel_column("household"),)), + Slice("person", (support_channel_column("person"),)), + ), + weights=WeightTransition("household", "importance", mass="declared"), + mass="declared", + # The reused production harmonization kernel owns this phase name; + # composing a different ASEC source does not make its operation a + # different one, and its contract is deliberately not re-versioned. + params={"phase": ASSEMBLY_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "preparation", + CREATE_NODE, + "preparation", + US_STACK_PREPARATION_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ), + ) + + +def composed_population_graph( + columns: Sequence[Owned], + *, + sample_fraction: float, + sample_seed: int, + geography: bool = True, +) -> Graph: + """Declare the composed development population, never a release pool.""" + nodes = composed_population_nodes( + columns, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + if not geography: + return Graph(country="us", sources=COMPOSED_SOURCES, nodes=nodes) + return Graph( + country="us", + sources=(*COMPOSED_SOURCES, *LOOKUP_SOURCES), + nodes=( + *nodes, + *us_geography_nodes( + columns, base=HARMONIZE_NODE, context_producer=HARMONIZE_NODE + ), + ), + ) + + +def composed_population_registry(*, geography: bool = True) -> KernelRegistry: + """Register exactly the kernels this graph runs; two of three are reused.""" + registry = KernelRegistry() + registry.register(USComposedPopulationCreateKernel()) + registry.register(USSpineHarmonizeKernel()) + if geography: + register_us_geography_kernels(registry) + return registry + + +__all__ = [ + "ACS_NATIVE_SOURCE_NAME", + "COHORT_COLUMN", + "COMPOSED_DEPENDENCIES", + "COMPOSED_PHASE", + "COMPOSED_SOURCES", + "COMPOSED_STAGE", + "CREATE_NODE", + "GEOGRAPHY_PHASE", + "HARMONIZE_NODE", + "NATIVE_IDENTITY_COLUMNS", + "US_COMPOSED_SOURCE_ORIGIN_TYPE", + "ComposedPopulationResult", + "USComposedPopulationCreateKernel", + "bind_composed_source_origin", + "bridge_prepared_string_storage", + "compose_from_sources", + "composed_population_graph", + "composed_population_nodes", + "composed_population_registry", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py new file mode 100644 index 000000000..c1732455a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py @@ -0,0 +1,278 @@ +"""Identity-bound metadata for real US operators in the population graph. + +Population cells, weights and strata remain executor-owned. This typed artifact +carries the metadata and prior mass records that existing US operators require, +so they cannot silently reload context from an unrelated mutable checkpoint. +Reconstruction uses only the node's declared table views; it grants no access to +undeclared columns or a live population object. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import asdict + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA, Frame, MassChangeRecord +from microcosm.graph import ArtifactType, KernelContext +from microcosm.graph.canonical import canonical_json + +US_FRAME_CONTEXT_TYPE = ArtifactType("microcosm.us.frame_context", 1) + +_FIELDS = {"schema_version", "entities", "metadata", "mass_log", "weight_sources"} +_ENTITY_FIELDS = {"columns", "rows", "id_dtype", "ordered_ids_sha256"} +_MASS_FIELDS = {"entity", "old_total", "new_total", "declared_factor", "reason"} + +# These five receipts are read as authority by the existing US operators. +# Operational observations belong in run receipts, outside this keyed edge. +# An additional normative receipt requires an explicit contract revision here. +US_NORMATIVE_METADATA_KEYS = frozenset( + { + "us_spine_assembly_manifest", + "us_stacked_spine_manifest", + "us_puf_clone_attachment_manifest", + "us_late_producer_transition_authority", + "acs_pums_earnings_universe_application", + } +) +_RUN_FIELDS = frozenset( + { + "created_at", + "updated_at", + "timestamp", + "started_at", + "finished_at", + "elapsed_seconds", + "wall_time_seconds", + "duration_seconds", + "peak_rss", + "hostname", + "host", + "pid", + "run_id", + "path", + "checkpoint_dir", + "output_dir", + "timings", + "cache_hit", + "cache_hits", + } +) + + +def _normative_metadata(metadata: Mapping[str, object]) -> object: + unknown = set(metadata) - US_NORMATIVE_METADATA_KEYS + if unknown: + raise ValueError( + f"US graph context has undeclared metadata: {sorted(unknown)}." + ) + + def validate(value: object) -> None: + if isinstance(value, Mapping): + if set(value) & _RUN_FIELDS: + raise ValueError("US graph context cannot contain run-level metadata.") + for item in value.values(): + validate(item) + elif isinstance(value, list | tuple): + for item in value: + validate(item) + + validate(metadata) + return _json_data(metadata) + + +def _json_data(value: object) -> object: + """Normalize NumPy metadata without coercing unknown objects to strings.""" + if isinstance(value, np.generic): + return _json_data(value.item()) + if isinstance(value, np.ndarray): + return _json_data(value.tolist()) + if isinstance(value, Mapping): + if any(not isinstance(key, str) for key in value): + raise ValueError("US graph context metadata requires string mapping keys.") + return {key: _json_data(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_json_data(item) for item in value] + # canonical_json validates scalar types and refuses nonfinite numbers. + return value + + +def _row_identity(table: pd.DataFrame, entity: str) -> dict[str, object]: + column = US_SCHEMA.entity_id_column(entity) + if column not in table: + raise ValueError(f"US graph context {entity} identity column is missing.") + ids = table[column] + if not pd.api.types.is_integer_dtype(ids.dtype) or ids.isna().any(): + raise ValueError(f"US graph context {entity} identity must be integral.") + if len(ids) and (int(ids.min()) < 0 or int(ids.max()) > np.iinfo(np.int64).max): + raise ValueError(f"US graph context {entity} identity exceeds int64 bounds.") + values = ids.to_numpy(dtype=" bytes: + """Encode current US metadata and row identities, without population cells.""" + if frame.schema != US_SCHEMA: + raise ValueError("US graph context requires the US entity schema.") + entities = {} + for entity in US_SCHEMA.entities: + table = frame.table(entity) + if entity in US_SCHEMA.group_entities and ( + not isinstance(table.index, pd.RangeIndex) + or table.index.start != 0 + or table.index.stop != len(table) + or table.index.step != 1 + or table.index.name is not None + ): + # Version 1 carries group IDs, not pandas index descriptors. An + # identity FILTER uses Frame.select, which resets group indices. + # Refuse an unsupported representation before emitting authority + # that downstream person-only nodes cannot independently inspect. + raise ValueError( + f"US graph context {entity} requires a default unnamed RangeIndex." + ) + if any(not isinstance(column, str) for column in table.columns): + raise ValueError("US graph context column names must be strings.") + entities[entity] = { + "columns": list(table.columns), + **_row_identity(table, entity), + } + document = { + "schema_version": US_FRAME_CONTEXT_TYPE.schema_version, + "entities": entities, + "metadata": _normative_metadata(frame.metadata), + "mass_log": [_json_data(asdict(record)) for record in frame.mass_log], + "weight_sources": { + entity: frame.weights_for(entity).kind.value + for entity in frame.weighted_entities + }, + } + return canonical_json(document) + + +def _decode(payload: bytes) -> dict[str, object]: + def unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"Duplicate US graph context field {key!r}.") + result[key] = value + return result + + try: + document = json.loads(payload, object_pairs_hook=unique_pairs) + canonical_json(document) + except (UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError) as error: + raise ValueError( + "US graph context must be finite, unambiguous JSON." + ) from error + if ( + not isinstance(document, dict) + or set(document) != _FIELDS + or type(document["schema_version"]) is not int + or document["schema_version"] != US_FRAME_CONTEXT_TYPE.schema_version + ): + raise ValueError("US graph context has an unsupported schema.") + entities = document["entities"] + if not isinstance(entities, dict) or set(entities) != set(US_SCHEMA.entities): + raise ValueError("US graph context must describe exactly the US entities.") + for entity, value in entities.items(): + if not isinstance(value, dict) or set(value) != _ENTITY_FIELDS: + raise ValueError(f"US graph context {entity} identity is malformed.") + columns = value["columns"] + if ( + not isinstance(columns, list) + or any(not isinstance(column, str) or not column for column in columns) + or len(set(columns)) != len(columns) + or US_SCHEMA.entity_id_column(entity) not in columns + or type(value["rows"]) is not int + or value["rows"] < 0 + ): + raise ValueError(f"US graph context {entity} schema is malformed.") + if not isinstance(document["metadata"], dict): + raise ValueError("US graph context metadata must be an object.") + _normative_metadata(document["metadata"]) + weight_sources = document["weight_sources"] + if ( + not isinstance(weight_sources, dict) + or not weight_sources + or set(weight_sources) - set(US_SCHEMA.entities) + or any(not isinstance(kind, str) for kind in weight_sources.values()) + ): + raise ValueError("US graph context weight sources are malformed.") + if not isinstance(document["mass_log"], list): + raise ValueError("US graph context mass_log must be an array.") + return document + + +def _mass_records(raw: list[object]) -> tuple[MassChangeRecord, ...]: + records = [] + for value in raw: + if not isinstance(value, dict) or set(value) != _MASS_FIELDS: + raise ValueError("US graph context mass record is malformed.") + if value["entity"] not in US_SCHEMA.entities or not isinstance( + value["reason"], str + ): + raise ValueError("US graph context mass record has invalid authority.") + for field in ("old_total", "new_total", "declared_factor"): + number = value[field] + if field == "declared_factor" and number is None: + continue + if type(number) not in (int, float) or number < 0: + raise ValueError("US graph context mass record is invalid.") + records.append(MassChangeRecord(**value)) + return tuple(records) + + +def us_frame_from_context( + context: KernelContext, *, artifact_alias: str = "frame_context" +) -> Frame: + """Restore an isolated US operator view from declared cells and typed context. + + The complete row identity of each entity must match the artifact. Columns + omitted by the node stay omitted. Callers needing a narrower row population + must declare a structural graph stage and publish its own context first. + """ + artifact = context.artifacts.get(artifact_alias) + if artifact is None or artifact.type != US_FRAME_CONTEXT_TYPE: + raise ValueError("US graph context artifact has a missing or incorrect type.") + document = _decode(artifact.payload) + if set(context.tables) != set(US_SCHEMA.entities): + raise ValueError("US graph context requires declared views for every entity.") + tables = {} + for entity in US_SCHEMA.entities: + table = context.tables[entity] + declared = document["entities"][entity] + actual = _row_identity(table, entity) + if any(declared[key] != value for key, value in actual.items()): + raise ValueError(f"US graph context {entity} identity does not match.") + if set(table.columns) - set(declared["columns"]): + raise ValueError(f"US graph context {entity} includes unknown columns.") + ordered = [column for column in declared["columns"] if column in table] + tables[entity] = table.loc[:, ordered].copy(deep=True) + weights = {} + for entity, kind in document["weight_sources"].items(): + value = context.weights.get(entity) + if value is None or value.kind.value != kind: + raise ValueError( + f"US graph context has no matching {entity} weight source." + ) + weights[entity] = value + return Frame( + tables, + US_SCHEMA, + weights, + context.strata.copy(deep=True), + mass_log=_mass_records(document["mass_log"]), + metadata=document["metadata"], + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py new file mode 100644 index 000000000..416c79650 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py @@ -0,0 +1,277 @@ +"""Project authenticated survey geography into declared household cells. + +This nonstructural node qualifies observed inputs only. Shared atomic operators +own location assignment; the country runner owns receiving-Population and warm +replay admission. Preparation bytes and descriptive receipts grant no authority. +""" + +from __future__ import annotations + +import json + +import pandas as pd + +from microcosm.frame import WeightKind +from microcosm.graph import ( + ArtifactInput, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + source_hash, +) +from microcosm.graph.canonical import canonical_json + +from . import graph_survey_population as source_graph + +NODE = "survey_population.observed_geography" +REF = "us.survey_population.observed_geography@1" +PHASE = "us.survey_population.observed_geography.v1" +OUTPUT_COLUMNS = ( + "survey_geography_origin_key", + "survey_observed_state", + "survey_observed_puma", +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_GEOGRAPHY_PROJECTION_" + reason) + + +def _qualifier(): + # Declaration-only imports need not load retained source owners. + from . import current_survey_geography + + return current_survey_geography + + +def current_survey_geography_node( + *, + preparation_sha256, + projection_receipt_sha256, + population, + node_id=NODE, +): + """Declare three new observed columns; digest arguments authenticate nothing.""" + _require( + type(population) is str + and 0 < len(population) <= 256 + and type(node_id) is str + and 0 < len(node_id) <= 256 + and population != node_id, + "NODE_NAMES", + ) + return Node( + id=node_id, + kernel=REF, + population=population, + inputs=(Slice("household", source_graph._provenance_columns("household")),), + outputs=tuple(Owned("household", name, "string") for name in OUTPUT_COLUMNS), + params={ + "phase": PHASE, + "preparation_sha256": source_graph._digest(preparation_sha256), + "projection_receipt_sha256": source_graph._digest( + projection_receipt_sha256 + ), + }, + artifact_inputs=( + ArtifactInput( + "preparation", + source_graph.CREATE_NODE, + "preparation", + source_graph.PREPARATION_TYPE, + ), + ), + description="Project qualified observed state, PUMA and stable source draw keys.", + ) + + +def _check_context(context, *, payload, source_frame, receipt_sha256): + expected = current_survey_geography_node( + preparation_sha256=source_graph._sha(payload), + projection_receipt_sha256=receipt_sha256, + population=context.node.population, + node_id=context.node.id, + ) + _require( + context.node == expected and dict(context.params) == dict(expected.params), + "DECLARATION", + ) + _require( + not context.sources + and set(context.artifacts) == {"preparation"} + and set(context.tables) == {"household"}, + "CONTEXT_ROSTER", + ) + source_graph._artifact( + context, "preparation", source_graph.PREPARATION_TYPE, payload + ) + table = context.tables["household"] + columns = ("household_id", *source_graph._provenance_columns("household")) + _require(tuple(table.columns) == columns, "CONTEXT_COLUMNS") + try: + pd.testing.assert_frame_equal( + table, + source_frame.table("household").loc[:, list(columns)], + check_exact=True, + ) + except (AssertionError, ValueError, TypeError): + raise ValueError("SURVEY_GEOGRAPHY_PROJECTION_HOUSEHOLD_ORIGIN") from None + _require( + set(context.weights) == {"household"} + and context.weights["household"].kind is WeightKind.IMPORTANCE + and len(context.weights["household"].values) == len(table), + "ALLOCATED_WEIGHTS_REQUIRED", + ) + + +def _check_projection(qualifier, projection, *, payload, receipt_sha256): + _require( + type(projection) is qualifier.CurrentSurveyGeographyValues + and qualifier.COLUMNS == OUTPUT_COLUMNS + and type(projection.receipt) is bytes + and 0 < len(projection.receipt) <= qualifier.MAX_RECEIPT_BYTES + and source_graph._sha(projection.receipt) == receipt_sha256, + "PROJECTION_RECEIPT", + ) + document = json.loads(projection.receipt) + _require( + document["protocol"] == qualifier.PROTOCOL + and document["preparation_sha256"] == source_graph._sha(payload) + and document["columns"] == list(OUTPUT_COLUMNS) + and document["households"] == len(projection.household) + and document["projection_sha256"] + == qualifier._projection_digest(projection.household) + and document["source_admission_issued"] is False + and document["population_admission_issued"] is False + and document["release_eligible"] is False, + "QUALIFIED_PROJECTION", + ) + return document + + +class CurrentSurveyGeographyKernel(KernelBase): + """Retain the live preparation; independently qualify every executed output.""" + + ref = REF + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=source_graph.STAGE_DEPENDENCIES, + ) + + def __init__(self, preparation): + self._preparation = preparation + + def implementation_hash(self): + qualifier = _qualifier() + demographics = qualifier.demographics + # Preserve the source stage's maintained dependency fence, and bind the + # additional actual projection functions and their demographic owners. + return source_graph._sha( + canonical_json( + { + "source_stage": source_graph._Kernel.implementation_hash(self), + "projection": source_hash( + type(self), + qualifier, + qualifier.qualify_current_survey_geography, + demographics, + demographics.qualify_current_asec_demographics, + demographics.demographic, + demographics.demographic.load_authenticated_asec_demographic_source, + demographics.demographic._snapshot, + demographics.household, + demographics.source_csv_builtin, + qualifier.dtype_for_token, + dependencies=self.capabilities.dependencies, + ), + } + ) + ) + + def run(self, context): + preparation = self._preparation + owner = source_graph._source_owner() + _require( + type(preparation) is owner.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + payload, state = entry[1], entry[2] + receipt_sha256 = source_graph._digest( + context.params.get("projection_receipt_sha256") + ) + _check_context( + context, + payload=payload, + source_frame=state.frame, + receipt_sha256=receipt_sha256, + ) + qualifier = _qualifier() + projection = qualifier.qualify_current_survey_geography(preparation) + document = _check_projection( + qualifier, projection, payload=payload, receipt_sha256=receipt_sha256 + ) + _require( + projection.household.index.tolist() + == context.tables["household"].household_id.tolist(), + "PROJECTION_HOUSEHOLD_ORDER", + ) + receipt = { + "phase": PHASE, + "preparation_sha256": source_graph._sha(payload), + "projection_receipt_sha256": receipt_sha256, + "source_projection": document, + "population_admission_issued": False, + "release_eligible": False, + } + receipt_bytes = canonical_json(receipt) + result = KernelResult( + columns={ + ("household", name): projection.household[name].copy(deep=True) + for name in OUTPUT_COLUMNS + }, + receipt=receipt, + ) + # Finish source-owner I/O before rechecking mutable source views, + # projected context and every detached returned column/receipt. + final_entry = preparation._checked() + _require( + self._preparation is preparation + and final_entry is entry + and owner._ISSUED.get(id(preparation)) is entry + and preparation.payload == payload, + "FINAL_ISSUANCE", + ) + owner._pure_final(state) + _check_context( + context, + payload=payload, + source_frame=state.frame, + receipt_sha256=receipt_sha256, + ) + _check_projection( + qualifier, projection, payload=payload, receipt_sha256=receipt_sha256 + ) + _require( + set(result.columns) == {("household", name) for name in OUTPUT_COLUMNS} + and result.frame is result.keep is result.expand is result.weights is None + and result.strata is None + and not result.artifacts + and canonical_json(result.receipt) == receipt_bytes, + "FINAL_RESULT", + ) + returned = pd.DataFrame( + {name: result.columns[("household", name)] for name in OUTPUT_COLUMNS} + ) + _require( + qualifier._projection_digest(returned) == document["projection_sha256"], + "FINAL_OUTPUT", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py new file mode 100644 index 000000000..73f4c0b67 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py @@ -0,0 +1,431 @@ +"""Private health fragment for the one country enrichment host. + +The host retains the actual checked PUF run, preparation, kernel registry and +source files. This fragment never issues a run or creates a second receiving +branch. Its final ordinary attachment preserves the incoming population version. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + ArtifactValue, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph import population as population_ops + +from . import current_survey_health_coverage as health +from . import current_survey_health_source as source +from . import graph_full_puf_enrichment as physical +from .graph_survey_population import SOURCE_NAME + +qualify_health_coverage = source.qualify_current_survey_health +require = health.require +SOURCE_NODE = "survey_health.source" +RAW_NODE = "survey_health.raw_columns" +RECODE_PREFIX = "survey_health.recode." +ATTACH_NODE = "survey_health.attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_health_projection", 1) +COLUMNS_TYPE = ArtifactType("microcosm.us.current_survey_health_columns", 1) +ATTACHMENT_TYPE = ArtifactType("microcosm.us.current_survey_health_attachment", 1) + + +def _json(value): + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _table_bytes(table): + return table.to_json(orient="table").encode() + + +def health_coverage_seal(qualified): + require(type(qualified) is source.QualifiedSurveyHealthCoverage, "QUALIFIED_TYPE") + require( + qualified.projection + == qualified.raw.reset_index().to_json(orient="table", index=False).encode() + and qualified.evidence["projection_sha256"] + == hashlib.sha256(qualified.projection).hexdigest(), + "PROJECTION_BINDING", + ) + return ( + source.source._frame_identity(qualified.source_frame), + physical._table_stamp(qualified.origins), + physical._table_stamp(qualified.raw), + qualified.projection, + _json(qualified.evidence), + ) + + +def _source_edge(): + return ArtifactInput( + "health_projection", SOURCE_NODE, "projection", PROJECTION_TYPE + ) + + +def _column_artifact(node_id, alias): + return ArtifactInput(alias, node_id, "columns", COLUMNS_TYPE) + + +def _dtype(series): + return population_ops.token_for_dtype(series.dtype) + + +def _outputs(table): + return tuple(Owned("person", name, _dtype(table[name])) for name in table) + + +def _completed(qualified): + return pd.concat( + [ + health.source_columns(qualified.raw), + *(health.recode_field(qualified.raw, f.output) for f in health.FIELDS), + ], + axis=1, + ) + + +def _params(qualified): + return { + "protocol": health.PROTOCOL, + "projection_sha256": hashlib.sha256(qualified.projection).hexdigest(), + "source_evidence": _json(qualified.evidence).decode(), + } + + +def health_coverage_nodes(qualified, *, receiving_version, after): + health_coverage_seal(qualified) + require( + type(receiving_version) is str + and receiving_version + and type(after) is ArtifactInput, + "FRAGMENT_INPUT", + ) + params = _params(qualified) + raw = health.source_columns(qualified.raw) + create = Node( + SOURCE_NODE, + "us.survey_health.source@1", + structural=StructuralDelta.CREATE, + sources=(SOURCE_NAME,), + outputs=(Owned("person", "health_native_person_id", "int64"),), + params=params, + artifact_inputs=(after,), + artifact_outputs=(ArtifactOutput("projection", PROJECTION_TYPE),), + description="Borrow original ACS/ASEC person support and retain authenticated current-coverage source projection.", + ) + literal = Node( + RAW_NODE, + "us.survey_health.raw_columns@1", + population=SOURCE_NODE, + outputs=_outputs(raw), + params=params, + artifact_inputs=(_source_edge(),), + artifact_outputs=(ArtifactOutput("columns", COLUMNS_TYPE),), + description="Expose literal coverage, allocation and edit codes with survey observation years.", + ) + recodes = [] + for field in health.FIELDS: + inputs = [ + health.SOURCE_PREFIX + "source", + health.SOURCE_PREFIX + field.asec, + health.SOURCE_PREFIX + "I_" + field.asec, + ] + if field.acs: + inputs += [ + health.SOURCE_PREFIX + field.acs, + health.SOURCE_PREFIX + "F" + field.acs + "P", + ] + recodes.append( + Node( + RECODE_PREFIX + field.output, + "us.survey_health.recode@1", + population=SOURCE_NODE, + inputs=(Slice("person", tuple(inputs)),), + outputs=_outputs(health.recode_field(qualified.raw, field.output)), + params={**params, "field": field.output}, + artifact_inputs=( + _source_edge(), + _column_artifact(RAW_NODE, "health_raw_columns"), + ), + artifact_outputs=(ArtifactOutput("columns", COLUMNS_TYPE),), + description=f"Current coverage: {field.asec}; ACS {field.acs or field.acs_gap}. Missing codes remain unknown.", + ) + ) + identity = tuple( + f("person") + for f in ( + health.provenance.support_source_id_column, + health.provenance.support_clone_index_column, + health.provenance.spine_source_id_column, + health.provenance.support_channel_column, + ) + ) + attach = Node( + ATTACH_NODE, + "us.survey_health.attach@1", + population=receiving_version, + inputs=(Slice("person", identity),), + outputs=_outputs(_completed(qualified)), + params=params, + artifact_inputs=( + after, + _source_edge(), + _column_artifact(RAW_NODE, "health_raw_columns"), + *( + _column_artifact(n.id, "health_field_" + str(i)) + for i, n in enumerate(recodes) + ), + ), + artifact_outputs=(ArtifactOutput("attachment", ATTACHMENT_TYPE),), + description="Attach only exact source-person observations and explicit unknowns to both PUF clones; preserve all incoming fields.", + ) + return (create, literal, *recodes, attach) + + +def _expected_artifacts(qualified): + return { + (SOURCE_NODE, "projection"): qualified.projection, + (RAW_NODE, "columns"): _table_bytes(health.source_columns(qualified.raw)), + **{ + (RECODE_PREFIX + f.output, "columns"): _table_bytes( + health.recode_field(qualified.raw, f.output) + ) + for f in health.FIELDS + }, + } + + +def _check_artifacts(node, artifacts, qualified): + require(set(artifacts) == {a.name for a in node.artifact_inputs}, "ARTIFACT_ROSTER") + expected = _expected_artifacts(qualified) + for edge in node.artifact_inputs: + value = artifacts[edge.name] + require( + type(value) is ArtifactValue + and value.type == edge.type + and type(value.payload) is bytes, + "ARTIFACT_TYPE", + ) + if (edge.producer, edge.artifact) in expected: + require( + value.payload == expected[edge.producer, edge.artifact], + "ARTIFACT_PAYLOAD", + ) + # The incoming amount edge is authenticated by the owning host, which + # retains its node/store keys. The fragment never treats its JSON as an issuer. + + +def _result(qualified, node, people): + if node.id == SOURCE_NODE: + original = source.source._copy_source(qualified.source_frame) + # A projection branch keeps original support/design and structural IDs; + # its coverage cells are introduced visibly by the following nodes. + tables = {} + for entity in original.entities: + columns = [original.schema.entity_id_column(entity)] + if entity == original.schema.person_entity: + columns += [ + original.schema.membership_column(e) + for e in original.schema.group_entities + ] + tables[entity] = original.table(entity).loc[:, columns].copy() + require( + np.array_equal( + qualified.origins.index.to_numpy(), original.person.person_id.to_numpy() + ), + "CREATE_ORIGIN_AXIS", + ) + tables["person"]["health_native_person_id"] = ( + qualified.origins.native_person_id.to_numpy(copy=True) + ) + projected = Frame( + tables, + original.schema, + dict(original._weights), + original.strata, + metadata=original.metadata, + mass_log=original.mass_log, + ) + return KernelResult( + frame=projected, artifacts={"projection": qualified.projection} + ) + if node.id == RAW_NODE: + table = health.source_columns(qualified.raw) + elif node.id.startswith(RECODE_PREFIX): + table = health.recode_field(qualified.raw, node.params["field"]) + else: + require(node.id == ATTACH_NODE, "NODE_ID") + columns = health.attach_columns( + qualified.origins, SimpleNamespace(person=people), _completed(qualified) + ) + return KernelResult( + columns=columns, + artifacts={ + "attachment": _json( + { + "protocol": health.PROTOCOL, + "projection_sha256": hashlib.sha256( + qualified.projection + ).hexdigest(), + "receiving_version": node.population, + "rows": len(people), + "source_admission_issued": False, + "release_eligible": False, + } + ) + }, + ) + require( + people is not None + and np.array_equal(people.person_id.to_numpy(), table.index.to_numpy()), + "SOURCE_PERSON_AXIS", + ) + return KernelResult( + columns={("person", c): table[c].copy() for c in table}, + artifacts={"columns": _table_bytes(table)}, + ) + + +def expected_health_population(node_id, incoming, *, qualified, node, artifacts): + """Reconstruct the complete output, independently of executor cache results.""" + require(node_id == node.id, "EXPECTED_NODE") + _check_artifacts(node, artifacts, qualified) + result = _result( + qualified, node, None if incoming is None else incoming.frame.person + ) + if node.structural is StructuralDelta.CREATE: + require(incoming is None, "CREATE_INCOMING") + return population_ops.Population.from_frame(result.frame, node.id) + require(incoming is not None, "ORDINARY_INCOMING") + return population_ops.patch(incoming, node, result) + + +class _HealthKernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def __init__(self, qualified, nodes, require_current, *, create=False, ref): + self.ref, self.qualified, self.nodes = ref, qualified, nodes + self.require_current, self.seal = ( + require_current, + health_coverage_seal(qualified), + ) + if create: + self.capabilities = replace( + self.capabilities, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + health, + source, + physical, + source.source, + source.housing, + source.records, + source.asec, + source.source_csv_builtin, + sys.modules[health.__package__ + ".cps_carried"], + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + self.require_current() + require(health_coverage_seal(self.qualified) == self.seal, "QUALIFIED_CHANGED") + require( + context.node in self.nodes and context.node.kernel == self.ref, + "KERNEL_NODE", + ) + _check_artifacts(context.node, context.artifacts, self.qualified) + if context.node.id.startswith(RECODE_PREFIX): + expected = health.source_columns(self.qualified.raw) + actual = context.tables["person"].set_index("person_id") + for selection in context.node.inputs: + for column in selection.columns: + require( + population_ops.storage_equal(actual[column], expected[column]), + "RECODE_SOURCE_SLICE", + ) + result = _result(self.qualified, context.node, context.tables.get("person")) + result_seal = ( + physical._table_stamp( + pd.concat( + [v.rename(c) for (_e, c), v in result.columns.items()], axis=1 + ) + ) + if result.columns + else None + ) + frame_seal = ( + source.source._frame_identity(result.frame) + if result.frame is not None + else None + ) + artifacts = tuple(sorted(result.artifacts.items())) + self.require_current() + require(health_coverage_seal(self.qualified) == self.seal, "QUALIFIED_CHANGED") + require( + ( + physical._table_stamp( + pd.concat( + [v.rename(c) for (_e, c), v in result.columns.items()], axis=1 + ) + ) + if result.columns + else None + ) + == result_seal + and ( + source.source._frame_identity(result.frame) + if result.frame is not None + else None + ) + == frame_seal + and tuple(sorted(result.artifacts.items())) == artifacts, + "RESULT_CHANGED", + ) + return result + + +def health_coverage_kernels(qualified, *, receiving_version, after, require_current): + nodes = health_coverage_nodes( + qualified, receiving_version=receiving_version, after=after + ) + require(callable(require_current), "HOST_CALLBACK") + return tuple( + _HealthKernel( + qualified, + nodes, + require_current, + create=ref == "us.survey_health.source@1", + ref=ref, + ) + for ref in dict.fromkeys(n.kernel for n in nodes) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py new file mode 100644 index 000000000..b6d9cfd66 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py @@ -0,0 +1,616 @@ +"""Graph training, draw and attachment of current survey financial predictors. + +The ASEC fitting branch leaves the source CREATE before importance allocation; +its household-derived person weights remain DESIGN. No PUF clone is a donor. +All fitted totals, prior-target conditioning and final source-origin joins are +visible typed graph dependencies. Cold and replay results require the same +materialized verifier, independently of cache or node receipt labels. +""" + +from __future__ import annotations + +import sys + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input, qrf +from microcosm.fit.graph_legacy_apply_matrix import ( + MATRIX_APPLY_STATE_TYPE, + decode_matrix_apply_state, +) +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph import population as population_ops + +from . import current_survey_predictors as values +from . import survey_population_replay as replay + +host = values.host +require = values.require +PROJECTION_NODE = "survey_predictors.source_projection" +DONOR_NODE = "survey_predictors.asec_design_donor" +DONOR_COLUMNS_NODE = "survey_predictors.asec_current_columns" +FIT_PREFIX = "survey_predictors.fit" +APPLY_PREFIX = "survey_predictors.apply" +ATTACH_NODE = "survey_predictors.attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_predictor_projection", 1) + + +def _inputs(frame): + result = [] + for entity in US_SCHEMA.entities: + structural = {US_SCHEMA.entity_id_column(entity)} + if entity == "person": + structural.update( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + result.append( + Slice(entity, tuple(c for c in frame.table(entity) if c not in structural)) + ) + return tuple(result) + + +def _projection_edges(): + return ( + ArtifactInput("projection", PROJECTION_NODE, "projection", PROJECTION_TYPE), + ArtifactInput( + "matrix", PROJECTION_NODE, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ), + ) + + +def _attach_edges(): + result = list(_projection_edges()) + for i in range(len(values.TARGETS)): + result.extend( + ( + ArtifactInput( + f"raw_{i:03d}", + f"{APPLY_PREFIX}.{i:03d}", + "raw_draw", + codec.RAW_TARGET_TYPE, + ), + ArtifactInput( + f"state_{i:03d}", + f"{APPLY_PREFIX}.{i:03d}", + "apply_state", + MATRIX_APPLY_STATE_TYPE, + ), + ) + ) + return tuple(result) + + +def _geography_edge(): + return ArtifactInput( + "geography_validation", + "geography.gate", + "validation", + host.survey_budget.geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + ) + + +def _params(qualified, host_pins, n_estimators): + require(type(n_estimators) is int and n_estimators > 0, "TREE_COUNT") + values.feature_columns(qualified.demographic_conditioning) + geography_enabled = qualified.geography_config_payload is not None + require( + (type(qualified.geography_validation) is bytes) + if geography_enabled + else qualified.geography_validation is None, + "GEOGRAPHY_VALIDATION_VALUES", + ) + names = {e.name for e in host.current_survey_host_edges()} + if geography_enabled: + names.add(_geography_edge().name) + require( + type(host_pins) is dict and set(host_pins) == names, + "HOST_PINS", + ) + for pin in host_pins.values(): + require( + type(pin) is dict + and set(pin) == {"producer_key", "artifact_key", "payload_sha256"} + and all(codec._hash(v) for v in pin.values()), + "HOST_PIN_DIGESTS", + ) + return { + "protocol": values.PROTOCOL, + "source_projection_sha256": codec.sha(qualified.projection), + "host_edges": codec.encode_json(host_pins).decode(), + "seed": values.SEED, + "n_estimators": n_estimators, + "demographic_conditioning": qualified.demographic_conditioning, + "geography_config_sha256": None + if qualified.geography_config_payload is None + else codec.sha(qualified.geography_config_payload), + } + + +def current_survey_predictor_nodes( + qualified, clone_frame, *, host_pins, n_estimators=100 +): + require( + type(qualified) is values.QualifiedSurveyPredictors, "QUALIFIED_VALUES_TYPE" + ) + params = _params(qualified, host_pins, n_estimators) + predictors = values.feature_columns(qualified.demographic_conditioning) + projection = Node( + PROJECTION_NODE, + CurrentSurveyPredictorProjectionKernel.ref, + # This ordinary node must live outside CREATE: its typed allocation + # evidence is downstream of that version's structural allocation node. + population=DONOR_NODE, + inputs=_inputs(qualified.donor_frame), + params=params, + artifact_inputs=host.current_survey_host_edges(), + artifact_outputs=( + ArtifactOutput("projection", PROJECTION_TYPE), + ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE), + ), + ) + donor = Node( + DONOR_NODE, + CurrentSurveyPredictorDonorFilterKernel.ref, + base=host.survey_graph.CREATE_NODE, + structural=StructuralDelta.FILTER, + inputs=_inputs(qualified.source_frame), + mass="free", + params=params, + # Branch from the authenticated CREATE before its weight transition. + # The later projection depends on this FILTER, never conversely. + artifact_inputs=( + host.current_survey_host_edges()[0], + *( + (_geography_edge(),) + if qualified.geography_config_payload is not None + else () + ), + ), + description="Select native ASEC whole households before allocation; retain original design weights for survey financial fitting.", + ) + columns = Node( + DONOR_COLUMNS_NODE, + CurrentSurveyPredictorDonorColumnsKernel.ref, + population=DONOR_NODE, + inputs=_inputs(qualified.donor_frame), + params=params, + outputs=tuple( + Owned("person", c, "float64") for c in (*predictors, *values.TARGETS) + ), + artifact_inputs=_projection_edges(), + ) + fit = legacy_qrf_train_nodes( + FIT_PREFIX, + population=DONOR_NODE, + entity="person", + predictors=predictors, + targets=values.TARGETS, + seed=values.SEED, + phase=values.PHASE, + n_estimators=n_estimators, + zero_atol=0, + ) + apply = legacy_qrf_apply_matrix_nodes( + APPLY_PREFIX, + population=host.survey_clone.COMBINED_CLONE_NODE, + fit_nodes=fit, + matrix_producer=PROJECTION_NODE, + seed=values.SEED, + phase=values.PHASE, + ) + attach = Node( + ATTACH_NODE, + CurrentSurveyPredictorAttachKernel.ref, + population=host.survey_clone.COMBINED_CLONE_NODE, + inputs=_inputs(clone_frame), + params=params, + outputs=tuple( + Owned("person", c, "float64", rewrite=(c in clone_frame.person)) + for c in values.OUTPUTS + ), + artifact_inputs=_attach_edges(), + ) + return (projection, donor, columns, *fit, *apply, attach) + + +class _Kernel(host._CurrentSurveyKernel): + def __init__( + self, + preparation, + allocated_population, + clone_population, + *, + host_pins, + n_estimators=100, + demographic_conditioning=False, + geography_config=None, + ): + self.preparation = preparation + self.allocated_population = allocated_population + self.clone_population = clone_population + self.host_pins = codec.decode_json(codec.encode_json(host_pins)) + self.n_estimators = n_estimators + values.feature_columns(demographic_conditioning) + self.demographic_conditioning = demographic_conditioning + self.geography_config = geography_config + self.geography_config_payload = host.survey_budget._config_payload( + geography_config + ) + + def implementation_hash(self): + demographics = values.observed_geography.demographics + return codec.sha( + codec.encode_json( + { + "host": super().implementation_hash(), + "predictors": source_hash( + sys.modules[__name__], + values, + values.leaves, + values.universe, + values.observed_geography, + demographics, + demographics.qualify_current_asec_demographics, + demographics.demographic, + demographics.demographic.load_authenticated_asec_demographic_source, + demographics.demographic._snapshot, + demographics.household, + demographics.source_csv_builtin, + population_ops, + replay, + qrf, + model_input, + sys.modules[decode_matrix_apply_state.__module__], + sys.modules[legacy_qrf_train_nodes.__module__], + # The optional postclone geography admission invokes + # the budget owner's complete reconstruction closure. + *host.survey_budget._modules(), + dependencies=self.capabilities.dependencies, + ), + } + ) + ) + + def _qualified(self, context): + require(not context.sources, "UNDECLARED_SOURCE") + require( + host.survey_budget._config_payload(self.geography_config) + == self.geography_config_payload, + "GEOGRAPHY_CONFIG_CHANGED", + ) + result = values.qualify_current_survey_predictors( + self.preparation, + self.allocated_population, + self.clone_population, + demographic_conditioning=self.demographic_conditioning, + geography_config=self.geography_config, + ) + require( + result.geography_config_payload == self.geography_config_payload, + "GEOGRAPHY_CONFIG_CHANGED", + ) + nodes = current_survey_predictor_nodes( + result, + self.clone_population.frame, + host_pins=self.host_pins, + n_estimators=self.n_estimators, + ) + expected = {n.id: n for n in nodes} + require(context.node == expected.get(context.node.id), "NODE_DECLARATION") + require( + set(context.artifacts) == {e.name for e in context.node.artifact_inputs}, + "ARTIFACT_ROSTER", + ) + if context.node.id == DONOR_NODE: + edge = host.current_survey_host_edges()[0] + require(edge.name == "preparation", "PREPARATION_EDGE") + value = host.shared.artifact(context, edge.name, edge.type) + require( + self.host_pins[edge.name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + } + and value.payload == self.preparation.payload, + "DONOR_PREPARATION_BINDING", + ) + if result.geography_config_payload is not None: + edge = _geography_edge() + value = host.shared.artifact(context, edge.name, edge.type) + require( + self.host_pins[edge.name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + } + and type(value.payload) is bytes + and value.payload == result.geography_validation, + "GEOGRAPHY_VALIDATION_BINDING", + ) + elif context.node.id != PROJECTION_NODE: + projection = host.shared.artifact(context, "projection", PROJECTION_TYPE) + matrix = host.shared.artifact( + context, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ) + host.shared.siblings(context, ("projection", "matrix")) + require( + projection.payload == result.projection + and matrix.payload == result.matrix, + "SOURCE_ARTIFACT_BYTES", + ) + return result + + +class CurrentSurveyPredictorProjectionKernel(_Kernel): + ref = "us.survey_predictors.source_projection@1" + + def run(self, context): + qualified = self._qualified(context) + host._current_context_frame(context, qualified.donor_frame) + loaded = {} + for edge in host.current_survey_host_edges(): + value = host.shared.artifact(context, edge.name, edge.type) + require( + self.host_pins[edge.name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + }, + "HOST_EDGE_PIN", + ) + loaded[edge.name] = value.payload + host.shared.siblings(context, ("allocation", "frame_context")) + require( + loaded["preparation"] == self.preparation.payload + and codec.sha(loaded["allocation"]) + == qualified.evidence["allocation_sha256"] + and codec.sha(loaded["frame_context"]) + == codec.decode_json(loaded["allocation"])["output_context_sha256"], + "HOST_EDGE_BYTES", + ) + return KernelResult( + artifacts={"projection": qualified.projection, "matrix": qualified.matrix}, + receipt=qualified.evidence, + ) + + +class CurrentSurveyPredictorDonorFilterKernel(_Kernel): + ref = "us.survey_predictors.asec_design_donor@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=host._CurrentSurveyKernel.capabilities.dependencies, + ) + + def run(self, context): + self._qualified(context) + # CREATE carries raw ACS NIU blanks. The qualified feature frame has + # separately receipted materialized zeros and is not this context. + original = self.preparation._checked()[2].frame + host._current_context_frame(context, original) + person = original.person + keep = ( + person[values.provenance.support_channel_column("person")] + .eq("asec") + .to_numpy() + ) + return KernelResult( + keep=pd.Series( + keep, + index=pd.Index(person.person_id.to_numpy(), name="person_id"), + dtype=bool, + ), + receipt={ + "selection": "ASEC_native_whole_households", + "fit_weight_kind": "design", + "release_eligible": False, + }, + ) + + +class CurrentSurveyPredictorDonorColumnsKernel(_Kernel): + ref = "us.survey_predictors.asec_current_columns@1" + + def run(self, context): + qualified = self._qualified(context) + host._current_context_frame(context, qualified.donor_frame) + return KernelResult( + columns={ + ("person", c): qualified.donor_columns[c] + for c in ( + *values.feature_columns(qualified.demographic_conditioning), + *values.TARGETS, + ) + }, + receipt=qualified.evidence, + ) + + +def read_current_survey_draws( + matrix, + matrix_producer_key, + raw_draws, + apply_states, + *, + demographic_conditioning=False, +): + predictors = values.feature_columns(demographic_conditioning) + require( + codec._hash(matrix_producer_key) + and type(raw_draws) is tuple + and type(apply_states) is tuple + and len(raw_draws) == len(apply_states) == len(values.TARGETS), + "DRAW_CHAIN_ROSTER", + ) + prepared = model_input.decode_recipient_matrix(matrix) + require( + prepared.entity == "person" and tuple(prepared.features.columns) == predictors, + "MATRIX_FEATURE_ROSTER", + ) + result = pd.DataFrame(index=prepared.features.index) + models = [] + for i, target in enumerate(values.TARGETS): + packet = decode_matrix_apply_state(apply_states[i]) + require( + packet["matrix_sha256"] == codec.sha(matrix) + and packet["matrix_producer_key"] == matrix_producer_key, + "DRAW_MATRIX_BINDING", + ) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + require( + chain.entity == "person" + and tuple(chain.predictors) == predictors + and tuple(chain.targets) == values.TARGETS + and tuple(chain.completed_targets) == values.TARGETS[: i + 1] + and chain.recipient_index == qrf._index_identity(prepared.features.index) + and application["seed"] == values.SEED + and application["raw_targets"] + == [ + {"target": name, "sha256": codec.sha(raw)} + for name, raw in zip( + values.TARGETS[: i + 1], raw_draws[: i + 1], strict=True + ) + ], + "DRAW_CHAIN_BINDING", + ) + current_models = application["models"] + require( + current_models[:i] == models and len(current_models) == i + 1, + "DRAW_MODEL_HISTORY", + ) + models = current_models + values_array = codec.read_raw_target( + raw_draws[i], target=target, index=prepared.features.index + ) + require(np.isfinite(values_array).all(), "DRAW_UNKNOWN") + result[target] = values_array + return result + + +class CurrentSurveyPredictorAttachKernel(_Kernel): + ref = "us.survey_predictors.attach@1" + + def run(self, context): + qualified = self._qualified(context) + host._current_context_frame(context, self.clone_population.frame) + matrix = host.shared.artifact( + context, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ) + raw, states = [], [] + for i in range(len(values.TARGETS)): + r, s = f"raw_{i:03d}", f"state_{i:03d}" + host.shared.siblings(context, (r, s)) + raw.append(host.shared.artifact(context, r, codec.RAW_TARGET_TYPE).payload) + states.append( + host.shared.artifact(context, s, MATRIX_APPLY_STATE_TYPE).payload + ) + drawn = read_current_survey_draws( + matrix.payload, + matrix.producer_key, + tuple(raw), + tuple(states), + demographic_conditioning=self.demographic_conditioning, + ) + columns = values.complete_predictor_columns( + qualified, self.clone_population.frame, drawn + ) + return KernelResult( + columns=columns, + receipt={ + **qualified.evidence, + "raw_sha256": [codec.sha(r) for r in raw], + "all_output_cells_available": True, + "ACS_financial_origin": "modeled", + "paired_draws": "source_origin_join", + "host_weights_changed": False, + }, + ) + + +def verify_materialized_current_survey_predictors( + preparation, + allocated_population, + clone_population, + *, + population, + projection, + matrix, + matrix_producer_key, + raw_draws, + apply_states, + host_pins, + n_estimators=100, + demographic_conditioning=False, + geography_config=None, +): + """Use actual executor-observed Population and authenticated typed artifacts. + + Call only after graph store/type/producer-key checks, both cold and replay. + Requalifies the current source owners, then compares every column, identity, + owner, metadata value, weight and mass ledger of the expected full result. + """ + qualified = values.qualify_current_survey_predictors( + preparation, + allocated_population, + clone_population, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + require( + projection == qualified.projection and matrix == qualified.matrix, + "MATERIALIZED_SOURCE", + ) + drawn = read_current_survey_draws( + matrix, + matrix_producer_key, + raw_draws, + apply_states, + demographic_conditioning=demographic_conditioning, + ) + attach = current_survey_predictor_nodes( + qualified, + clone_population.frame, + host_pins=host_pins, + n_estimators=n_estimators, + )[-1] + expected = population_ops.patch( + clone_population, + attach, + KernelResult( + columns=values.complete_predictor_columns( + qualified, clone_population.frame, drawn + ) + ), + ) + replay.same_replayed_population(expected, population) + return { + **qualified.evidence, + "all_output_cells_available": True, + "paired_draws": "source_origin_join", + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_property.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_property.py new file mode 100644 index 000000000..f870e67d6 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_property.py @@ -0,0 +1,1125 @@ +"""Opt-in property components on the complete initial survey clone frame. + +The existing country host retains source authority and issues its financial +run. These nodes borrow that owner's sources; neither options nor descriptive +projections are an admission token. Legacy tax leaves and their CAP chain remain +unchanged. Broad property receipts and retirement-account earnings are not tax +rental income or retirement withdrawals. +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass, replace + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import graph_signed_reconciliation as signed_graph +from microcosm.fit import model_input, qrf +from microcosm.fit.graph_legacy_qrf import LegacyQRFApplyKernel +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph import population as population_ops +from microcosm.graph.kernel import KernelContext +from microcosm.graph.keys import opaque_artifact_key + +from . import current_property_completion_routing as completion +from . import current_property_income_sources as sources +from . import current_survey_predictors as shared +from . import graph_current_survey_predictors as financial +from . import graph_property_income as model +from . import survey_population_replay as replay +from .property_income_constants import ( + PROPERTY_COMPONENTS, + PROPERTY_DRAW_COLUMNS, + PROPERTY_REPORTED_TOTAL, +) + +host = shared.host +PROTOCOL = "microcosm.us.current-survey-property.v1" +PREFIX = "survey_property" +PROJECTION_NODE = PREFIX + ".source_projection" +DONOR_NODE = PREFIX + ".asec_eligible_donor" +DONOR_COLUMNS_NODE = PREFIX + ".asec_donor_columns" +RECIPIENT_NODE = PREFIX + ".acs_eligible_recipient" +RECIPIENT_COLUMNS_NODE = PREFIX + ".acs_recipient_columns" +ATTACH_NODE = PREFIX + ".attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_property_projection", 1) +CAP_LIMITATION = "legacy_CAP_conditions_on_legacy_INT_DIV; reconciled_component_consistency_not_claimed" +LEGACY_DIFFERENCE = "property_legacy_interest_draw_minus_reconciled_interest" +BASIS_DIAGNOSTICS = ( + "property_interest_component_discrepancy", + "property_reported_minus_component_total", +) + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_PROPERTY_" + reason) + + +@dataclass(frozen=True, kw_only=True) +class PropertyIncomeOptions: + scales: tuple[float, float, float, float] + atol: float + rtol: float + n_estimators: int + completion_routing: bool = False + + def __post_init__(self): + # Reuse the real numeric declaration's domain checks, including finite + # positive scales and finite nonnegative tolerances; no fitted model. + require(type(self.completion_routing) is bool, "COMPLETION_ROUTING_OPTION") + self.nodes((PROPERTY_REPORTED_TOTAL,)) + + def nodes(self, features): + return model.property_income_nodes( + PREFIX, + donor_population=DONOR_NODE, + recipient_population=RECIPIENT_NODE, + features=features, + seed=shared.SEED, + n_estimators=self.n_estimators, + scales=self.scales, + atol=self.atol, + rtol=self.rtol, + ) + + def to_bytes(self): + return codec.encode_json(self.document()) + + def document(self): + require(type(self.completion_routing) is bool, "COMPLETION_ROUTING_OPTION") + value = self.model_document() + if self.completion_routing: + value["completion_routing"] = True + return value + + def model_document(self): + """Numerical settings; completion diagnostics never alter these.""" + return { + "scales": self.scales, + "atol": self.atol, + "rtol": self.rtol, + "n_estimators": self.n_estimators, + } + + +def _features(qualified): + return ( + *shared.feature_columns(qualified.shared_predictors.demographic_conditioning), + PROPERTY_REPORTED_TOTAL, + ) + + +def _diagnostics(): + return signed_graph._diagnostics(PROPERTY_COMPONENTS, PREFIX + "_reconciliation") + + +def owned_columns(): + adjustments, active, residual, objective = _diagnostics() + floats = ( + *PROPERTY_COMPONENTS, + *PROPERTY_DRAW_COLUMNS, + PROPERTY_REPORTED_TOTAL, + *adjustments, + residual, + objective, + LEGACY_DIFFERENCE, + *BASIS_DIAGNOSTICS, + ) + return ( + *tuple(Owned("person", c, "float64") for c in floats), + *tuple(Owned("person", c, "boolean") for c in active), + Owned("person", "property_anchor_known", "bool"), + Owned("person", "property_components_known", "bool"), + ) + + +def _seal_json(value): + if type(value) is bytes: + return {"bytes_sha256": codec.sha(value)} + if type(value) is tuple: + return [_seal_json(x) for x in value] + if type(value) is dict: + return {k: _seal_json(v) for k, v in value.items()} + return value + + +def _projection_values(qualified): + require(type(qualified) is sources.QualifiedPropertyIncomeSources, "QUALIFIED_TYPE") + require( + qualified.donor_frame is not None + and qualified.recipient_frame is not None + and qualified.recipient_matrix is not None, + "EMPTY_MODEL_BRANCH", + ) + donor_matrix = model_input.encode_recipient_matrix( + qualified.donor_columns, + entity="person", + entity_ids=qualified.donor_columns.index.to_numpy(dtype=" FiscalCalibrationBounds: + """Read bounded, canonical typed-ID and exact float64 reference arrays.""" + _require(type(payload) is bytes and 0 < len(payload) <= MAX_BYTES, "BOUNDS_PAYLOAD") + try: + raw = json.loads(payload) + except (ValueError, UnicodeError): + raise ValueError("FISCAL_CALIBRATION_BOUNDS_JSON") from None + _require(type(raw) is dict and set(raw) == _FIELDS, "BOUNDS_FIELDS") + _require(canonical_json(raw) == payload, "BOUNDS_CANONICAL") + _require(raw["protocol"] == BOUNDS_PROTOCOL, "BOUNDS_PROTOCOL") + _require(_digest(raw["budget_sha256"]), "BUDGET_DIGEST") + _require( + type(raw["population"]) is str and 0 < len(raw["population"]) <= 256, + "POPULATION", + ) + ids = measurement._read_array(raw["household_ids"], {" 0 and len(set(ids.tolist())) == len(ids), "HOUSEHOLD_IDS") + vectors = { + name: measurement._read_array(raw[name], {"= 0) for v in vectors.values()), + "REFERENCE_VALUES", + ) + _require( + all( + len(vectors[n]) == len(ids) + for n in ("original_design", "incoming", "row_upper") + ), + "REFERENCE_ALIGNMENT", + ) + _require(np.any(vectors["incoming"] > 0), "EMPTY_POSITIVE_SUPPORT") + groups = group_bounds.GroupedUpperBounds( + ids, + measurement._read_array(raw["group_indices"], {" bytes: + """Freeze explicitly supplied original references without issuing authority. + + ``row_upper`` is already the original source-reference cap, not an inferred + multiple of incoming or cloned DESIGN values. Its provenance and arithmetic + belong to the original budget owner. Full uint64 IDs and row order survive. + """ + _require( + type(original_design_weights) is Weights + and original_design_weights.kind is WeightKind.DESIGN, + "DESIGN_KIND", + ) + _require( + type(incoming_weights) is Weights + and incoming_weights.kind is WeightKind.IMPORTANCE, + "IMPORTANCE_KIND", + ) + _require( + type(grouped_upper_bounds) is group_bounds.GroupedUpperBounds, "GROUP_TYPE" + ) + _require( + type(household_ids) is np.ndarray + and household_ids.dtype.str in {" 0), + "FIXED_ZERO_SUPPORT", + ) + + +def verify_fiscal_calibration_bounds( + payload, + *, + household_ids, + incoming_weights, + original_design_weights, +) -> FiscalCalibrationBounds: + """Compare numeric bytes to a host's live references; not source admission. + + The host must obtain original_design_weights from retained Population + anchors and separately authenticate the original group and row caps. + """ + bounds = decode_fiscal_calibration_bounds(payload) + _require(_same_array(bounds.household_ids, household_ids), "HOUSEHOLD_ALIGNMENT") + _require( + type(incoming_weights) is Weights + and incoming_weights.kind is WeightKind.IMPORTANCE, + "IMPORTANCE_KIND", + ) + _require( + _same_array(bounds.incoming.values, incoming_weights.values), "INCOMING_ANCHOR" + ) + _require( + type(original_design_weights) is Weights + and original_design_weights.kind is WeightKind.DESIGN + and _same_array(bounds.original_design.values, original_design_weights.values), + "DESIGN_ANCHOR", + ) + return bounds + + +def _measurement_node(raw): + params = raw["params"] + result = measurement.fiscal_measurement_node( + measurement._registry(json.loads(params["registry"])), + population=raw["population"], + node_id=raw["id"], + input_columns={s["entity"]: tuple(s["columns"]) for s in raw["inputs"]}, + contract_targets=json.loads(params["contract_targets"]), + advertised_scopes=measurement._scopes(json.loads(params["advertised_scopes"])), + geography_vintage=params["geography_vintage"], + period=params["period"], + model_outputs=tuple(params["model_outputs"]), + ) + _require( + canonical_json(normative(result)) == canonical_json(raw), "MEASUREMENT_NODE" + ) + return result + + +def fiscal_dense_calibration_node( + *, + measurement_node: Node, + bounds_node: str, + epochs: int = 256, + learning_rate: float = 0.02, + node_id: str = "us.fiscal_dense_calibration", +) -> Node: + """Declare one weight-only, fixed-zero grouped solve on the measured parent.""" + _require(type(epochs) is int and 1 <= epochs <= 10000, "EPOCHS") + _require( + type(learning_rate) in (int, float) + and np.isfinite(learning_rate) + and 0 < learning_rate <= 1, + "LEARNING_RATE", + ) + raw = canonical_json(normative(measurement_node)) + _require(len(raw) <= 2 * measurement.MAX_DECLARATION_BYTES, "DECLARATION_SIZE") + verified = _measurement_node(json.loads(raw)) + return Node( + node_id, + FiscalDenseCalibrationKernel.ref, + base=verified.population, + inputs=verified.inputs, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + params={ + "measurement_node": raw.decode(), + "epochs": epochs, + "learning_rate": learning_rate, + "method": "adam", + "seed": 0, + "grouped_preserve_zeros": True, + "l0_lambda": 0.0, + "l1_lambda": 0.0, + "l2_lambda": 0.0, + "l2_anchor": "initial", + "weight_anchor": "explicit_original_sampling_references", + "cap_enforcement": "numeric_group_projection_and_final_row_refusal", + }, + artifact_inputs=( + ArtifactInput( + "measurement", verified.id, "measurement", measurement.MEASUREMENT_TYPE + ), + ArtifactInput("bounds", bounds_node, "numeric_bounds", BOUNDS_TYPE), + ), + artifact_outputs=( + ArtifactOutput("diagnostics", DIAGNOSTICS_TYPE), + ArtifactOutput("origin_diagnostics", ORIGIN_TYPE), + ), + ) + + +def _artifact(context, name, type_, output): + value = context.artifacts[name] + _require( + value.type == type_ + and value.key == opaque_artifact_key(value.producer_key, output), + "ARTIFACT_EDGE", + ) + return value.payload + + +class FiscalDenseCalibrationKernel(KernelBase): + """Use the existing solver and independently check the compiled measurement. + + ``target_snapshots`` is optional host-owned instrumentation, held only on + this kernel instance. Observer configuration is absent from the graph and + cache identity. Sinks finish before the existing final measurement, ID and + weight checks; their exceptions propagate. A required cache hit does not + execute this kernel or emit optimizer snapshots. + """ + + ref = "us.fiscal_dense_calibration@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.REWEIGHT, + consumes_se=False, + dependencies=("numpy", "pandas", "scipy", "torch"), + ) + + def __init__(self, *, target_snapshots: TargetSnapshotObserver | None = None): + if target_snapshots is not None and not isinstance( + target_snapshots, TargetSnapshotObserver + ): + raise TypeError("target_snapshots must be a TargetSnapshotObserver or None") + self._target_snapshots = target_snapshots + + def implementation_hash(self): + return _sha( + canonical_json( + { + "adapter": source_hash( + sys.modules[__name__], + measurement, + group_bounds, + target_module, + measurement.TargetSpec, + measurement.CalibrationHierarchy, + calibration_kernels.diagnostics_module.assemble_target_loss_attribution, + dependencies=self.capabilities.dependencies, + ), + "solver": calibration_kernels.CALIBRATE_ADAM.implementation_hash(), + } + ) + ) + + def run(self, context): + params = context.params + _require( + tuple(a.name for a in context.node.artifact_inputs) + == ("measurement", "bounds"), + "ALIASES", + ) + measured_node = _measurement_node(json.loads(params["measurement_node"])) + expected = fiscal_dense_calibration_node( + measurement_node=measured_node, + bounds_node=context.node.artifact_inputs[1].producer, + epochs=params["epochs"], + learning_rate=params["learning_rate"], + node_id=context.node.id, + ) + _require( + normative(context.node) == normative(expected) + and dict(params) == dict(expected.params), + "DECLARATION", + ) + _require( + context.weights["household"].kind is WeightKind.IMPORTANCE, + "IMPORTANCE_REQUIRED", + ) + _require(not context.sources, "UNDECLARED_SOURCE") + raw_measurement = _artifact( + context, "measurement", measurement.MEASUREMENT_TYPE, "measurement" + ) + measured_context = replace( + context, node=measured_node, params=measured_node.params, artifacts={} + ) + measured = measurement.verify_fiscal_measurement( + raw_measurement, + context=measured_context, + expected_sha256=_sha(raw_measurement), + ) + raw_bounds = _artifact(context, "bounds", BOUNDS_TYPE, "numeric_bounds") + bounds = decode_fiscal_calibration_bounds(raw_bounds) + _require( + bounds.document["population"] == context.node.base, "BOUNDS_POPULATION" + ) + _require( + _same_array(bounds.household_ids, measured.household_ids), + "HOUSEHOLD_ALIGNMENT", + ) + _require( + _same_array(bounds.incoming.values, context.weights["household"].values), + "INCOMING_ANCHOR", + ) + table = pd.DataFrame({"household_id": measured.household_ids.copy()}) + work = calibration_kernels._frame_from_context( + replace( + context, + tables={"household": table}, + weights={"household": bounds.incoming}, + ), + "household", + ) + targets = [] + for i, spec in enumerate(measured.registry): + # The public callable API materializes only this one sparse row. + def row(_frame, position=i): + return measured.matrix[position : position + 1].toarray().ravel() + + targets.append( + replace(spec.to_target(), entity="household", measure=row, filter=None) + ) + result = calibrate( + work, + TargetSet(targets), + weight_entity="household", + method="adam", + seed=0, + epochs=params["epochs"], + learning_rate=params["learning_rate"], + mass="free", + max_weight_ratio=None, + target_records=None, + l0_lambda=0.0, + l1_lambda=0.0, + l2_lambda=0.0, + l2_anchor="initial", + grouped_upper_bounds=bounds.grouped, + grouped_preserve_zeros=True, + target_snapshots=self._target_snapshots, + ) + _require( + not result.skipped + and result.problem.names + == tuple(s.to_target().row_name for s in measured.registry), + "TARGET_ALIGNMENT", + ) + _require( + _same_array(result.problem.target_vector, measured.target_values) + and result.problem.matrix.shape == measured.matrix.shape + and all( + _same_array( + getattr(result.problem.matrix, k), getattr(measured.matrix, k) + ) + for k in ("indptr", "indices", "data") + ), + "MATRIX_IDENTITY", + ) + _require( + _same_array(result.initial_weights, bounds.incoming.values), + "SOLVER_INITIAL", + ) + weights = result.frame.weights_for("household") + _require(weights.kind is WeightKind.CALIBRATED, "CALIBRATED_KIND") + check_fiscal_calibration_weights(bounds, weights.values) + accepted = weights.values.tobytes() + _require( + np.isfinite(result.loss_trajectory).all() + and np.isfinite(result.final_loss), + "NONFINITE_LOSS", + ) + initial_estimates = measured.matrix @ bounds.incoming.values + final_estimates = measured.matrix @ weights.values + _require( + np.isfinite(initial_estimates).all() + and np.isfinite(final_estimates).all() + and np.isfinite(final_estimates - measured.target_values).all(), + "NONFINITE_RESIDUAL", + ) + _require( + _same_array( + np.asarray([d.initial_estimate for d in result.diagnostics]), + initial_estimates, + ) + and _same_array( + np.asarray([d.final_estimate for d in result.diagnostics]), + final_estimates, + ), + "DIAGNOSTIC_ESTIMATES", + ) + build = { + "measurement_sha256": _sha(raw_measurement), + "numeric_bounds_sha256": _sha(raw_bounds), + "budget_sha256": bounds.document["budget_sha256"], + "original_design_sha256": _sha(bounds.original_design.values.tobytes()), + "incoming_sha256": _sha(bounds.incoming.values.tobytes()), + "accepted_weight_sha256": _sha(accepted), + "constraint_digest": bounds.grouped.digest, + "matrix_matches_measurement": True, + "numeric_measurement_entity": "household", + "original_measurement_entities": { + s.name: s.entity for s in measured.registry + }, + "weight_anchor": params["weight_anchor"], + "cap_enforcement": params["cap_enforcement"], + } + diagnostics = canonical_json( + diagnostics_payload( + result, + target_registry=measured.registry, + build=build, + ) + ) + origin = canonical_json( + { + "protocol": "microcosm.us.fiscal-calibration-origin-diagnostics.v1", + **build, + "household_ids": measurement._array(bounds.household_ids), + "accepted_weights": measurement._array(weights.values), + "group_totals": measurement._array( + bounds.grouped.totals(weights.values) + ), + "group_upper": measurement._array(bounds.grouped.absolute_bounds), + "group_diagnostics": bounds.grouped.diagnostics( + weights.values, + result.options["grouped_upper_bounds"][ + "last_corrected_group_count" + ], + ), + "fixed_zero_rows": int(np.count_nonzero(bounds.incoming.values == 0)), + "active_rows": int(np.count_nonzero(weights.values > 0)), + "binding_row_caps": int( + np.count_nonzero(weights.values == bounds.row_upper) + ), + "all_row_caps_satisfied": True, + "release_eligible": False, + "source_admission": "required_from_complete_parent_owner", + } + ) + _require( + len(diagnostics) <= MAX_BYTES and len(origin) <= MAX_BYTES, + "DIAGNOSTIC_SIZE", + ) + output = KernelResult( + weights=weights, + artifacts={"diagnostics": diagnostics, "origin_diagnostics": origin}, + receipt={ + **build, + "diagnostics_sha256": _sha(diagnostics), + "origin_diagnostics_sha256": _sha(origin), + "release_eligible": False, + "source_admission": "required_from_complete_parent_owner", + "scope": "one_numeric_dense_fiscal_solve_no_ancestry_admission", + }, + ) + check_fiscal_calibration_weights( + decode_fiscal_calibration_bounds(raw_bounds), weights.values + ) + _require(weights.values.tobytes() == accepted, "FINAL_WEIGHT_BYTES") + _require( + _same_array( + work.table("household").household_id.to_numpy(), bounds.household_ids + ) + and _same_array( + result.frame.table("household").household_id.to_numpy(), + bounds.household_ids, + ), + "FINAL_HOUSEHOLD_ALIGNMENT", + ) + measurement.verify_fiscal_measurement( + raw_measurement, + context=measured_context, + expected_sha256=_sha(raw_measurement), + ) + return output diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py new file mode 100644 index 000000000..f2761f07f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py @@ -0,0 +1,775 @@ +"""Declared fiscal measurements on one supplied US population, without fitting. + +V1 supports shared-interpreter value variables, sums and same-entity predicates. +Every target also has an explicit national/state/CD household scope. Specialized +providers, counterfactuals, band interpretation and monetary-binding receipts +are refused. This is not full fiscal-registry coverage or a population issuer. +The host owns source/target activation and complete-enrichment ancestry. + +Computed roots use the existing real US adapter only after its static closure +is explicitly present, with no missing cells or default allowlist. Optional +leaf policies bind producer intent or planned literal assumptions; assumption +execution remains unsupported until a complete-parent host admits it. +Measurements and model outputs live only in private tables and a CSR artifact. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import asdict, dataclass + +import numpy as np +import pandas as pd +from scipy import sparse + +from microcosm.build import target_materialization +from microcosm.calibrate import TargetRegistry, TargetSpec, matrix +from microcosm.calibrate import target as target_math +from microcosm.calibrate.hierarchy import CalibrationHierarchy, HierarchyGeography +from microcosm.frame import US_SCHEMA, Frame +from microcosm.frame.adapters import policyengine_us +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + SeedSource, + Slice, + source_hash, +) +from microcosm.graph.canonical import canonical_json, normative +from microcosm.graph.executor import _context_digest + +from . import asec_engine_evaluation as runtime +from . import fiscal_leaf_policy as leaf_policies + +MEASUREMENT_TYPE = ArtifactType("microcosm.us.declared_fiscal_measurement", 1) +PROTOCOL = "microcosm.us.declared-fiscal-measurement.v1" +MAX_BYTES = 64 * 1024**2 +MAX_DECLARATION_BYTES = 1024**2 +_BINDING_KEYS = {"value_variable", "value_expression", "filters", "from_entity"} +_LEVELS = {"national", "state", "congressional_district"} + + +def _require(condition, reason): + if not condition: + raise ValueError("FISCAL_MEASUREMENT_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _json(value): + return canonical_json(value).decode() + + +def _registry_document(registry): + return {"country": registry.country, "specs": [asdict(s) for s in registry]} + + +def _registry(raw): + _require(set(raw) == {"country", "specs"}, "REGISTRY_FIELDS") + specs = [] + for item in raw["specs"]: + hierarchy = item.get("hierarchy") + specs.append( + TargetSpec( + **{ + **item, + "hierarchy": None + if hierarchy is None + else CalibrationHierarchy.from_dict(hierarchy), + } + ) + ) + result = TargetRegistry(specs, country=raw["country"]) + _require(result.country == "us" and len(result) > 0, "US_REGISTRY_REQUIRED") + _require(_json(_registry_document(result)) == _json(raw), "REGISTRY_CANONICAL") + return result + + +@dataclass(frozen=True) +class FiscalGeographyScope: + """An explicit interpretation of a target hierarchy's geographic scope. + + The national scope uses no column/value; state and CD scopes use the named + integer household geography columns. The host admits the boundary vintage. + """ + + geography: HierarchyGeography + column: str | None = None + value: int | None = None + + def __post_init__(self): + level = self.geography.level + _require(level in _LEVELS, "GEOGRAPHY_LEVEL") + expected = { + "national": None, + "state": "state_fips", + "congressional_district": "congressional_district_geoid", + }[level] + _require(self.column == expected, "GEOGRAPHY_COLUMN") + _require( + self.value is None if expected is None else type(self.value) is int, + "GEOGRAPHY_VALUE", + ) + + +def _scopes(raw): + scopes = tuple( + FiscalGeographyScope( + HierarchyGeography(**r["geography"]), r["column"], r["value"] + ) + for r in raw + ) + _require({s.geography.level for s in scopes} == _LEVELS, "ADVERTISED_LEVELS") + keys = [(s.geography.level, s.geography.id) for s in scopes] + _require(len(set(keys)) == len(keys), "DUPLICATE_SCOPE") + _require( + len({(s.column, s.value) for s in scopes}) == len(scopes), + "DUPLICATE_SCOPE_PREDICATE", + ) + _require( + sum(s.geography.level == "national" for s in scopes) == 1, "NATIONAL_SCOPE" + ) + return scopes + + +def _model_contract(roots): + if not roots: + return {"evaluation": "none", "roots": []} + identity = runtime.engine_runtime_identity() + index = policyengine_us.PolicyEngineUSVariableMetadataIndex() + contracts = [] + for root in roots: + closure = index.variable_dependency_closure(root) + contracts.append( + { + "closure": asdict(closure), + "entity": index.variable_metadata(root).entity, + "leaves": { + leaf: index.variable_metadata(leaf).entity + for leaf in closure.input_leaves + }, + } + ) + return {"evaluation": "real_baseline", "runtime": identity, "roots": contracts} + + +def _resolved_model_contract(roots, leaf_policy=None): + # Validate malformed policies before importing any engine metadata/runtime. + document = ( + None + if leaf_policy is None + else leaf_policies.fiscal_leaf_policy_document(leaf_policy) + ) + if document is not None: + _require(document["roots"] == list(roots), "POLICY_ROOTS") + model = _model_contract(roots) + if document is None: + return model + leaves = {} + for contract in model["roots"]: + for name, entity in contract["leaves"].items(): + _require(name not in leaves or leaves[name] == entity, "LEAF_ENTITY") + leaves[name] = entity + document = leaf_policies.classify_fiscal_leaf_policy( + leaf_policy, + period=document["period"], + roots=roots, + leaves=leaves, + ) + assumptions = [ + name + for name, entry in document["entries"].items() + if entry["kind"] == "assumption" + ] + for name in assumptions: + affected = [ + contract["closure"]["root"] + for contract in model["roots"] + if name in contract["leaves"] + ] + _require( + document["entries"][name]["affected_roots"] == affected, + "ASSUMPTION_AFFECTED_ROOTS:" + name, + ) + records = {} + if assumptions: + index = policyengine_us.PolicyEngineUSVariableMetadataIndex() + records = leaf_policies.fiscal_assumption_engine_records( + document, + metadata={ + name: asdict(index.variable_metadata(name)) for name in assumptions + }, + defaults=policyengine_us.PolicyEngineUSEngine().default_values(assumptions), + ) + model["leaf_policy"] = { + "sha256": leaf_policy.sha256, + "document": document, + "engine_records": records, + } + return model + + +def fiscal_measurement_node( + registry: TargetRegistry, + *, + population: str, + input_columns: dict[str, tuple[str, ...]], + contract_targets: dict, + advertised_scopes: tuple[FiscalGeographyScope, ...], + geography_vintage: str, + period: int, + model_outputs: tuple[str, ...] = (), + leaf_policy: leaf_policies.FiscalLeafPolicy | None = None, + node_id: str = "us.fiscal_measurement", +) -> Node: + """Declare a bounded measurement stage; supplied targets are not activated here. + + Use the same model_outputs on FiscalMeasurementKernel. Full US structural + person membership is retained through its required input slice. Unread + group tables are reconstructed as sorted IDs only, using Frame's exact + membership contract; none of their values is inferred or used. + Every required input leaf is checked against this declaration before model + evaluation. None retains the strict all-producer contract. A policy with + assumptions may be declared for inspection but cannot execute in this + kernel: complete-parent collision admission has not been implemented. + Neither this node nor its artifact admits a complete parent. + """ + _require(type(period) is int and period > 0, "PERIOD") + _require(type(geography_vintage) is str and bool(geography_vintage), "VINTAGE") + _require( + type(model_outputs) is tuple + and all(type(r) is str and r for r in model_outputs) + and len(set(model_outputs)) == len(model_outputs), + "MODEL_OUTPUTS", + ) + if leaf_policy is not None: + policy_document = leaf_policies.fiscal_leaf_policy_document(leaf_policy) + _require(policy_document["period"] == period, "POLICY_PERIOD") + _require(policy_document["roots"] == list(model_outputs), "POLICY_ROOTS") + registry = _registry(_registry_document(registry)) + scopes = _scopes([asdict(s) for s in advertised_scopes]) + entities = (US_SCHEMA.person_entity, *US_SCHEMA.group_entities) + _require(set(input_columns) <= set(entities), "INPUT_ENTITIES") + columns = {entity: tuple(input_columns.get(entity, ())) for entity in entities} + _require(bool(columns["person"]), "DECLARED_PERSON_PROJECTION") + _require( + all(len(set(names)) == len(names) for names in columns.values()), + "DUPLICATE_INPUT", + ) + _require( + {s.column for s in scopes if s.column} <= set(columns["household"]), + "DECLARED_GEOGRAPHY", + ) + keys = {(s.geography.level, s.geography.id) for s in scopes} + target_scopes = set() + used_bindings = set() + measurements = {} + for spec in registry: + _require( + spec.period == period and spec.hierarchy is not None, + "TARGET_PERIOD_HIERARCHY", + ) + key = (spec.hierarchy.geography.level, spec.hierarchy.geography.id) + _require(key in keys, "TARGET_SCOPE") + target_scopes.add(key) + _require( + "monetary_binding" not in spec.metadata, "UNSUPPORTED_MONETARY_BINDING" + ) + name = spec.metadata.get("contract_target_id") + _require(name in contract_targets, "MISSING_BINDING") + used_bindings.add(name) + target = contract_targets[name] + _require( + set(target) == {"bindings"} and set(target["bindings"]) == {"policyengine"}, + "BINDING_SHAPE", + ) + binding = target["bindings"]["policyengine"] + coordinate = (spec.entity, spec.measure) + _require( + coordinate not in measurements + or measurements[coordinate] == _json(binding), + "CONFLICTING_MEASUREMENT_BINDINGS", + ) + measurements[coordinate] = _json(binding) + _require(set(binding) <= _BINDING_KEYS, "UNSUPPORTED_BINDING") + _require( + ("value_variable" in binding) != ("value_expression" in binding), + "VALUE_BINDING", + ) + _require( + binding.get("from_entity", spec.entity) == spec.entity, "BINDING_ENTITY" + ) + _require(spec.entity in entities, "TARGET_ENTITY") + _require( + spec.measure != f"{spec.entity}_id" + and not ( + spec.entity == "person" + and spec.measure in {f"person_{e}_id" for e in US_SCHEMA.group_entities} + ), + "STRUCTURAL_MEASURE_COLLISION", + ) + _require(spec.measure not in columns[spec.entity], "INPUT_MEASURE_COLLISION") + _require(spec.measure not in model_outputs, "MODEL_MEASURE_COLLISION") + for predicate in binding.get("filters", ()): + _require( + set(predicate) + in ({"variable", "operator", "value"}, {"variable", "equals"}), + "PREDICATE_FIELDS", + ) + _require(used_bindings == set(contract_targets), "UNUSED_BINDINGS") + _require(target_scopes == keys, "UNCOVERED_ADVERTISED_SCOPE") + model = _resolved_model_contract(model_outputs, leaf_policy) + entries = {} if leaf_policy is None else model["leaf_policy"]["document"]["entries"] + supplied = {name for names in columns.values() for name in names} + for name, entry in entries.items(): + if entry["kind"] == "assumption": + _require(name not in supplied, "ASSUMPTION_COLUMN_COLLISION:" + name) + for contract in model["roots"]: + root = contract["closure"]["root"] + _require(root not in columns[contract["entity"]], "FORMULA_INPUT_COLLISION") + _require( + not ( + set(contract["closure"]["formula_nodes"]) + & {name for names in columns.values() for name in names} + ), + "FORMULA_INPUT_COLLISION", + ) + for leaf, entity in contract["leaves"].items(): + if leaf not in entries or entries[leaf]["kind"] == "producer": + _require(leaf in columns[entity], "UNDECLARED_MODEL_LEAF:" + leaf) + params = { + "registry": _json(_registry_document(registry)), + "contract_targets": _json(contract_targets), + "advertised_scopes": _json([asdict(s) for s in scopes]), + "geography_vintage": geography_vintage, + "period": period, + "model_outputs": model_outputs, + "model_contract": _json(model), + } + _require(len(_json(params).encode()) <= MAX_DECLARATION_BYTES, "DECLARATION_BOUND") + return Node( + node_id, + FiscalMeasurementKernel.ref, + population=population, + inputs=tuple(Slice(e, columns[e]) for e in entities if columns[e]), + params=params, + artifact_outputs=(ArtifactOutput("measurement", MEASUREMENT_TYPE),), + ) + + +class _Adapter: + def __init__(self, frame): + self.tables = {e: frame.table(e).copy(deep=True) for e in frame.entities} + + def column(self, entity, variable): + if variable == f"{entity}_count": + return np.ones(len(self.tables[entity]), dtype=np.float64) + return self.tables[entity][variable].to_numpy(copy=True) + + def has_column(self, entity, variable): + return variable in self.tables[entity] + + def require_known_predicate(self, entity, variable): + # A missing comparison input is unknown, even when NumPy would turn + # the comparison into False or treat NaN as a nonzero target mask. + _require( + not pd.isna(self.column(entity, variable)).any(), + "MISSING_PREDICATE_INPUT:" + entity + "." + variable, + ) + + def set_column(self, entity, variable, values): + _require(variable not in self.tables[entity], "MEASUREMENT_COLUMN_COLLISION") + array = np.asarray(values) + _require(array.shape == (len(self.tables[entity]),), "MEASUREMENT_ALIGNMENT") + self.tables[entity][variable] = array + + +def _array(array): + array = np.ascontiguousarray(array) + return {"dtype": array.dtype.str, "hex": array.tobytes().hex()} + + +def _read_array(raw, dtypes): + _require(type(raw) is dict and set(raw) == {"dtype", "hex"}, "ARRAY_FIELDS") + _require(raw["dtype"] in dtypes and type(raw["hex"]) is str, "ARRAY_DTYPE") + try: + values = np.frombuffer(bytes.fromhex(raw["hex"]), dtype=raw["dtype"]).copy() + except (ValueError, TypeError): + raise ValueError("FISCAL_MEASUREMENT_ARRAY_ENCODING") from None + _require(_array(values) == raw, "ARRAY_CANONICAL") + return values + + +@dataclass(frozen=True) +class FiscalMeasurement: + """Decoded numbers and declarations only; no source or release authority.""" + + document: dict + registry: TargetRegistry + household_ids: np.ndarray + matrix: sparse.csr_array + target_values: np.ndarray + + +def decode_fiscal_measurement(payload: bytes) -> FiscalMeasurement: + """Decode bounded, canonical numerical bytes without loading models/files.""" + _require(type(payload) is bytes and 0 < len(payload) <= MAX_BYTES, "PAYLOAD_BOUND") + raw = json.loads(payload) + _require(canonical_json(raw) == payload, "CANONICAL") + _require( + set(raw) + == { + "protocol", + "node", + "declaration", + "projection_sha256", + "household_ids", + "indptr", + "indices", + "data", + "target_values", + "support", + }, + "PAYLOAD_FIELDS", + ) + _require(raw["protocol"] == PROTOCOL, "PROTOCOL") + registry = _registry(json.loads(raw["declaration"]["registry"])) + ids = _read_array(raw["household_ids"], {" 0 and len(set(ids.tolist())) == len(ids), "HOUSEHOLD_IDS") + indptr = _read_array(raw["indptr"], {"= 0) + and indptr[-1] == len(indices) == len(data), + "CSR_SHAPE", + ) + _require( + np.all((indices >= 0) & (indices < len(ids))) and np.isfinite(data).all(), + "CSR_VALUES", + ) + problem = sparse.csr_array((data, indices, indptr), shape=(len(registry), len(ids))) + _require(problem.has_canonical_format and np.all(data != 0), "CSR_CANONICAL") + _require( + type(raw["support"]) is list and len(raw["support"]) == len(registry), + "SUPPORT_SHAPE", + ) + for spec, row in zip(registry, raw["support"], strict=True): + _require( + type(row) is dict + and set(row) + == { + "target", + "positive_weight_positive_rows", + "positive_weight_negative_rows", + }, + "SUPPORT_FIELDS", + ) + _require(row["target"] == spec.to_target().row_name, "SUPPORT_TARGET") + for key in ("positive_weight_positive_rows", "positive_weight_negative_rows"): + _require( + type(row[key]) is int and 0 <= row[key] <= len(ids), "SUPPORT_COUNT" + ) + return FiscalMeasurement(raw, registry, ids, problem, values) + + +def verify_fiscal_measurement(payload, *, context, expected_sha256): + """Bind a graph-selected artifact to its exact declared input projection. + + The host must obtain expected_sha256 from the matching actual graph record; + an arbitrary supplied hash cannot establish producer or parent authority. + """ + _require(_sha(payload) == expected_sha256, "ARTIFACT_DIGEST") + result = decode_fiscal_measurement(payload) + _require( + _json(result.document["node"]) == _json(normative(context.node)), + "NODE_BINDING", + ) + _require( + _json(result.document["declaration"]) == _json(dict(context.params)), + "DECLARATION_BINDING", + ) + _require( + result.document["projection_sha256"] == _context_digest(context).hex(), + "PROJECTION_BINDING", + ) + expected = context.tables["household"]["household_id"].to_numpy() + _require( + result.household_ids.dtype == expected.dtype + and np.array_equal(result.household_ids, expected), + "HOUSEHOLD_ALIGNMENT", + ) + masks = _geography_masks( + context.tables["household"], + _scopes(json.loads(context.params["advertised_scopes"])), + ) + weights = context.weights["household"].values + for i, spec in enumerate(result.registry): + row = result.matrix[i : i + 1] + mask = masks[spec.hierarchy.geography.level, spec.hierarchy.geography.id] + _require(mask[row.indices].all(), "GEOGRAPHIC_ROW_SUPPORT") + _require( + _support(spec, row, weights) == result.document["support"][i], + "SUPPORT_BINDING", + ) + return result + + +def _geography_masks(household, scopes): + masks = {} + for column in ("state_fips", "congressional_district_geoid"): + values = household[column] + _require( + values.dtype.kind in "iu" and not values.isna().any(), "GEOGRAPHY_DTYPE" + ) + _require( + np.array_equal( + household.congressional_district_geoid.to_numpy() // 100, + household.state_fips.to_numpy(), + ), + "STATE_CD_MAPPING", + ) + for scope in scopes: + mask = ( + np.ones(len(household), dtype=bool) + if scope.column is None + else household[scope.column].to_numpy() == scope.value + ) + _require(mask.any(), "EMPTY_ADVERTISED_GEOGRAPHY:" + scope.geography.id) + masks[scope.geography.level, scope.geography.id] = mask + for level in ("state", "congressional_district"): + covered = np.sum( + [mask for (scope_level, _), mask in masks.items() if scope_level == level], + axis=0, + ) + _require(np.all(covered == 1), "UNCOVERED_POPULATION_GEOGRAPHY") + return masks + + +def _support(spec, row, weights): + positive = int(np.count_nonzero((row.data > 0) & (weights[row.indices] > 0))) + negative = int(np.count_nonzero((row.data < 0) & (weights[row.indices] > 0))) + _require( + spec.value == 0 or (positive > 0 if spec.value > 0 else negative > 0), + "UNSUPPORTED_TARGET:" + spec.name, + ) + return { + "target": spec.to_target().row_name, + "positive_weight_positive_rows": positive, + "positive_weight_negative_rows": negative, + } + + +class FiscalMeasurementKernel(KernelBase): + """Materialize declared measures and compile the real household CSR system.""" + + ref = "us.declared_fiscal_measurement@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + dependencies=("numpy", "pandas", "scipy"), + ) + + def __init__(self, *, model_outputs=(), leaf_policy=None): + self.model_outputs = model_outputs + if leaf_policy is not None: + leaf_policies.fiscal_leaf_policy_document(leaf_policy) + self.leaf_policy = leaf_policy + + def implementation_hash(self): + return _sha( + canonical_json( + { + "implementation": source_hash( + sys.modules[__name__], + target_materialization, + matrix, + target_math, + TargetSpec, + Frame, + policyengine_us, + # The adapter delegates table/weight materialization + # to this helper's separate defining module. + policyengine_us.engine_tables, + runtime, + leaf_policies, + _context_digest, + dependencies=self.capabilities.dependencies, + ), + "model": _resolved_model_contract( + self.model_outputs, self.leaf_policy + ), + } + ) + ) + + def run(self, context: KernelContext) -> KernelResult: + params = context.params + _require(params["model_outputs"] == self.model_outputs, "KERNEL_MODEL_OUTPUTS") + registry = _registry(json.loads(params["registry"])) + scopes = _scopes(json.loads(params["advertised_scopes"])) + expected = fiscal_measurement_node( + registry, + population=context.node.population, + input_columns={s.entity: s.columns for s in context.node.inputs}, + contract_targets=json.loads(params["contract_targets"]), + advertised_scopes=scopes, + geography_vintage=params["geography_vintage"], + period=params["period"], + model_outputs=self.model_outputs, + leaf_policy=self.leaf_policy, + node_id=context.node.id, + ) + _require(context.node.normative() == expected.normative(), "NODE_DECLARATION") + model = json.loads(params["model_contract"]) + _require( + not any( + entry["kind"] == "assumption" + for entry in model.get("leaf_policy", {}) + .get("document", {}) + .get("entries", {}) + .values() + ), + "ASSUMPTION_PARENT_ADMISSION_UNSUPPORTED", + ) + projection = _context_digest(context).hex() + tables = {e: t.copy(deep=True) for e, t in context.tables.items()} + for entity in US_SCHEMA.group_entities: + if entity not in tables: + tables[entity] = pd.DataFrame( + { + f"{entity}_id": np.unique( + tables["person"][f"person_{entity}_id"].to_numpy() + ), + } + ) + frame = Frame( + tables, + US_SCHEMA, + {"household": context.weights["household"]}, + strata=context.strata.copy(), + ) + adapter = _Adapter(frame) + if self.model_outputs: + for contract in model["roots"]: + for leaf, entity in contract["leaves"].items(): + values = frame.table(entity)[leaf] + _require(not values.isna().any(), "MISSING_MODEL_INPUT:" + leaf) + if pd.api.types.is_numeric_dtype(values.dtype): + _require( + np.isfinite(values.to_numpy()).all(), + "NONFINITE_MODEL_INPUT", + ) + outputs = policyengine_us.PolicyEngineUSEngine().materialize( + frame, self.model_outputs, params["period"] + ) + _require(set(outputs) == set(self.model_outputs), "MODEL_RESULT_ROSTER") + for contract in model["roots"]: + output = np.asarray(outputs[contract["closure"]["root"]]) + _require( + output.dtype.kind in "biuf" and np.isfinite(output).all(), + "MODEL_RESULT_VALUES", + ) + adapter.set_column( + contract["entity"], + contract["closure"]["root"], + outputs[contract["closure"]["root"]], + ) + bindings = json.loads(params["contract_targets"]) + predicates = { + (spec.entity, predicate["variable"]) + for spec in registry + for predicate in bindings[spec.metadata["contract_target_id"]]["bindings"][ + "policyengine" + ].get("filters", ()) + } + for entity, variable in sorted(predicates): + adapter.require_known_predicate(entity, variable) + prepared = target_materialization.materialize_target_bindings( + adapter, + registry, + bindings, + period=params["period"], + ) + _require( + not prepared.skipped, "UNMATERIALIZED_TARGETS:" + str(prepared.skipped) + ) + # TargetSpec.filter is evaluated by the CSR compiler, independently + # of contract binding predicates, and may use a prepared measure. + for entity, variable in sorted( + {(s.entity, s.filter) for s in registry if s.filter is not None} + ): + adapter.require_known_predicate(entity, variable) + work = Frame( + adapter.tables, US_SCHEMA, {"household": context.weights["household"]} + ) + problem = matrix.build_constraint_matrix( + work, registry.to_target_set(), weight_entity="household" + ) + _require( + not problem.skipped + and problem.names == tuple(s.to_target().row_name for s in registry), + "TARGET_ALIGNMENT", + ) + household = frame.table("household") + masks = _geography_masks(household, scopes) + rows, support = [], [] + incoming = problem.initial_weights.values + for i, spec in enumerate(registry): + mask = masks[spec.hierarchy.geography.level, spec.hierarchy.geography.id] + row = problem.matrix[i : i + 1].multiply(mask).tocsr() + row.eliminate_zeros() + rows.append(row) + support.append(_support(spec, row, incoming)) + csr = sparse.vstack(rows, format="csr") + _require(csr.nnz * 48 + frame.n("household") * 16 < MAX_BYTES, "MATRIX_BOUND") + payload = canonical_json( + { + "protocol": PROTOCOL, + "node": normative(context.node), + "declaration": dict(params), + "projection_sha256": projection, + "household_ids": _array(household.household_id.to_numpy()), + "indptr": _array(csr.indptr.astype(" str: + return implementation_hash("geography") + + +class USGeographyLookupKernel(_USGeographyKernel): + ref = "us.production.geography_lookup@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": GEOGRAPHY_PHASE}: + raise ValueError("Unsupported US geography lookup parameters.") + ladder_bytes = load_source_bytes( + "raw-bytes-v1", context.sources["us_puma_ladder_2020"] + ) + crosswalk_bytes = load_source_bytes( + "raw-bytes-v1", context.sources["us_cd_crosswalk_117_119"] + ) + ladder, expected = _lookups(ladder_bytes, crosswalk_bytes) + return KernelResult( + artifacts={"ladder": ladder_bytes, "crosswalk": crosswalk_bytes}, + receipt={ + "phase": GEOGRAPHY_PHASE, + "implementation": implementation_manifest("geography"), + "district_vintage": CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + "district_count": len(expected), + "puma_count": len(ladder), + "ladder_sha256": hashlib.sha256(ladder_bytes).hexdigest(), + "crosswalk_sha256": hashlib.sha256(crosswalk_bytes).hexdigest(), + }, + ) + + +class USGeographyBoundaryKernel(_USGeographyKernel): + """Preserve all rows while opening the geography rewrite version.""" + + ref = "us.production.geography_boundary@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": GEOGRAPHY_PHASE}: + raise ValueError("US geography boundary requires its registered phase.") + person = context.tables["person"] + return KernelResult( + receipt={"implementation": implementation_manifest("geography")}, + keep=pd.Series( + True, index=pd.Index(person["person_id"], name="person_id"), dtype=bool + ), + ) + + +class USGeographyAssignKernel(_USGeographyKernel): + ref = "us.production.geography_assign@2" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + seed_source=SeedSource.PARAM, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != { + "phase": GEOGRAPHY_PHASE, + "seed": 0, + "assign_tract": False, + }: + raise ValueError( + "US geography requires the declared seed and joint county/CD contract." + ) + artifact = context.artifacts.get("frame_context") + if artifact is None or artifact.type != US_FRAME_CONTEXT_TYPE: + raise ValueError("US geography requires a typed frame context.") + document = _decode(artifact.payload) + _mass_records(document["mass_log"]) + household = context.tables["household"] + declared = document["entities"]["household"] + if any( + declared[key] != value + for key, value in _row_identity(household, "household").items() + ): + raise ValueError("US geography household identity does not match context.") + if set(household.columns) - set(declared["columns"]): + raise ValueError("US geography has undeclared household input columns.") + if ( + document["weight_sources"].get("household") + != context.weights["household"].kind.value + ): + raise ValueError( + "US geography requires the matching household weight source." + ) + ladder, expected = _lookups( + context.artifacts["ladder"].payload, context.artifacts["crosswalk"].payload + ) + table = assign_us_puma_ladder( + household, + ladder, + seed=0, + assign_tract=False, + expected_congressional_district_vintage=CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + ) + columns = {} + for owned in context.node.outputs: + if ( + owned.entity != "household" + or owned.column not in US_PUMA_LADDER_COLUMNS + ): + raise ValueError("US geography has an unsupported owned output.") + original = table[owned.column] + converted = original.astype(dtype_for_token(owned.dtype)) + if not original.isna().equals(converted.isna()) or not np.array_equal( + original.to_numpy(), converted.to_numpy() + ): + raise ValueError( + "US geography storage conversion changed values or missingness." + ) + table[owned.column] = converted + columns[(owned.entity, owned.column)] = pd.Series( + converted.array, + index=pd.Index(table["household_id"], name="household_id"), + ) + if set(column for _, column in columns) != set(US_PUMA_LADDER_COLUMNS): + raise ValueError( + "US geography must own exactly its three household outputs." + ) + # This operator changes only these three cells on each household. The + # graph boundary preserves every row, weight and membership. Carry the + # authenticated descriptors for untouched entities without copying + # their population tables into this household-only operator. + for column in US_PUMA_LADDER_COLUMNS: + if column not in declared["columns"]: + declared["columns"].append(column) + return KernelResult( + columns=columns, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": GEOGRAPHY_PHASE, + "implementation": implementation_manifest("geography"), + "seed": 0, + "assign_tract": False, + "district_vintage": CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + "target_district_count": len(expected), + "summary": us_puma_ladder_assignment_summary( + table, + ladder, + weight_values=context.weights["household"].values, + ), + }, + ) + + +class USGeographyGateKernel(_USGeographyKernel): + ref = "us.production.geography_gate@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + role=KernelRole.GATE, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": GEOGRAPHY_PHASE}: + raise ValueError("US geography gate requires its registered phase.") + gate = us_puma_ladder_gate( + context.tables["household"], + context.weights["household"].values, + assign_tract=False, + ) + ladder, _ = _lookups( + context.artifacts["ladder"].payload, context.artifacts["crosswalk"].payload + ) + joint_gate = us_puma_ladder_joint_support_gate( + context.tables["household"], + ladder, + assign_tract=False, + expected_congressional_district_vintage=CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + ) + return KernelResult( + receipt={ + "outcome": "pass" if gate.passed and joint_gate.passed else "fail", + "evidence": GateReport((gate, joint_gate)).to_manifest(), + "scope": "geography_assignment_integrity", + } + ) + + +def us_geography_nodes( + columns: Sequence[Owned], *, base: str, context_producer: str +) -> tuple[Node, ...]: + """Append a declared production geography stage after source assembly. + + This checks assignment integrity, not CD calibration or release readiness. + """ + inventory = {(owned.entity, owned.column): owned for owned in columns} + if len(inventory) != len(columns): + raise ValueError("US geography input column declarations repeat coordinates.") + if ("person", "age") not in inventory: + raise ValueError("US geography requires the assembled person age column.") + if ("household", "state_fips") not in inventory: + raise ValueError("US geography requires household state_fips.") + input_columns = ("state_fips",) + ( + ("puma",) if ("household", "puma") in inventory else () + ) + boundary = f"{GEOGRAPHY_PHASE}.boundary" + lookup = f"{GEOGRAPHY_PHASE}.lookup" + output_types = { + "puma": "string", + CONGRESSIONAL_DISTRICT_GEOID_COLUMN: "int64", + "county_fips": "string", + } + outputs = [] + for column, dtype in output_types.items(): + incumbent = inventory.get(("household", column)) + if incumbent: + if incumbent.dtype not in ( + {"int64", "Int64"} if dtype == "int64" else {"string"} + ): + raise ValueError(f"Unsupported incumbent geography dtype: {column}.") + dtype = incumbent.dtype + outputs.append(Owned("household", column, dtype, rewrite=incumbent is not None)) + return ( + Node( + id=lookup, + kernel=USGeographyLookupKernel.ref, + population=base, + sources=tuple(source.name for source in LOOKUP_SOURCES), + params={"phase": GEOGRAPHY_PHASE}, + artifact_outputs=( + ArtifactOutput("ladder", US_PUMA_LOOKUP_TYPE), + ArtifactOutput("crosswalk", US_CD_CROSSWALK_TYPE), + ), + ), + Node( + id=boundary, + kernel=USGeographyBoundaryKernel.ref, + base=base, + structural=StructuralDelta.FILTER, + inputs=(Slice("person", ("age",)),), + params={"phase": GEOGRAPHY_PHASE}, + ), + Node( + id=GEOGRAPHY_PHASE, + kernel=USGeographyAssignKernel.ref, + population=boundary, + inputs=(Slice("household", input_columns),), + outputs=tuple(outputs), + params={"phase": GEOGRAPHY_PHASE, "seed": 0, "assign_tract": False}, + artifact_inputs=( + ArtifactInput( + "frame_context", + context_producer, + "frame_context", + US_FRAME_CONTEXT_TYPE, + ), + ArtifactInput("ladder", lookup, "ladder", US_PUMA_LOOKUP_TYPE), + ArtifactInput("crosswalk", lookup, "crosswalk", US_CD_CROSSWALK_TYPE), + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ), + Node( + id=f"{GEOGRAPHY_PHASE}.gate", + kernel=USGeographyGateKernel.ref, + population=boundary, + inputs=(Slice("household", ("state_fips", *US_PUMA_LADDER_COLUMNS)),), + params={"phase": GEOGRAPHY_PHASE}, + artifact_inputs=( + ArtifactInput("ladder", lookup, "ladder", US_PUMA_LOOKUP_TYPE), + ArtifactInput("crosswalk", lookup, "crosswalk", US_CD_CROSSWALK_TYPE), + ), + ), + ) + + +def register_us_geography_kernels(registry: KernelRegistry) -> None: + for kernel in ( + USGeographyLookupKernel(), + USGeographyBoundaryKernel(), + USGeographyAssignKernel(), + USGeographyGateKernel(), + ): + registry.register(kernel) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py new file mode 100644 index 000000000..c8bbdf3c0 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py @@ -0,0 +1,230 @@ +"""Non-authoritative HU transport bound to CREATE and selected graph ancestry. + +Parsing these bytes never issues AuthenticatedHousingUniverse or Frame authority. +The graph supplies the producer and preparation-receipt custody; this module +checks its native bytes and the exact int64 promotion of selected carried cells. +""" + +from __future__ import annotations + +import struct +from dataclasses import InitVar, dataclass + +import numpy as np +import pandas as pd + +from microcosm.graph import ArtifactType + +from . import asec_housing_universe as hu +from . import asec_housing_universe_source as source + +US_ASEC_HOUSING_UNIVERSE_TYPE = ArtifactType( + "microcosm.us.asec_bound_housing_universe", 1 +) +HOUSEHOLD_EVIDENCE_COLUMNS = ( + *("asec_" + name for name in ("H_SEQ",) + hu.CONTEXT_COLUMNS), + *source.ATTACHED_COLUMNS, +) +_TOKEN = object() +_HEADER_KEYS = frozenset( + { + "schema_version", + "artifact_kind", + "source_authentication", + "release_eligible", + "source_period_kind", + "zero_origin_policy", + "zero_origin_evidence", + "household_rows", + "columns", + "codebook", + "dictionaries", + "definition_sha256", + "source", + "implementation", + "limitations", + } +) + + +ACCEPTED_PREPARED_RECEIPT_SCHEMAS = ("microcosm.us.asec_prepared_receipt.v3",) +ACCEPTED_PREPARED_SOURCE_KINDS = ("us_asec_prepared_current_money_v3",) + + +@dataclass(frozen=True) +class BoundHousingUniverse: + """Immutable native buffers with graph bindings, no full-source authority.""" + + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + hu._require(_token is _TOKEN, "BOUND_HU_CONSTRUCTOR") + + def array(self, name: str) -> np.ndarray: + hu._require(name in hu.COLUMNS, "BOUND_HU_COLUMN") + return np.frombuffer( + self.buffers[hu.COLUMNS.index(name)], + dtype=" BoundHousingUniverse: + """Validate transport against a graph-provided receipt, not a source decoder.""" + try: + hu._require( + type(payload) is bytes + and len(hu.MAGIC) + 4 + 32 < len(payload) <= hu.PAYLOAD_MAX_BYTES, + "BOUND_HU_SIZE", + ) + binding = prepared_receipt["housing_universe"] + hu._require( + prepared_receipt["schema"] in ACCEPTED_PREPARED_RECEIPT_SCHEMAS + and prepared_receipt["source_kind"] in ACCEPTED_PREPARED_SOURCE_KINDS + and prepared_receipt["release_eligible"] is False + and binding["parent_kind"] == "HousingStatusAttachedAsec" + and binding["source_period_kind"] == "interview_household_universe" + and binding["native_dtype"] == "uint8" + and binding["graph_dtype"] == "int64" + and binding["aliases"] == list(source.ATTACHED_COLUMNS), + "BOUND_HU_PREPARATION_SCHEMA", + ) + hu._require(hu._sha(payload) == binding["payload_sha256"], "BOUND_HU_PAYLOAD") + hu._require(payload.startswith(hu.MAGIC), "BOUND_HU_MAGIC") + hu._require(hu._sha(payload[:-32]) == payload[-32:].hex(), "BOUND_HU_CHECKSUM") + size = struct.unpack_from(" None: + """Compare graph int64 rows with native evidence; no cast or authority upgrade.""" + hu._require(type(body) is BoundHousingUniverse, "BOUND_HU_TYPE") + hu._require( + type(positions) is np.ndarray + and positions.dtype == np.dtype("int64") + and positions.shape == (len(household),) + and bool( + ((positions >= 0) & (positions < len(body.array("household_id")))).all() + ) + and (len(positions) < 2 or bool((np.diff(positions) > 0).all())), + "BOUND_HU_POSITIONS", + ) + hu._require(household.columns.is_unique, "BOUND_HU_TABLE_COLUMNS") + for name, column in ( + ("household_id", "household_id"), + *((n, "asec_" + n) for n in ("H_SEQ",) + hu.CONTEXT_COLUMNS), + *zip(hu.DERIVED_COLUMNS, source.ATTACHED_COLUMNS, strict=True), + ): + hu._require(column in household, "BOUND_HU_CARRIED_COLUMN") + values = household[column] + hu._require( + values.dtype == np.dtype("int64") + and np.array_equal(values.to_numpy(), body.array(name)[positions]), + "BOUND_HU_CARRIED_VALUES", + ) + if income_years is not None: + hu._require( + type(income_years) is np.ndarray + and income_years.dtype == np.dtype("int64") + and np.array_equal(income_years, body.array("income_year")[positions]), + "BOUND_HU_COHORTS", + ) + + +__all__ = [ + "BoundHousingUniverse", + "HOUSEHOLD_EVIDENCE_COLUMNS", + "US_ASEC_HOUSING_UNIVERSE_TYPE", + "bind_housing_universe", + "verify_graph_housing_rows", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py new file mode 100644 index 000000000..e715b47de --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py @@ -0,0 +1,532 @@ +"""Reviewed, path-independent implementation identities for the US source graph. + +Whole modules remain the unit of code identity. The packaged inventory is a +reviewed dependency fence, not a Python sandbox or inferred call-graph proof. +Its import/use/resource expressions prevent silently extending a reviewed route +to an unbound helper. Updating that inventory requires a scope review. +""" + +from __future__ import annotations + +import ast +import csv +import hashlib +import importlib.metadata as importlib_metadata +import importlib.util +import inspect +import json +import sys +from collections.abc import Mapping +from functools import lru_cache +from pathlib import Path +from types import MappingProxyType + +import pandas as pd + +STAGE_DEPENDENCIES = MappingProxyType( + { + "asec_codec_v4": ("numpy", "pandas", "h5py"), + "acs_codec_2024": ("numpy", "pandas", "microunit", "PyYAML", "pyarrow"), + "acs_housing_universe_2024": ( + "numpy", + "pandas", + "microunit", + "PyYAML", + "pyarrow", + ), + # Full authenticated source catalogues, one household draw, selected + # native construction and one allocation. This deliberately excludes + # legacy prepared-source income observations and engine resources. + "authenticated_survey_population_v1": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + "assembly_prepare": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + "assembly_harmonize": ("numpy", "pandas"), + "geography": ("numpy", "pandas"), + # The ASEC prepared current-money slice. PolicyEngine-US is deliberately + # absent: its version is a normative parameter of the evaluation node and + # a pinned field of every admitted closure contract, so engine drift + # refuses there. Putting it here instead would make a secrets-free lane + # without the engine extra unable to hash any stage of this graph. + "asec_prepared_v3": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + # The composed prepared-ASEC/native-ACS population. It runs both source + # scopes in one CREATE, so its module inventory is their union and its + # dependency set is theirs (identical in both). PolicyEngine-US is + # absent for the same reason it is absent from the prepared scope: no + # engine node lives in this graph. + "composed_population_v1": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + # Binding the carried prepared-ASEC evidence to the composed ASEC rows + # and deriving the reviewed corrected leaves and reported observations + # over them. It runs no source loader of its own, but its resolution and + # every verifier it reuses live inside the composing scope, so its module + # inventory is that scope's plus the two binding modules and its + # dependency set is identical. + "composed_asec_binding_v1": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + } +) +_CODECS = MappingProxyType( + { + "us-asec-raw-stage-v4": ( + "asec_codec_v4", + "microcosm.build.us_runtime.graph_sources:load_graph_asec", + ), + "us-acs-native-2024-v1": ( + "acs_codec_2024", + "microcosm.build.us_runtime.graph_sources:load_graph_acs", + ), + "us-asec-prepared-current-money-v3": ( + "asec_prepared_v3", + "microcosm.build.us_runtime.asec_prepared_source:load_graph_asec_prepared", + ), + "raw-bytes-v1": ("geography", "microcosm.graph.codecs:load_raw_bytes"), + "us-acs-housing-universe-2024-v1": ( + "acs_housing_universe_2024", + "microcosm.build.us_runtime.acs_housing_universe_source:load_graph_acs_housing_universe", + ), + } +) +_RESOURCE_CALLS = frozenset( + { + "open", + "read_bytes", + "read_text", + "read_csv", + "read_parquet", + "read_hdf", + "read_excel", + "read_json", + "read_table", + "read_feather", + "File", + "loadtxt", + "genfromtxt", + "load", + "safe_load", + "files", + "joinpath", + "import_module", + "__import__", + "exec", + "eval", + } +) + + +# These local lazy entry points are dependencies even though no ImportFrom +# binds their names. Record references AND complete calls in their caller scope: +# passing the dispatcher around must not hide its use, and changing a selected +# symbol must require review. This adds tripwires; it grants no scope exemption. +_LOCAL_IMPORT_DISPATCHERS = { + "microcosm.build/us_runtime/graph_sources.py": frozenset( + { + "_stacked_alias", + "__getattr__", + "assemble_stacked_spine", + "prepare_stacked_spine", + "harmonize_stacked_spine_weights", + } + ), +} + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _digest(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _package_roots() -> dict[str, Path]: + # Resolve packages, never import leaf modules (including optional adapters) + # merely to hash their files. Editable and wheel installs use the same names. + roots = {} + for package in ( + "microcosm.build", + "microcosm.frame", + "microcosm.graph", + "microunit", + ): + spec = importlib.util.find_spec(package) + if spec is None and package == "microunit": + continue # Only the ACS/CREATE scopes require this dependency. + locations = () if spec is None else spec.submodule_search_locations or () + if len(locations) != 1: + raise ValueError( + f"US implementation package inventory unavailable: {package}." + ) + roots[package] = Path(next(iter(locations))).resolve() + return roots + + +@lru_cache(maxsize=256) +def _dependency_details(payload: bytes, name: str, covered: tuple[str, ...]) -> dict: + """Static tripwire cached by exact bytes, never by path, mtime or version.""" + tree = ast.parse(payload) + imports, aliases, unbound_imports = set(), {}, set() + package, relative = name.split("/", 1) + module = package + "." + relative.removesuffix(".py").replace("/", ".") + parent = module.rpartition(".")[0] + for node in ast.walk(tree): + if isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + if alias.name == "*": + raise ValueError("Unclassified wildcard dependency import.") + target = ( + alias.name + if isinstance(node, ast.Import) + else ( + importlib.util.resolve_name( + "." * node.level + (node.module or ""), parent + ) + if node.level + else node.module + ) + ) + is_stdlib = target.split(".")[0] in sys.stdlib_module_names + if not is_stdlib: + imports.add(target) + is_bound = target in covered or any( + target == item[1:] or target.startswith(item[1:] + ".") + for item in covered + if item.startswith("*") + ) + if not is_stdlib and not is_bound: + aliases[alias.asname or alias.name.split(".")[0]] = target + unbound_imports.add(f"import:{target}:{alias.name}") + resources, uses = set(), set(unbound_imports) + dispatchers = _LOCAL_IMPORT_DISPATCHERS.get(name, ()) + + class Visitor(ast.NodeVisitor): + scope = "" + + def visit_FunctionDef(self, node): + previous = self.scope + self.scope = f"{previous}.{node.name}" + self.generic_visit(node) + self.scope = previous + + visit_AsyncFunctionDef = visit_FunctionDef # noqa: N815 - AST visitor API + visit_ClassDef = visit_FunctionDef # noqa: N815 - AST visitor API + + def visit_Name(self, node): + if isinstance(node.ctx, ast.Load) and node.id in dispatchers: + resources.add( + f"{self.scope}:{ast.dump(node, include_attributes=False)}" + ) + if isinstance(node.ctx, ast.Load) and node.id in aliases: + uses.add(f"{self.scope}:{aliases[node.id]}:{node.id}") + + def visit_Call(self, node): + name = ( + node.func.attr + if isinstance(node.func, ast.Attribute) + else getattr(node.func, "id", None) + ) + # Bind reflective selection too, including the dispatcher's + # getattr(import_module(...), name) target expression. + if ( + name in _RESOURCE_CALLS + or name in dispatchers + or (dispatchers and name == "getattr") + ): + resources.add( + f"{self.scope}:{ast.dump(node, include_attributes=False)}" + ) + self.generic_visit(node) + + Visitor().visit(tree) + return { + "imports": sorted(imports), + "unbound_uses": sorted(uses), + "resource_accesses": sorted(resources), + } + + +def _covered_imports(name: str, inventory: dict) -> tuple[str, ...]: + """Omit a symbol tripwire only if every scope using this file binds it.""" + scopes = [spec for spec in inventory["stages"].values() if name in spec["modules"]] + modules = set.intersection(*(set(spec["modules"]) for spec in scopes)) + dependencies = set.intersection(*(set(spec["dependencies"]) for spec in scopes)) + covered = set() + for module in modules: + package, relative = module.split("/", 1) + resolved = package + "." + relative.removesuffix(".py").replace("/", ".") + covered.add(resolved.removesuffix(".__init__")) + covered.update( + "*" + ("yaml" if dependency == "PyYAML" else dependency) + for dependency in dependencies + ) + return tuple(sorted(covered)) + + +def _dependency_contract(payload: bytes, name: str, covered: tuple[str, ...]) -> dict: + details = _dependency_details(payload, name, covered) + return { + "imports": details["imports"], + "unbound_uses_sha256": _digest(_canonical(details["unbound_uses"])), + "resource_accesses_sha256": _digest(_canonical(details["resource_accesses"])), + } + + +def _inventory(roots: Mapping[str, Path]) -> tuple[dict, bytes]: + payload = ( + roots["microcosm.build"] / "us_runtime/graph_implementation_inventory.json" + ).read_bytes() + value = json.loads(payload) + if value["schema"] != "microcosm.us.implementation-inventory.v1": + raise ValueError("Unsupported US implementation inventory schema.") + if set(value["stages"]) != set(STAGE_DEPENDENCIES): + raise ValueError("US implementation stage dependency inventory differs.") + for name, dependencies in STAGE_DEPENDENCIES.items(): + if value["stages"][name]["dependencies"] != list(dependencies): + raise ValueError(f"US implementation dependency inventory differs: {name}.") + imports = { + name for contract in value["contracts"].values() for name in contract["imports"] + } + if set(value["import_classifications"]) != imports: + raise ValueError("US implementation import classification inventory differs.") + return value, payload + + +def _validate_package_roster(package: str, root: Path, inventory: dict) -> None: + actual = {path.relative_to(root).as_posix() for path in root.rglob("*.py")} + expected = inventory["package_rosters"][package] + if len(expected) != len(set(expected)) or actual != set(expected): + raise ValueError(f"US implementation Python inventory differs: {package}.") + if package == "microunit": + # py.typed is packaging metadata. Everything else must be explicitly + # bound, including an added non-Python rule/data file. + actual_resources = { + path.relative_to(root).as_posix() + for path in root.rglob("*") + if path.is_file() + and path.suffix != ".py" + and "__pycache__" not in path.parts + and path.name != "py.typed" + } + if actual_resources != set(inventory["package_resources"][package]): + raise ValueError("US implementation resource inventory differs: microunit.") + + +def _projections() -> dict: + from . import operator_boundary + + def columns(value): + if isinstance(value, Mapping): + return {key: columns(item) for key, item in sorted(value.items())} + if not isinstance(value, (tuple, list, frozenset, set)) or not all( + isinstance(item, str) for item in value + ): + raise ValueError( + "US source registry projection requires string column sets." + ) + return sorted(value) + + return { + name: columns(getattr(operator_boundary, name)) + for name in ( + "PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES", + "FORMULA_OWNED_SOURCE_COLUMNS", + ) + } + + +def _loader_ref(loader: object, expected: str, roots: Mapping[str, Path]) -> None: + actual = ( + f"{getattr(loader, '__module__', '')}:{getattr(loader, '__qualname__', '')}" + ) + module, _ = expected.split(":") + package = next(name for name in roots if module.startswith(name + ".")) + path = roots[package] / ( + module.removeprefix(package + ".").replace(".", "/") + ".py" + ) + # A correctly named function imported from another checkout is not the + # implementation whose file bytes this manifest promises. + source = inspect.getsourcefile(loader) if inspect.isfunction(loader) else None + if actual != expected or source is None or Path(source).resolve() != path.resolve(): + raise ValueError( + f"US source codec loader differs from declared reference: {expected}." + ) + + +def validate_source_codecs(codecs) -> None: + """Bind US registry names to real loaders; generic graph codec APIs stay unchanged.""" + roots = _package_roots() + # Explicit declared selectors keep this registry inspection source-blind. + try: + loaders = ( + codecs.get("us-asec-raw-stage-v4"), + codecs.get("us-acs-native-2024-v1"), + codecs.get("us-asec-prepared-current-money-v3"), + codecs.get("raw-bytes-v1"), + codecs.get("us-acs-housing-universe-2024-v1"), + ) + except Exception as error: + raise ValueError("US source codec loader missing.") from error + for (_, (_, expected)), loader in zip(_CODECS.items(), loaders, strict=True): + _loader_ref(loader, expected, roots) + + +def implementation_manifest(stage: str) -> dict: + """Return small auditable identities, never source data or absolute paths.""" + if stage not in STAGE_DEPENDENCIES: + raise ValueError(f"Unclassified US implementation stage: {stage}.") + roots = _package_roots() + inventory, inventory_bytes = _inventory(roots) + spec = inventory["stages"][stage] + modules, resources = {}, {} + packages = {name.split("/", 1)[0] for name in spec["modules"]} + if not packages <= roots.keys(): + raise ValueError("US implementation dependency package inventory unavailable.") + for package in packages - {"microcosm.build"}: + _validate_package_roster(package, roots[package], inventory) + if len(spec["modules"]) != len(set(spec["modules"])): + raise ValueError("Duplicate US implementation module inventory.") + for name in spec["modules"]: + package, relative = name.split("/", 1) + path = roots[package] / relative + if not path.is_file(): + raise ValueError(f"US implementation module inventory missing: {name}.") + payload = path.read_bytes() + actual = _dependency_contract(payload, name, _covered_imports(name, inventory)) + if actual != inventory["contracts"][name]: + raise ValueError(f"Unclassified US dependency/resource contract: {name}.") + modules[name] = _digest(payload) + for name in spec["resources"]: + package, relative = name.split("/", 1) + resources[name] = _digest((roots[package] / relative).read_bytes()) + manifest = { + "schema": "microcosm.us.implementation-manifest.v1", + "stage": stage, + "inventory_sha256": _digest(inventory_bytes), + "modules": modules, + "resources": resources, + "dependencies": { + name: importlib_metadata.version(name) for name in STAGE_DEPENDENCIES[stage] + }, + "projections": _projections() if spec["source_boundary_projection"] else {}, + "runtime_options": ( + { + "pandas.mode.string_storage": pd.get_option("mode.string_storage"), + "pandas.future.infer_string": pd.get_option("future.infer_string"), + "acs_reader_resolved_string_storage": pd.StringDtype().storage, + } + if stage + in { + "acs_codec_2024", + "assembly_prepare", + "acs_housing_universe_2024", + "authenticated_survey_population_v1", + "composed_population_v1", + "composed_asec_binding_v1", + } + else {} + ), + "codecs": {}, + } + if stage in { + "asec_prepared_v3", + "acs_housing_universe_2024", + "authenticated_survey_population_v1", + "composed_population_v1", + "composed_asec_binding_v1", + }: + # These stages parse pinned official CSV members with the stdlib reader. + # csv's field size limit is process state, so it belongs in the identity + # of each stage that reads under it. Whole-module and inventory hashes + # separately record code changes, including additions of new stages. + manifest["runtime_options"]["csv.field_size_limit"] = csv.field_size_limit() + if stage in { + "assembly_prepare", + "composed_population_v1", + "composed_asec_binding_v1", + }: + bound = ( + ("asec_codec_v4", "acs_codec_2024") + if stage == "assembly_prepare" + # The composed CREATE reads the prepared directory, not the + # raw-stage checkpoint, so it binds that codec's identity instead. + # The binding stage reads no source, but every artifact it admits + # was produced under those two codecs, so it binds them too rather + # than claiming an identity narrower than its actual inputs. + else ("asec_prepared_v3", "acs_codec_2024") + ) + for name, (codec_stage, loader) in _CODECS.items(): + if codec_stage in bound: + manifest["codecs"][name] = { + "loader": loader, + "implementation_sha256": implementation_hash(codec_stage), + } + elif stage == "authenticated_survey_population_v1": + # The standalone factory registers one real closure over an operational + # capture directory. Its whole module belongs to this stage. The legacy + # five-codec registry and its strict validator retain their contract. + manifest["codecs"] = { + "us-survey-population-source-v1": { + "loader": ( + "microcosm.build.us_runtime.graph_survey_population:" + "survey_population_source_codecs..load" + ) + } + } + elif stage in ( + "asec_codec_v4", + "acs_codec_2024", + "geography", + "asec_prepared_v3", + "acs_housing_universe_2024", + ): + manifest["codecs"] = { + name: {"loader": loader} + for name, (codec_stage, loader) in _CODECS.items() + if codec_stage == stage + } + return manifest + + +def implementation_hash(stage: str) -> str: + return _digest( + b"microcosm.us.scoped-implementation.v1\0" + + _canonical(implementation_manifest(stage)) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json new file mode 100644 index 000000000..f3832710b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json @@ -0,0 +1,2466 @@ +{ + "contracts": { + "microcosm.build/cd_benchmark/canonical.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/cd_benchmark/origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.canonical" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/frame_checkpoint.py": { + "imports": [ + "h5py", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "6e84799b53beb6b39981b770f65412a60e3808f6416bda934c0017ad36196e13", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/frame_sampling.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/gates.py": { + "imports": [ + "microcosm.build.ledger_targets", + "microcosm.build.us_runtime.stacked_spine", + "microcosm.calibrate.registry", + "microcosm.calibrate.solve", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "06b1202277975ae894fc8ece00dbac4b51129c78ab41d44d45aa86969f8e940e" + }, + "microcosm.build/outer_stage_runtime.py": { + "imports": [ + "microcosm.build.frame_checkpoint", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "a0d0bd839844e63cf2581413187e20748db921b657051a72a022ea6185176fd0", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/serialization_dtypes.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/survey_allocation.py": { + "imports": [ + "microcosm.frame", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/survey_domain_sample.py": { + "imports": [ + "microcosm.build.survey_allocation", + "microcosm.frame", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/table_identity.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/_asec_current_money_codec.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_housing_universe.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_housing_universe_source.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.acs_housing_universe", + "microcosm.build.us_runtime.acs_inputs", + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.acs_sources", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.source_csv_builtin", + "microcosm.frame", + "microcosm.graph.canonical", + "microcosm.graph.population", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "a8d2536f4bb75f36e40995185901ce8867ce1c01478efba980a75d0bd778b3f7", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_inputs.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_native_coverage_binding.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.frame" + ], + "resource_accesses_sha256": "493c1a38451cc17c161232077fcea53211dc46f91695d6ec2485d755da52b291", + "unbound_uses_sha256": "937ac46755604aef55c18570df2520f92e51dd0486b93e2316a160cdaa67ad6a" + }, + "microcosm.build/us_runtime/acs_person_coverage_authentication.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.source_csv_builtin", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "27da902db7790fcf13f79f383532fb1eaffea49c118f54f76870c36cad264004", + "unbound_uses_sha256": "dc548c6590471125d90a86717034d6ed275b6e98115676512a623e4b9f7dc323" + }, + "microcosm.build/us_runtime/acs_person_coverage_columns.py": { + "imports": [ + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.source_csv_builtin", + "pandas" + ], + "resource_accesses_sha256": "94d488b944fbeaa44d07120dd12f7f1b01e7e20faddf2e3daac77d905c2eca88", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_population_catalogue.py": { + "imports": [ + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "8899ffc02d4808966ab99d6897940046a3d422acb99014c0f3cb44591978c775", + "unbound_uses_sha256": "c175b4f5a567b65b62146020a7b0ae39aa9d32d57bdf0839618b13d1003721a1" + }, + "microcosm.build/us_runtime/acs_pums.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.frame", + "microcosm.frame.units", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "3de7aae8590dfd67cfd50c129c5fca3b3ba9eb8b2444ed59f84f92ff94a735ee", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_sources.py": { + "imports": [ + "microcosm.build.us_runtime.acs_pums" + ], + "resource_accesses_sha256": "1f6355e38c0cc1c36391097afe9917115275b0e5585a9429af6007419dbee036", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/alimony.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.source_manifest", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "905273f67cd9edd6939cb6467fd9466c837f90603e4fff9365c09b42326ce7b9", + "unbound_uses_sha256": "c07a32776a6c50eb55aa704fa8671cf2e5a029c88ccd2c86e418ed0310d6fd43" + }, + "microcosm.build/us_runtime/asec_2024_native_population.py": { + "imports": [ + "microcosm.build.cd_benchmark", + "microcosm.build.us_runtime", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "466e46803aecf67189f31031cdb4eb9287af37669e5fa0b36610788d62564e46", + "unbound_uses_sha256": "71463df4f645cd2a98f537a212187b4d1e52bffcc4d2317defe6968d7f287608" + }, + "microcosm.build/us_runtime/asec_checkpoint.py": { + "imports": [ + "microcosm.build.frame_checkpoint", + "microcosm.build.outer_stage_runtime", + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.reported_coverage_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_coverage_authentication.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.education_assistance_source", + "pandas" + ], + "resource_accesses_sha256": "f1826cc233390a0a2b3e97a9905d95d51df58edb6bcee521e8893be26d37969d", + "unbound_uses_sha256": "f03f18df090746827bf57705d6bcc5bc28fcaaf07c16832c8450ac3f28c1c492" + }, + "microcosm.build/us_runtime/asec_current_money.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_graph_resources.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money" + ], + "resource_accesses_sha256": "eaea1354548803c511b49d0f3c3d9d88bb50700553056a34e88a2c314de2282e", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_resources.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "2a2ed530efb66ac76ba8757f6bd5bd7492e586fe2f28c6d2eddada8aed722f78", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_selection.py": { + "imports": [ + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.graph", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_source.py": { + "imports": [ + "microcosm.build", + "microcosm.build.outer_stage_runtime", + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_resources", + "microcosm.build.us_runtime.asec_person_income_source", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "10c92df9a22413a1f5bdee648fc1a79edefc13d469c1f915c14b2c3f90eb605d", + "unbound_uses_sha256": "edeb60d6acaab0ae263edf0537a6de3887a07bbd00382f545191e41cb1a1e278" + }, + "microcosm.build/us_runtime/asec_current_money_units.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_source", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.frame", + "microunit", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "5bebc4222c02f5c2f4e165df30b178cea156109dfbff6d6d88e5aed190e34e9c", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_engine_evaluation.py": { + "imports": [ + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "ef73c9051031f964f8066b33572925566d6e8e7757d633429d48f67951d462ec", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_household_coverage_fields.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "57a3c65a298ad7f131570a5873ebb67b964c566b1f6368636b53e4a4c16c39f3", + "unbound_uses_sha256": "7a523e77a972d9ee60ba7ad78d95c13660d89df0920c6e82ecb98f38292c7cc8" + }, + "microcosm.build/us_runtime/asec_household_observations.py": { + "imports": [ + "h5py", + "microcosm.build.outer_stage_runtime", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "c44acab8834834c2e938718ae7f8da07bc2a3ee1e147c4f5d1ca40c7654d1180", + "unbound_uses_sha256": "17ce33d5d0b830d29e75dbee4a5c87d6f5b08415ac1d421f23aa46bb73b1c8f8" + }, + "microcosm.build/us_runtime/asec_housing_status.py": { + "imports": [ + "microcosm.build.us_runtime.asec_housing_status_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "11b2b6becb66ad05e2cc9d4728cb56c348701971f0dcf4cd491d56a8c813b23f", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_housing_status_source.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_units", + "microcosm.build.us_runtime.asec_housing_status", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "bebe07e6f115689ca38ae68c2d7f69df4ab09cfaad3326e622f9f7b8a9dc89a0", + "unbound_uses_sha256": "ec593873f78e53325d93fc7f8cd3422f3935c0bec2a60120738bd3e606e065fc" + }, + "microcosm.build/us_runtime/asec_housing_universe.py": { + "imports": [ + "microcosm.build.us_runtime.asec_housing_status", + "microcosm.build.us_runtime.asec_housing_universe_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "ac86369b087151b3b538064a5d1dcfaa533bd2adf1948c1132fb6ace4a2365d0", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_housing_universe_source.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_units", + "microcosm.build.us_runtime.asec_housing_status", + "microcosm.build.us_runtime.asec_housing_status_source", + "microcosm.build.us_runtime.asec_housing_universe", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "0f622ceee1c600d15eb314ecf7b88121f6c707b9cf8703c2965af95240ec73ee", + "unbound_uses_sha256": "a8cb1277b205b614033078bca03e9c055d190134adc1172bcae5294c76d2737e" + }, + "microcosm.build/us_runtime/asec_income_observations.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.build.us_runtime.education_assistance_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "92fb0be32eb3167e2c7fdcff8ba929ef0ad3b8a49d50eb5c19a5e7431d84fe23", + "unbound_uses_sha256": "21636adcde1832a321b968f7959d8c75331d08831f9ed093003a26eb450acecd" + }, + "microcosm.build/us_runtime/asec_original_household_weights.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "8c98f1d18560b79086a6ce71868f5800b3d6b8a214bb20446cafe3e3204e6180", + "unbound_uses_sha256": "d01a0598bee95a0bc72aa37ad70ec054cc52e17e07f426f9cb3d25ca464d1c9a" + }, + "microcosm.build/us_runtime/asec_person_coverage_source.py": { + "imports": [ + "microcosm.build.us_runtime", + "pandas" + ], + "resource_accesses_sha256": "d048b0a585f81a8902dc11ef2fa5da96c8874f77802885fdaa08c7e69764994b", + "unbound_uses_sha256": "5a52b6865fa982890fc246f9fcf2568fb77fa725623e9759dd3342e26fae2ae7" + }, + "microcosm.build/us_runtime/asec_person_income_source.py": { + "imports": [ + "microcosm.build", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_resources", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "ef03627f20e0d35ccaacd16397281e6bbe5b1f765a55f58cc4702a3fe2dfe2a3", + "unbound_uses_sha256": "d3ed60989ff82d6e0c96de69dcd77106689c19ecdb7f96c57a4f6d15bd0d415d" + }, + "microcosm.build/us_runtime/asec_population_catalogue.py": { + "imports": [ + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "2622beb3c86b2a9e1cea2db1c6b34a1ec2d2cf29824c3d3e276ea41baeb4f93b", + "unbound_uses_sha256": "138ac0967786b481d52704c91d5d5cefc4e39f92d7cab4154215bc77a981ef6e" + }, + "microcosm.build/us_runtime/asec_prepared_source.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_checkpoint", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_units", + "microcosm.build.us_runtime.asec_housing_status_source", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "d53265b60115118b47ecd636a9fb01a878791d7c9fb1deff86ea21e35947b9c6", + "unbound_uses_sha256": "57f904134e8a69655e1b36357fd288a9ab69966fe10f27dd4f7607c80fba93af" + }, + "microcosm.build/us_runtime/asec_student_controls.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "d20705bfb2110cfc573f493e988e0dbc47b5441e2bda83ef5425110adab2d4e8", + "unbound_uses_sha256": "983062f97faeff20a1761770968d967cc871520de8decbbd98c8018067475796" + }, + "microcosm.build/us_runtime/congressional_district_geography.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/congressional_district_vintage.py": { + "imports": [ + "microcosm.calibrate.geography_constants", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "cba8c434f995a8a867f6b67951909b2038954efc13cf25c04ea958ffac6a8f61", + "unbound_uses_sha256": "d49a89e818222a896afaaed38a0f448e33b38a5656f2eb9591e1bd9195eed687" + }, + "microcosm.build/us_runtime/cps_carried.py": { + "imports": [ + "microcosm.build.us_runtime.alimony", + "microcosm.build.us_runtime.public_assistance_type_source", + "microcosm.build.us_runtime.reported_coverage_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "d2c7c444dbfd99bf73f141320b4a1d4a6eb409a07dbc84d36984f1bc5c95e288" + }, + "microcosm.build/us_runtime/cps_carried_current.py": { + "imports": [ + "microcosm.build.us_runtime.alimony", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.cps_carried", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/education_assistance_source.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "27781733e25a9a4e98fb2ee8fe937391e7a301cf22f3ed1b2faa9bf8524831a0", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/geography_ladder.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "539fedb954503afc210fa2e9ad841638d8fc2fc1ab6e5070d21827cb896246fc", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_acs_housing_universe.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "microcosm.graph.keys", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "fb7b9295aae7f0e4db8fc053fa9fdb192fdb5176f4db5d0aa80bb0c2e61d8a86" + }, + "microcosm.build/us_runtime/graph_asec_income.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "79c5158f94d6ba24554bdfe39ec48a02debebd761b4b67ae40caa22a7c471dd2" + }, + "microcosm.build/us_runtime/graph_asec_prepared.py": { + "imports": [ + "microcosm.build.frame_sampling", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_graph_resources", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.asec_engine_evaluation", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.cps_carried_current", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_housing_universe", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.frame", + "microcosm.frame.adapters.policyengine_us", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "1c0575822ea8e45ece5c94da38b4593df28eb7d96df04cece468174f1a38ff2d" + }, + "microcosm.build/us_runtime/graph_composed_asec_binding.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_asec_prepared", + "microcosm.build.us_runtime.graph_composed_contracts", + "microcosm.build.us_runtime.graph_composed_population", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_housing_universe", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.stacked_spine", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_composed_asec_measures.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.cps_carried_current", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_asec_prepared", + "microcosm.build.us_runtime.graph_composed_asec_binding", + "microcosm.build.us_runtime.graph_composed_contracts", + "microcosm.build.us_runtime.graph_composed_population", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_geography", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_composed_contracts.py": { + "imports": [ + "microcosm.graph" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_composed_population.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_asec_prepared", + "microcosm.build.us_runtime.graph_composed_contracts", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_geography", + "microcosm.build.us_runtime.graph_housing_universe", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.build.us_runtime.stacked_spine", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_context.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_geography.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.build.us_runtime.congressional_district_vintage", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.puma_ladder", + "microcosm.graph", + "microcosm.graph.canonical", + "microcosm.graph.codecs", + "microcosm.graph.kernel", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_housing_universe.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.graph", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "457bc0cc87c6883ba9423769b70238b0e4ea060bca16c94d03379de01ac882c6" + }, + "microcosm.build/us_runtime/graph_implementation.py": { + "imports": [ + "microcosm.build.us_runtime", + "pandas" + ], + "resource_accesses_sha256": "454f9e3c1f7f1b840429f7c14c16dac67ab8219df0c9c32cf5807352b5c338ce", + "unbound_uses_sha256": "b45c8251bea93efe1847308338b21c6f8e74852564ad1a76ba5c384bd6de99a1" + }, + "microcosm.build/us_runtime/graph_sources.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.acs_housing_universe_source", + "microcosm.build.us_runtime.acs_inputs", + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.asec_checkpoint", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "microcosm.graph.codecs", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "627540b59d564b58991a32dc38aed03b9e2c852a3e9326792f21ae5beef19ebb", + "unbound_uses_sha256": "e9036a4821c20237bd98219383026cf440cdb29ac1b78f7b1bafd9be99b1dd30" + }, + "microcosm.build/us_runtime/graph_survey_population.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_combined_clone", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.build.us_runtime.support_provenance", + "microcosm.build.us_runtime.survey_catalogue_selection", + "microcosm.build.us_runtime.survey_population_domains", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.artifact_edges", + "microcosm.graph.codecs", + "microcosm.graph.executor", + "microcosm.graph.keys", + "microcosm.graph.manifest", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "5a6799ab18a561e0d4fdc4e08b11656a6100216df59e268e27e0ad4d38b1c994" + }, + "microcosm.build/us_runtime/native_household_origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "740699f0e8648d061a7057844a4aabf04bb5ce2df7118fa4256dc2d215820c93", + "unbound_uses_sha256": "d4cc434fc5fa5a17c6fa46592dbeb8019d061927ff218c618036288221e76503" + }, + "microcosm.build/us_runtime/operator_boundary.py": { + "imports": [ + "microcosm.build.us_runtime.adult_care", + "microcosm.build.us_runtime.child_support", + "microcosm.build.us_runtime.childcare", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.build.us_runtime.cps_carried", + "microcosm.build.us_runtime.disability_benefits", + "microcosm.build.us_runtime.education_inputs", + "microcosm.build.us_runtime.eligibility_inputs", + "microcosm.build.us_runtime.energy_subsidy", + "microcosm.build.us_runtime.geography_ladder", + "microcosm.build.us_runtime.hours_worked", + "microcosm.build.us_runtime.housing_inputs", + "microcosm.build.us_runtime.immigration", + "microcosm.build.us_runtime.medicare_take_up", + "microcosm.build.us_runtime.operator_column_contracts", + "microcosm.build.us_runtime.pregnancy", + "microcosm.build.us_runtime.prior_year_income", + "microcosm.build.us_runtime.qbi_inputs", + "microcosm.build.us_runtime.relationship_inputs", + "microcosm.build.us_runtime.retirement_contributions", + "microcosm.build.us_runtime.retirement_distributions", + "microcosm.build.us_runtime.scf_wealth", + "microcosm.build.us_runtime.weeks_unemployed", + "microcosm.build.us_runtime.wic_claim", + "microcosm.build.us_runtime.workers_compensation", + "microcosm.frame", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "f3e52e09d1c5b005d16f083e716966bf60087b72eba71a149a8128faa7551617" + }, + "microcosm.build/us_runtime/operator_column_contracts.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/puf_support.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.acs_income_universe", + "microcosm.build.us_runtime.operator_column_contracts", + "microcosm.build.us_runtime.puf_e01000_reconciliation", + "microcosm.build.us_runtime.puf_interest_components", + "microcosm.build.us_runtime.qbi_inputs", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.frame.adapters.policyengine_us", + "microcosm.frame.schema", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "851b8041c69b1b721dcdc02d1ce312976a87e45b66fc4aca59d0ec538ab62b44", + "unbound_uses_sha256": "3b7e8d1e071256065b1c8a43c7fea2f9f39378b10d1613f76bfa9cbbf0cff6ad" + }, + "microcosm.build/us_runtime/puma_ladder.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.build.us_runtime.geography_ladder", + "microcosm.build.us_runtime.puma_ladder_sources", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "056fbb3baf3f3008aa654612e700957b3ecb795b8da9460be5e9a884930c36a7", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/puma_ladder_sources.py": { + "imports": [ + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/reported_coverage_source.py": { + "imports": [ + "microcosm.build.us_runtime.education_assistance_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4b87bba3a98f67416ff34353df950e2638c86d56170d03d372f62578149f3e2e", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/source_csv_builtin.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/spine_assembly.py": { + "imports": [ + "microcosm.build.us_runtime.operator_column_contracts", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/stacked_spine.py": { + "imports": [ + "microcosm.build.frame_sampling", + "microcosm.build.gates", + "microcosm.build.serialization_dtypes", + "microcosm.build.source_manifest", + "microcosm.build.table_identity", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.acs_income_universe", + "microcosm.build.us_runtime.acs_transfer", + "microcosm.build.us_runtime.late_producer_dag", + "microcosm.build.us_runtime.multispine_pool", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.post_transfer_calibration", + "microcosm.build.us_runtime.puf_aggregate_records", + "microcosm.build.us_runtime.puf_capital_gains_tail", + "microcosm.build.us_runtime.puf_interest_components", + "microcosm.build.us_runtime.puf_qrf_chain", + "microcosm.build.us_runtime.puf_support", + "microcosm.build.us_runtime.spine_assembly", + "microcosm.build.us_runtime.support_provenance", + "microcosm.build.us_runtime.us_late_overlap_ownership", + "microcosm.build.us_runtime.us_late_producer_registry", + "microcosm.fit", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "3839407286d44ddbf3f207c63dcb92e1c0b45e2eec2e88a09c25ebb3a4711430", + "unbound_uses_sha256": "062e621661c109d963f2356bcaa67a3da0906c8e66af1dcd81122d7ce05bd454" + }, + "microcosm.build/us_runtime/support_provenance.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.cps_carried", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "bfcbcc3d41a567ee0d5b015c6715c3fb659937713db37d3b2d74e5fa2ddeead3" + }, + "microcosm.build/us_runtime/survey_catalogue_selection.py": { + "imports": [ + "microcosm.build.survey_domain_sample", + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "00c2ba97f0fb73d788c8f9c6e42ab90a67c5a2f7fe947d2232441aa6b03ae27c" + }, + "microcosm.build/us_runtime/survey_observed_age.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/survey_population_domains.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/survey_population_preparation.py": { + "imports": [ + "microcosm.build", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "ccfed1c1acff5a1c50b538424dc129930f69c374d9841e233b3fb593a440c3aa", + "unbound_uses_sha256": "29c09f6fcb25ce8bc6111dd0dc76f3501d9a6b635929e94b80835889d296ef91" + }, + "microcosm.frame/__init__.py": { + "imports": [ + "microcosm.frame.accounting", + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "microcosm.frame.units", + "microcosm.frame.weights" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/accounting.py": { + "imports": [ + "microcosm.frame.bundle", + "numpy", + "pandas", + "pandas.api.types" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/adapters/__init__.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/adapters/_policyengine_us_source_index.py": { + "imports": [ + "microcosm.frame.schema", + "yaml" + ], + "resource_accesses_sha256": "a64f5650fc19608459f4e1c034e1f7e692de8ca77e4e2e111f8dd40559ce481f", + "unbound_uses_sha256": "6da25f88c6cadb5b1522da2e977a71e749dd54c6cb1d58fb1c6224161e9e42be" + }, + "microcosm.frame/adapters/axiom.py": { + "imports": [ + "axiom_rules_engine", + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "9f0928dfa308893a0724f2e22e2676993aa7084c83115c4fdaecc341e515bd76" + }, + "microcosm.frame/adapters/policyengine_uk.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "numpy", + "policyengine_uk", + "policyengine_uk.data" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "366f98c5d3507744c7f9667e7323821c41358a47c56bd83f7ca9c677756c7094" + }, + "microcosm.frame/adapters/policyengine_us.py": { + "imports": [ + "microcosm.frame.adapters._policyengine_us_source_index", + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "microcosm.frame.units", + "numpy", + "pandas", + "policyengine_us", + "policyengine_us.data" + ], + "resource_accesses_sha256": "b87a963f56b28cef5b93690cec90f1ff2e935075283e553fdf7af693f8bf3204", + "unbound_uses_sha256": "175c1d0a9621cee21f400b0c3e4618a40eb2fdb795adbf42b16488ea2dcf5b23" + }, + "microcosm.frame/bundle.py": { + "imports": [ + "microcosm.frame.schema", + "microcosm.frame.weights", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/kernels.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.rules", + "microcosm.frame.schema", + "microcosm.graph", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/materialize.py": { + "imports": [ + "microcosm.frame.bundle", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/rules.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.schema", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "bb3739bfa9c1bdb290ce2e1136dfd73bc4f2c40715a822ea87e3c91ef51a5760", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/schema.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/units.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.schema", + "microcosm.frame.weights", + "microunit", + "microunit.units", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "e40d25e078da274aaaa43639b66ac70df5a8b87b6b637f46d00069e73bff3e73" + }, + "microcosm.frame/weights.py": { + "imports": [ + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/__init__.py": { + "imports": [ + "microcosm.graph.codecs", + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.executor", + "microcosm.graph.explain", + "microcosm.graph.kernel", + "microcosm.graph.keys", + "microcosm.graph.manifest", + "microcosm.graph.population", + "microcosm.graph.randomness", + "microcosm.graph.serialize", + "microcosm.graph.store", + "microcosm.graph.view" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/artifact_edges.py": { + "imports": [ + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.kernel", + "microcosm.graph.keys" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/availability.py": { + "imports": [ + "microcosm.graph.decl", + "microcosm.graph.kernel" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/canonical.py": { + "imports": [ + "microcosm.graph.decl" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/codecs.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4adeae91a0658854557dc4d2bf01d09ad531726de7cec890f00258a2dbe81da2", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/decl.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/errors.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/executor.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.artifact_edges", + "microcosm.graph.availability", + "microcosm.graph.canonical", + "microcosm.graph.codecs", + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.kernel", + "microcosm.graph.keys", + "microcosm.graph.manifest", + "microcosm.graph.population", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/explain.py": { + "imports": [ + "microcosm.graph.availability", + "microcosm.graph.decl", + "microcosm.graph.manifest", + "microcosm.graph.population", + "microcosm.graph.view" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/kernel.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.decl", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "313aceccf09abe24eace2f6d432f41960afb9ed7e80ebe890a10cc41557d87d3", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/keys.py": { + "imports": [ + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.kernel" + ], + "resource_accesses_sha256": "5950b5602a721f5334ac91d4027a61707cf58600a827a6f1a4e3b73e690f9748", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/manifest.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.artifact_edges", + "microcosm.graph.availability", + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.kernel", + "microcosm.graph.population", + "microcosm.graph.store", + "pandas" + ], + "resource_accesses_sha256": "13c805cbaa1c2fd026afc6574c4c08dd71143a68f787d616852d5efcbb151c82", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/population.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.kernel", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/randomness.py": { + "imports": [ + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/serialize.py": { + "imports": [ + "microcosm.graph.canonical", + "microcosm.graph.decl" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/store.py": { + "imports": [ + "microcosm.frame", + "microcosm.frame.bundle", + "microcosm.graph.errors", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "99afee26ca660c678dcbd7117eb2e64d70a0343ead9a602fa5819222aca02bc3", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/view.py": { + "imports": [ + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.manifest" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/__init__.py": { + "imports": [ + "microunit.core", + "microunit.diagnostics", + "microunit.registry", + "microunit.rule_helpers", + "microunit.tax_unit_construction" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/core.py": { + "imports": [ + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/diagnostics.py": { + "imports": [ + "microunit.core", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/registry.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/rule_helpers.py": { + "imports": [ + "yaml" + ], + "resource_accesses_sha256": "d880e849f439db697d177523c8c7e92775b109fc42d2f71372e2702dd9c19749", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/tax_unit_construction.py": { + "imports": [ + "microunit.rule_helpers", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/__init__.py": { + "imports": [ + "microunit.units.medicaid", + "microunit.units.passthrough", + "microunit.units.programs", + "microunit.units.snap", + "microunit.units.spm", + "microunit.units.tax" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/_helpers.py": { + "imports": [ + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/medicaid.py": { + "imports": [ + "microunit.core", + "microunit.units.programs", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/passthrough.py": { + "imports": [ + "microunit.core", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/programs.py": { + "imports": [ + "microunit.core", + "microunit.units.spm", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/snap.py": { + "imports": [ + "microunit.core", + "microunit.units._helpers", + "microunit.units.programs", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/spm.py": { + "imports": [ + "microunit.core", + "microunit.units._helpers", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/tax.py": { + "imports": [ + "microunit.core", + "microunit.tax_unit_construction", + "microunit.units._helpers", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + "import_classifications": { + "axiom_rules_engine": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "h5py": "versioned dependency in explicit stages", + "microcosm.build": "package namespace import of explicitly bound frame_checkpoint; other package exports are inactive in this slice", + "microcosm.build.cd_benchmark": "package-relative original source-member identity alias; origin and its pure canonical helper are explicitly whole-module bound in authenticated_survey_population_v1", + "microcosm.build.cd_benchmark.canonical": "whole-module pure canonical JSON and source digest validation in authenticated_survey_population_v1; no benchmark targets or data", + "microcosm.build.cd_benchmark.origin": "whole-module original source-member identity in authenticated_survey_population_v1; no benchmark targets or data", + "microcosm.build.frame_checkpoint": "whole-module in explicit stages", + "microcosm.build.frame_sampling": "whole-module in explicit stages", + "microcosm.build.gates": "whole-module in explicit stages", + "microcosm.build.ledger_targets": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.outer_stage_runtime": "whole-module in explicit stages", + "microcosm.build.serialization_dtypes": "whole-module in explicit stages", + "microcosm.build.source_manifest": "inactive legacy alimony source-stage declaration; scoped-use and resource-access tripwires retained", + "microcosm.build.survey_allocation": "whole-module pure domain share and original-anchor contracts in authenticated_survey_population_v1", + "microcosm.build.survey_domain_sample": "whole-module deterministic per-domain household selection in authenticated_survey_population_v1", + "microcosm.build.table_identity": "whole-module in explicit stages", + "microcosm.build.us_runtime": "package-relative live aliases for original money/schema/checkpoint/household/person-income/housing verifiers, separate student controls and graph context are explicitly module-bound in asec_prepared_v3; other shared-module or optional routes remain inactive with scoped-use tripwires; v2 adds package-relative HU definition/source aliases and non-authoritative transport, each explicitly whole-module bound. The authenticated survey stage additionally binds its exact catalogue, selected-native, original-source, pure-selection and preparation aliases; optional PUF clone validation remains downstream and bound by the clone operator identity.", + "microcosm.build.us_runtime._asec_current_money_codec": "whole-module in asec_prepared_v3; original canonical full-source replay unchanged", + "microcosm.build.us_runtime.acs_housing_universe": "Whole-module in acs_housing_universe_2024; only source capture/projection/native Frame and typed custody are live. Other stages retain scoped-use tripwires for registry availability.", + "microcosm.build.us_runtime.acs_housing_universe_source": "Whole-module in acs_housing_universe_2024; only source capture/projection/native Frame and typed custody are live. Other stages retain scoped-use tripwires for registry availability.", + "microcosm.build.us_runtime.acs_income_universe": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.acs_inputs": "whole-module in explicit stages", + "microcosm.build.us_runtime.acs_pums": "whole-module in explicit stages", + "microcosm.build.us_runtime.acs_sources": "Whole-module in acs_housing_universe_2024; only source capture/projection/native Frame and typed custody are live. Other stages retain scoped-use tripwires for registry availability.", + "microcosm.build.us_runtime.acs_transfer": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.adult_care": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.alimony": "whole-module in asec_prepared_v3 for pure ASEC split; source manifest and other nominal routes inactive", + "microcosm.build.us_runtime.asec_checkpoint": "whole-module in explicit stages", + "microcosm.build.us_runtime.asec_current_money": "whole-module in asec_prepared_v3; immutable recipe source identity", + "microcosm.build.us_runtime.asec_current_money_graph_resources": "whole-module in asec_prepared_v3; additive status loader leaves original T1/money resource identity unchanged", + "microcosm.build.us_runtime.asec_current_money_resources": "whole-module in asec_prepared_v3; original recipe and installed microunit verification unchanged", + "microcosm.build.us_runtime.asec_current_money_selection": "whole-module in asec_prepared_v3; selected body has provenance but no full-source authority", + "microcosm.build.us_runtime.asec_current_money_source": "whole-module in asec_prepared_v3; authenticated original source verification", + "microcosm.build.us_runtime.asec_current_money_units": "whole-module in asec_prepared_v3; real microunit with separate authenticated student controls", + "microcosm.build.us_runtime.asec_engine_evaluation": "whole-module in asec_prepared_v3; pinned complete installed formula closures and empty defaults allowlist", + "microcosm.build.us_runtime.asec_housing_status": "whole-module in asec_prepared_v3; status definition and sealed authenticated artifact", + "microcosm.build.us_runtime.asec_housing_status_source": "whole-module in asec_prepared_v3; housing source remains original T1", + "microcosm.build.us_runtime.asec_housing_universe": "Whole-module in asec_prepared_v3; native HU source definition/ownership and non-authoritative graph transport plus selected-row validation; no ACS or selected-Frame authority.", + "microcosm.build.us_runtime.asec_housing_universe_source": "Whole-module in asec_prepared_v3; native HU source definition/ownership and non-authoritative graph transport plus selected-row validation; no ACS or selected-Frame authority.", + "microcosm.build.us_runtime.asec_person_income_source": "whole-module in asec_prepared_v3; unchanged T1 restoration verifier closure", + "microcosm.build.us_runtime.asec_prepared_source": "whole-module in asec_prepared_v3; registry-only import in other graph_sources scopes retains scoped-use tripwire", + "microcosm.build.us_runtime.asec_student_controls": "whole-module in asec_prepared_v3; separate S authority before tax units, never part of original T1 verifier", + "microcosm.build.us_runtime.child_support": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.childcare": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.congressional_district_geography": "whole-module in explicit stages", + "microcosm.build.us_runtime.congressional_district_vintage": "whole-module in explicit stages", + "microcosm.build.us_runtime.cps_carried": "whole-module in asec_prepared_v3 for split constants; nominal consumer route inactive", + "microcosm.build.us_runtime.cps_carried_current": "whole-module in asec_prepared_v3; typed corrected money amounts and declared raw routing only", + "microcosm.build.us_runtime.disability_benefits": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.education_assistance_source": "whole-module in explicit stages", + "microcosm.build.us_runtime.education_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.eligibility_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.energy_subsidy": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.geography_ladder": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_asec_income": "Whole-module in asec_prepared_v3; PAW source evidence, bound reported-income accounting or shared typed graph declarations. No source authority is minted from graph transport.", + "microcosm.build.us_runtime.graph_asec_prepared": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_combined_clone": "optional downstream clone declaration/registration; clone kernels retain their own implementation identities; no donor or target resource in authenticated_survey_population_v1", + "microcosm.build.us_runtime.graph_composed_asec_binding": "whole-module in composed_asec_binding_v1; composed-arm resolution, its typed binding/arm-row transport and the declared row mask. Reads source provenance by charter and mints no source authority.", + "microcosm.build.us_runtime.graph_composed_contracts": "Pure composed producer IDs and artifact types; whole module bound by both legacy composed stages and explicit PUF extension hashes. No resources or operation dispatch.", + "microcosm.build.us_runtime.graph_composed_population": "whole-module in composed_population_v1 and composed_asec_binding_v1; the composing CREATE's declarations, node ids and typed origin binder. Adds no source route of its own.", + "microcosm.build.us_runtime.graph_context": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_geography": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_housing_universe": "Whole-module in asec_prepared_v3; native HU source definition/ownership and non-authoritative graph transport plus selected-row validation; no ACS or selected-Frame authority.", + "microcosm.build.us_runtime.graph_implementation": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_sources": "whole-module in explicit graph stages; prepared slice uses frame column declarations and validated codec registration only; authenticated survey preparation uses only frame column declarations and lossless storage canonicalization, not legacy source codecs or assembly treatment", + "microcosm.build.us_runtime.hours_worked": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.housing_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.immigration": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.late_producer_dag": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.medicare_take_up": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.multispine_pool": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.operator_boundary": "whole-module in explicit stages", + "microcosm.build.us_runtime.operator_column_contracts": "Pure shared column declarations bound by every source/assembly scope that consumes them; no runtime operator or resource loading.", + "microcosm.build.us_runtime.post_transfer_calibration": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.pregnancy": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.prior_year_income": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.public_assistance_type_source": "inactive legacy cps_carried consumer route; split constants used by corrected slice do not call this reader", + "microcosm.build.us_runtime.puf_aggregate_records": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_capital_gains_tail": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.puf_e01000_reconciliation": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_interest_components": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_qrf_chain": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_support": "whole-module in explicit stages", + "microcosm.build.us_runtime.puma_ladder": "whole-module in explicit stages", + "microcosm.build.us_runtime.puma_ladder_sources": "whole-module in explicit stages", + "microcosm.build.us_runtime.qbi_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.relationship_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.reported_coverage_source": "whole-module in explicit stages", + "microcosm.build.us_runtime.retirement_contributions": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.retirement_distributions": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.scf_wealth": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.source_csv_builtin": "Whole-module in acs_housing_universe_2024; stdlib-only checked CSV builtin capture, without source or resource access.", + "microcosm.build.us_runtime.spine_assembly": "whole-module in explicit stages", + "microcosm.build.us_runtime.stacked_spine": "whole-module in explicit stages", + "microcosm.build.us_runtime.support_provenance": "whole-module in explicit stages", + "microcosm.build.us_runtime.survey_catalogue_selection": "whole-module pure complete-catalogue selection plan in authenticated_survey_population_v1", + "microcosm.build.us_runtime.survey_population_domains": "whole-module raw source-record classification and prespecified development allocation declaration in authenticated_survey_population_v1", + "microcosm.build.us_runtime.us_late_overlap_ownership": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.us_late_producer_registry": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.weeks_unemployed": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.wic_claim": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.workers_compensation": "live registry projection; other shared-module routes inactive", + "microcosm.calibrate.geography_constants": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.calibrate.registry": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.calibrate.solve": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.fit": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.frame": "whole-module in explicit stages", + "microcosm.frame.accounting": "whole-module in explicit stages", + "microcosm.frame.adapters._policyengine_us_source_index": "whole-module in explicit stages", + "microcosm.frame.adapters.policyengine_us": "whole-module in explicit stages", + "microcosm.frame.bundle": "whole-module in explicit stages", + "microcosm.frame.materialize": "whole-module in explicit stages", + "microcosm.frame.rules": "whole-module in explicit stages", + "microcosm.frame.schema": "whole-module in explicit stages", + "microcosm.frame.units": "whole-module in explicit stages", + "microcosm.frame.weights": "whole-module in explicit stages", + "microcosm.graph": "whole-module in explicit stages", + "microcosm.graph.artifact_edges": "whole-module in explicit stages", + "microcosm.graph.availability": "whole-module in explicit stages", + "microcosm.graph.canonical": "whole-module in explicit stages", + "microcosm.graph.codecs": "whole-module in explicit stages", + "microcosm.graph.decl": "whole-module in explicit stages", + "microcosm.graph.errors": "whole-module in explicit stages", + "microcosm.graph.executor": "whole-module in explicit stages", + "microcosm.graph.explain": "whole-module in explicit stages", + "microcosm.graph.kernel": "whole-module in explicit stages", + "microcosm.graph.keys": "whole-module in explicit stages", + "microcosm.graph.manifest": "whole-module in explicit stages", + "microcosm.graph.population": "whole-module in explicit stages", + "microcosm.graph.randomness": "whole-module in explicit stages", + "microcosm.graph.serialize": "whole-module in explicit stages", + "microcosm.graph.store": "whole-module in explicit stages", + "microcosm.graph.view": "whole-module in explicit stages", + "microunit": "whole-module in explicit stages", + "microunit.core": "whole-module in explicit stages", + "microunit.diagnostics": "whole-module in explicit stages", + "microunit.registry": "whole-module in explicit stages", + "microunit.rule_helpers": "whole-module in explicit stages", + "microunit.tax_unit_construction": "whole-module in explicit stages", + "microunit.units": "whole-module in explicit stages", + "microunit.units._helpers": "whole-module in explicit stages", + "microunit.units.medicaid": "whole-module in explicit stages", + "microunit.units.passthrough": "whole-module in explicit stages", + "microunit.units.programs": "whole-module in explicit stages", + "microunit.units.snap": "whole-module in explicit stages", + "microunit.units.spm": "whole-module in explicit stages", + "microunit.units.tax": "whole-module in explicit stages", + "numpy": "versioned dependency in explicit stages", + "pandas": "versioned dependency in explicit stages", + "pandas.api.types": "versioned dependency in explicit stages", + "policyengine_uk": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "policyengine_uk.data": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "policyengine_us": "live evaluation only, explicitly pinned installed runtime and parameter identity; inactive in other shared adapter routes", + "policyengine_us.data": "live evaluation dataset adapter only, bound by pinned installed PE-US runtime identity; inactive in other stages", + "yaml": "versioned dependency in explicit stages" + }, + "package_resources": { + "microunit": [ + "data/dependent_gross_income_limit.yaml" + ] + }, + "package_rosters": { + "microcosm.frame": [ + "__init__.py", + "accounting.py", + "adapters/__init__.py", + "adapters/_policyengine_us_source_index.py", + "adapters/axiom.py", + "adapters/policyengine_uk.py", + "adapters/policyengine_us.py", + "bundle.py", + "kernels.py", + "materialize.py", + "rules.py", + "schema.py", + "units.py", + "weights.py" + ], + "microcosm.graph": [ + "__init__.py", + "artifact_edges.py", + "availability.py", + "canonical.py", + "codecs.py", + "decl.py", + "errors.py", + "executor.py", + "explain.py", + "kernel.py", + "keys.py", + "manifest.py", + "population.py", + "randomness.py", + "serialize.py", + "store.py", + "view.py" + ], + "microunit": [ + "__init__.py", + "core.py", + "diagnostics.py", + "registry.py", + "rule_helpers.py", + "tax_unit_construction.py", + "units/__init__.py", + "units/_helpers.py", + "units/medicaid.py", + "units/passthrough.py", + "units/programs.py", + "units/snap.py", + "units/spm.py", + "units/tax.py" + ] + }, + "reviewed_base": "c71e31654faf0e3f874e68e8c100b09c7eb9955f", + "schema": "microcosm.us.implementation-inventory.v1", + "scope_notes": { + "asec_prepared_engine": "PE-US/Core are live only in the evaluation kernel. That kernel adds their dependencies and verifies pinned installed implementation/parameter bytes in its implementation identity before cache lookup. The common prepared scope can be hashed without engine extras; its local adapter/source-index modules and empty-defaults resource are bound.", + "asec_prepared_v3": "Full P/H/T1 restoration, unchanged money recipe, separate S authentication/attachment before tax units, housing authentication against original T1, lossless dtype promotion, typed whole-household selection, corrected CPS leaves and four admitted PE outputs. Graph/Frame/microunit rosters are bound conservatively. Only declared source codec registration and frame column declarations run from graph_sources; ACS and stack assembly routes remain inactive. Alimony's pure ASEC split and cps_carried's split constants are bound, while their legacy consumer/source-manifest routes are inactive. V2 additionally authenticates HU from original T1, attaches it after housing status, preserves the pre-HU parent signature and carries native HU bytes plus selected-row ancestry with exact int64 promotion; pure ACS HU remains inactive. V3 additionally issues separate PAW_VAL/PAW_YN observations from the same official members and a derived six-column reported-income branch. The 33-field body, T1/S definitions, selected-money v1 and selection v2 payload contract remain sealed; historical v2 artifacts are preserved, but v3 runtime keys invalidate old caches.", + "authenticated_survey_population_v1": "Complete ACS and ASEC catalogues authenticate original source records before one domain draw. Selected native issuers produce original DESIGN anchors, which are stacked without share normalization; the graph applies share/inclusion probability once. Explicit narrow source closure includes pure origin canonical encoding and the original household/person custody helpers, not the legacy prepared ASEC resource union. Original source JSONs and microunit YAML are the only packaged data. The isolated real source-codec loader is declared separately from the unchanged global five codecs. The runner reconstructs source authority on cold and warm invocations and checks actual populations, design ancestry, owners, weight kinds, mass ledgers and typed artifact descriptors. Optional PUF support cloning follows over both survey arms under existing clone implementation identities. No donor fit/draw, policy engine, calibration or target measurement is part of this stage. Initial source and graph snapshots remain preliminary pending integrated invented execution and independent review.", + "bound_modules": "Whole bytes of explicitly named modules; helper-only parsing never imports adapters.", + "composed_asec_binding_v1": "Binding the carried prepared-ASEC evidence to the composed ASEC rows and deriving the reviewed corrected leaves and reported observations over them. This stage reads no source; every artifact it admits was produced by composed_population_v1, and every verifier, selector and derivation it reuses already lives in that scope, so its module inventory is exactly that scope's plus the two binding modules and its dependency set is identical. Because the inventory is a superset of composed_population_v1 and therefore of each of its parents, no existing module's covered-import set moves. It binds the same two codecs the composing CREATE binds rather than claiming an identity narrower than its actual inputs.", + "composed_population_v1": "The composed prepared-ASEC/native-ACS population. One CREATE runs both reviewed source scopes, so this stage's module inventory is exactly the union of assembly_prepare, asec_prepared_v3 and geography plus the composing module, and its dependency set is theirs. Because the union is a superset of each parent, no existing module's covered-import set and therefore no existing stage identity moves. It binds the prepared codec and the ACS codec it actually reads; the raw-stage codec is not bound because this stage never reads it. Sampling, stacking, provenance and importance allocation remain in the unchanged stacked_spine/spine_assembly modules, and harmonization reuses the assembly_harmonize scope through its own kernel. No engine, clone, transfer or calibration route is active in this scope.", + "current_money_resources": "Original source recipe and consumers declaration bytes remain unchanged to preserve authenticated T1/money identities. The additive graph consumer resource is separately pinned and supersedes only implementation status for the graph slice and microunit; all old nominal consumer routes remain pending.", + "occupied_housing_review_base": "HU v1 source definition and attachment are exact approved 4b26e59b additions on d4bddcb6ac2aba52903cd8dace90330cee4a21e1; only the v2 prepared graph/transport wiring is new. Original source-verifier modules and resources are unchanged.", + "operator_boundary": "Resolved output/formula registries projected; provider bodies are not called by source construction.", + "other_imports": "Canonical non-stdlib import modules are explicitly classified. Ordinary symbols from a module/dependency bound in every using scope need no inventory edit. Only unbound-helper uses retain a lexical-scope fingerprint; resource-access expressions retain their own fingerprint. Expanded AST details are audit output, not packaged data. Neither fingerprint is a proof of dynamic closure.", + "resources": "Microunit rule YAML is bound. Default packaged geography/source-stage resources belong to inactive legacy entrypoints; graph lookups are explicit SourceRefs." + }, + "stages": { + "acs_codec_2024": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "acs_housing_universe_2024": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/acs_housing_universe.py", + "microcosm.build/us_runtime/acs_housing_universe_source.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/acs_sources.py", + "microcosm.build/us_runtime/graph_acs_housing_universe.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/source_csv_builtin.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/acs_2024_1yr_sources.json", + "microcosm.build/us_runtime/acs_2024_housing_universe.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "asec_codec_v4": { + "dependencies": [ + "numpy", + "pandas", + "h5py" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py" + ], + "resources": [], + "source_boundary_projection": true + }, + "asec_prepared_v3": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/alimony.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_graph_resources.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_selection.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_current_money_units.py", + "microcosm.build/us_runtime/asec_engine_evaluation.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_housing_status.py", + "microcosm.build/us_runtime/asec_housing_status_source.py", + "microcosm.build/us_runtime/asec_housing_universe.py", + "microcosm.build/us_runtime/asec_housing_universe_source.py", + "microcosm.build/us_runtime/asec_income_observations.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_prepared_source.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/cps_carried.py", + "microcosm.build/us_runtime/cps_carried_current.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_asec_income.py", + "microcosm.build/us_runtime/graph_asec_prepared.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_housing_universe.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_engine_defaults_v1.json", + "microcosm.build/us_runtime/asec_current_money_graph_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microcosm.build/us_runtime/asec_income_observations_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "assembly_harmonize": { + "dependencies": [ + "numpy", + "pandas" + ], + "modules": [ + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py" + ], + "resources": [], + "source_boundary_projection": false + }, + "assembly_prepare": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "authenticated_survey_population_v1": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/cd_benchmark/canonical.py", + "microcosm.build/cd_benchmark/origin.py", + "microcosm.build/frame_checkpoint.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/survey_allocation.py", + "microcosm.build/survey_domain_sample.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/acs_housing_universe.py", + "microcosm.build/us_runtime/acs_housing_universe_source.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_native_coverage_binding.py", + "microcosm.build/us_runtime/acs_person_coverage_authentication.py", + "microcosm.build/us_runtime/acs_person_coverage_columns.py", + "microcosm.build/us_runtime/acs_population_catalogue.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/acs_sources.py", + "microcosm.build/us_runtime/asec_2024_native_population.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_coverage_authentication.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_household_coverage_fields.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_original_household_weights.py", + "microcosm.build/us_runtime/asec_person_coverage_source.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_population_catalogue.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_acs_housing_universe.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/graph_survey_population.py", + "microcosm.build/us_runtime/native_household_origin.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/source_csv_builtin.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.build/us_runtime/survey_catalogue_selection.py", + "microcosm.build/us_runtime/survey_observed_age.py", + "microcosm.build/us_runtime/survey_population_domains.py", + "microcosm.build/us_runtime/survey_population_preparation.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/acs_2024_1yr_sources.json", + "microcosm.build/us_runtime/acs_2024_housing_universe.json", + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "composed_asec_binding_v1": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/alimony.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_graph_resources.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_selection.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_current_money_units.py", + "microcosm.build/us_runtime/asec_engine_evaluation.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_housing_status.py", + "microcosm.build/us_runtime/asec_housing_status_source.py", + "microcosm.build/us_runtime/asec_housing_universe.py", + "microcosm.build/us_runtime/asec_housing_universe_source.py", + "microcosm.build/us_runtime/asec_income_observations.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_prepared_source.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/congressional_district_geography.py", + "microcosm.build/us_runtime/congressional_district_vintage.py", + "microcosm.build/us_runtime/cps_carried.py", + "microcosm.build/us_runtime/cps_carried_current.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/geography_ladder.py", + "microcosm.build/us_runtime/graph_asec_income.py", + "microcosm.build/us_runtime/graph_asec_prepared.py", + "microcosm.build/us_runtime/graph_composed_asec_binding.py", + "microcosm.build/us_runtime/graph_composed_asec_measures.py", + "microcosm.build/us_runtime/graph_composed_contracts.py", + "microcosm.build/us_runtime/graph_composed_population.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_geography.py", + "microcosm.build/us_runtime/graph_housing_universe.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/puma_ladder.py", + "microcosm.build/us_runtime/puma_ladder_sources.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_engine_defaults_v1.json", + "microcosm.build/us_runtime/asec_current_money_graph_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microcosm.build/us_runtime/asec_income_observations_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "composed_population_v1": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/alimony.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_graph_resources.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_selection.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_current_money_units.py", + "microcosm.build/us_runtime/asec_engine_evaluation.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_housing_status.py", + "microcosm.build/us_runtime/asec_housing_status_source.py", + "microcosm.build/us_runtime/asec_housing_universe.py", + "microcosm.build/us_runtime/asec_housing_universe_source.py", + "microcosm.build/us_runtime/asec_income_observations.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_prepared_source.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/congressional_district_geography.py", + "microcosm.build/us_runtime/congressional_district_vintage.py", + "microcosm.build/us_runtime/cps_carried.py", + "microcosm.build/us_runtime/cps_carried_current.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/geography_ladder.py", + "microcosm.build/us_runtime/graph_asec_income.py", + "microcosm.build/us_runtime/graph_asec_prepared.py", + "microcosm.build/us_runtime/graph_composed_contracts.py", + "microcosm.build/us_runtime/graph_composed_population.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_geography.py", + "microcosm.build/us_runtime/graph_housing_universe.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/puma_ladder.py", + "microcosm.build/us_runtime/puma_ladder_sources.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_engine_defaults_v1.json", + "microcosm.build/us_runtime/asec_current_money_graph_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microcosm.build/us_runtime/asec_income_observations_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "geography": { + "dependencies": [ + "numpy", + "pandas" + ], + "modules": [ + "microcosm.build/gates.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/congressional_district_geography.py", + "microcosm.build/us_runtime/congressional_district_vintage.py", + "microcosm.build/us_runtime/geography_ladder.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_geography.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/puma_ladder.py", + "microcosm.build/us_runtime/puma_ladder_sources.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py" + ], + "resources": [], + "source_boundary_projection": false + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_national_age_counts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_national_age_counts.py new file mode 100644 index 000000000..c30472fa5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_national_age_counts.py @@ -0,0 +1,451 @@ +"""Disjoint household age counts, and the composition that calibrates them. + +This is an ordinary source-blind population operator. It declares exactly what +it reads — each person's age and household membership, and the household ids — +and owns one int64 count column per declared age band. It opens no file, reads +no source-channel or spine column, imports no rules engine, calculates nothing +from a model, and never touches weights: these are unweighted head counts, and +the graph executor owns every weight in the build. + +The graph contract splits that read surface in two, and the declaration follows +it rather than working around it. ``age`` is an owned cell, so it travels in an +input :class:`~microcosm.graph.Slice`. Entity ids and person-to-group +memberships are structural: the executor puts them in every context table and +no node owns them, so a ``Slice`` cannot name them. They are declared instead +in the node's normative parameters, where they still enter the node key, and +the kernel refuses a context whose household view carries anything beyond the +declared id column. + +Every included person is counted exactly once. Group-quarters people are +included because the activated target universe is the whole resident +population, and their household membership is preserved rather than dropped. +The bands are declared, contiguous from zero, disjoint, and closed by a single +open top band, so an age outside every band is impossible by construction and a +person can never fall into two. + +Refusals are explicit; nothing is silently dropped. An unknown, non-integer, +negative or non-finite age refuses the node, as does a membership that names no +household row, a duplicated household id, or a household row that no person +belongs to. The last one matches the population contract itself, which requires +each group row to be referenced by at least one person — a memberless household +is a broken population, not a zero-count household. + +:func:`national_age_calibration_nodes` composes this operator with the existing +`demographic_calibration_node`, unchanged. It emits no US frame-context +artifact: a context written before reweighting must not be relabelled after it, +so consumers read the executor-owned calibrated population and the diagnostics +edge instead. +""" + +from __future__ import annotations + +import json +import sys + +import numpy as np +import pandas as pd +from pandas.api.types import ( + is_bool_dtype, + is_complex_dtype, + is_integer_dtype, + is_numeric_dtype, +) + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + StructuralDelta, + source_hash, +) + +from .demographic_calibration_graph import demographic_calibration_node +from .national_age_activation import NATIONAL_AGE_ACTIVATION, AgeBand + +__all__ = [ + "AGE_COUNT_SCHEMA_VERSION", + "SUPPORTED_AGE_CONVENTION", + "NationalAgeCountKernel", + "national_age_count_node", + "national_age_calibration_nodes", +] + +#: Revision of the band-schema parameter's canonical form. +AGE_COUNT_SCHEMA_VERSION = 1 + +#: The one convention this operator implements: the person's ``age`` column is +#: read verbatim as completed years observed at that person's own source +#: interview. No birth date is consulted, no one is aged forward, and no claim +#: is made that the person was observed in the target period. +SUPPORTED_AGE_CONVENTION = "observed_interview_age_completed_years" +if SUPPORTED_AGE_CONVENTION != NATIONAL_AGE_ACTIVATION.age_convention_id: + raise ValueError("The age-count operator and target activation must agree on age.") + +# Operational input envelope for this development connection, not an assertion +# that every source age within it is observed or valid. Outliers refuse rather +# than silently entering the open top band. +MAX_SUPPORTED_AGE = 120 + +_AGE_COLUMN = "age" +_PERSON = US_SCHEMA.person_entity +_HOUSEHOLD = "household" +_HOUSEHOLD_ID = US_SCHEMA.entity_id_column(_HOUSEHOLD) +_PERSON_HOUSEHOLD_ID = US_SCHEMA.membership_column(_HOUSEHOLD) + +#: Every person column the kernel reads: the owned age cell plus the structural +#: household membership. Declared normatively so the read surface is in the key. +_PERSON_COLUMNS = (_AGE_COLUMN, _PERSON_HOUSEHOLD_ID) +#: Every household column the kernel reads: the structural id, and nothing else. +_HOUSEHOLD_COLUMNS = (_HOUSEHOLD_ID,) + +_PARAMS = frozenset( + { + "bands", + "person_columns", + "household_columns", + "age_convention", + "schema_version", + "maximum_age", + } +) + + +def _canonical(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def _bands_param(bands: tuple[AgeBand, ...]) -> str: + return _canonical([[band.column, band.low, band.high] for band in bands]) + + +def _bands_from_param(text: object) -> tuple[tuple[str, int, int | None], ...]: + """Parse and fully re-validate the declared band schema. + + The kernel trusts nothing it did not check here: the schema must be the + canonical form of a contiguous, disjoint, zero-based partition closed by + one open top band, with distinct column names. + """ + + if not isinstance(text, str) or len(text.encode()) > 65_536: + raise ValueError("A bounded canonical age-band declaration is required.") + rows = json.loads(text) + if not isinstance(rows, list) or not rows: + raise ValueError("The age-band declaration must be a nonempty list.") + bands: list[tuple[str, int, int | None]] = [] + expected_low = 0 + for index, row in enumerate(rows): + if ( + not isinstance(row, list) + or len(row) != 3 + or not isinstance(row[0], str) + or not row[0] + or type(row[1]) is not int + ): + raise ValueError("Each age band declares a column, a low and a high.") + column, low, high = row + last = index == len(rows) - 1 + if (high is None) is not last: + raise ValueError("Exactly the final age band is an open interval.") + if high is not None and (type(high) is not int or high < low): + raise ValueError("A closed age band needs an integer high at or above low.") + if low != expected_low: + raise ValueError("Age bands must be contiguous and start at zero.") + expected_low = None if high is None else high + 1 + bands.append((column, low, high)) + if len({column for column, _, _ in bands}) != len(bands): + raise ValueError("Age bands must own distinct count columns.") + if _canonical([[c, lo, hi] for c, lo, hi in bands]) != text: + raise ValueError("The age-band declaration must be in canonical form.") + return tuple(bands) + + +def _node_from_schema(schema: str, *, population: str, node_id: str) -> Node: + """The single declaration site, shared by the factory and the kernel.""" + + bands = _bands_from_param(schema) + return Node( + id=node_id, + kernel=NationalAgeCountKernel.ref, + population=population, + inputs=(Slice(_PERSON, (_AGE_COLUMN,)),), + outputs=tuple(Owned(_HOUSEHOLD, column, "int64") for column, _, _ in bands), + params={ + "bands": schema, + "person_columns": _PERSON_COLUMNS, + "household_columns": _HOUSEHOLD_COLUMNS, + "age_convention": SUPPORTED_AGE_CONVENTION, + "schema_version": AGE_COUNT_SCHEMA_VERSION, + "maximum_age": MAX_SUPPORTED_AGE, + }, + ) + + +def national_age_count_node( + *, + population: str, + bands: tuple[AgeBand, ...] = NATIONAL_AGE_ACTIVATION.bands, + node_id: str = "national.age_counts", +) -> Node: + """Declare the household age-count operator over one population version. + + Args: + population: The population version whose rows the counts live in. + bands: The activated bands, in order. Their canonical form enters the + node key, so an altered schema is a different node. + node_id: The node's id. + + Returns: + The node owning one int64 count column per band. + """ + + return _node_from_schema( + _bands_param(bands), population=population, node_id=node_id + ) + + +class NationalAgeCountKernel(KernelBase): + """Count each person once into their household's declared age band.""" + + ref = "us.national_age_counts@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.NONE, + consumes_se=False, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self) -> str: + return source_hash( + sys.modules[__name__], dependencies=self.capabilities.dependencies + ) + + def run(self, context: KernelContext) -> KernelResult: + params = context.params + if set(params) != _PARAMS: + raise ValueError("The age-count node declares exactly its band schema.") + if params["age_convention"] != SUPPORTED_AGE_CONVENTION: + raise ValueError( + "This operator implements only " + f"{SUPPORTED_AGE_CONVENTION!r}; it cannot honour " + f"{params['age_convention']!r}." + ) + if params["schema_version"] != AGE_COUNT_SCHEMA_VERSION: + raise ValueError("Unsupported age-band schema version.") + if ( + params["person_columns"] != _PERSON_COLUMNS + or params["household_columns"] != _HOUSEHOLD_COLUMNS + ): + raise ValueError( + "This operator reads exactly " + f"{_PERSON_COLUMNS} on {_PERSON!r} and {_HOUSEHOLD_COLUMNS} on " + f"{_HOUSEHOLD!r}." + ) + bands = _bands_from_param(params["bands"]) + expected = _node_from_schema( + params["bands"], + population=context.node.population or "", + node_id=context.node.id, + ) + if context.node.normative() != expected.normative(): + raise ValueError("Age-count node differs from its complete declaration.") + + household = context.tables[_HOUSEHOLD] + person = context.tables[_PERSON] + # The household view is exactly its id column: no weight, no source + # channel, no other attribute is visible to this operator. + if tuple(household.columns) != _HOUSEHOLD_COLUMNS: + raise ValueError( + f"The household view must carry exactly {_HOUSEHOLD_COLUMNS}, " + f"got {tuple(household.columns)}." + ) + missing = [column for column in _PERSON_COLUMNS if column not in person] + if missing: + raise ValueError(f"The person view is missing {missing}.") + household_ids = _ids(household, _HOUSEHOLD_ID, what="household id") + index = pd.Index(household_ids, name=_HOUSEHOLD_ID) + if index.has_duplicates: + raise ValueError("Household ids must be unique.") + # Align by id, never by row order: neither axis is assumed sequential, + # sorted, or shared between the two tables. + positions = index.get_indexer( + _ids(person, _PERSON_HOUSEHOLD_ID, what="household membership") + ) + if np.any(positions < 0): + missing = np.unique(person[_PERSON_HOUSEHOLD_ID].to_numpy()[positions < 0])[ + :5 + ] + raise ValueError( + "Every person's household membership must resolve to a household " + f"row; unresolved ids include {missing.tolist()}." + ) + + age = _ages(person) + band_of_person = np.full(len(person), -1, dtype=np.int64) + for ordinal, (_, low, high) in enumerate(bands): + inside = age >= low if high is None else (age >= low) & (age <= high) + if np.any(band_of_person[inside] >= 0): + raise ValueError("Declared age bands overlap.") + band_of_person[inside] = ordinal + if np.any(band_of_person < 0): + raise ValueError("Declared age bands do not cover every observed age.") + + counts = np.bincount( + band_of_person * len(index) + positions, + minlength=len(bands) * len(index), + ).astype(np.int64, copy=False) + counts = counts.reshape(len(bands), len(index)) + if int(counts.sum()) != len(person): + raise ValueError("Every person must be counted into exactly one band.") + people = counts.sum(axis=0) + if np.any(people == 0): + # The population contract requires every group row to be referenced + # by a person, so this is a broken population, not an empty house. + raise ValueError( + "Every household row must contain at least one person; " + f"{int((people == 0).sum())} household(s) contain none." + ) + return KernelResult( + columns={ + (_HOUSEHOLD, column): pd.Series( + counts[ordinal], index=index, dtype="int64" + ) + for ordinal, (column, _, _) in enumerate(bands) + }, + receipt={ + "scope": "national_age_counts_development", + "age_convention": SUPPORTED_AGE_CONVENTION, + "bands": params["bands"], + "people_counted": int(counts.sum()), + "households": int(len(index)), + "maximum_observed_age": int(age.max()) if len(age) else None, + "minimum_observed_age": int(age.min()) if len(age) else None, + "maximum_supported_age": MAX_SUPPORTED_AGE, + "consumes_weights": False, + "universe": "every person on the population, group quarters included", + "release_eligible": False, + }, + ) + + +def _ids(table: pd.DataFrame, column: str, *, what: str) -> np.ndarray: + series = table[column] + if ( + is_bool_dtype(series.dtype) + or is_complex_dtype(series.dtype) + or not is_numeric_dtype(series.dtype) + ): + raise ValueError(f"The {what} column must be an integer column.") + if series.isna().any(): + raise ValueError(f"The {what} column contains missing values.") + values = series.to_numpy(dtype="float64", na_value=np.nan) + if not np.all(np.isfinite(values) & (values == np.floor(values))): + raise ValueError(f"The {what} column must hold whole numbers.") + # Preserve exact integer ids without a float round trip, and refuse casts + # that would wrap or saturate into another household's identity. The open + # float upper bound matters: float64 cannot represent int64's maximum. + if is_integer_dtype(series.dtype): + limits = np.iinfo(np.int64) + outside = ((series < limits.min) | (series > limits.max)).any() + else: + outside = np.any(values < -(2**63)) or np.any(values >= 2**63) + if outside: + raise ValueError(f"The {what} column must fit signed int64 exactly.") + return series.to_numpy(dtype="int64") + + +def _ages(person: pd.DataFrame) -> np.ndarray: + """Ages as nonnegative whole years; anything else refuses the node.""" + + series = person[_AGE_COLUMN] + if ( + is_bool_dtype(series.dtype) + or is_complex_dtype(series.dtype) + or not is_numeric_dtype(series.dtype) + ): + raise ValueError("Person age must be a real numeric column.") + if series.isna().any(): + raise ValueError( + f"{int(series.isna().sum())} person age(s) are unknown; an age-count " + "node refuses rather than dropping people." + ) + values = series.to_numpy(dtype="float64", na_value=np.nan) + if not np.all(np.isfinite(values)): + raise ValueError("Person age must be finite.") + if np.any(values < 0): + raise ValueError("Person age must not be negative.") + if np.any(values > MAX_SUPPORTED_AGE): + raise ValueError( + f"Person age exceeds the declared maximum of {MAX_SUPPORTED_AGE}." + ) + if not np.all(values == np.floor(values)): + raise ValueError("Person age must be a whole number of completed years.") + return values + + +def national_age_calibration_nodes( + *, + base: str, + registry, + epochs: int, + learning_rate: float, + max_weight_ratio: float, + max_initial_weight_ratio: float, + bands: tuple[AgeBand, ...] = NATIONAL_AGE_ACTIVATION.bands, + count_node_id: str = "national.age_counts", + calibration_node_id: str = "national.demographic_calibration", +) -> tuple[Node, Node]: + """Compose the age-count operator with the existing calibration node. + + The count node lives in ``base``'s population version and owns the columns; + the unchanged `demographic_calibration_node` reweights over ``base`` and + reads exactly those columns, so the count node is its predecessor. + + Args: + base: The population version carrying household importance weights. + registry: The activated :class:`TargetRegistry`. Its measures must be + exactly the declared band columns, in order — a registry and a band + schema that disagree are refused here rather than half-wired. + epochs: Solver epochs. + learning_rate: Solver learning rate. + max_weight_ratio: The executor's cap against original design weights. + max_initial_weight_ratio: The solver's cap against incoming weights. + bands: The activated bands, in order. + count_node_id: Id of the count node. + calibration_node_id: Id of the calibration node. + + Returns: + ``(count_node, calibration_node)`` in dependency order. + + Raises: + ValueError: If the registry's measures are not the band columns. + """ + + measures = tuple(spec.measure for spec in registry) + columns = tuple(band.column for band in bands) + if measures != columns: + raise ValueError( + "The registry's measures must be exactly the declared band columns " + f"in order; got {measures} against {columns}." + ) + return ( + national_age_count_node(population=base, bands=bands, node_id=count_node_id), + demographic_calibration_node( + registry, + base=base, + node_id=calibration_node_id, + epochs=epochs, + learning_rate=learning_rate, + max_weight_ratio=max_weight_ratio, + max_initial_weight_ratio=max_initial_weight_ratio, + ), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py new file mode 100644 index 000000000..3d52c78a7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py @@ -0,0 +1,415 @@ +"""Graph source producers and population-grain native household lineage. + +Only source kernels open original members. Consumers preserve the executor's +typed producer edges when decoding transport; a nominal artifact type by itself +is not source authentication. These nodes issue no benchmark approval or score. +""" + +from __future__ import annotations + +import json +import tempfile +from collections.abc import Sequence +from pathlib import Path + +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + SourceRef, +) +from microcosm.graph.artifact_edges import value_from_descriptor +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + +from . import native_household_origin as native +from .graph_native_origin_implementation import ( + DEPENDENCIES, + implementation_hash, + implementation_manifest, +) + +SOURCE_TYPE = ArtifactType("microcosm.us.native_household_origins", 1) +BINDING_TYPE = ArtifactType("microcosm.us.population_household_origins", 1) +SOURCE_OUTPUT = (ArtifactOutput("origins", SOURCE_TYPE),) +BINDING_OUTPUT = (ArtifactOutput("binding", BINDING_TYPE),) +PREFIX = "native_household_origins" +ACS_NODE = PREFIX + ".acs" +ASEC_NODE = PREFIX + ".asec" +ACS_SOURCE = "native_origin_acs" +ASEC_SOURCES = tuple(f"native_origin_asec_{year}" for year in (2022, 2023, 2024)) +SOURCES = ( + SourceRef(ACS_SOURCE, native.acs.ACS_HU_CODEC), + *(SourceRef(name, "raw-bytes-v1") for name in ASEC_SOURCES), +) + + +class _Kernel(KernelBase): + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=DEPENDENCIES, + ) + + def implementation_hash(self): + return implementation_hash() + + +class NativeACSOriginKernel(_Kernel): + """Authenticate original ACS archive members through the accepted codec.""" + + ref = "us.native_household_origin.acs@1" + + def run(self, context): + native._require( + tuple(context.node.sources) == (ACS_SOURCE,) + and not context.node.inputs + and not context.node.artifact_inputs + and context.node.artifact_outputs == SOURCE_OUTPUT + and not context.params, + "ACS_ORIGIN_DECLARATION", + ) + with tempfile.TemporaryDirectory(prefix="graph-native-acs-") as tmp: + root = Path(tmp).resolve() + snapshots = root / "snapshots" + snapshots.mkdir() + source = native.produce_acs_native_origins( + context.sources[ACS_SOURCE], + snapshot_root=snapshots, + output_dir=root / "projection", + ) + return _source_result(source) + + +class NativeASECOriginKernel(_Kernel): + """Authenticate the registered original HOUSEHOLD CSV projections.""" + + ref = "us.native_household_origin.asec@1" + + def run(self, context): + native._require( + tuple(context.node.sources) == ASEC_SOURCES + and not context.node.inputs + and not context.node.artifact_inputs + and context.node.artifact_outputs == SOURCE_OUTPUT + and not context.params, + "ASEC_ORIGIN_DECLARATION", + ) + source = native.produce_asec_native_origins( + { + year: context.sources[name] + for year, name in zip((2022, 2023, 2024), ASEC_SOURCES, strict=True) + } + ) + return _source_result(source) + + +def _source_result(source): + document = source.document + return KernelResult( + artifacts={"origins": source.payload}, + receipt={ + "phase": "native_household_origin_source", + "arm": document["arm"], + "households": len(document["records"]), + "source_projection_sha256": native._sha(source.payload), + "source_receipt_sha256": document["source_receipt_sha256"], + "implementation": implementation_manifest(), + "release_eligible": False, + }, + ) + + +def _artifact(context, name, expected_type, output): + value = context.artifacts[name] + native._require( + value.type == expected_type + and value.key == opaque_artifact_key(value.producer_key, output), + "ORIGIN_ARTIFACT_EDGE", + ) + return value + + +class PopulationOriginBindingKernel(_Kernel): + """Bind carried source evidence to every current row and clone membership.""" + + ref = "us.native_household_origin.bind@1" + + def run(self, context): + node = context.node + native._require( + not node.sources + and not node.outputs + and node.artifact_outputs == BINDING_OUTPUT + and set(context.params) == {"columns", "parent_binding"}, + "ORIGIN_BIND_DECLARATION", + ) + expected = { + entity: [US_SCHEMA.entity_id_column(entity)] + for entity in US_SCHEMA.entities + } + expected[US_SCHEMA.person_entity].extend( + US_SCHEMA.membership_column(group) for group in US_SCHEMA.group_entities + ) + for entity, column, _dtype in context.params["columns"]: + expected[entity].append(column) + native._require( + set(context.tables) == set(expected) + and all( + set(context.tables[e]) == set(columns) + for e, columns in expected.items() + ), + "ORIGIN_BIND_COLUMN_INVENTORY", + ) + # Graph context supplies IDs and memberships implicitly. All remaining + # source cells must be declared by the integrating graph. The returned + # content digest must also be verified against the complete materialized + # population: this projected context cannot discover omitted columns. + tables = { + e: context.tables[e].loc[:, columns] for e, columns in expected.items() + } + frame = Frame(tables, US_SCHEMA, dict(context.weights), context.strata) + aliases = {"acs", "asec"} + if context.params["parent_binding"]: + aliases.add("parent_binding") + native._require( + set(context.artifacts) == aliases, "ORIGIN_BIND_ARTIFACT_ROSTER" + ) + values = [ + _artifact(context, arm, SOURCE_TYPE, "origins") for arm in ("acs", "asec") + ] + # Typed immutable transport was content-checked by the executor. This + # does not reopen sources or mint an independent source-authentication + # claim; the concrete producer keys are carried into the new artifact. + sources = [] + for arm, value in zip(("acs", "asec"), values, strict=True): + native._require( + native._source_document(value.payload)["arm"] == arm, + "ORIGIN_SOURCE_ARM", + ) + sources.append( + native.AuthenticatedNativeOriginSource( + value.payload, _token=native._TOKEN + ) + ) + parent = None + if "parent_binding" in aliases: + value = _artifact(context, "parent_binding", BINDING_TYPE, "binding") + doc = json.loads(value.payload) + native._require( + canonical_json(doc) == value.payload + and doc["schema"] == native.BINDING_SCHEMA, + "ORIGIN_PARENT_PAYLOAD", + ) + parent = native.PopulationOriginBinding( + value.payload, _token=native._BOUND_TOKEN + ) + result = native.bind_population_origins(frame, sources=sources, parent=parent) + document = result.document + document["graph_parent_edges"] = { + alias: { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": native._sha(value.payload), + } + for alias, value in context.artifacts.items() + } + document["population_node"] = node.population + payload = canonical_json(document) + return KernelResult( + artifacts={"binding": payload}, + receipt={ + "phase": "native_household_origin_binding", + "summary": result.summary, + "binding_sha256": native._sha(payload), + "parent_artifact_keys": { + alias: value.key for alias, value in context.artifacts.items() + }, + "implementation": implementation_manifest(), + "release_eligible": False, + }, + ) + + +def native_origin_source_nodes(*, population: str) -> tuple[Node, ...]: + """Declare source-only producers at an explicit graph population version. + + The graph requires the version to disambiguate structural branches. These + kernels declare no population columns and change no rows. + """ + return ( + Node( + ACS_NODE, + NativeACSOriginKernel.ref, + population=population, + sources=(ACS_SOURCE,), + artifact_outputs=SOURCE_OUTPUT, + ), + Node( + ASEC_NODE, + NativeASECOriginKernel.ref, + population=population, + sources=ASEC_SOURCES, + artifact_outputs=SOURCE_OUTPUT, + ), + ) + + +def native_origin_binding_node( + columns: Sequence[Owned], + *, + population: str, + node_id: str, + parent_binding_node: str | None = None, +) -> Node: + """Bind the exact declared full population; optionally extend native ancestry.""" + grouped = {entity: [] for entity in US_SCHEMA.entities} + for cell in columns: + grouped[cell.entity].append(cell.column) + native._require( + all(grouped.values()) and all(len(v) == len(set(v)) for v in grouped.values()), + "ORIGIN_COLUMN_DECLARATION", + ) + artifacts = [ + ArtifactInput("acs", ACS_NODE, "origins", SOURCE_TYPE), + ArtifactInput("asec", ASEC_NODE, "origins", SOURCE_TYPE), + ] + if parent_binding_node is not None: + artifacts.append( + ArtifactInput( + "parent_binding", parent_binding_node, "binding", BINDING_TYPE + ) + ) + return Node( + node_id, + PopulationOriginBindingKernel.ref, + population=population, + inputs=tuple( + Slice(entity, tuple(values)) for entity, values in grouped.items() + ), + artifact_inputs=tuple(artifacts), + artifact_outputs=BINDING_OUTPUT, + params={ + "columns": tuple((c.entity, c.column, c.dtype) for c in columns), + "parent_binding": parent_binding_node is not None, + }, + ) + + +def register_native_origin_kernels(registry: KernelRegistry) -> None: + """Register only the three new kernels, without editing upstream identities.""" + for kernel in ( + NativeACSOriginKernel(), + NativeASECOriginKernel(), + PopulationOriginBindingKernel(), + ): + registry.register(kernel) + + +def verify_materialized_population_origins( + manifest, + store, + *, + population_node: str, + binding_node: str, + parent_binding_node: str | None = None, +) -> native.PopulationOriginBinding: + """Verify graph evidence against its complete materialized population. + + This is a mandatory integration boundary: a kernel cannot know whether its + declared slices omitted another population column. The manifest must come + from the reviewed graph and store; matching local receipts do not constitute + an external producer authorization or approve any benchmark evaluation. + No original source is reopened, and no kernel is executed here. + """ + expected_implementation = implementation_hash() + + def read(node_id, output, expected_type, expected_kernel): + receipt = manifest.nodes[node_id] + descriptor = receipt.typed_artifacts["outputs"][output] + payload = store.load_bytes(receipt.opaque_artifacts[output]) + value = value_from_descriptor(payload, descriptor) + native._require( + receipt.kernel_ref == expected_kernel + and receipt.kernel_impl_hash == expected_implementation + and descriptor["producer"] == node_id + and descriptor["artifact"] == output + and value.producer_key == receipt.key + and value.key == receipt.opaque_artifacts[output] + and value.type == expected_type, + "ORIGIN_MATERIALIZED_PRODUCER", + ) + return value + + try: + values = { + arm: read(node_id, "origins", SOURCE_TYPE, kernel.ref) + for arm, node_id, kernel in ( + ("acs", ACS_NODE, NativeACSOriginKernel), + ("asec", ASEC_NODE, NativeASECOriginKernel), + ) + } + sources = [ + native.AuthenticatedNativeOriginSource(value.payload, _token=native._TOKEN) + for value in values.values() + ] + parent = None + if parent_binding_node is not None: + values["parent_binding"] = read( + parent_binding_node, + "binding", + BINDING_TYPE, + PopulationOriginBindingKernel.ref, + ) + parent = native.PopulationOriginBinding( + values["parent_binding"].payload, _token=native._BOUND_TOKEN + ) + # The native parent's own projection must also describe its actual + # complete population. This boundary supports the first full clone + # (or one native household selection), not an implicit ancestry DAG. + verify_materialized_population_origins( + manifest, + store, + population_node=parent.document["population_node"], + binding_node=parent_binding_node, + ) + artifact = read( + binding_node, "binding", BINDING_TYPE, PopulationOriginBindingKernel.ref + ) + actual = native.PopulationOriginBinding( + artifact.payload, _token=native._BOUND_TOKEN + ) + document = actual.document + frame = manifest.population(population_node) + native.verify_population_origin_binding(frame, actual) + expected = native.bind_population_origins( + frame, sources=sources, parent=parent + ).document + expected["population_node"] = population_node + expected["graph_parent_edges"] = { + alias: { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": native._sha(value.payload), + } + for alias, value in values.items() + } + native._require( + document == expected and canonical_json(document) == artifact.payload, + "ORIGIN_MATERIALIZED_LINEAGE", + ) + return actual + except native.NativeOriginError: + raise + except (ValueError, TypeError, KeyError, IndexError, OverflowError): + raise native.NativeOriginError("ORIGIN_MATERIALIZED_CONTRACT") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_origin_implementation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_origin_implementation.py new file mode 100644 index 000000000..26977b24c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_origin_implementation.py @@ -0,0 +1,111 @@ +"""Closed additional identity for native-origin nodes; upstream bytes stay fixed. + +The shared v1 service and inventory are unchanged. Their complete composition +and ACS-source scopes are validated before adding this independently classified +inventory. A new origin helper or registry changes only these extension keys. +""" + +from __future__ import annotations + +import csv +import hashlib +import json + +from . import native_household_origin as native +from .graph_implementation import ( + STAGE_DEPENDENCIES, + _canonical, + _covered_imports, + _dependency_contract, + _package_roots, +) +from .graph_implementation import implementation_manifest as upstream_manifest + +STAGE = "native_household_origin_v1" +BASE_STAGES = ("composed_asec_binding_v1", "acs_housing_universe_2024") +DEPENDENCIES = STAGE_DEPENDENCIES[BASE_STAGES[0]] +INVENTORY_FILE = "us_runtime/native_origin_graph_inventory.json" +INVENTORY_SCHEMA = "microcosm.us.native-origin-extension-inventory.v1" +EXTRA_MODULES = ( + "microcosm.build/cd_benchmark/canonical.py", + "microcosm.build/cd_benchmark/origin.py", + "microcosm.build/us_runtime/native_household_origin.py", + "microcosm.build/us_runtime/graph_native_household_origin.py", + "microcosm.build/us_runtime/graph_native_origin_implementation.py", +) + + +def implementation_manifest(): + """Verify all live classified dependencies without opening any survey file.""" + bases = {stage: upstream_manifest(stage) for stage in BASE_STAGES} + roots = _package_roots() + payload = (roots["microcosm.build"] / INVENTORY_FILE).read_bytes() + inventory = json.loads(payload) + native._require( + set(inventory) + == { + "schema", + "stage", + "base_stages", + "dependencies", + "extra_modules", + "contracts", + "import_classifications", + } + and inventory["schema"] == INVENTORY_SCHEMA + and inventory["stage"] == STAGE + and inventory["base_stages"] == list(BASE_STAGES) + and inventory["dependencies"] == list(DEPENDENCIES) + and inventory["extra_modules"] == list(EXTRA_MODULES) + and set(inventory["contracts"]) == set(EXTRA_MODULES), + "ORIGIN_IMPLEMENTATION_INVENTORY", + ) + imports = { + name + for contract in inventory["contracts"].values() + for name in contract["imports"] + } + native._require( + set(inventory["import_classifications"]) == imports + and all( + isinstance(label, str) and bool(label.strip()) + for label in inventory["import_classifications"].values() + ), + "ORIGIN_IMPORT_CLASSIFICATION", + ) + modules = set(EXTRA_MODULES) + for base in bases.values(): + modules.update(base["modules"]) + scope = { + "stages": { + STAGE: {"modules": sorted(modules), "dependencies": list(DEPENDENCIES)} + } + } + hashes = {} + for name in EXTRA_MODULES: + package, relative = name.split("/", 1) + code = (roots[package] / relative).read_bytes() + actual = _dependency_contract(code, name, _covered_imports(name, scope)) + native._require( + actual == inventory["contracts"][name], "ORIGIN_UNCLASSIFIED_DEPENDENCY" + ) + hashes[name] = hashlib.sha256(code).hexdigest() + return { + "schema": "microcosm.us.native-origin-extension-implementation.v1", + "stage": STAGE, + "inventory_sha256": hashlib.sha256(payload).hexdigest(), + "modules": hashes, + "upstream_implementations": bases, + "dependencies": dict(bases[BASE_STAGES[0]]["dependencies"]), + "acs_source_implementation": native.acs._implementation(), + "asec_household_member_registry": native.asec_member_registry(), + "csv_field_size_limit": csv.field_size_limit(), + } + + +def implementation_hash(): + """Identity of the additional implementation, not a replacement upstream key.""" + return hashlib.sha256( + b"microcosm.us.native-origin-extension-implementation.v1\0" + + _canonical(implementation_manifest()) + ).hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py new file mode 100644 index 000000000..030132146 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py @@ -0,0 +1,339 @@ +"""Joint property-income draws and visible reconciliation on original persons. + +This source-blind fragment consumes already qualified donor/recipient branches. +The country host owns source admission, original design-weight mapping, unknown +exclusions and later clone attachment. No source, tax treatment or release +authority follows from executing these numerical operations. +""" + +from __future__ import annotations + +import sys +from dataclasses import replace + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import qrf +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_nodes, + legacy_qrf_train_nodes, +) +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.fit.graph_signed_reconciliation import signed_reconciliation_node +from microcosm.frame import WeightKind +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + source_hash, +) +from microcosm.graph.keys import opaque_artifact_key + +from .property_income_constants import ( + PROPERTY_COMPONENTS, + PROPERTY_DRAW_COLUMNS, + PROPERTY_REPORTED_TOTAL, +) + +PROTOCOL = "microcosm.us.property-income-model.v1" +DRAW_SUMMARY_TYPE = ArtifactType("microcosm.us.property_income_joint_draws", 1) + + +def require(condition, reason): + if not condition: + raise ValueError("PROPERTY_INCOME_GRAPH_" + reason) + + +def _features(features): + codec.names(features, "property predictors") + require( + PROPERTY_REPORTED_TOTAL in features + and not set(features).intersection( + (*PROPERTY_COMPONENTS, *PROPERTY_DRAW_COLUMNS) + ), + "FEATURE_ROSTER", + ) + return features + + +def _draw_node(prefix, population, features, seed, phase): + edges = [] + for i in range(len(PROPERTY_COMPONENTS)): + producer = f"{prefix}.apply.{i:03d}" + edges.extend( + ( + ArtifactInput(f"raw_{i}", producer, "raw_draw", codec.RAW_TARGET_TYPE), + ArtifactInput( + f"state_{i}", producer, "apply_state", codec.APPLY_STATE_TYPE + ), + ) + ) + edges.append( + ArtifactInput( + "training", + f"{prefix}.fit.{len(PROPERTY_COMPONENTS) - 1:03d}", + "training_state", + codec.TRAINING_STATE_TYPE, + ) + ) + return Node( + prefix + ".draws", + PropertyIncomeDrawColumnsKernel.ref, + population=population, + inputs=(Slice("person", _features(features)),), + outputs=tuple(Owned("person", c, "float64") for c in PROPERTY_DRAW_COLUMNS), + params={"prefix": prefix, "features": features, "seed": seed, "phase": phase}, + artifact_inputs=tuple(edges), + artifact_outputs=(ArtifactOutput("summary", DRAW_SUMMARY_TYPE),), + description="Materialize the complete joint draw on original recipient persons; retain model and ordered raw-draw history.", + ) + + +def property_income_nodes( + prefix, + *, + donor_population, + recipient_population, + features, + seed, + n_estimators, + scales, + atol, + rtol, +): + """Declare four fits, four draws, draw placement and signed reconciliation. + + Donor and recipient populations must be separate original-person branches. + The aggregate feature is a qualified ASEC reported total on the donor and + adjusted ACS INTP on the recipient. Unknown anchors must be handled upstream. + Scales and tolerances are required modeling parameters, with no defaults. + """ + require( + type(prefix) is str + and bool(prefix) + and type(seed) is int + and seed >= 0 + and type(n_estimators) is int + and n_estimators > 0 + and donor_population != recipient_population, + "DECLARATION_PARAMETERS", + ) + features = _features(features) + phase = PROTOCOL + ":" + prefix + fits = legacy_qrf_train_nodes( + prefix + ".fit", + population=donor_population, + entity="person", + predictors=features, + targets=PROPERTY_COMPONENTS, + seed=seed, + n_estimators=n_estimators, + zero_atol=0, + phase=phase, + ) + applies = legacy_qrf_apply_nodes( + prefix + ".apply", + population=recipient_population, + fit_nodes=fits, + seed=seed, + phase=phase, + ) + fits = tuple(replace(node, kernel=PropertyIncomeTrainKernel.ref) for node in fits) + draws = _draw_node(prefix, recipient_population, features, seed, phase) + reconcile = signed_reconciliation_node( + prefix + ".reconcile", + population=recipient_population, + entity="person", + anchor=PROPERTY_REPORTED_TOTAL, + draws=PROPERTY_DRAW_COLUMNS, + components=PROPERTY_COMPONENTS, + nonnegative=(True, True, True, False), + scales=scales, + atol=atol, + rtol=rtol, + diagnostic_prefix=prefix + "_reconciliation", + ) + return (*fits, *applies, draws, reconcile) + + +class PropertyIncomeTrainKernel(LegacyQRFTrainKernel): + """The existing weighted QRF, restricted to the qualified property basis.""" + + ref = "us.property_income.train@1" + + def implementation_hash(self): + return codec.sha( + codec.encode_json( + { + "fragment": source_hash(sys.modules[__name__]), + "fit": LegacyQRFTrainKernel().implementation_hash(), + "components": PROPERTY_COMPONENTS, + "aggregate": PROPERTY_REPORTED_TOTAL, + } + ) + ) + + def run(self, context): + require( + context.node.kernel == self.ref + and context.params == context.node.params + and tuple(context.params["targets"]) == PROPERTY_COMPONENTS, + "TRAINING_DECLARATION", + ) + _features(context.params["predictors"]) + require( + set(context.tables) == {"person"} + and context.weights["person"].kind is WeightKind.DESIGN, + "ORIGINAL_DESIGN_WEIGHTS", + ) + table = context.tables["person"] + columns = (*context.params["predictors"], *PROPERTY_COMPONENTS) + require( + all(table[c].dtype == np.dtype("float64") for c in columns) + and np.isfinite(table.loc[:, list(columns)].to_numpy()).all(), + "KNOWN_DONOR_VALUES", + ) + require( + (table.loc[:, list(PROPERTY_COMPONENTS[:3])].to_numpy() >= 0).all() + and np.array_equal( + table.loc[:, list(PROPERTY_COMPONENTS)].sum(axis=1).to_numpy(), + table[PROPERTY_REPORTED_TOTAL].to_numpy(), + ), + "RESOLVED_DONOR_COMPONENTS", + ) + # Delegate the actual fit and its validation to the shared implementation. + # The executable graph retains the stricter country kernel identity. + return LegacyQRFTrainKernel().run( + replace( + context, node=replace(context.node, kernel=LegacyQRFTrainKernel.ref) + ), + ) + + +class PropertyIncomeDrawColumnsKernel(KernelBase): + ref = "us.property_income.draw_columns@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + codec, + qrf, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + expected = _draw_node( + population=context.node.population, **dict(context.params) + ) + require(context.node.normative() == expected.normative(), "DRAW_DECLARATION") + require( + set(context.tables) == {"person"} + and not context.sources + and set(context.artifacts) + == {edge.name for edge in expected.artifact_inputs}, + "DRAW_CONTEXT", + ) + table = context.tables["person"] + require( + table.person_id.dtype == np.dtype("int64") and table.person_id.is_unique, + "PERSON_IDENTITY", + ) + require( + all( + table[c].dtype == np.dtype("float64") + for c in context.params["features"] + ) + and np.isfinite( + table.loc[:, list(context.params["features"])].to_numpy() + ).all(), + "KNOWN_RECIPIENT_FEATURES", + ) + edge_map = {e.name: e for e in expected.artifact_inputs} + + def payload(name, kind): + value = codec.artifact(context, name, kind) + require( + value.key + == opaque_artifact_key(value.producer_key, edge_map[name].artifact), + "ARTIFACT_KEY", + ) + return value.payload + + training, trained = codec.read_training( + payload("training", codec.TRAINING_STATE_TYPE) + ) + state = trained.to_dict() + require( + state["entity"] == "person" + and tuple(state["predictors"]) == context.params["features"] + and tuple(state["targets"]) == PROPERTY_COMPONENTS + and tuple(state["completed_targets"]) == PROPERTY_COMPONENTS, + "COMPLETE_TRAINING_HISTORY", + ) + raw, columns = [], {} + index = pd.Index(table.person_id.to_numpy(copy=True), name="person_id") + for i, (target, column) in enumerate( + zip(PROPERTY_COMPONENTS, PROPERTY_DRAW_COLUMNS, strict=True) + ): + raw_name, state_name = f"raw_{i}", f"state_{i}" + require( + context.artifacts[raw_name].producer_key + == context.artifacts[state_name].producer_key, + "DRAW_SIBLING_PRODUCER", + ) + raw.append(payload(raw_name, codec.RAW_TARGET_TYPE)) + application, chain = codec.read_application( + payload(state_name, codec.APPLY_STATE_TYPE) + ) + require( + chain.entity == "person" + and tuple(chain.predictors) == context.params["features"] + and tuple(chain.targets) == PROPERTY_COMPONENTS + and tuple(chain.completed_targets) == PROPERTY_COMPONENTS[: i + 1] + and chain.recipient_index == qrf._index_identity(table.index) + and application["seed"] == context.params["seed"] + and application["models"] == training["models"][: i + 1] + and application["raw_targets"] + == [ + {"target": name, "sha256": codec.sha(value)} + for name, value in zip( + PROPERTY_COMPONENTS[: i + 1], raw, strict=True + ) + ], + "ORDERED_JOINT_DRAW_HISTORY", + ) + values = codec.read_raw_target(raw[-1], target=target, index=table.index) + columns["person", column] = pd.Series( + values.copy(), index=index, name=column + ) + return KernelResult( + columns=columns, + artifacts={ + "summary": codec.encode_json( + { + "protocol": PROTOCOL, + "rows": len(table), + "components": PROPERTY_COMPONENTS, + "models": training["models"], + "raw_sha256": [codec.sha(value) for value in raw], + "allocation": "one_joint_draw_per_input_person", + } + ) + }, + receipt={"protocol": PROTOCOL, "rows": len(table)}, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income_receipts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income_receipts.py new file mode 100644 index 000000000..1911a590e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income_receipts.py @@ -0,0 +1,251 @@ +"""Verify property model receipts against authenticated population/artifact inputs. + +This is a source-blind numerical check, not a producer or source issuer. Callers +must authenticate store payloads and the typed producer/population dependency +closure before supplying model bytes: decoding a pickle is not authentication. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import qrf, qrf_target +from microcosm.fit.graph_legacy_qrf import ( + LegacyQRFApplyKernel, + legacy_qrf_apply_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import WeightKind +from microcosm.graph import Node +from microcosm.graph.canonical import canonical_json, normative +from microcosm.graph.population import Population + +from . import graph_property_income as property_graph +from .property_income_constants import PROPERTY_COMPONENTS, PROPERTY_REPORTED_TOTAL + + +def _require(condition, reason): + if not condition: + raise ValueError("PROPERTY_MODEL_RECEIPTS_" + reason) + + +def _declarations(nodes, donor, recipient): + _require( + isinstance(nodes, tuple) + and all(type(node) is Node for node in nodes) + and len({node.id for node in nodes}) == len(nodes), + "NODE_ROSTER", + ) + fits = tuple( + n for n in nodes if n.kernel == property_graph.PropertyIncomeTrainKernel.ref + ) + applies = tuple(n for n in nodes if n.kernel == LegacyQRFApplyKernel.ref) + _require(len(fits) == len(applies) == len(PROPERTY_COMPONENTS), "MODEL_ROSTER") + # Declaration order must describe the ordered chain; other fragment nodes + # (source projection, attachment, reconciliation) are the host's concern. + first = fits[0] + _require(first.id.endswith(".fit.000"), "FIT_PREFIX") + prefix = first.id.removesuffix(".fit.000") + params = first.params + features = property_graph._features(params.get("predictors")) + seed, trees = params.get("seed"), params.get("n_estimators") + _require( + bool(prefix) + and type(seed) is int + and seed >= 0 + and type(trees) is int + and trees > 0 + and donor.version != recipient.version, + "PARAMETERS", + ) + phase = property_graph.PROTOCOL + ":" + prefix + expected_fits = legacy_qrf_train_nodes( + prefix + ".fit", + population=donor.version, + entity="person", + predictors=features, + targets=PROPERTY_COMPONENTS, + seed=seed, + n_estimators=trees, + zero_atol=0, + phase=phase, + ) + expected_applies = legacy_qrf_apply_nodes( + prefix + ".apply", + population=recipient.version, + fit_nodes=expected_fits, + seed=seed, + phase=phase, + ) + expected_fits = tuple( + replace(node, kernel=property_graph.PropertyIncomeTrainKernel.ref) + for node in expected_fits + ) + _require( + all( + canonical_json(normative(actual)) == canonical_json(normative(expected)) + for actual, expected in zip( + (*fits, *applies), (*expected_fits, *expected_applies), strict=True + ) + ), + "DECLARATION", + ) + return fits, applies, features, seed, trees, phase + + +def _table(population, columns): + table = population.frame.person + _require( + table.person_id.dtype == np.dtype("int64") + and table.person_id.is_unique + and all(c in table and table[c].dtype == np.dtype("float64") for c in columns) + and np.isfinite(table.loc[:, list(columns)].to_numpy()).all(), + "PERSON_IDENTITY_OR_VALUES", + ) + return table + + +def verify_property_model_receipts( + nodes, donor_population, recipient_population, artifacts +): + """Return the published eight fit/apply receipts after numerical validation. + + ``nodes`` is the ordered property fragment (additional non-model nodes may + be present); populations are the exact original donor/recipient branches. + ``artifacts[(node_id, output_name)]`` holds caller-authenticated bytes. + All four target fits are checked against the actual design donor and + chain-start state without fitting again. Applications are replayed using + those models and the current recipient features, because the legacy apply + checkpoint carries an index but no independent feature digest. + + The caller owns population identity, source qualification and lifetime + checks. This function preserves the supplied frames and does not confer + authority on detached populations, arbitrary model bytes or its result. + """ + _require( + type(donor_population) is Population + and type(recipient_population) is Population + and isinstance(artifacts, Mapping), + "INPUT_TYPES", + ) + fits, applies, features, seed, trees, phase = _declarations( + nodes, donor_population, recipient_population + ) + required = { + (node.id, output.name) + for node in (*fits, *applies) + for output in node.artifact_outputs + } + _require( + all(key in artifacts and type(artifacts[key]) is bytes for key in required), + "ARTIFACT_ROSTER", + ) + donor = _table(donor_population, (*features, *PROPERTY_COMPONENTS)) + recipient = _table(recipient_population, features).loc[:, list(features)].copy() + weights = donor_population.frame.resolve_weights("person") + _require(weights.kind is WeightKind.DESIGN, "ORIGINAL_DESIGN_WEIGHTS") + _require( + (donor.loc[:, list(PROPERTY_COMPONENTS[:3])].to_numpy() >= 0).all() + and np.array_equal( + donor.loc[:, list(PROPERTY_COMPONENTS)].sum(axis=1).to_numpy(), + donor[PROPERTY_REPORTED_TOTAL].to_numpy(), + ), + "RESOLVED_DONOR_COMPONENTS", + ) + model_frame = codec.model_frame( + SimpleNamespace(weights={"person": weights}), fits[0].inputs[0], donor + ) + model = qrf.RegimeGatedQRF( + seed=seed, n_estimators=trees, zero_atol=0, max_samples_leaf=None + ) + application_state = model.start_chain( + model_frame, list(features), list(PROPERTY_COMPONENTS), weights="design" + ) + before = qrf_target.LegacyQRFTrainingState.from_chain(application_state) + receipts, history, raw_history = {}, [], [] + raw = pd.DataFrame(index=recipient.index) + for i, (fit, apply, target) in enumerate( + zip(fits, applies, PROPERTY_COMPONENTS, strict=True) + ): + payload = artifacts[fit.id, "model"] + packet, after = codec.read_training(artifacts[fit.id, "training_state"]) + _require( + len(packet["models"]) == i + 1 + and packet["models"][:i] == history + and packet["models"][-1]["sha256"] == codec.sha(payload), + "TRAINING_HISTORY", + ) + fitted = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=packet["models"][-1]["sha256"] + ) + _require( + fitted.training_state == before + and fitted.next_training_state == after + and fitted.donor_sha256 + == qrf_target._consumed_values_sha256( + model_frame.person, (*features, *PROPERTY_COMPONENTS[:i], target) + ), + "TRAINING_DONOR", + ) + history.append( + { + "target": target, + "sha256": codec.sha(payload), + "training_id": fitted.training_id, + } + ) + _require(packet["models"] == history, "TRAINING_HISTORY") + before = after + application, chain = codec.read_application(artifacts[apply.id, "apply_state"]) + raw_payload = artifacts[apply.id, "raw_draw"] + actual_raw = codec.read_raw_target( + raw_payload, target=target, index=recipient.index + ) + raw_history.append({"target": target, "sha256": codec.sha(raw_payload)}) + _require( + application["seed"] == seed + and application["models"] == history + and application["raw_targets"] == raw_history + and qrf_target.LegacyQRFTrainingState.from_chain(chain) == after + and chain.recipient_index == qrf._index_identity(recipient.index), + "APPLICATION_HISTORY", + ) + result = qrf_target.apply_target( + fitted, recipient, raw, state=application_state + ) + _require( + result.state == chain + and codec.encode_raw_target( + result.raw_draw, target=target, index=recipient.index + ) + == raw_payload, + "APPLICATION_VALUES", + ) + application_state = result.state + raw[target] = actual_raw + receipts[fit.id] = { + "phase": phase, + "target": target, + "training_id": fitted.training_id, + "model_sha256": codec.sha(payload), + "donor_rows": len(donor), + "entity": "person", + "weight_kind": "design", + "regime": fitted.regime, + } + receipts[apply.id] = { + "phase": phase, + "target": target, + "recipient_rows": len(recipient), + "entity": "person", + "model_sha256": codec.sha(payload), + "raw_sha256": codec.sha(raw_payload), + "regime": fitted.regime, + } + return receipts diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py new file mode 100644 index 000000000..43fb40247 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py @@ -0,0 +1,510 @@ +"""Strict deterministic property splits and a numerical completeness gate. + +The host owns source qualification, clone pairing and full-Frame lifetime checks. +These operations grant no source or release authority and never complete unknown +components. The receiving FILTER opens a real population version and ledger entry. +""" + +from __future__ import annotations + +import hashlib +import sys + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + KernelRole, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph.decl import ROWS_ALL +from microcosm.graph.keys import opaque_artifact_key + +from . import cps_carried +from .property_income_constants import PROPERTY_COMPONENTS + +PROTOCOL = "microcosm.us.property-tax-leaves.v1" +RECEIVING_NODE = "survey_property.tax_receiving" +TAX_LEAVES_NODE = "survey_property.tax_leaves" +GATE_NODE = "survey_property.tax_leaf_gate" +TAX_LEAF_COLUMNS = ( + "taxable_interest_income", + "tax_exempt_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", +) +DIAGNOSTICS_TYPE = ArtifactType("microcosm.us.property_tax_leaf_diagnostics", 1) +VERIFICATION_TYPE = ArtifactType("microcosm.us.property_tax_leaf_verification", 1) +_INPUTS = (PROPERTY_COMPONENTS[0], PROPERTY_COMPONENTS[2], PROPERTY_COMPONENTS[1]) + + +def _require(condition, reason): + if not condition: + raise ValueError("PROPERTY_TAX_LEAVES_" + reason) + + +def _parameters(atol, rtol): + _require( + all(type(v) is float and np.isfinite(v) and v >= 0 for v in (atol, rtol)), + "TOLERANCES", + ) + fractions = ( + cps_carried.TAXABLE_INTEREST_FRACTION, + cps_carried.QUALIFIED_DIVIDEND_FRACTION, + ) + _require( + all(type(v) is float and np.isfinite(v) and 0 < v < 1 for v in fractions), + "FRACTIONS", + ) + return { + "protocol": PROTOCOL, + "taxable_interest_fraction": fractions[0], + "qualified_dividend_fraction": fractions[1], + "complements": "total_minus_computed_primary", + "unknown": "nan", + "retirement_interest": "auxiliary_unchanged", + "atol": atol, + "rtol": rtol, + } + + +def _person(person): + _require( + type(person) is pd.DataFrame + and person.columns.is_unique + and {"person_id", *_INPUTS} <= set(person), + "PERSON_COLUMNS", + ) + _require( + person.person_id.dtype == np.dtype("int64") and person.person_id.is_unique, + "PERSON_IDENTITY", + ) + for name in _INPUTS: + _require(person[name].dtype == np.dtype("float64"), "INPUT_DTYPE:" + name) + values = person[name].to_numpy(copy=False) + _require(not np.isinf(values).any(), "INFINITE_COMPONENT:" + name) + if name != PROPERTY_COMPONENTS[1]: + _require(not (values < 0).any(), "NEGATIVE_COMPONENT:" + name) + return pd.Index(person.person_id.to_numpy(copy=True), name="person_id") + + +def split_property_tax_leaves(person: pd.DataFrame) -> pd.DataFrame: + """Split individually known O/D amounts; retain NaN independently per family. + + Returned float64 columns are keyed by exact integer person IDs. Source age, + net anchors, donor eligibility, retirement earnings, and old tax predictions + never establish knownness or select the amount of either split. + """ + index = _person(person) + params = _parameters(0.0, 0.0) + columns = {} + for component, primary, complement, fraction in ( + ( + PROPERTY_COMPONENTS[0], + *TAX_LEAF_COLUMNS[:2], + params["taxable_interest_fraction"], + ), + ( + PROPERTY_COMPONENTS[2], + *TAX_LEAF_COLUMNS[2:], + params["qualified_dividend_fraction"], + ), + ): + amount = person[component].to_numpy(copy=False) + known = np.isfinite(amount) + first = np.full(len(person), np.nan, dtype=np.float64) + second = first.copy() + first[known] = amount[known] * fraction + second[known] = amount[known] - first[known] + _require( + np.isfinite(first[known]).all() and np.isfinite(second[known]).all(), + "UNSTABLE_SPLIT", + ) + columns[primary], columns[complement] = first, second + return pd.DataFrame(columns, index=index) + + +def _slices(frame, anticipated_outputs=()): + _require(type(frame) is Frame and frame.schema.person_entity == "person", "FRAME") + _require(not frame.links, "UNSUPPORTED_LINKS") + _require(type(anticipated_outputs) is tuple, "ANTICIPATED_OUTPUTS") + dtypes = { + entity: {c: str(frame.table(entity)[c].dtype) for c in frame.table(entity)} + for entity in frame.entities + } + seen = set() + for owned in anticipated_outputs: + _require( + type(owned) is Owned + and owned.entity in dtypes + and not owned.column.endswith("_id") + and owned.rows == ROWS_ALL, + "ANTICIPATED_OUTPUT", + ) + coordinate = (owned.entity, owned.column) + _require(coordinate not in seen, "DUPLICATE_ANTICIPATED_OUTPUT") + seen.add(coordinate) + existing = dtypes[owned.entity].get(owned.column) + _require( + (existing is None and not owned.rewrite) + or (owned.rewrite and existing == owned.dtype), + "CONFLICTING_ANTICIPATED_OUTPUT", + ) + dtypes[owned.entity][owned.column] = owned.dtype + if not anticipated_outputs: + _person(frame.person) + else: + _require( + frame.person.person_id.dtype == np.dtype("int64") + and frame.person.person_id.is_unique, + "PERSON_IDENTITY", + ) + _require( + all(dtypes["person"].get(c) == "float64" for c in _INPUTS), + "ANTICIPATED_INPUT_DTYPE", + ) + _require(len(frame.person) > 0, "EMPTY_RECEIVING_FRAME") + result = [] + for entity in frame.entities: + table = frame.table(entity) + columns = tuple(c for c in dtypes[entity] if not c.endswith("_id")) + _require(bool(columns), "NO_READABLE_ENTITY_COLUMN:" + entity) + if entity != "person": + _require( + type(table.index) is pd.RangeIndex + and table.index.equals(pd.RangeIndex(len(table))) + and table.index.name is None, + "UNSUPPORTED_GROUP_INDEX:" + entity, + ) + ids = table[frame.schema.entity_id_column(entity)] + members = frame.person[frame.schema.membership_column(entity)] + _require(set(ids) == set(members), "ORPHAN_GROUP:" + entity) + result.append(Slice(entity, columns)) + for name in TAX_LEAF_COLUMNS: + _require( + dtypes["person"].get(name) == "float64", + "INCUMBENT_LEAF:" + name, + ) + return tuple(result) + + +def _nodes(base, slices, projection, reconciliation, atol, rtol): + _require( + type(projection) is type(reconciliation) is ArtifactInput + and projection.name == "projection" + and reconciliation.name == "reconciliation", + "ARTIFACT_EDGES", + ) + params = _parameters(atol, rtol) + edges = (projection, reconciliation) + receiving = Node( + RECEIVING_NODE, + PropertyTaxReceivingKernel.ref, + base=base, + structural=StructuralDelta.FILTER, + inputs=slices, + params={"protocol": PROTOCOL, "mode": "all_rows_supported_shape"}, + artifact_inputs=edges, + description="Open a tax-rebase version with all original rows; refuse orphan groups, links and group-index normalization.", + ) + rebase = Node( + TAX_LEAVES_NODE, + PropertyTaxLeavesKernel.ref, + population=RECEIVING_NODE, + inputs=(Slice("person", (*_INPUTS, *TAX_LEAF_COLUMNS)),), + outputs=tuple( + Owned("person", c, "float64", rewrite=True) for c in TAX_LEAF_COLUMNS + ), + params=params, + artifact_inputs=edges, + artifact_outputs=(ArtifactOutput("diagnostics", DIAGNOSTICS_TYPE),), + description="Rebase four tax leaves from individually known ordinary interest and dividends; preserve unknowns and retirement-account earnings.", + ) + gate = Node( + GATE_NODE, + PropertyTaxLeafGateKernel.ref, + population=RECEIVING_NODE, + inputs=(Slice("person", (*_INPUTS, *TAX_LEAF_COLUMNS)),), + params=params, + artifact_inputs=( + *edges, + ArtifactInput("rebase", TAX_LEAVES_NODE, "diagnostics", DIAGNOSTICS_TYPE), + ), + artifact_outputs=(ArtifactOutput("verification", VERIFICATION_TYPE),), + description="Recompute all four leaves, partition sums and individual knownness; missing member inputs keep completeness false.", + ) + return receiving, rebase, gate + + +def property_tax_leaf_nodes( + frame: Frame, + *, + population: str, + projection: ArtifactInput, + reconciliation: ArtifactInput, + atol: float, + rtol: float, + anticipated_outputs: tuple[Owned, ...] = (), +) -> tuple[Node, Node, Node]: + """Declare an explicit receiving version, four rewrites and numeric gate. + + The host must bind/recheck this complete input Frame. FILTER-all cannot + preserve orphan groups, link tables or nondefault group indices; refuse + those shapes before execution. Group tables need a declared non-ID column + so the receiving kernel can check their actual IDs through KernelContext. + """ + return _nodes( + population, + _slices(frame, anticipated_outputs), + projection, + reconciliation, + atol, + rtol, + ) + + +def _bindings(context): + _require( + set(context.artifacts) == {e.name for e in context.node.artifact_inputs}, + "ARTIFACT_ROSTER", + ) + result = {} + for edge in context.node.artifact_inputs: + value = context.artifacts[edge.name] + _require( + value.type == edge.type + and value.key == opaque_artifact_key(value.producer_key, edge.artifact), + "ARTIFACT_BINDING", + ) + if edge.name != "rebase": + result[edge.name] = { + "producer": edge.producer, + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": hashlib.sha256(value.payload).hexdigest(), + "type": (edge.type.name, edge.type.schema_version), + } + return result + + +def _diagnostics(person, split, params, bindings): + interest = np.isfinite(person[PROPERTY_COMPONENTS[0]].to_numpy()) + dividend = np.isfinite(person[PROPERTY_COMPONENTS[2]].to_numpy()) + return { + "protocol": PROTOCOL, + "parameters": dict(params), + "input_artifacts": bindings, + "person_ids": [str(value) for value in person.person_id], + "interest_known": interest.tolist(), + "dividend_known": dividend.tolist(), + "known_leaf_origin": "derived_maintained_fraction_and_subtraction_complement", + "unknown_leaf_origin": "unknown_component_no_completion", + "leaf_knownness_groups": (TAX_LEAF_COLUMNS[:2], TAX_LEAF_COLUMNS[2:]), + "input_sha256": { + c: hashlib.sha256( + person[c].to_numpy().astype(" KernelResult: + node = context.node + edges = {e.name: e for e in node.artifact_inputs} + _require(set(edges) == {"projection", "reconciliation"}, "ARTIFACT_EDGES") + expected = _nodes( + node.base, + node.inputs, + edges["projection"], + edges["reconciliation"], + 0.0, + 0.0, + )[0] + _require( + node.normative() == expected.normative() + and not context.sources + and dict(context.params) == dict(node.params), + "RECEIVING_DECLARATION", + ) + _bindings(context) + _require( + {s.entity for s in node.inputs} == set(context.tables) + and "person" in context.tables, + "RECEIVING_TABLES", + ) + person = context.tables["person"] + ids = _person(person) + _require(len(ids) > 0, "EMPTY_RECEIVING_FRAME") + for entity, table in context.tables.items(): + if entity == "person": + continue + _require( + table.index.equals(pd.RangeIndex(len(table))) + and table.index.name is None, + "UNSUPPORTED_GROUP_INDEX:" + entity, + ) + _require( + set(table[entity + "_id"]) == set(person["person_" + entity + "_id"]), + "ORPHAN_GROUP:" + entity, + ) + return KernelResult( + keep=pd.Series(True, index=ids), + receipt={ + "protocol": PROTOCOL, + "persons": len(ids), + "all_rows_retained": True, + "population_ledger_transition": "filter_conserve", + "source_authority": False, + }, + ) + + +class PropertyTaxLeavesKernel(_Kernel): + ref = "us.property_tax_leaves@1" + + def run(self, context: KernelContext) -> KernelResult: + bindings = self._check(context, False) + person = context.tables["person"] + split = split_property_tax_leaves(person) + document = _diagnostics(person, split, context.params, bindings) + return KernelResult( + columns={("person", c): split[c] for c in TAX_LEAF_COLUMNS}, + artifacts={"diagnostics": codec.encode_json(document)}, + receipt=document, + ) + + +class PropertyTaxLeafGateKernel(_Kernel): + ref = "us.property_tax_leaf_gate@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + role=KernelRole.GATE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + bindings = self._check(context, True) + person = context.tables["person"] + split = split_property_tax_leaves(person) + for name in TAX_LEAF_COLUMNS: + _require( + name in person and person[name].dtype == np.dtype("float64"), + "LEAF_DTYPE", + ) + _require( + np.array_equal( + person[name].to_numpy().view("uint64"), + split[name].to_numpy().view("uint64"), + ), + "LEAF_BITS:" + name, + ) + for component, names in ( + (PROPERTY_COMPONENTS[0], TAX_LEAF_COLUMNS[:2]), + (PROPERTY_COMPONENTS[2], TAX_LEAF_COLUMNS[2:]), + ): + amount = person[component].to_numpy() + known = np.isfinite(amount) + with np.errstate(over="ignore", invalid="ignore"): + summed = split.loc[:, list(names)].to_numpy()[known].sum(axis=1) + _require(np.isfinite(summed).all(), "CONSERVATION_OVERFLOW") + error = np.abs(summed - amount[known]) + scale = amount[known] + relative = np.divide( + error, scale, out=np.zeros_like(error), where=scale != 0 + ) + _require( + ( + (error <= context.params["atol"]) + | ((scale != 0) & (relative <= context.params["rtol"])) + ).all(), + "PARTITION_CONSERVATION", + ) + document = _diagnostics(person, split, context.params, bindings) + _require( + context.artifacts["rebase"].payload == codec.encode_json(document), + "DIAGNOSTIC_PAYLOAD", + ) + document = {**document, "numeric_verified": True} + return KernelResult( + artifacts={"verification": codec.encode_json(document)}, + receipt={ + "outcome": "pass" if document["complete"] else "evidence_absent", + "evidence": document, + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py new file mode 100644 index 000000000..086c0032e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py @@ -0,0 +1,635 @@ +"""One source-backed canonical PUF55 donor CREATE, shared by both routes. + +All ordinary returns reach one nine-predictor model Frame. An eight-predictor +training Slice can omit the transported SS total without constructing another +donor cohort. Technical model persons are not source persons. The emitted +receipts describe construction; the eventual host must verify actual source +keys, producer implementations and typed ancestry before trusting any model. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from collections.abc import Mapping +from dataclasses import fields, is_dataclass +from functools import partial +from pathlib import Path +from types import FunctionType + +import numpy as np + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + StructuralDelta, + codecs, + source_hash, +) +from microcosm.graph import canonical as graph_canonical +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import Population + +from . import full_puf_enrichment as full +from . import graph_full_puf_enrichment as physical +from . import puf55_canonical_donor as projection +from . import puf55_route_finalization as numerical +from . import puf59_canonical as canonical +from . import puf59_canonical_artifact as envelope +from . import puf_full_source as source +from . import puf_full_source_graph as source_graph +from . import puf_interest_components as interest +from . import puf_raw_source as raw + +CANONICAL_DONOR_NODE = "survey_puf55.canonical_donor" +CANONICAL_DONOR_TYPE = ArtifactType("microcosm.us.puf59_canonical_return", 2) +DONOR_PROJECTION_TYPE = ArtifactType("microcosm.us.puf55_donor_projection", 1) +PHASE = "us.survey_puf55.canonical_donor.v1" +MAX_PROJECTION_BYTES = 128 * 1024 + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_CANONICAL_SOURCE_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _pin(pin): + _require(type(pin) is raw.SourcePin, "PIN_TYPE") + values = tuple(getattr(pin, field.name) for field in fields(raw.SourcePin)) + _require( + all(type(value) is str for value in values[:3]) + and type(pin.bytes) is int + and all(type(value) is str for value in values[4:7]) + and type(pin.delivered_header) is tuple + and all(type(value) is str for value in pin.delivered_header) + and type(pin.data_records) is int, + "PIN_FIELDS", + ) + return values + + +def _definition(definition): + """Pure check against reconstructed literals, never caller-made authority.""" + _require(type(definition) is raw.PufRawSourceDefinition, "DEFINITION_TYPE") + payload, route, digest = definition.canonical, definition.route, definition.sha256 + _require( + type(payload) is bytes + and type(route) is str + and route in ("packaged", "test_fixture") + and type(digest) is str + and _sha(payload) == digest + and canonical_json(raw._thawed(definition.document)) == payload, + "DEFINITION_BYTES", + ) + rebuilt = raw._definition_from_document(json.loads(payload), route=route) + actual = ( + route, + payload, + digest, + _pin(definition.main), + _pin(definition.demographic), + ) + _require( + actual + == ( + rebuilt.route, + rebuilt.canonical, + rebuilt.sha256, + _pin(rebuilt.main), + _pin(rebuilt.demographic), + ), + "DEFINITION_PINS", + ) + return actual + + +def _fresh_definition(fixture_definition): + packaged_bytes = raw._packaged_bytes() + _require(type(packaged_bytes) is bytes, "PACKAGED_BYTES") + packaged = raw._definition_from_document( + json.loads(packaged_bytes), route="packaged" + ) + _definition(packaged) + if fixture_definition is None: + return packaged, _sha(packaged_bytes) + _definition(fixture_definition) + document = json.loads(fixture_definition.canonical) + _require( + fixture_definition.route == "test_fixture" + and document.get("authority") == "invented_fixture_nonauthority" + and not {fixture_definition.main.sha256, fixture_definition.demographic.sha256} + & {packaged.main.sha256, packaged.demographic.sha256}, + "FIXTURE_AUTHORITY", + ) + # Reproduce the maintained fixture contract with the freshly read packaged + # pins, without consulting or accepting its process-global _PACKAGED cache. + return raw._definition_from_document(document, route="test_fixture"), _sha( + packaged_bytes + ) + + +def _recipe(seed, growth_scheme): + _require(type(seed) is int and 0 <= seed < 2**64, "SEED") + _require( + type(growth_scheme) is str and growth_scheme in ("family_observed", "cpi_only"), + "GROWTH_SCHEME", + ) + qbi, growth = canonical.qbi, canonical.growth + parameters = qbi._json(qbi.model_parameters()).encode() + _require(_sha(parameters) == qbi.PARAMETERS_SHA256, "QBI_PARAMETERS") + _require( + _sha(growth._RECIPE_JSON.encode()) == growth.RECIPE_SHA256 + and canonical_json(growth.growth_recipe()) == canonical_json(growth._RECIPE), + "GROWTH_RECIPE", + ) + runtime_bands = interest.puf_e19200_agi_bands_runtime_identity() + return { + "phase": PHASE, + "seed": seed, + "growth_scheme": growth_scheme, + "source_statistical_year": 2015, + "money_year": 2024, + "qbi_parameters_sha256": qbi.PARAMETERS_SHA256, + "growth_recipe_sha256": growth.RECIPE_SHA256, + "interest_asset_sha256": canonical.INTEREST_ASSET_SHA256, + "interest_band_facts_sha256": canonical.INTEREST_BAND_FACTS_SHA256, + "interest_runtime_sha256": runtime_bands["sha256"], + "profile": full.PUF55_SURVEY_SS.value, + "ordinary_cohort": "complete; zero design weights retained", + } + + +def _params(definition, seed, growth_scheme, packaged_sha256): + return { + **_recipe(seed, growth_scheme), + "definition": definition.params_text, + "definition_sha256": definition.sha256, + "packaged_definition_bytes_sha256": packaged_sha256, + "source_pins": canonical_json( + (_pin(definition.main), _pin(definition.demographic)) + ).decode("ascii"), + } + + +def _node(definition, params): + profile = full.require_puf_output_profile(full.PUF55_SURVEY_SS) + _require(len(profile.predictors) == 9 and len(profile.targets) == 55, "PROFILE") + return Node( + CANONICAL_DONOR_NODE, + CanonicalPuf55DonorKernel.ref, + structural=StructuralDelta.CREATE, + sources=(definition.main.source_name, definition.demographic.source_name), + outputs=tuple( + Owned("tax_unit", name, "float64") + for name in (*profile.predictors, *profile.targets) + ), + params=params, + artifact_outputs=( + ArtifactOutput("full_return_source", source_graph.FULL_RETURN_SOURCE_TYPE), + ArtifactOutput("canonical_donor", CANONICAL_DONOR_TYPE), + ArtifactOutput("donor_projection", DONOR_PROJECTION_TYPE), + ), + description="Construct one complete source-pinned modeled PUF55 donor; nine/eight training Slices share its cohort.", + ) + + +def canonical_puf55_donor_node( + *, seed=578, growth_scheme="family_observed", fixture_definition=None +): + definition, packaged_sha = _fresh_definition(fixture_definition) + return _node(definition, _params(definition, seed, growth_scheme, packaged_sha)) + + +def _modules(): + # Include actual consumed codecs, numerical source/recipe owners and Frame + # construction/storage helpers. No survey issuer or country engine is used. + modules = ( + sys.modules[__name__], + raw, + source, + source_graph, + canonical, + canonical.qbi, + canonical.growth, + envelope, + projection, + interest, + full, + full.support, + numerical, + full.codec, + full.qrf_target, + physical, + physical.operator_column_contracts, + physical.population_ops, + physical.store_ops, + codecs, + graph_canonical, + sys.modules[full.Frame.__module__], + sys.modules[full.support.EntitySchema.__module__], + sys.modules[full.support.Weights.__module__], + ) + return tuple(dict.fromkeys(modules)) + + +def _marker(value, depth=0): + """Snapshot borrowed code/configuration as an immutable object graph. + + Functions can close over a namespace that contains the same function (for + example generated class annotation machinery). Record an object's local + reference before descending, so a back edge is explicit rather than an + unbounded expansion. Each new call captures all reachable contents again; + the memo is neither a source cache nor an exemption for mutable objects. + """ + seen = {} + + def visit(item, level): + kind = type(item) + if kind in (type(None), bool, int, str, bytes, float): + _require(level <= 16, "LIVE_DEPTH") + return kind, item.hex() if kind is float else item + previous = seen.get(id(item)) + if previous is not None: + _require(previous[0] is item, "LIVE_REFERENCE") + return "reference", previous[1] + _require(level <= 16, "LIVE_DEPTH") + reference = len(seen) + # Keep the object alive while traversing borrowed mappings/dataclasses, + # preventing an identity from being recycled during this snapshot. + seen[id(item)] = (item, reference) + if kind in (tuple, list): + contents = tuple(visit(child, level + 1) for child in item) + elif kind in (set, frozenset): + contents = frozenset(visit(child, level + 1) for child in item) + elif isinstance(item, Mapping): + contents = tuple( + (visit(k, level + 1), visit(v, level + 1)) for k, v in item.items() + ) + elif kind is np.ndarray and not item.dtype.hasobject: + contents = item.dtype.str, item.shape, item.tobytes() + elif is_dataclass(item) and not isinstance(item, type): + contents = tuple( + (field.name, visit(getattr(item, field.name), level + 1)) + for field in fields(item) + ) + elif kind is FunctionType: + # Keep identity as well as immutable code/default/closure contents; + # comparing retained mutable functions to themselves is not a seal. + # Marshal bytes also encode interpreter reference-sharing state; + # retaining a returned literal can change them without a code edit. + # Code objects and their constant graph are immutable. Keep the + # object alive, retain its identity, and snapshot public fields; + # mutable function defaults/closures still use visit below. + code_object = item.__code__ + code = ( + code_object, + id(code_object), + code_object.co_argcount, + code_object.co_posonlyargcount, + code_object.co_kwonlyargcount, + code_object.co_nlocals, + code_object.co_stacksize, + code_object.co_flags, + code_object.co_code, + code_object.co_consts, + code_object.co_names, + code_object.co_varnames, + code_object.co_freevars, + code_object.co_cellvars, + code_object.co_filename, + code_object.co_name, + code_object.co_qualname, + code_object.co_firstlineno, + code_object.co_linetable, + code_object.co_exceptiontable, + ) + defaults = visit(item.__defaults__, level + 1) + kwdefaults = visit(item.__kwdefaults__, level + 1) + closure = [] + for cell in item.__closure__ or (): + try: + cell_value = cell.cell_contents + except ValueError: + closure.append(("empty_cell",)) + else: + closure.append(("cell", visit(cell_value, level + 1))) + contents = ( + item, + code, + defaults, + kwdefaults, + tuple(closure), + ) + else: + contents = id(item) + return "object", reference, kind, contents + + return visit(value, depth) + + +def _live(): + result = [] + for module in _modules(): + result.append(("module", module, module.__name__, module.__file__)) + for name, value in vars(module).items(): + if name.startswith("__") or (module is raw and name == "_PACKAGED"): + continue + if isinstance(value, type) and value.__module__ == module.__name__: + members = [] + for member, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if type(function) is FunctionType: + members.append((member, _marker(function))) + elif not member.startswith("__"): + members.append((member, _marker(function))) + result.append((module.__name__, name, value, tuple(members))) + else: + result.append((module.__name__, name, _marker(value))) + result.append( + ( + "rng", + tuple( + _marker(value) + for value in ( + np.random.default_rng, + np.random.Generator, + np.random.PCG64, + np.random.SeedSequence, + ) + ), + ) + ) + return tuple(result) + + +def _frame_seal(frame): + return physical._population_stamp( + Population.from_frame(frame, CANONICAL_DONOR_NODE) + ) + + +class CanonicalPuf55DonorKernel(KernelBase): + ref = "us.survey_puf55.canonical_donor@1" + capabilities = Capabilities( + Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.CREATE, + dependencies=("numpy", "pandas"), + ) + + def __init__( + self, *, seed=578, growth_scheme="family_observed", fixture_definition=None + ): + definition, packaged_sha = _fresh_definition(fixture_definition) + self._definition = definition + self.source_codecs = raw.puf_raw_source_codecs(definition) + self.source_refs = raw.us_puf_raw_source_refs(definition) + params = canonical_json(_params(definition, seed, growth_scheme, packaged_sha)) + # Retain primitive pins and immutable baselines before any future borrow. + self._state = ( + definition, + _definition(definition), + params, + packaged_sha, + seed, + growth_scheme, + self.implementation_hash(), + _live(), + ) + self._check_state(self._state) + + def implementation_hash(self): + # The executor calls this on required replay too. Resource reads cannot + # live only in run(), which is deliberately skipped on a cache hit. + state = getattr(self, "_state", None) + if state is not None: + self._check_state(state) + packaged = raw._packaged_bytes() + asset = interest.puf_e19200_interest_components_asset_identity() + code = source_hash(*_modules(), dependencies=self.capabilities.dependencies) + if state is not None: + self._check_state(state) + _require( + type(packaged) is bytes + and asset["asset_sha256"] == canonical.INTEREST_ASSET_SHA256 + and asset["agi_bands"] + == interest.puf_e19200_agi_bands_runtime_identity()["agi_bands"], + "IMPLEMENTATION_RESOURCES", + ) + return _sha( + canonical_json( + { + "code": code, + "packaged_definition_bytes_sha256": _sha(packaged), + "interest_asset": asset, + "csv_acceptance": raw.csv_acceptance_profile(), + } + ) + ) + + def _check_state(self, state): + definition, definition_seal, params, packaged_sha, seed, scheme, _, live = state + _require( + self._state is state + and self._definition is definition + and _definition(definition) == definition_seal + and canonical_json(_params(definition, seed, scheme, packaged_sha)) + == params + and self.source_refs == raw.us_puf_raw_source_refs(definition) + and _live() == live, + "LIVE_STATE_CHANGED", + ) + expected = (definition.main, definition.demographic) + _require( + type(self.source_codecs) is codecs.SourceCodecRegistry + and not self.source_codecs.names(), + "CODEC_MODE", + ) + _require( + set(self.source_codecs.bytes_names()) == {p.codec for p in expected}, + "CODEC_ROSTER", + ) + for pin in expected: + loader = self.source_codecs.get(pin.codec) + _require( + type(loader) is partial + and loader.func is raw.load_pinned_puf_bytes + and loader.args == () + and set(loader.keywords) == {"pin"} + and loader.keywords["pin"] is pin + and _pin(loader.keywords["pin"]) == _pin(pin), + "CODEC_BINDING", + ) + + def _context(self, context, state, expected_paths=None): + definition, _, params, *_ = state + _require( + context.node == _node(definition, json.loads(params)) + and canonical_json(dict(context.params)) == params + and not context.tables + and not context.weights + and not context.artifacts + and len(context.strata) == 0 + and set(context.sources) + == {definition.main.source_name, definition.demographic.source_name}, + "DECLARATION_OR_CONTEXT", + ) + paths = tuple( + (p.source_name, context.sources[p.source_name]) + for p in (definition.main, definition.demographic) + ) + _require( + all(type(path) is type(Path()) and path.is_absolute() for _, path in paths), + "SOURCE_PATH", + ) + _require( + expected_paths is None or paths == expected_paths, "SOURCE_PATH_CHANGED" + ) + return paths + + def _read_sources(self, paths, state): + definition = state[0] + self._check_state(state) + return tuple( + codecs.load_source_bytes(pin.codec, path, registry=self.source_codecs) + for pin, (_, path) in zip( + (definition.main, definition.demographic), paths, strict=True + ) + ) + + def _read_resources(self, state): + packaged_bytes = raw._packaged_bytes() + asset = interest.puf_e19200_interest_components_asset_identity() + implementation = self.implementation_hash() + # All file/metadata reads precede these pure comparisons. + self._check_state(state) + _require( + type(packaged_bytes) is bytes + and _sha(packaged_bytes) == state[3] + and implementation == state[6], + "SOURCE_CODE_OR_RESOURCE_CHANGED", + ) + _require( + asset["asset_sha256"] == canonical.INTEREST_ASSET_SHA256 + and asset["agi_bands"] + == interest.puf_e19200_agi_bands_runtime_identity()["agi_bands"], + "INTEREST_RESOURCE", + ) + + def run(self, context): + state = self._state + self._check_state(state) + paths = self._context(context, state) + self._read_resources(state) + buffers = self._read_sources(paths, state) + definition, _, params, _, seed, scheme, *_ = state + decoded = source.decode_full_puf_source(*buffers, definition) + raw_payload = source.encode_full_puf_source(decoded) + constructed = canonical.construct_canonical_puf59( + decoded, + interest_bands=interest.US_PUF_E19200_AGI_BANDS, + interest_asset_sha256=canonical.INTEREST_ASSET_SHA256, + seed=seed, + growth_scheme=scheme, + ) + canonical_payload = envelope.encode_canonical_puf59( + constructed, expected_growth_scheme=scheme + ) + donor, evidence = projection.canonical_puf55_donor_from_artifact( + canonical_payload, + expected_artifact_sha256=_sha(canonical_payload), + expected_growth_scheme=scheme, + profile=full.PUF55_SURVEY_SS, + ) + profile = full.require_puf_output_profile(full.PUF55_SURVEY_SS) + selected = full._validated_model_donor(donor, profile=profile) + frame = numerical._model_donor_frame(selected) + ids = decoded.status["RECID"][decoded.ordinary] + _require( + np.array_equal(donor.index.to_numpy(), ids) + and np.array_equal(frame.table("tax_unit").index.to_numpy(), ids) + and frame.weights_for("tax_unit").values.tobytes() + == ( + decoded.status["S006"][decoded.ordinary].astype(np.float64) / 100 + ).tobytes() + and all(dtype == np.dtype("float64") for dtype in selected.dtypes), + "ORDINARY_COHORT_OR_CONVERSION", + ) + frame_sha = _frame_seal(frame) + receipt = { + **json.loads(params), + "protocol": PHASE, + "source_route": definition.route, + "source_rows": len(decoded.status["RECID"]), + "ordinary_rows": len(ids), + "zero_weight_rows": int((frame.weights_for("tax_unit").values == 0).sum()), + "full_return_source_sha256": _sha(raw_payload), + "canonical_donor_sha256": _sha(canonical_payload), + "projection": evidence, + "model_frame_sha256": frame_sha, + "source_recid_sha256": _sha(ids.astype(" list[str]: + return sorted(set(globals()) | _STACKED_ALIASES) + + +ASEC_CODEC = "us-asec-raw-stage-v4" +ACS_CODEC = "us-acs-native-2024-v1" +ASEC_PREPARED_CODEC = "us-asec-prepared-current-money-v3" +ASSEMBLY_PHASE = "assemble_stacked_spine" +US_STACK_PREPARATION_TYPE = ArtifactType("microcosm.us.stacked_spine_preparation", 1) +US_SOURCE_DEPENDENCIES = STAGE_DEPENDENCIES["assembly_prepare"] + + +@dataclass(frozen=True) +class GraphAssemblyResult: + frame: Frame + receipt: object + dtype_transitions: tuple[dict[str, str], ...] + + +def _canonical_assembly(frame: Frame, sources: tuple[Frame, ...]): + """Represent source-declared integer/bool/string absence with nullable storage. + + Legacy union assembly fills absent non-float columns with object None. + The graph has no arbitrary-object dtype. Promotion is determined by the + actual source dtype, never by guessing from the assembled observations. + Values and the missing mask must be identical after the explicit boundary. + """ + transitions = [] + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table: + original = table[column] + if not pd.api.types.is_object_dtype(original.dtype): + continue + dtypes = [ + source.table(entity)[column].dtype + for source in sources + if column in source.table(entity) + ] + if not dtypes: + continue # graph-added provenance strings are handled below + if all(pd.api.types.is_bool_dtype(dtype) for dtype in dtypes): + target = "boolean" + elif all(pd.api.types.is_integer_dtype(dtype) for dtype in dtypes): + target = "Int64" + elif all(isinstance(dtype, pd.StringDtype) for dtype in dtypes): + target = "string" + else: + continue + promoted = original.astype(dtype_for_token(target)) + if not original.isna().equals(promoted.isna()): + raise ValueError( + f"US graph dtype promotion changed nulls: {entity}.{column}." + ) + observed = original.notna() + if not np.array_equal( + original[observed].to_numpy(), promoted[observed].to_numpy() + ): + raise ValueError( + f"US graph dtype promotion changed values: {entity}.{column}." + ) + table[column] = promoted + transitions.append( + {"entity": entity, "column": column, "from": "object", "to": target} + ) + canonicalize_frame_string_dtypes(frame, boundary="US graph assembly", in_place=True) + # Build checkpoint strings use NumPy NaN; the graph column codec requires + # nullable pandas strings. This is an explicit physical storage boundary. + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table: + original = table[column] + if isinstance( + original.dtype, pd.StringDtype + ) and original.dtype != dtype_for_token("string"): + promoted = original.astype(dtype_for_token("string")) + if not original.isna().equals(promoted.isna()): + raise ValueError( + f"US graph string promotion changed nulls: {entity}.{column}." + ) + observed = original.notna() + if not np.array_equal( + original[observed].to_numpy(), promoted[observed].to_numpy() + ): + raise ValueError( + f"US graph string promotion changed values: {entity}.{column}." + ) + table[column] = promoted + transitions.append( + { + "entity": entity, + "column": column, + "from": "string[python,nan]", + "to": "string[python,pd.NA]", + } + ) + return tuple(transitions) + + +def load_graph_asec(path: Path, *, store=None) -> Frame: + return _decode_graph_asec(path, store=store) + + +def _decode_graph_asec(path: Path, *, store=None) -> Frame: + del store + frame, _binding = load_asec_raw_stage_checkpoint_v4(path) + return frame + + +def load_graph_acs(path: Path, *, store=None) -> Frame: + return _decode_graph_acs(path, store=store) + + +def _decode_graph_acs(path: Path, *, store=None) -> Frame: + del store + expected = {"csv_hus.zip", "csv_pus.zip"} + if not path.is_dir() or {entry.name for entry in path.iterdir()} != expected: + raise ValueError("US ACS graph source must contain exactly the two archives.") + if any(not (path / name).is_file() for name in expected): + raise ValueError("US ACS graph source archives must be regular files.") + source = AcsPumsSource(path / "csv_hus.zip", path / "csv_pus.zip", vintage=2024) + raw, _receipt = build_acs_pums_unit_frame(source) + mapped = map_acs_native_inputs(raw) + assert_operator_free_source_frame( + mapped.frame, label="US graph ACS source", native_inputs=mapped.native_inputs + ) + return mapped.frame + + +def us_source_codecs() -> SourceCodecRegistry: + codecs = SourceCodecRegistry() + codecs.register(ASEC_CODEC, load_graph_asec) + codecs.register(ACS_CODEC, load_graph_acs) + # The prepared current-money directory source. Its loader lives with the + # preparation it names, so registering it here adds no import cycle and no + # second preparation path. + codecs.register(ASEC_PREPARED_CODEC, load_graph_asec_prepared) + codecs.register_bytes("raw-bytes-v1", load_raw_bytes) + codecs.register(ACS_HU_CODEC, load_graph_acs_housing_universe) + validate_source_codecs(codecs) + return codecs + + +def assembly_from_sources( + asec_path: Path, acs_path: Path, *, sample_fraction: float, sample_seed: int +): + """The direct-call parity oracle and the CREATE kernel share this operation.""" + asec = load_graph_asec(asec_path) + acs = load_graph_acs(acs_path) + result = _stacked_alias("assemble_stacked_spine")( + asec, acs, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + transitions = _canonical_assembly(result.frame, (asec, acs)) + return GraphAssemblyResult(result.frame, result.receipt, transitions) + + +def frame_column_declarations(frame: Frame) -> tuple[Owned, ...]: + """Inventory data cells; the graph owns IDs and membership implicitly.""" + if frame.schema != US_SCHEMA: + raise ValueError("US graph columns require the US schema.") + return tuple( + Owned(entity, column, token_for_dtype(table[column].dtype)) + for entity in US_SCHEMA.entities + for table in (frame.table(entity),) + for column in table.columns + if column != US_SCHEMA.entity_id_column(entity) + and not ( + entity == US_SCHEMA.person_entity + and column + in { + US_SCHEMA.membership_column(group) for group in US_SCHEMA.group_entities + } + ) + ) + + +def _source_implementation_hash() -> str: + """Compatibility spelling for the new preparation identity only.""" + # This runs before cache lookup, including resume=require. A renamed or + # substituted public loader cannot claim the declared codec implementation. + us_source_codecs() + return implementation_hash("assembly_prepare") + + +class USAssemblyCreateKernel(KernelBase): + ref = "us.production.spine_prepare@2" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.CREATE, + dependencies=US_SOURCE_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return _source_implementation_hash() + + def run(self, context: KernelContext) -> KernelResult: + if set(context.params) != {"phase", "sample_fraction", "sample_seed"}: + raise ValueError("US assembly parameters have an unsupported contract.") + if context.params["phase"] != ASSEMBLY_PHASE: + raise ValueError("US assembly has an incorrect outer phase.") + asec = load_graph_asec(context.sources["asec_raw_stage"]) + acs = load_graph_acs(context.sources["acs_native"]) + result = _stacked_alias("prepare_stacked_spine")( + asec, + acs, + sample_fraction=context.params["sample_fraction"], + sample_seed=context.params["sample_seed"], + ) + transitions = _canonical_assembly(result.frame, (asec, acs)) + if frame_column_declarations(result.frame) != context.node.outputs: + raise ValueError( + "US assembly source columns differ from the declared inventory." + ) + return KernelResult( + frame=result.frame, + receipt={ + "phase": ASSEMBLY_PHASE, + "implementation": implementation_manifest("assembly_prepare"), + "preparation": result.receipt, + "dtype_transitions": transitions, + }, + artifacts={ + "frame_context": encode_us_frame_context(result.frame), + "preparation": canonical_json(_json_data(result.receipt)), + }, + ) + + +class USSpineHarmonizeKernel(KernelBase): + ref = "us.production.spine_harmonize@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.REWEIGHT, + dependencies=STAGE_DEPENDENCIES["assembly_harmonize"], + ) + + def implementation_hash(self) -> str: + return implementation_hash("assembly_harmonize") + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": ASSEMBLY_PHASE}: + raise ValueError( + "US spine harmonization has an unsupported phase contract." + ) + bound = context.artifacts["frame_context"] + prepared = context.artifacts["preparation"] + if ( + bound.type != US_FRAME_CONTEXT_TYPE + or prepared.type != US_STACK_PREPARATION_TYPE + or bound.producer_key != prepared.producer_key + ): + raise ValueError( + "US spine preparation/context must share a typed producer." + ) + document = _decode(bound.payload) + if document["weight_sources"] != {"household": "design"}: + raise ValueError( + "US spine harmonization requires its explicit DESIGN anchor." + ) + for entity in ("person", "household"): + if any( + document["entities"][entity][key] != value + for key, value in _row_identity(context.tables[entity], entity).items() + ): + raise ValueError(f"US spine {entity} context identity differs.") + # This is canonical, producer-owned JSON, not a self-authored declaration. + import json + + preparation = json.loads(prepared.payload) + if canonical_json(preparation) != prepared.payload: + raise ValueError("US spine preparation must be canonical JSON.") + if preparation["assembly_metadata"] != document["metadata"]: + raise ValueError("US spine preparation and context metadata differ.") + result = _stacked_alias("harmonize_stacked_spine_weights")( + household=context.tables["household"], + weights=context.weights["household"], + preparation=preparation, + ) + # The graph's declared mass is person mass by source stratum. Compute + # it with the same Frame reducer from minimal declared membership views. + tables = { + "person": context.tables["person"][["person_id", "person_household_id"]], + "household": context.tables["household"][["household_id"]], + } + schema = EntitySchema(group_entities=("household",)) + before = Frame( + tables, schema, {"household": context.weights["household"]}, context.strata + ) + after = Frame(tables, schema, {"household": result.weights}, context.strata) + before_mass, after_mass = before.stratum_mass(), after.stratum_mass() + document["metadata"] = _normative_metadata(result.metadata) + document["mass_log"] = [ + _json_data(asdict(item)) for item in result.legacy_mass_log + ] + document["weight_sources"] = {"household": "importance"} + return KernelResult( + weights=result.weights, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": ASSEMBLY_PHASE, + "implementation": implementation_manifest("assembly_harmonize"), + "assembly": result.receipt, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + "stratum_before": before_mass.to_dict(), + "stratum_after": after_mass.to_dict(), + }, + }, + ) + + +def us_assembly_graph( + columns: Sequence[Owned], *, sample_fraction: float, sample_seed: int +) -> Graph: + """Construct only the assembly development graph, never a release pool.""" + return Graph( + country="us", + sources=( + SourceRef( + "asec_raw_stage", ASEC_CODEC, "Operator-free ASEC raw-stage checkpoint." + ), + SourceRef( + "acs_native", ACS_CODEC, "2024 ACS one-year national PUMS archives." + ), + ), + nodes=( + Node( + id=f"{ASSEMBLY_PHASE}.prepare", + kernel=USAssemblyCreateKernel.ref, + outputs=tuple(columns), + structural=StructuralDelta.CREATE, + sources=("asec_raw_stage", "acs_native"), + params={ + "phase": ASSEMBLY_PHASE, + "sample_fraction": sample_fraction, + "sample_seed": sample_seed, + }, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("preparation", US_STACK_PREPARATION_TYPE), + ), + ), + Node( + id=ASSEMBLY_PHASE, + kernel=USSpineHarmonizeKernel.ref, + base=f"{ASSEMBLY_PHASE}.prepare", + structural=StructuralDelta.REWEIGHT, + inputs=( + Slice("household", (support_channel_column("household"),)), + Slice("person", (support_channel_column("person"),)), + ), + weights=WeightTransition("household", "importance", mass="declared"), + mass="declared", + params={"phase": ASSEMBLY_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", + f"{ASSEMBLY_PHASE}.prepare", + "frame_context", + US_FRAME_CONTEXT_TYPE, + ), + ArtifactInput( + "preparation", + f"{ASSEMBLY_PHASE}.prepare", + "preparation", + US_STACK_PREPARATION_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + ), + ), + ) + + +def us_assembly_registry() -> KernelRegistry: + kernels = KernelRegistry() + kernels.register(USAssemblyCreateKernel()) + kernels.register(USSpineHarmonizeKernel()) + return kernels diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_age_artifact.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_age_artifact.py new file mode 100644 index 000000000..b59f3f4df --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_age_artifact.py @@ -0,0 +1,221 @@ +"""Unweighted age counts as an ordered artifact, without population columns. + +The actual NationalAgeCountKernel performs the measurement. Its household +roster is the sorted unique membership of the declared person projection; +Frame requires every household to be referenced. Downstream admission must +also compare the artifact's ordered IDs with the complete receiving population. +This artifact is a measurement, not source authority or target activation. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import dataclass, replace + +import numpy as np +import pandas as pd + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + KernelBase, + KernelResult, + source_hash, +) + +from . import graph_national_age_counts as ages +from . import national_age_activation as activation + +COUNTS_TYPE = ArtifactType("microcosm.us.survey_household_age_counts", 1) +MAGIC = b"MCUSAGE1\n" +MAX_BYTES = 64 * 1024**2 +MAX_HEADER_BYTES = 65_536 +MAX_PEOPLE = 10_000_000 +_COLUMNS = tuple(b.column for b in activation.NATIONAL_AGE_ACTIVATION.bands) +_WIDTH = 1 + len(_COLUMNS) +_FIELDS = frozenset( + {"protocol", "population", "columns", "rows", "people", "age_convention"} +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_AGE_ARTIFACT_" + reason) + + +def _json(value): + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _shape(rows, people): + _require(type(rows) is int and type(people) is int, "INTEGER_SHAPE") + _require(0 < rows <= people <= MAX_PEOPLE, "SHAPE") + _require( + len(MAGIC) + 4 + MAX_HEADER_BYTES + rows * _WIDTH * 8 <= MAX_BYTES, "LIMIT" + ) + + +def _validate(header: dict, values: np.ndarray): + _require(type(header) is dict and set(header) == _FIELDS, "HEADER") + _require( + header["protocol"] == "microcosm.us.survey-household-age-counts.v1", "PROTOCOL" + ) + _require( + type(header["population"]) is str and 0 < len(header["population"]) <= 256, + "POPULATION", + ) + _require(header["columns"] == list(_COLUMNS), "COLUMNS") + _require(header["age_convention"] == ages.SUPPORTED_AGE_CONVENTION, "CONVENTION") + _shape(header["rows"], header["people"]) + _require( + type(values) is np.ndarray + and values.dtype == np.dtype(" ids[:-1]), "ORDERED_IDS") + _require(np.all(counts >= 0) and np.all(counts <= header["people"]), "COUNTS") + _require(np.all(np.any(counts > 0, axis=1)), "EMPTY_HOUSEHOLD") + # The shape and per-cell limits above bound this sum well below int64 max. + _require(int(counts.sum(dtype=np.int64)) == header["people"], "PEOPLE_TOTAL") + + +@dataclass(frozen=True) +class SurveyAgeCountValues: + """Decoded values only. Typed ancestry and current-population checks are separate.""" + + population: str + household_ids: np.ndarray + counts: pd.DataFrame + people: int + + +def decode_survey_age_counts(payload: bytes) -> SurveyAgeCountValues: + _require( + type(payload) is bytes and len(MAGIC) + 4 < len(payload) <= MAX_BYTES, "PAYLOAD" + ) + _require(payload.startswith(MAGIC), "MAGIC") + offset = len(MAGIC) + length = int.from_bytes(payload[offset : offset + 4], "big") + _require( + 0 < length <= MAX_HEADER_BYTES and offset + 4 + length <= len(payload), + "HEADER_LIMIT", + ) + offset += 4 + raw = payload[offset : offset + length] + try: + header = json.loads(raw) + except (ValueError, UnicodeError): + raise ValueError("SURVEY_AGE_ARTIFACT_HEADER_JSON") from None + _require(type(header) is dict and set(header) == _FIELDS, "HEADER") + _require(_json(header) == raw, "CANONICAL_HEADER") + _shape(header["rows"], header["people"]) + offset += length + _require(len(payload) - offset == header["rows"] * _WIDTH * 8, "BYTE_COUNT") + values = np.frombuffer(payload, dtype=" bytes: + """Project numbers from bounded bytes; this pure operation authenticates nothing.""" + document = _document(payload) + ids = document.get("household_ids") + groups = document.get("group_indices") + records = document.get("origins") + _require( + type(ids) is list + and 0 < len(ids) <= numeric.MAX_ROWS + and type(groups) is list + and len(groups) == len(ids) + and type(records) is list + and 0 < len(records) <= budgets.MAX_GROUPS + and type(document.get("group_count")) is int + and document["group_count"] == len(records) + and len(ids) == 2 * len(records), + "COMPLETE_CLONE_SHAPE", + ) + _require( + all(type(i) is int and -(2**63) <= i < 2**63 for i in ids) + and len(set(ids)) == len(ids), + "HOUSEHOLD_IDS", + ) + positions = {i: position for position, i in enumerate(ids)} + incoming, row_upper = [None] * len(ids), [None] * len(ids) + upper, seen = [], set() + for group, record in enumerate(records): + _require(type(record) is dict, "ORIGIN_RECORD") + # Validate before duplicating per-origin tokens into per-row output. + numeric._vector([record.get("design_bound_float64_hex")], 1) + numeric._vector([record.get("upper_float64_hex")], 1) + members, weights = ( + record.get("members"), + record.get("incoming_clone_float64_bytes"), + ) + _require( + type(members) is list + and len(members) == 2 + and type(weights) is list + and len(weights) == 2, + "COMPLETE_ROLES", + ) + roles = set() + for member, weight_bytes in zip(members, weights, strict=True): + _require( + type(member) is list + and len(member) == 2 + and type(member[0]) is int + and member[0] in positions + and member[0] not in seen + and type(member[1]) is int + and member[1] in (0, 1), + "MEMBERSHIP", + ) + seen.add(member[0]) + roles.add(member[1]) + position = positions[member[0]] + _require( + type(groups[position]) is int and groups[position] == group, + "GROUP_ORDER", + ) + _require( + type(weight_bytes) is str + and len(weight_bytes) == 16 + and all(c in "0123456789abcdef" for c in weight_bytes), + "WEIGHT_BYTES", + ) + value = np.frombuffer(bytes.fromhex(weight_bytes), dtype=np.float64)[0] + incoming[position] = float(value).hex() + row_upper[position] = record.get("design_bound_float64_hex") + _require(roles == {0, 1}, "COMPLETE_ROLES") + upper.append(record.get("upper_float64_hex")) + _require(seen == set(ids), "COMPLETE_MEMBERSHIP") + output = graph._bounded_json( + { + "protocol": numeric.BOUNDS_PROTOCOL, + "budget_sha256": _sha(payload), + "population": clone.COMBINED_CLONE_NODE, + "household_ids": ids, + "group_indices": groups, + "group_upper_hex": upper, + "row_upper_hex": row_upper, + "incoming_hex": incoming, + }, + numeric.MAX_BYTES, + ) + _require(len(output) <= numeric.MAX_BYTES, "NUMERIC_LIMIT") + numeric.decode_numeric_survey_bounds(output) + return output + + +def survey_sampling_budget_node(*, budget_sha256): + _require(numeric._digest(budget_sha256), "BUDGET_DIGEST") + return Node( + BUDGET_NODE, + SurveySamplingBudgetKernel.ref, + population=clone.COMBINED_CLONE_NODE, + # This provenance input forces the actual ownership claim to precede + # the budget transport. It is not an imputation feature. + inputs=(Slice("household", (support_clone_index_column("household"),)),), + params={"budget_sha256": budget_sha256, "authority": "country_runner_required"}, + artifact_inputs=( + ArtifactInput( + "preparation", graph.CREATE_NODE, "preparation", graph.PREPARATION_TYPE + ), + ArtifactInput( + "allocation", graph.ALLOCATION_NODE, "allocation", graph.ALLOCATION_TYPE + ), + ), + artifact_outputs=( + ArtifactOutput("budget", budgets.BUDGET_TYPE), + ArtifactOutput("numeric_bounds", numeric.BOUNDS_TYPE), + ), + ) + + +class SurveySamplingBudgetKernel(KernelBase): + """Carry immutable caller values with checked source and clone projections.""" + + ref = "us.survey_sampling_budget@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def __init__(self, payload): + _document(payload) + self._payload = payload + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + budgets, + numeric, + numeric.group_bounds, + support_clone_index_column, + graph, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + payload = self._payload + document = _document(payload) + expected = survey_sampling_budget_node(budget_sha256=_sha(payload)) + _require(context.node.normative() == expected.normative(), "DECLARATION") + _require(dict(context.params) == dict(expected.params), "PARAMETERS") + _require( + set(context.tables) == set(context.weights) == {"household"} + and not context.sources + and set(context.artifacts) == {"preparation", "allocation"}, + "CONTEXT", + ) + for edge in expected.artifact_inputs: + value = context.artifacts[edge.name] + _require( + value.type == edge.type + and value.key == opaque_artifact_key(value.producer_key, edge.artifact) + and _sha(value.payload) == document[edge.name + "_sha256"], + "SOURCE_EDGE", + ) + numbers_raw = numeric_survey_budget_payload(payload) + numbers = numeric.decode_numeric_survey_bounds(numbers_raw) + household = context.tables["household"] + role_column = support_clone_index_column("household") + _require(set(household.columns) == {"household_id", role_column}, "PROJECTION") + _require( + tuple(household.household_id) == numbers.grouped.household_ids, + "ORDERED_IDS", + ) + roles = { + member[0]: member[1] + for record in document["origins"] + for member in record["members"] + } + _require( + tuple(household[role_column]) + == tuple(roles[i] for i in household.household_id), + "CLONE_ROLES", + ) + weights = context.weights["household"] + _require( + weights.kind is WeightKind.IMPORTANCE + and weights.values.dtype == numbers.incoming.dtype + and weights.values.tobytes() == numbers.incoming.tobytes(), + "ACTUAL_INCOMING_WEIGHTS", + ) + result = KernelResult( + artifacts={"budget": payload, "numeric_bounds": numbers_raw}, + receipt={ + "budget_sha256": _sha(payload), + "numeric_bounds_sha256": _sha(numbers_raw), + "constraint_digest": numbers.grouped.digest, + "group_count": numbers.grouped.group_count, + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + ) + _require( + type(self._payload) is bytes and self._payload == payload, "FINAL_PAYLOAD" + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_calibration.py new file mode 100644 index 000000000..b3d839f8c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_calibration.py @@ -0,0 +1,417 @@ +"""Source-blind age calibration over typed numeric budget and count artifacts. + +These kernels do not authenticate a survey. The country runner must reconstruct +the source-qualified budget and independently admit the complete population on +cold execution, cached restoration and final return. Frozen incoming weights +are numeric artifact values, never replacement original DESIGN anchors. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import dataclass, replace + +import numpy as np + +from microcosm.calibrate import calibrate, diagnostics_payload, group_bounds +from microcosm.frame import WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + SeedSource, + StructuralDelta, + WeightTransition, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + +from . import demographic_calibration_graph as demographic +from . import graph_survey_age_artifact as ages +from . import survey_age_activation as age_activation + +BOUNDS_TYPE = ArtifactType("microcosm.us.survey_calibration_numeric_bounds", 1) +BOUNDS_PROTOCOL = "microcosm.us.survey-calibration-numeric-bounds.v1" +MAX_BYTES = 64 * 1024**2 +MAX_ROWS = MAX_BYTES // 128 +CALIBRATION_NODE = "survey.age_calibration" +_FIELDS = frozenset( + { + "protocol", + "budget_sha256", + "population", + "household_ids", + "group_indices", + "group_upper_hex", + "row_upper_hex", + "incoming_hex", + } +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_CALIBRATION_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _digest(value): + return ( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +def _vector(raw, length): + _require(type(raw) is list and len(raw) == length, "VECTOR_LENGTH") + _require(all(type(v) is str and 0 < len(v) <= 32 for v in raw), "FLOAT_TOKEN") + try: + values = np.array([float.fromhex(v) for v in raw], dtype=np.float64) + except (ValueError, OverflowError): + raise ValueError("SURVEY_CALIBRATION_FLOAT_TOKEN") from None + _require( + np.all(np.isfinite(values) & (values >= 0)) + and all(float(v).hex() == text for v, text in zip(values, raw, strict=True)), + "FLOAT_VALUES", + ) + return values + + +@dataclass(frozen=True) +class NumericSurveyBounds: + """Decoded numbers only; neither a source certificate nor a live budget.""" + + budget_sha256: str + population: str + grouped: group_bounds.GroupedUpperBounds + incoming: np.ndarray + row_upper: np.ndarray + + +def decode_numeric_survey_bounds(payload: bytes) -> NumericSurveyBounds: + _require(type(payload) is bytes and 0 < len(payload) <= MAX_BYTES, "PAYLOAD") + try: + document = json.loads(payload) + except (ValueError, UnicodeError): + raise ValueError("SURVEY_CALIBRATION_JSON") from None + _require(type(document) is dict and set(document) == _FIELDS, "FIELDS") + _require(canonical_json(document) == payload, "CANONICAL") + _require(document["protocol"] == BOUNDS_PROTOCOL, "PROTOCOL") + _require(_digest(document["budget_sha256"]), "BUDGET_DIGEST") + _require( + type(document["population"]) is str and 0 < len(document["population"]) <= 256, + "POPULATION", + ) + ids, indices = document["household_ids"], document["group_indices"] + _require(type(ids) is list and 0 < len(ids) <= MAX_ROWS, "ROW_COUNT") + _require( + all(type(v) is int and -(2**63) <= v < 2**63 for v in ids) + and all(a < b for a, b in zip(ids, ids[1:], strict=False)), + "ORDERED_IDS", + ) + _require(type(indices) is list and len(indices) == len(ids), "GROUP_INDICES") + raw_upper = document["group_upper_hex"] + _require(type(raw_upper) is list and 0 < len(raw_upper) <= len(ids), "GROUP_COUNT") + _require( + all(type(v) is int and 0 <= v < len(raw_upper) for v in indices), + "GROUP_INDICES", + ) + upper = _vector(raw_upper, len(raw_upper)) + incoming = _vector(document["incoming_hex"], len(ids)) + row_upper = _vector(document["row_upper_hex"], len(ids)) + grouped = group_bounds.GroupedUpperBounds(ids, indices, upper) + grouped.check(incoming, positive=False) + _require(np.all(incoming <= row_upper), "INITIAL_ROW_CAP") + _require(np.any(incoming > 0), "EMPTY_POSITIVE_SUPPORT") + return NumericSurveyBounds( + document["budget_sha256"], document["population"], grouped, incoming, row_upper + ) + + +def check_numeric_survey_weights(bounds, values): + """Independent numerical checks, including strict retained zero support.""" + _require(type(bounds) is NumericSurveyBounds, "BOUNDS_TYPE") + _require( + type(values) is np.ndarray + and values.dtype == np.dtype("float64") + and values.shape == bounds.incoming.shape, + "WEIGHT_ARRAY", + ) + bounds.grouped.check(values, positive=False) + _require(np.all(values <= bounds.row_upper), "ROW_REFERENCE_CAP") + _require(np.array_equal(values == 0, bounds.incoming == 0), "FIXED_ZERO_SUPPORT") + + +def survey_age_calibration_node( + registry, *, base, budget_node, count_node, epochs, learning_rate +): + frozen = demographic._registry_from_json(demographic._registry_json(registry)) + _require( + tuple(s.measure for s in frozen) == ages._COLUMNS + and all(s.metadata["evidence_scope"] == "invented" for s in frozen), + "INVENTED_AGE_REGISTRY_REQUIRED", + ) + return _survey_age_calibration_node( + frozen, + base=base, + budget_node=budget_node, + count_node=count_node, + epochs=epochs, + learning_rate=learning_rate, + ) + + +def survey_age_development_node( + registry, + *, + activation_binding, + base, + budget_node, + count_node, + epochs, + learning_rate, +): + """Declare source-documented numbers; the country runner activates sources.""" + frozen = age_activation.validate_survey_age_registry(registry, activation_binding) + return _survey_age_calibration_node( + frozen, + base=base, + budget_node=budget_node, + count_node=count_node, + epochs=epochs, + learning_rate=learning_rate, + activation_binding=activation_binding, + ) + + +def _survey_age_calibration_node( + frozen, + *, + base, + budget_node, + count_node, + epochs, + learning_rate, + activation_binding=None, +): + _require(type(epochs) is int and 1 <= epochs <= 1000, "EPOCHS") + _require( + type(learning_rate) in (int, float) + and np.isfinite(learning_rate) + and 0 < learning_rate <= 1, + "LEARNING_RATE", + ) + activation_params = {} + if activation_binding is not None: + declaration = age_activation.declaration_from_binding(activation_binding) + activation_params["activation"] = canonical_json( + age_activation.activation_binding(declaration) + ).decode() + return Node( + CALIBRATION_NODE, + SurveyAgeCalibrationKernel.ref, + base=base, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + params={ + **activation_params, + "registry": demographic._registry_json(frozen), + "epochs": epochs, + "learning_rate": learning_rate, + "weight_anchor": "source_qualified_sampling_reference", + "cap_enforcement": "mandatory_country_runner_cold_cache_and_final", + "grouped_preserve_zeros": True, + "method": "adam", + "seed": 0, + "l2_lambda": 0.0, + "l2_anchor": "initial", + "initial_anchor_meaning": "frozen_incoming_clone_weights", + }, + artifact_inputs=( + ArtifactInput("bounds", budget_node, "numeric_bounds", BOUNDS_TYPE), + ArtifactInput("counts", count_node, "counts", ages.COUNTS_TYPE), + ), + artifact_outputs=(ArtifactOutput("diagnostics", demographic.DIAGNOSTICS_TYPE),), + ) + + +def calibration_receipt_scope(params): + """Identify numerical evidence scope without claiming source admission.""" + return ( + "source_documented_survey_age_calibration_numbers_only" + if "activation" in params + else "invented_survey_age_calibration_numbers_only" + ) + + +def calibration_activation_binding(params): + """Decode the bounded canonical-text profile without acquiring its sources.""" + if "activation" not in params: + return None + value = params["activation"] + _require(type(value) is str and 0 < len(value.encode()) <= 65536, "ACTIVATION_TEXT") + try: + binding = json.loads(value) + encoded = canonical_json(binding).decode() + except (ValueError, TypeError, RecursionError, OverflowError): + raise ValueError("SURVEY_CALIBRATION_ACTIVATION_JSON") from None + _require(encoded == value, "ACTIVATION_CANONICAL") + age_activation.declaration_from_binding(binding) + return binding + + +def _artifact(context, name, type_, output): + value = context.artifacts[name] + _require( + value.type == type_ + and value.key == opaque_artifact_key(value.producer_key, output) + and type(value.payload) is bytes, + "ARTIFACT_EDGE", + ) + return value.payload + + +class SurveyAgeCalibrationKernel(KernelBase): + """Real fixed-zero grouped Adam on a minimal numeric work Frame.""" + + ref = "us.survey_age_calibration@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.REWEIGHT, + consumes_se=False, + dependencies=("numpy", "pandas", "scipy", "torch"), + ) + + def implementation_hash(self): + return _sha( + canonical_json( + { + "adapter": source_hash( + sys.modules[__name__], + group_bounds, + dependencies=self.capabilities.dependencies, + ), + "age_artifact": ages.SurveyAgeCountArtifactKernel().implementation_hash(), + "activation_declaration": source_hash( + age_activation, + age_activation.age, + age_activation.survey_age_requests, + ), + "solver": demographic.DemographicCalibrationKernel().implementation_hash(), + } + ) + ) + + def run(self, context): + aliases = context.node.artifact_inputs + _require(tuple(a.name for a in aliases) == ("bounds", "counts"), "ALIASES") + registry = demographic._registry_from_json(context.params["registry"]) + factory = survey_age_calibration_node + profile_options = {} + if "activation" in context.params: + factory = survey_age_development_node + profile_options["activation_binding"] = calibration_activation_binding( + context.params + ) + expected = factory( + registry, + **profile_options, + base=context.node.base, + budget_node=aliases[0].producer, + count_node=aliases[1].producer, + epochs=context.params["epochs"], + learning_rate=context.params["learning_rate"], + ) + _require(context.node.normative() == expected.normative(), "DECLARATION") + _require(dict(context.params) == dict(expected.params), "PARAMETERS") + _require( + not context.tables and not context.weights and not context.sources, + "CONTEXT", + ) + bounds_raw = _artifact(context, "bounds", BOUNDS_TYPE, "numeric_bounds") + counts_raw = _artifact(context, "counts", ages.COUNTS_TYPE, "counts") + bounds = decode_numeric_survey_bounds(bounds_raw) + counts = ages.decode_survey_age_counts(counts_raw) + _require( + bounds.population == counts.population == context.node.base + and tuple(counts.household_ids) == bounds.grouped.household_ids, + "COMPLETE_ORDERED_POPULATION", + ) + table = counts.counts.copy(deep=True) + table.insert(0, "household_id", counts.household_ids.copy()) + incoming = Weights(bounds.incoming.copy(), WeightKind.IMPORTANCE) + work = demographic.kernels_module._frame_from_context( + replace( + context, tables={"household": table}, weights={"household": incoming} + ), + "household", + ) + result = calibrate( + work, + registry.to_target_set(), + weight_entity="household", + method="adam", + seed=0, + epochs=context.params["epochs"], + learning_rate=context.params["learning_rate"], + mass="free", + max_weight_ratio=None, + grouped_upper_bounds=bounds.grouped, + grouped_preserve_zeros=True, + l2_lambda=context.params["l2_lambda"], + l2_anchor=context.params["l2_anchor"], + ) + _require(not result.skipped, "SKIPPED_TARGETS") + weights = result.frame.weights_for("household") + accepted = weights.values.tobytes() + check_numeric_survey_weights(bounds, weights.values) + anchors = { + "budget_sha256": bounds.budget_sha256, + "numeric_bounds_sha256": _sha(bounds_raw), + "counts_sha256": _sha(counts_raw), + "accepted_weight_sha256": _sha(accepted), + "constraint_digest": bounds.grouped.digest, + "weight_anchor": expected.params["weight_anchor"], + "cap_enforcement": expected.params["cap_enforcement"], + "fixed_zero_rows": int(np.count_nonzero(bounds.incoming == 0)), + } + payload = canonical_json( + diagnostics_payload(result, target_registry=registry, build=anchors) + ) + output = KernelResult( + weights=weights, + artifacts={"diagnostics": payload}, + receipt={ + **anchors, + "diagnostics_sha256": _sha(payload), + "scope": calibration_receipt_scope(expected.params), + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + ) + # Validation follows diagnostics and return-object materialization. + fresh = decode_numeric_survey_bounds(bounds_raw) + check_numeric_survey_weights(fresh, weights.values) + _require(weights.values.tobytes() == accepted, "FINAL_WEIGHT_BYTES") + _require( + tuple(result.frame.table("household").household_id) + == fresh.grouped.household_ids, + "FINAL_HOUSEHOLD_IDS", + ) + return output diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py new file mode 100644 index 000000000..a262f2682 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py @@ -0,0 +1,1266 @@ +"""Graph declarations for a source-authenticated, selected survey population. + +The catalogue draw precedes native construction. CREATE therefore materializes +the selected combined population, and a separate REWEIGHT applies source shares +and inverse inclusion. Neither decoded evidence nor these declarations grants +source, population, or release authority. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections.abc import Sequence +from dataclasses import asdict, dataclass +from fractions import Fraction +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA, MassChange, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Ownership, + Slice, + StructuralDelta, + WeightTransition, +) + +from . import graph_context +from .graph_context import US_FRAME_CONTEXT_TYPE +from .support_provenance import ( + spine_source_id_column, + support_channel_column, + support_clone_index_column, + support_source_id_column, +) +from .survey_catalogue_selection import CatalogueSelectionPlan, SelectedHousehold +from .survey_population_domains import HouseholdKey, Source + +PHASE = "us.authenticated_survey_population.v1" +SOURCE_NAME = "survey_population_source" +SOURCE_CODEC = "us-survey-population-source-v1" +CREATE_NODE = "survey_population.create" +ALLOCATION_NODE = "survey_population.allocate" +CREATE_REF = "us.survey_population.create@1" +ALLOCATION_REF = "us.survey_population.allocate@1" +PREPARATION_TYPE = ArtifactType("microcosm.us.survey_population_preparation", 2) +ALLOCATION_TYPE = ArtifactType("microcosm.us.survey_population_allocation", 1) +PREPARATION_MAX_BYTES = 64 * 1024**2 +ALLOCATION_MAX_BYTES = 64 * 1024**2 +STAGE = "authenticated_survey_population_v1" +STAGE_DEPENDENCIES = ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", +) +_HOUSEHOLD_ORIGIN_FIELDS = frozenset( + { + "household_id", + "source", + "source_year", + "survey_year", + "raw_native_id", + "selected_receiving_household_id", + "original_anchor", + } +) + + +@dataclass(frozen=True, slots=True) +class AllocationInstruction: + """Exact arithmetic values, without source or population authority.""" + + household_id: int + key: HouseholdKey + selected_receiving_household_id: int + original_anchor: Fraction + share: Fraction + inclusion_probability: Fraction + + @property + def importance_weight(self): + return self.original_anchor * self.share / self.inclusion_probability + + +class SurveyPopulationGraphError(ValueError): + """Static graph refusal without source values or local paths.""" + + +def _require(condition, code): + if not condition: + raise SurveyPopulationGraphError(code) + + +def _digest(value): + _require( + type(value) is str + and len(value) == 64 + and not set(value) - set("0123456789abcdef"), + "DIGEST", + ) + return value + + +def _draw_parameters(fraction, seed): + _require(type(fraction) is Fraction and 0 < fraction <= 1, "FRACTION") + _require(type(seed) is int and 0 <= seed < 2**64, "SEED") + return (fraction.numerator, fraction.denominator), seed + + +def _fraction_pair(value): + _require( + type(value) in (list, tuple) + and len(value) == 2 + and all(type(v) is int for v in value) + and value[1] > 0, + "FRACTION_PAIR", + ) + number = Fraction(*value) + _require((number.numerator, number.denominator) == tuple(value), "FRACTION_PAIR") + return number + + +def allocation_instructions(plan, household_origins): + """Join checked preparation values; this pure join authenticates nothing. + + The runner obtains both arguments from a freshly issued preparation. Keeping + this operation separate lets the allocation kernel hold immutable values + instead of a source Frame or a mutable decoded preparation document. + """ + _require(type(plan) is CatalogueSelectionPlan, "SELECTION_PLAN") + _require(type(plan.selected) is tuple and plan.selected, "EMPTY_SELECTION") + _require(type(household_origins) in (list, tuple), "HOUSEHOLD_ORIGINS") + _require(len(household_origins) == len(plan.selected), "ORIGIN_COUNT") + # A prospective record bound precedes the two maps and output tuple. The + # streaming transport encoder enforces its exact bound separately. + _require(len(plan.selected) <= ALLOCATION_MAX_BYTES // 128, "ALLOCATION_LIMIT") + selected = {} + for row in plan.selected: + _require( + type(row) is SelectedHousehold and type(row.key) is HouseholdKey, + "SELECTED_TYPE", + ) + _require( + type(row.key.source) is Source + and type(row.key.source_year) is int + and row.key.source_year == 2024 + and type(row.key.survey_year) is int + and row.key.survey_year == (2024 if row.key.source is Source.ACS else 2025) + and type(row.key.native_id) is str + and 0 < len(row.key.native_id) <= 128, + "SELECTED_KEY", + ) + _require(row.key not in selected, "SELECTED_DUPLICATE") + _require( + type(row.original_design_weight) is Fraction + and row.original_design_weight >= 0 + and type(row.share) is Fraction + and 0 < row.share <= 1 + and type(row.inclusion_probability) is Fraction + and 0 < row.inclusion_probability <= 1, + "SELECTED_ARITHMETIC", + ) + selected[row.key] = row + output, seen_ids, seen_native, seen_receiving = [], set(), set(), set() + for origin in household_origins: + _require( + type(origin) is dict and set(origin) == _HOUSEHOLD_ORIGIN_FIELDS, + "ORIGIN_FIELDS", + ) + _require( + type(origin["source"]) is str and origin["source"] in {"acs", "asec"}, + "ORIGIN_SOURCE", + ) + _require( + all( + type(origin[field]) is int + for field in ( + "household_id", + "source_year", + "survey_year", + "selected_receiving_household_id", + ) + ), + "ORIGIN_INTEGER", + ) + _require( + all( + 0 <= origin[field] < 2**63 + for field in ("household_id", "selected_receiving_household_id") + ), + "ORIGIN_INTEGER", + ) + _require(type(origin["raw_native_id"]) is str, "ORIGIN_NATIVE_KEY") + key = HouseholdKey( + Source(origin["source"]), + origin["source_year"], + origin["survey_year"], + origin["raw_native_id"], + ) + row = selected.get(key) + _require(row is not None, "ORIGIN_NATIVE_KEY") + anchor = _fraction_pair(origin["original_anchor"]) + _require(anchor == row.original_design_weight, "ORIGIN_ANCHOR") + hh_id = origin["household_id"] + receiving = (key.source, origin["selected_receiving_household_id"]) + _require( + hh_id not in seen_ids + and key not in seen_native + and receiving not in seen_receiving, + "ORIGIN_DUPLICATE", + ) + seen_ids.add(hh_id) + seen_native.add(key) + seen_receiving.add(receiving) + instruction = AllocationInstruction( + hh_id, key, receiving[1], anchor, row.share, row.inclusion_probability + ) + _finite_weight(anchor) + _finite_weight(instruction.importance_weight) + output.append(instruction) + return tuple(output) + + +def _finite_weight(value): + try: + number = float(value) + except (OverflowError, ValueError): + raise SurveyPopulationGraphError("FLOAT_WEIGHT") from None + _require( + math.isfinite(number) and number >= 0 and (value == 0 or number > 0), + "FLOAT_WEIGHT", + ) + return number + + +def _bounded_json(value, limit): + """Canonical UTF-8 encoding with a bound checked before every append.""" + _require(type(limit) is int and 0 < limit <= 64 * 1024**2, "TRANSPORT_LIMIT") + result = bytearray() + encoder = json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ) + try: + for piece in encoder.iterencode(value): + # All variable source tokens admitted to allocation are <=128 chars; + # this pre-encoding bound also protects against unexpected strings. + _require(len(piece) <= limit - len(result), "TRANSPORT_LIMIT") + encoded = piece.encode("utf-8") + _require(len(encoded) <= limit - len(result), "TRANSPORT_LIMIT") + result.extend(encoded) + except SurveyPopulationGraphError: + raise + except (TypeError, ValueError, UnicodeError): + raise SurveyPopulationGraphError("TRANSPORT_ENCODING") from None + return bytes(result) + + +def _source_owner(): + # The owner remains independent of this graph adapter. Declaration-only + # imports do not load source capture owners or their runtime resources. + from . import survey_population_preparation + + return survey_population_preparation + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _checked_preparation(preparation): + owner = _source_owner() + _require( + type(preparation) is owner.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + view = owner.AuthenticatedSurveyPopulationPreparation.checked_view(preparation) + payload, context = view.payload, view.context + _require( + type(payload) is bytes and 0 < len(payload) <= PREPARATION_MAX_BYTES, + "PREPARATION_BYTES", + ) + _require( + type(context) is bytes and 0 < len(context) <= PREPARATION_MAX_BYTES, + "CONTEXT_BYTES", + ) + return owner, view + + +def _artifact(context, name, expected_type, expected_payload): + from microcosm.graph.keys import opaque_artifact_key + + value = context.artifacts.get(name) + edges = [edge for edge in context.node.artifact_inputs if edge.name == name] + _require( + value is not None + and value.type == expected_type + and len(edges) == 1 + and edges[0].producer == CREATE_NODE + and edges[0].artifact == name + and value.key == opaque_artifact_key(value.producer_key, name) + and type(value.payload) is bytes + and value.payload == expected_payload, + "ARTIFACT_BINDING", + ) + return value + + +def _instruction_document(row): + def pair(value): + return [value.numerator, value.denominator] + + return { + "household_id": row.household_id, + "source": row.key.source.value, + "source_year": row.key.source_year, + "survey_year": row.key.survey_year, + "raw_native_id": row.key.native_id, + "selected_receiving_household_id": row.selected_receiving_household_id, + "original_anchor": pair(row.original_anchor), + "share": pair(row.share), + "inclusion_probability": pair(row.inclusion_probability), + "importance_multiplier": pair(row.share / row.inclusion_probability), + "importance_weight_float64_hex": _finite_weight(row.importance_weight).hex(), + } + + +def _allocation_payload( + instructions, *, preparation_sha256, input_context, output_context +): + """Stream rows individually; never construct a selected-population JSON tree.""" + metadata = { + "protocol": "microcosm.us.survey-population-allocation.v1", + "preparation_sha256": preparation_sha256, + "input_context_sha256": _sha(input_context), + "output_context_sha256": _sha(output_context), + "weight_transition": ["design", "importance"], + "release_eligible": False, + } + tail = b"]," + _bounded_json(metadata, 4096)[1:] + output = bytearray(b'{"households":[') + for index, row in enumerate(instructions): + raw = _bounded_json(_instruction_document(row), 4096) + separator = b"," if index else b"" + _require( + len(output) + len(separator) + len(raw) + len(tail) <= ALLOCATION_MAX_BYTES, + "ALLOCATION_LIMIT", + ) + output.extend(separator) + output.extend(raw) + _require(len(output) + len(tail) <= ALLOCATION_MAX_BYTES, "ALLOCATION_LIMIT") + output.extend(tail) + return bytes(output) + + +def _verify_allocation_view(frame, instructions): + _require(frame.weighted_entities == ("household",), "WEIGHT_SOURCES") + weights = frame.weights_for("household") + _require(weights.kind is WeightKind.DESIGN, "DESIGN_REQUIRED") + household = frame.table("household") + _require(len(household) == len(instructions), "HOUSEHOLD_COUNT") + _require(household.household_id.dtype == np.dtype("int64"), "HOUSEHOLD_ID_DTYPE") + _require( + tuple(household.household_id) == tuple(r.household_id for r in instructions), + "HOUSEHOLD_ORDER", + ) + _require( + tuple(household[support_channel_column("household")]) + == tuple(r.key.source.value for r in instructions), + "HOUSEHOLD_CHANNEL", + ) + _require( + tuple(household[spine_source_id_column("household")]) + == tuple(r.selected_receiving_household_id for r in instructions), + "RECEIVING_ID", + ) + for entity in US_SCHEMA.entities: + table = frame.table(entity) + _require( + table[support_clone_index_column(entity)].dtype == np.dtype("int64") + and bool(table[support_clone_index_column(entity)].eq(0).all()), + "PRECLONE_REQUIRED", + ) + expected = np.asarray( + [_finite_weight(r.original_anchor) for r in instructions], dtype="float64" + ) + _require( + weights.values.dtype == expected.dtype + and weights.values.tobytes() == expected.tobytes(), + "ORIGINAL_ANCHORS", + ) + + +def _allocation_output(frame, context_bytes, instructions, preparation_sha256): + _verify_allocation_view(frame, instructions) + values = np.asarray( + [_finite_weight(r.importance_weight) for r in instructions], dtype="float64" + ) + weights = Weights(values, WeightKind.IMPORTANCE) + allocated = frame.with_weights( + "household", + weights, + mass=MassChange( + factor=None, reason=f"{PHASE}: declared source shares and inverse inclusion" + ), + ) + context = graph_context._decode(context_bytes) + _require( + context["weight_sources"] == {"household": "design"}, "CONTEXT_WEIGHT_KIND" + ) + context["weight_sources"] = {"household": "importance"} + context["mass_log"] = [ + graph_context._json_data(asdict(r)) for r in allocated.mass_log + ] + new_context = _bounded_json(context, PREPARATION_MAX_BYTES) + payload = _allocation_payload( + instructions, + preparation_sha256=preparation_sha256, + input_context=context_bytes, + output_context=new_context, + ) + before, after = frame.stratum_mass(), allocated.stratum_mass() + receipt = { + "phase": PHASE, + "preparation_sha256": preparation_sha256, + "allocation_sha256": _sha(payload), + "release_eligible": False, + "frame_mass_log_append": [ + graph_context._json_data(asdict(r)) + for r in allocated.mass_log[len(frame.mass_log) :] + ], + "mass": { + "policy": "declared", + "before": float(before.sum()), + "after": float(after.sum()), + "stratum_before": before.to_dict(), + "stratum_after": after.to_dict(), + }, + } + return weights, new_context, payload, receipt, allocated + + +class _Kernel(KernelBase): + def implementation_hash(self): + from .graph_implementation import ( + STAGE_DEPENDENCIES as STAGES, + ) + from .graph_implementation import ( + implementation_hash, + ) + + _require(STAGES[STAGE] == STAGE_DEPENDENCIES, "STAGE_DEPENDENCIES") + return implementation_hash(STAGE) + + +class SurveyPopulationCreateKernel(_Kernel): + ref = CREATE_REF + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.CREATE, + dependencies=STAGE_DEPENDENCIES, + ) + + def __init__(self, preparation, *, source_dir): + self._preparation = preparation + self._source_dir = Path(source_dir).absolute() + + def run(self, context): + from .graph_sources import frame_column_declarations + + preparation = self._preparation + owner, view = _checked_preparation(preparation) + payload, context_bytes, plan = view.payload, view.context, view.selection_plan + expected, _ = survey_population_nodes( + frame_column_declarations(view.frame), + preparation_sha256=_sha(payload), + fraction=plan.fraction, + seed=plan.seed, + ) + _require( + context.node == expected and dict(context.params) == dict(expected.params), + "CREATE_DECLARATION", + ) + _require( + set(context.sources) == {SOURCE_NAME} + and Path(context.sources[SOURCE_NAME]) + == self._source_dir.resolve(strict=True), + "SOURCE_PATH", + ) + owner.verify_survey_population_preparation(preparation) + return KernelResult( + frame=view.frame, + artifacts={"frame_context": context_bytes, "preparation": payload}, + receipt={ + "phase": PHASE, + "preparation_sha256": _sha(payload), + "source_reconstruction": "fresh_before_graph_execution", + "release_eligible": False, + }, + ) + + +class SurveyPopulationAllocationKernel(_Kernel): + ref = ALLOCATION_REF + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.REWEIGHT, + dependencies=STAGE_DEPENDENCIES, + ) + + def __init__(self, instructions, *, preparation_bytes, context_bytes): + _require( + type(instructions) is tuple + and instructions + and all(type(row) is AllocationInstruction for row in instructions), + "INSTRUCTIONS", + ) + _require( + type(preparation_bytes) is bytes + and 0 < len(preparation_bytes) <= PREPARATION_MAX_BYTES, + "PREPARATION_BYTES", + ) + _require( + type(context_bytes) is bytes + and 0 < len(context_bytes) <= PREPARATION_MAX_BYTES, + "CONTEXT_BYTES", + ) + self._instructions = instructions + self._preparation_bytes = preparation_bytes + self._context_bytes = context_bytes + + def run(self, context): + node = context.node + digest = _sha(self._preparation_bytes) + _require( + node.id == ALLOCATION_NODE + and node.kernel == self.ref + and node.structural is StructuralDelta.REWEIGHT + and node.base == CREATE_NODE + and node.mass == "declared" + and not node.outputs + and not node.sources + and node.weights + == WeightTransition("household", "importance", mass="declared") + and node.inputs + == tuple(Slice(e, _provenance_columns(e)) for e in US_SCHEMA.entities) + and node.artifact_inputs + == ( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "preparation", CREATE_NODE, "preparation", PREPARATION_TYPE + ), + ) + and node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("allocation", ALLOCATION_TYPE), + ) + and dict(context.params) == {"phase": PHASE, "preparation_sha256": digest}, + "ALLOCATION_DECLARATION", + ) + _require( + set(context.artifacts) == {"frame_context", "preparation"}, "ARTIFACTS" + ) + bound = _artifact( + context, "frame_context", US_FRAME_CONTEXT_TYPE, self._context_bytes + ) + prepared = _artifact( + context, "preparation", PREPARATION_TYPE, self._preparation_bytes + ) + _require(bound.producer_key == prepared.producer_key, "ARTIFACT_PRODUCER") + original = graph_context.us_frame_from_context(context) + weights, new_context, payload, receipt, _allocated = _allocation_output( + original, bound.payload, self._instructions, digest + ) + return KernelResult( + weights=weights, + artifacts={"frame_context": new_context, "allocation": payload}, + receipt=receipt, + ) + + +def survey_population_source_codecs(*, snapshot_root): + """One real source codec, isolated from the legacy five-codec registry.""" + from microcosm.graph.codecs import SourceCodecRegistry + + snapshot_root = Path(snapshot_root) + + def load(path, *, store=None): + del store + owner = _source_owner() + fraction, seed = owner.read_survey_population_request(path) + preparation = owner.prepare_authenticated_survey_population( + path, snapshot_root=snapshot_root, fraction=fraction, seed=seed + ) + owner, view = _checked_preparation(preparation) + owner.verify_survey_population_preparation(preparation) + return view.frame + + codecs = SourceCodecRegistry() + codecs.register(SOURCE_CODEC, load) + return codecs + + +def _same_frame(expected, actual): + """Full semantic identity; mutable-storage source seals stay in their owner.""" + _require( + expected.schema == actual.schema + and expected.entities == actual.entities + and expected.links == actual.links + and expected.metadata == actual.metadata + and expected.mass_log == actual.mass_log + and expected.weighted_entities == actual.weighted_entities, + "MATERIALIZED_FRAME_CONTEXT", + ) + try: + for entity in expected.entities: + pd.testing.assert_frame_equal( + expected.table(entity), + actual.table(entity), + check_exact=True, + check_flags=True, + ) + for link in expected.links: + pd.testing.assert_frame_equal( + expected.link(link), + actual.link(link), + check_exact=True, + check_flags=True, + ) + pd.testing.assert_series_equal(expected.strata, actual.strata, check_exact=True) + except (AssertionError, ValueError, TypeError): + raise SurveyPopulationGraphError("MATERIALIZED_FRAME_VALUES") from None + for entity in expected.weighted_entities: + left, right = expected.weights_for(entity), actual.weights_for(entity) + _require( + left.kind is right.kind + and left.values.dtype == right.values.dtype + and left.values.tobytes() == right.values.tobytes(), + "MATERIALIZED_FRAME_WEIGHTS", + ) + + +def _verify_cloned_frame(before, actual, design_weights): + from . import puf_support + + _require( + before.schema == actual.schema + and before.entities == actual.entities + and before.links == actual.links == () + and before.metadata == actual.metadata + and before.mass_log == actual.mass_log + and actual.weighted_entities == ("household",), + "CLONE_CONTEXT", + ) + multiplier = puf_support._id_multiplier_for_frame(before) + try: + # Execute the real operator's row transform one entity at a time. This + # checks every carried cell, missing value, dtype, native key and remapped + # membership without constructing another full cloned Frame. + for entity in before.entities: + expected = puf_support._clone_preassembled_entity_table( + before.table(entity), + entity=entity, + schema=before.schema, + id_multiplier=multiplier, + ) + pd.testing.assert_frame_equal( + expected, actual.table(entity), check_exact=True, check_flags=True + ) + expected_strata = pd.concat([before.strata, before.strata], ignore_index=True) + pd.testing.assert_series_equal(expected_strata, actual.strata, check_exact=True) + except (AssertionError, ValueError, TypeError): + raise SurveyPopulationGraphError("CLONE_FRAME_VALUES") from None + weights = actual.weights_for("household") + expected_weights = np.tile(before.weights_for("household").values / 2, 2) + _require( + weights.kind is WeightKind.IMPORTANCE + and weights.values.tobytes() == expected_weights.tobytes(), + "CLONE_WEIGHTS", + ) + # The graph duplicates original design anchors along lineage. It halves + # current IMPORTANCE weights, never the original design evidence. + expected_design = np.tile(design_weights, 2) + puf_support.validate_puf_clone_attachment( + actual, boundary=PHASE, expected_fraction=1.0, expected_seed=0 + ) + return expected_design + + +def _check_design_anchors(population, expected): + _require(set(population.design_weights) == {"household"}, "DESIGN_ANCHOR_SOURCES") + actual = population.design_weights["household"] + _require( + type(actual) is np.ndarray + and actual.dtype == expected.dtype + and actual.shape == expected.shape + and actual.tobytes() == expected.tobytes(), + "DESIGN_ANCHOR_LINEAGE", + ) + + +def _check_population_state(population, *, version, owners, kind, ledger): + from microcosm.graph.population import MassRecord + + _require( + type(population.version) is str and population.version == version, + "POPULATION_VERSION", + ) + _require( + dict(population.owners) == owners + and all(type(value) is str for value in population.owners.values()), + "POPULATION_OWNERS", + ) + _require( + dict(population.weight_kind) == {"household": kind} + and type(population.weight_kind["household"]) is WeightKind, + "POPULATION_WEIGHT_KIND", + ) + _require( + type(population.mass_ledger) is tuple + and all(type(record) is MassRecord for record in population.mass_ledger) + and population.mass_ledger == ledger, + "POPULATION_MASS_LEDGER", + ) + + +def _final_artifact(manifest, store, *, node_id, name, type_, payload, capabilities): + from microcosm.graph.artifact_edges import descriptor + from microcosm.graph.keys import opaque_artifact_key + + node = manifest.node(node_id) + expected = descriptor( + producer=node_id, + artifact=name, + type_=type_, + producer_key=node.key, + capabilities=capabilities, + ) + _require( + node.typed_artifacts["outputs"].get(name) == expected, + "FINAL_ARTIFACT_DESCRIPTOR", + ) + key = opaque_artifact_key(node.key, name) + _require(node.opaque_artifacts.get(name) == key, "FINAL_ARTIFACT_KEY") + actual = store.load_bytes(key) + _require(type(actual) is bytes and actual == payload, "FINAL_ARTIFACT_BYTES") + + +def _check_node_states(manifest, expected): + from microcosm.graph.keys import _capabilities_projection + + _require(set(manifest.nodes) == set(expected), "MANIFEST_NODE_SET") + for node_id, state in expected.items(): + node = manifest.node(node_id) + _require( + node.key == state["key"] + and node.kernel_ref == state["ref"] + and type(node.capabilities) is Capabilities + and _capabilities_projection(node.capabilities) == state["capabilities"] + and node.kernel_impl_hash == state["implementation"] + and node.typed_artifacts == state["typed_artifacts"] + and type(node.seed) is int + and node.seed == state["seed"] + and node.frame_key == state["frame_key"] + and node.weight_key == state["weight_key"] + and node.artifacts == state["artifacts"] + and node.opaque_artifacts == state["opaque_artifacts"] + and node.receipt == state["receipt"] + and node.legacy_capabilities is False, + "MANIFEST_NODE_STATE", + ) + + +@dataclass(frozen=True) +class SurveyPopulationRunValues: + """Values retained from one completed run, without additional authority. + + Populations are the runner's detached observations, checked against the + complete attached manifest Frames before return, not decoded substitutes. + Downstream source-qualified boundaries must check the issued preparation + and complete populations themselves. Freezing this container does not seal + its contents, authenticate a copy, or admit a later graph descendant. + """ + + manifest: object + preparation: object + allocated_population: object + clone_population: object | None + compiled: object + store: object + kernels: object + sources: dict + + +def run_authenticated_survey_population( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed, + resume="auto", + clones=True, + return_values=False, +): + """Reconstruct sources, then execute and verify the real selected graph. + + Warm execution can skip graph kernels; it still reconstructs both complete + catalogues and the selected native populations. Returned manifests are + development build records. Decoding their bytes grants no source authority. + ``return_values=True`` returns the live composition values with the manifest; + it does not bypass any final checks or issue a downstream admission. + """ + from microcosm.graph import ( + ContentStore, + Graph, + KernelRegistry, + SourceRef, + compile_graph, + run_graph, + ) + from microcosm.graph.artifact_edges import typed_contracts + from microcosm.graph.executor import ( + _all_node_keys, + _expand_declared_payload, + _expand_rewrite_coordinates, + _input_writers, + _source_paths_and_keys, + _tolerance_writer_payload, + ) + from microcosm.graph.keys import ( + _capabilities_projection, + artifact_key, + frame_key, + opaque_artifact_key, + weights_key, + ) + from microcosm.graph.keys import seed as node_seed + from microcosm.graph.manifest import _freeze_json + from microcosm.graph.population import ( + _mass_record, + expand_lineage_receipt, + expand_writes_receipt, + mass_record_receipt, + weight_cap_receipt, + ) + + from .graph_sources import frame_column_declarations + + _draw_parameters(fraction, seed) + _require(type(clones) is bool, "CLONES_FLAG") + _require(type(return_values) is bool, "RETURN_VALUES_FLAG") + # Do not resolve source symlinks before their authenticated owner checks them. + source_dir, snapshot_root, store_root = ( + Path(path).absolute() for path in (source_dir, snapshot_root, store_root) + ) + owner = _source_owner() + preparation = owner.prepare_authenticated_survey_population( + source_dir, snapshot_root=snapshot_root, fraction=fraction, seed=seed + ) + owner, view = _checked_preparation(preparation) + # Resolve '..' only after the source owner has checked the raw path and + # refused symlinks. The executor uses this same canonical path identity. + source_dir = source_dir.resolve(strict=True) + payload, context_bytes, prepared_frame = view.payload, view.context, view.frame + instructions = allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + columns = frame_column_declarations(prepared_frame) + nodes = survey_population_nodes( + columns, preparation_sha256=_sha(payload), fraction=fraction, seed=seed + ) + _weights, allocated_context, allocation_payload, allocation_receipt, allocated = ( + _allocation_output(prepared_frame, context_bytes, instructions, _sha(payload)) + ) + design = np.array(prepared_frame.weights_for("household").values, copy=True) + kernels = KernelRegistry() + kernels.register(SurveyPopulationCreateKernel(preparation, source_dir=source_dir)) + kernels.register( + SurveyPopulationAllocationKernel( + instructions, preparation_bytes=payload, context_bytes=context_bytes + ) + ) + clone_nodes = () + if clones: + from . import graph_combined_clone as clone + from . import puf_support + from .graph_combined_clone import ( + register_us_combined_survey_clone_kernels, + us_combined_survey_clone_nodes, + ) + + clone_nodes = us_combined_survey_clone_nodes( + columns, base=ALLOCATION_NODE, source_channels=("acs", "asec") + ) + register_us_combined_survey_clone_kernels(kernels) + compiled = compile_graph( + Graph("us", (SourceRef(SOURCE_NAME, SOURCE_CODEC),), (*nodes, *clone_nodes)) + ) + store = ContentStore( + store_root, codecs=survey_population_source_codecs(snapshot_root=snapshot_root) + ) + observed = {} + expected_states = {} + expected_writer_receipts = {} + allocation_ledger = ( + _mass_record( + prepared_frame, + allocated, + nodes[1], + KernelResult(receipt=allocation_receipt), + "declared", + ), + ) + all_cells = tuple( + (entity, str(column)) + for entity in prepared_frame.entities + for column in prepared_frame.table(entity) + ) + + def observe(node_id, population): + owner.verify_survey_population_preparation(preparation) + if node_id == CREATE_NODE: + _same_frame(prepared_frame, population.frame) + expected_design = design + owners = dict.fromkeys(all_cells, CREATE_NODE) + kind, ledger = WeightKind.DESIGN, () + elif node_id == ALLOCATION_NODE: + _same_frame(allocated, population.frame) + expected_design = design + owners = dict.fromkeys(all_cells, ALLOCATION_NODE) + kind, ledger = WeightKind.IMPORTANCE, allocation_ledger + elif node_id in {node.id for node in clone_nodes}: + expected_design = _verify_cloned_frame(allocated, population.frame, design) + owners = dict.fromkeys(all_cells, clone_nodes[0].id) + if node_id == clone_nodes[1].id: + owners.update( + {(o.entity, o.column): node_id for o in clone_nodes[1].outputs} + ) + # Full cloned cells have been checked against the real operator row + # transform. Derive mass from these values, never the cached ledger. + kind = WeightKind.IMPORTANCE + ledger = ( + *allocation_ledger, + _mass_record( + allocated, + population.frame, + clone_nodes[0], + KernelResult(), + "conserve", + ), + ) + else: + raise SurveyPopulationGraphError("UNEXPECTED_POPULATION") + _check_design_anchors(population, expected_design) + state = dict( + version=compiled.versions[node_id], owners=owners, kind=kind, ledger=ledger + ) + _check_population_state(population, **state) + node = compiled.graph.node(node_id) + if node_id == CREATE_NODE: + receipt = { + "phase": PHASE, + "preparation_sha256": _sha(payload), + "source_reconstruction": "fresh_before_graph_execution", + "release_eligible": False, + } + elif node_id == ALLOCATION_NODE: + receipt = dict(allocation_receipt) + elif node_id == clone_nodes[0].id: + lineage, entity_facts = {}, {} + for entity in US_SCHEMA.entities: + lineage[entity], entity_facts[entity] = clone._entity_lineage( + allocated, population.frame, entity + ) + authority = puf_support.validate_puf_clone_attachment( + population.frame, boundary=PHASE, expected_fraction=1.0, expected_seed=0 + ) + receipt = clone.USCombinedSurveyCloneExpandKernel._receipt( + allocated, population.frame, ("acs", "asec"), authority, entity_facts + ) + receipt["expand"] = expand_lineage_receipt(lineage) + receipt["expand_declared"] = _expand_declared_payload(node) + receipt["expand_writes"] = expand_writes_receipt( + allocated, + population.frame, + node, + receipt, + rewrite_coordinates=_expand_rewrite_coordinates(compiled, node), + ) + else: + receipt = { + "phase": clone.COMBINED_CLONE_PHASE, + "claimed_cells": sorted( + f"{output.entity}.{output.column}" for output in node.outputs + ), + } + receipt["capabilities"] = dict(expected_nodes[node_id]["capabilities"]) + writers = _tolerance_writer_payload( + _input_writers(compiled, node_id, receipts=expected_writer_receipts) + ) + if writers: + receipt["capabilities"]["tolerance_writers"] = writers + if node.structural not in {StructuralDelta.NONE, StructuralDelta.CREATE}: + receipt["mass"] = { + **receipt.get("mass", {}), + **mass_record_receipt(ledger[-1]), + } + receipt.update(weight_cap_receipt(population, node)) + expected_nodes[node_id]["receipt"] = _freeze_json(receipt) + # Value-only input for the generic writer analysis. This is neither a + # returned NodeReceipt nor a source/Frame authority object. + expected_writer_receipts[node_id] = SimpleNamespace( + receipt=expected_nodes[node_id]["receipt"] + ) + _require(node_id not in observed, "DUPLICATE_POPULATION_OBSERVATION") + observed[node_id] = population + expected_states[node_id] = state + + # Derive expectations independently of returned/cache receipts. This is one + # additional streaming source-key pass, not another draw or native build. + _paths, source_keys = _source_paths_and_keys( + compiled, {SOURCE_NAME: source_dir}, store + ) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + expected_nodes = {} + for node in compiled.graph.nodes: + key = keys[node.id] + structural = node.structural is not StructuralDelta.NONE + cells = ( + all_cells + if structural + else tuple((output.entity, output.column) for output in node.outputs) + ) + weight_entity = ( + node.weights.entity + if node.weights is not None + else node.params.get("expand_weight_entity") + if node.structural is StructuralDelta.EXPAND + else None + ) + expected_nodes[node.id] = { + "key": key, + "ref": node.kernel, + "capabilities": _capabilities_projection( + kernels.get(node.kernel).capabilities + ), + "implementation": implementations[node.id], + "typed_artifacts": typed_contracts(compiled, node, keys, kernels), + "seed": node_seed(key), + "frame_key": frame_key(key) if structural else None, + "weight_key": weights_key(key, weight_entity) if weight_entity else None, + "artifacts": {(e, c): artifact_key(key, e, c) for e, c in cells}, + "opaque_artifacts": { + output.name: opaque_artifact_key(key, output.name) + for output in node.artifact_outputs + }, + } + manifest = run_graph( + compiled, + sources={SOURCE_NAME: source_dir}, + store=store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + _require(tuple(observed) == compiled.order, "POPULATION_OBSERVER_COVERAGE") + _check_node_states(manifest, expected_nodes) + # Explicit store reads are necessary: the observer sees real populations, + # while typed byte artifacts pass through a different materialization path. + for node_id, name, type_, raw, capabilities in ( + ( + CREATE_NODE, + "preparation", + PREPARATION_TYPE, + payload, + SurveyPopulationCreateKernel.capabilities, + ), + ( + CREATE_NODE, + "frame_context", + US_FRAME_CONTEXT_TYPE, + context_bytes, + SurveyPopulationCreateKernel.capabilities, + ), + ( + ALLOCATION_NODE, + "allocation", + ALLOCATION_TYPE, + allocation_payload, + SurveyPopulationAllocationKernel.capabilities, + ), + ( + ALLOCATION_NODE, + "frame_context", + US_FRAME_CONTEXT_TYPE, + allocated_context, + SurveyPopulationAllocationKernel.capabilities, + ), + ): + _final_artifact( + manifest, + store, + node_id=node_id, + name=name, + type_=type_, + payload=raw, + capabilities=capabilities, + ) + # Construct the optional value container before terminal checks. A caller + # still needs the downstream issuer's independent reconstruction; this is + # deliberately not another source certificate or a manifest decoder. + result = ( + SurveyPopulationRunValues( + manifest=manifest, + preparation=preparation, + allocated_population=observed[ALLOCATION_NODE], + clone_population=observed[clone_nodes[-1].id] if clone_nodes else None, + compiled=compiled, + store=store, + kernels=kernels, + sources={SOURCE_NAME: source_dir}, + ) + if return_values + else manifest + ) + # Finish all potentially expensive owner I/O before the final pure checks + # of graph-owned Frames. A callback during the last producer/source check + # must not mutate an already-observed derivative and escape detection. + _owner, final_view = _checked_preparation(preparation) + _require( + final_view.payload == payload + and final_view.context == context_bytes + and final_view.frame is prepared_frame, + "FINAL_PREPARATION_SEAL", + ) + _same_frame(prepared_frame, manifest.population(CREATE_NODE)) + _same_frame(allocated, manifest.population(ALLOCATION_NODE)) + for node_id in (CREATE_NODE, ALLOCATION_NODE): + _check_design_anchors(observed[node_id], design) + for node in clone_nodes: + cloned_design = _verify_cloned_frame( + allocated, manifest.population(compiled.versions[node.id]), design + ) + _check_design_anchors(observed[node.id], cloned_design) + for node_id, state in expected_states.items(): + # Observations are detached from executable/store populations. They can + # be retained for composition, so seal their values after the last I/O + # as well as the separately checked manifest Frames. + _same_frame( + manifest.population(compiled.versions[node_id]), observed[node_id].frame + ) + _check_population_state(observed[node_id], **state) + _require( + manifest.mass_ledger(compiled.versions[node_id]) == state["ledger"], + "FINAL_MASS_LEDGER", + ) + _check_node_states(manifest, expected_nodes) + return result + + +def _provenance_columns(entity): + return ( + support_channel_column(entity), + support_source_id_column(entity), + spine_source_id_column(entity), + support_clone_index_column(entity), + ) + + +def survey_population_nodes( + columns: Sequence[Owned], *, preparation_sha256: str, fraction: Fraction, seed: int +) -> tuple[Node, Node]: + """Declare selected CREATE then allocation, without an invented FILTER. + + The digest names checked preparation bytes; accepting it here authenticates + nothing. The country runner must reconstruct the preparation and verify the + materialized populations and artifacts on both cold and cached execution. + """ + _require(type(columns) in (tuple, list), "COLUMNS") + columns = tuple(columns) + _require(columns and all(type(column) is Owned for column in columns), "COLUMNS") + _require( + all( + column.entity in US_SCHEMA.entities + and column.rows == "all" + and column.ownership is Ownership.PRODUCED + and column.rewrite is False + for column in columns + ), + "CREATE_COLUMN_CONTRACT", + ) + inventory = {(column.entity, column.column): column for column in columns} + _require(len(inventory) == len(columns), "DUPLICATE_COLUMNS") + for entity in US_SCHEMA.entities: + for name in _provenance_columns(entity): + column = inventory.get((entity, name)) + _require(column is not None, "MISSING_PROVENANCE_COLUMN") + _require( + column.dtype + == ("string" if name == support_channel_column(entity) else "int64"), + "PROVENANCE_DTYPE", + ) + fraction_pair, seed = _draw_parameters(fraction, seed) + digest = _digest(preparation_sha256) + create = Node( + id=CREATE_NODE, + kernel=CREATE_REF, + structural=StructuralDelta.CREATE, + sources=(SOURCE_NAME,), + outputs=columns, + params={ + "phase": PHASE, + "preparation_sha256": digest, + "fraction": fraction_pair, + "sample_seed": seed, + }, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("preparation", PREPARATION_TYPE), + ), + ) + allocate = Node( + id=ALLOCATION_NODE, + kernel=ALLOCATION_REF, + base=CREATE_NODE, + structural=StructuralDelta.REWEIGHT, + inputs=tuple( + Slice(entity, _provenance_columns(entity)) for entity in US_SCHEMA.entities + ), + weights=WeightTransition("household", "importance", mass="declared"), + mass="declared", + params={"phase": PHASE, "preparation_sha256": digest}, + artifact_inputs=( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput("preparation", CREATE_NODE, "preparation", PREPARATION_TYPE), + ), + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("allocation", ALLOCATION_TYPE), + ), + ) + return create, allocate diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py new file mode 100644 index 000000000..bd02a95db --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py @@ -0,0 +1,648 @@ +"""Extend an issued postclone financial run with two PUF55 routes. + +The existing financial run is an explicit prerequisite. This host compiles and +executes the extension once; it does not claim that the upstream run's entire +lifetime used one execution. No donor or FILTER issuer is run in preparation. +Required replay is a separate explicit call over the same source paths/store. +""" + +from __future__ import annotations + +import weakref +from dataclasses import dataclass, replace +from pathlib import Path + +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ContentStore, + KernelRegistry, + artifact_edges, + codecs, + compile_graph, + run_graph, +) +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys +from microcosm.graph.serialize import graph_to_json + +from . import graph_puf55_route_attachment as attach + +require = attach.require +financial, recipient, canonical = attach.financial, attach.recipient, attach.canonical +physical, codec, population_ops = attach.physical, attach.codec, attach.population_ops +RUN_PROTOCOL = "microcosm.us.survey-puf55-checked-run.v1" +_ISSUED_RUNS = {} + + +@dataclass(frozen=True) +class SurveyPuf55Run: + """Actual result handle; constructor/copies do not issue run authority.""" + + financial_run: financial.AtomicSurveyFinancialRunValues + population: population_ops.Population + manifest: object + compiled: object + store: ContentStore + kernels: KernelRegistry + sources: tuple[tuple[str, Path], ...] + receipt: bytes + + def checked_view(self): + """Recheck this retained handle without fitting or graph execution.""" + return check_survey_puf55_run(self) + + +@dataclass(frozen=True) +class CheckedSurveyPuf55Run: + """Descriptive values; only the original run retains in-process authority. + + A downstream owner must check that run immediately before consumption and + after its last relevant I/O, before returning or exporting a successor. + Reconstructing this view from JSON or copying it cannot authorize a run. + Neither this view nor the run grants calibration or release admission. + """ + + payload: bytes + digest: str + population: population_ops.Population + + +@dataclass(frozen=True) +class _RunState: + boundary: attach.Boundary + population: population_ops.Population + population_stamp: str + expected: population_ops.Population + expected_stamp: str + manifest: object + manifest_bytes: str + manifest_populations: tuple + receipt: bytes + artifact_hashes: tuple + + +def _run_entry(run): + entry = _ISSUED_RUNS.get(id(run)) + require( + type(run) is SurveyPuf55Run and entry is not None and entry[0]() is run, + "UNISSUED_PUF55_RUN", + ) + return entry + + +def _forget_run(run, entry): + if _ISSUED_RUNS.get(id(run)) is entry: + _ISSUED_RUNS.pop(id(run)) + + +def _run_document(state): + """Describe checked ancestry without granting authority to portable bytes.""" + boundary = state.boundary + return codec.encode_json( + { + "protocol": RUN_PROTOCOL, + "graph_sha256": codec.sha(boundary.declaration.encode()), + "manifest_key": state.manifest.key, + "financial_run_sha256": codec.sha(boundary.entry[1]), + "receipt_sha256": codec.sha(state.receipt), + "node_count": len(boundary.compiled.order), + "node_keys": dict(boundary.keys), + "source_keys": dict(boundary.source_keys), + "artifact_payload_sha256": [list(row) for row in state.artifact_hashes], + "population_version": state.population.version, + "population_physical_sha256": state.population_stamp, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + + +def _pure_run(run, entry): + """Check the retained output after all external reads have completed.""" + require(_run_entry(run) is entry, "FINAL_PUF55_RUN_ISSUANCE") + state, boundary = entry[2], entry[2].boundary + require( + run.financial_run is boundary.run + and run.population is state.population + and run.manifest is state.manifest + and run.compiled is boundary.compiled + and run.store is boundary.store + and run.kernels is boundary.kernels + and run.sources is boundary.paths + and run.receipt == state.receipt, + "PUF55_RUN_BINDINGS_CHANGED", + ) + require( + physical._population_stamp(run.population) == state.population_stamp + and physical._population_stamp(state.expected) == state.expected_stamp, + "PUF55_RUN_POPULATION_CHANGED", + ) + require( + set(run.manifest.populations) + == set(run.manifest.mass_ledgers) + == set(run.compiled.versions.values()), + "PUF55_RUN_MANIFEST_ROSTER_CHANGED", + ) + require( + run.manifest.to_json() == state.manifest_bytes + and _heterogeneous_manifest_seals(run.manifest, run.compiled) + == state.manifest_populations, + "PUF55_RUN_MANIFEST_CHANGED", + ) + boundary.pure() + require( + graph_to_json(run.compiled.graph) == boundary.declaration + and run.compiled == compile_graph(run.compiled.graph), + "PUF55_RUN_DECLARATION_CHANGED", + ) + physical.replay.same_replayed_population(state.expected, run.population) + require(_run_document(state) == entry[1], "PUF55_RUN_DOCUMENT_CHANGED") + require(_run_entry(run) is entry, "FINAL_PUF55_RUN_ISSUANCE") + + +def check_survey_puf55_run(run): + """Requalify actual source/store ancestry, then seal the complete output. + + Does not fit, execute a graph, decode model pickles, or reconstruct the PUF + donor. The actual execution already verified those exact artifact bytes. + A failed check revokes this handle; restoring its fields cannot reissue it. + Downstream owners must recheck before consumption and after their last + relevant I/O before returning/exporting a successor. A successful check is + a point-in-time claim, not a lease permitting future unchecked mutation. + """ + entry = _run_entry(run) + try: + _pure_run(run, entry) + state, boundary = entry[2], entry[2].boundary + boundary.borrow() + loaded = financial._artifacts( + run.manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + boundary.borrow() + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items())) + == state.artifact_hashes, + "PUF55_RUN_ARTIFACT_CHANGED", + ) + result = CheckedSurveyPuf55Run(entry[1], codec.sha(entry[1]), run.population) + _pure_run(run, entry) + return result + except BaseException: + _forget_run(run, entry) + raise + + +def _issue_run(output, state): + """Retain only a fully verified execution, with no new baseline reads.""" + identifier = id(output) + require(identifier not in _ISSUED_RUNS, "PUF55_RUN_REISSUANCE") + + def forget(reference): + entry = _ISSUED_RUNS.get(identifier) + if entry is not None and entry[0] is reference: + _ISSUED_RUNS.pop(identifier) + + entry = (weakref.ref(output, forget), _run_document(state), state) + _ISSUED_RUNS[identifier] = entry + try: + _pure_run(output, entry) + except BaseException: + _forget_run(output, entry) + raise + return entry + + +def _registry(existing, donor): + require(type(existing) is codecs.SourceCodecRegistry, "UPSTREAM_CODEC_REGISTRY") + registry = codecs.SourceCodecRegistry() + for old in (existing, donor): + for name, loader in old.as_mapping().items(): + require( + name not in (*registry.names(), *registry.bytes_names()), + "CODEC_COLLISION", + ) + registry.register(name, loader) + for name, loader in old.as_bytes_mapping().items(): + require( + name not in (*registry.names(), *registry.bytes_names()), + "CODEC_COLLISION", + ) + registry.register_bytes(name, loader) + return registry + + +def _construct( + financial_run, donor_sources, *, fixture_definition, seed, n_estimators, zero_atol +): + """Construct actual declarations and bind actual source keys without running.""" + financial.check_atomic_survey_financial_run(financial_run) + upstream_count = len(financial_run.compiled.order) + require(type(seed) is int and 0 <= seed < 2**64, "SEED") + require(type(n_estimators) is int and n_estimators > 0, "TREE_COUNT") + qualified = recipient.values.qualify_puf55_survey_recipients(financial_run) + donor = canonical.CanonicalPuf55DonorKernel( + seed=seed, fixture_definition=fixture_definition + ) + require( + type(donor_sources) is dict + and set(donor_sources) == {s.name for s in donor.source_refs}, + "DONOR_SOURCE_ROSTER", + ) + donor_paths = { + name: Path(path).resolve(strict=True) for name, path in donor_sources.items() + } + require(not set(donor_paths) & set(financial_run.sources), "SOURCE_NAME_COLLISION") + boundary = attach.Boundary( + financial_run, + qualified, + donor, + donor_paths, + seed=seed, + n_estimators=n_estimators, + zero_atol=zero_atol, + ) + recipient_nodes = recipient.puf55_survey_recipient_nodes(qualified) + nodes = ( + *financial_run.compiled.graph.nodes, + *recipient_nodes, + boundary.donor_node, + boundary.nodes[0], + *(node for route in boundary.routes for node in (*route.fits, *route.applies)), + *boundary.nodes[1:], + ) + graph = replace( + financial_run.compiled.graph, + sources=(*financial_run.compiled.graph.sources, *donor.source_refs), + nodes=nodes, + ) + compiled = compile_graph(graph) + require( + len(compiled.order) == upstream_count + 6 + 110 * len(boundary.routes), + "NODE_COUNT", + ) + order = compiled.order + require( + order.index(financial.financial_output_node(financial_run)) + < order.index(recipient.PROJECTION_NODE) + < order.index(recipient.MATRIX_NODE) + < order.index(attach.FILTER_NODE) + < order.index(attach.MASK_NODE) + < order.index(attach.ATTACH_NODE), + "EXTENSION_ORDER", + ) + # Private registries preserve the original issued run's retained containers. + kernels = KernelRegistry() + for kernel in financial_run.kernels.as_mapping().values(): + kernels.register(kernel) + for kernel in ( + recipient.Puf55SurveyRecipientProjectionKernel(financial_run), + recipient.Puf55SurveyRecipientMatrixKernel(financial_run), + donor, + attach.SurveyPuf55KeepAllKernel(boundary), + attach.SurveyPuf55MaskKernel(boundary), + attach.SurveyPuf55AttachKernel(boundary), + ): + require(kernel.ref not in kernels.refs(), "KERNEL_COLLISION") + kernels.register(kernel) + for kernel in (LegacyQRFTrainKernel(), LegacyQRFApplyMatrixKernel()): + if kernel.ref not in kernels.refs(): + kernels.register(kernel) + else: + require(type(kernels.get(kernel.ref)) is type(kernel), "LEGACY_KERNEL_TYPE") + store = ContentStore( + financial_run.store.root, + codecs=_registry(financial_run.store.codecs, donor.source_codecs), + ) + paths, sources = _source_paths_and_keys( + compiled, {**financial_run.sources, **donor_paths}, store + ) + keys, implementations = _all_node_keys(compiled, kernels, sources) + # The preserved prefix must have the actual issued keys, not merely the + # same user-facing IDs. New donor sources are unused by the original nodes. + state = financial._run_entry(financial_run)[2] + require( + all(keys[n] == k for n, k in state.keys) + and all(implementations[n] == k for n, k in state.implementations), + "UPSTREAM_PRODUCER_KEYS", + ) + boundary.bind(compiled, store, kernels, paths, sources, keys, implementations) + boundary.borrow() + return boundary + + +def _heterogeneous_manifest_seals(manifest, compiled): + """Seal every attached version of this mixed-schema run, donor included. + + The survey helper stamps through ``source._frame_identity``, which requires + the six US_SCHEMA entity groups. This host attaches a tax_unit-only PUF + donor CREATE, so that helper cannot seal this manifest at all. The physical + seal is schema-aware and already seals exactly this donor frame elsewhere + (``canonical._frame_seal``). It records link *names* but never link bodies, + so a frame carrying link tables is refused here rather than sealed blind. + Reconstructed versions carry default owners and anchors derived from their + stored weights. This manifest-content seal does not recover independently + retained original owner/anchor/history evidence. + """ + seals = [] + for version in sorted(set(compiled.versions.values())): + frame = manifest.population(version) + require(not frame.links, "MANIFEST_SEAL_LINK_TABLES") + seals.append( + ( + version, + physical._population_stamp( + population_ops.Population.from_frame( + frame, version, mass_ledger=manifest.mass_ledger(version) + ) + ), + ) + ) + return tuple(seals) + + +def _check_replayed_survey_manifest(expected_manifest, actual_manifest, compiled): + """Compare an exact upstream survey roster across store representations. + + The caller keeps the original issued manifest and its physical lifetime + seals. These Frame/ledger views reconstruct the same default owners and + design anchors as the former manifest stamp; they do not recover true + execution ownership. Actual observed Populations are checked separately. + Manifest access may materialize lazy attachments, so the host brackets this + comparison with its retained-owner checks and keeps its final physical seal. + """ + versions = tuple(sorted(set(compiled.versions.values()))) + wanted = set(versions) + require( + expected_manifest.country + == actual_manifest.country + == compiled.graph.country + == "us", + "UPSTREAM_SURVEY_COUNTRY", + ) + require( + bool(versions) + and set(expected_manifest.nodes) == set(compiled.order) + and set(compiled.order) <= set(actual_manifest.nodes) + and set(expected_manifest.populations) == wanted + and set(expected_manifest.mass_ledgers) == wanted + and wanted <= set(actual_manifest.populations) + and wanted <= set(actual_manifest.mass_ledgers), + "UPSTREAM_SURVEY_ROSTER", + ) + for version in versions: + expected = expected_manifest.population(version) + actual = actual_manifest.population(version) + require(expected.schema == actual.schema == US_SCHEMA, "UPSTREAM_SURVEY_SCHEMA") + physical.replay.same_replayed_population( + population_ops.Population.from_frame( + expected, version, mass_ledger=expected_manifest.mass_ledger(version) + ), + population_ops.Population.from_frame( + actual, version, mass_ledger=actual_manifest.mass_ledger(version) + ), + ) + + +def _loaded_values(boundary, manifest, loaded, node): + """Only construct typed inputs from the already-checked real manifest.""" + record = manifest.node(node.id) + return { + edge.name: artifact_edges.value_from_descriptor( + loaded[edge.producer, edge.artifact], + record.typed_artifacts["inputs"][edge.name], + ) + for edge in node.artifact_inputs + } + + +def _reconstruct(boundary, observed, values, result): + """Use original retained upstream and independently source-derived donor.""" + run = boundary.run + # The original issuer retains final views per structural version. Its + # existing nineteen-node keys/types are checked above; the complete retained + # final views below bind the unchanged prefix, without inventing a second + # upstream observer history or re-executing a preparation graph. + physical.replay.same_replayed_population( + run.financial_population, observed[recipient.PROJECTION_NODE] + ) + physical.replay.same_replayed_population( + run.financial_population, observed[recipient.MATRIX_NODE] + ) + physical.replay.same_replayed_population( + boundary.expected, observed[attach.FILTER_NODE] + ) + donor_frame, donors, donor_seal = boundary.canonical_donor(values) + require( + attach._donor_seal(donor_frame, donors) == donor_seal, "DONOR_RESULT_CHANGED" + ) + donor_node = boundary.donor_node + # CREATE assigns every loaded column, including linkage, to this version. + donor_population = population_ops.Population.from_frame(donor_frame, donor_node.id) + physical.replay.same_replayed_population(donor_population, observed[donor_node.id]) + for route in boundary.routes: + for node in route.fits: + physical.replay.same_replayed_population( + donor_population, observed[node.id] + ) + for node in route.applies: + # Ordinary fit/apply nodes own no Population cells. Compiler order + # can put the independent mask before any apply, so compare with + # the preceding state of this version, not an assumed fixed order. + require(not node.outputs and node.weights is None, "CHAIN_POPULATION_WRITE") + current = {} + for node_id in boundary.compiled.order: + node = boundary.compiled.graph.node(node_id) + version = boundary.compiled.versions[node_id] + if node_id in run.compiled.order: + current[version] = observed[node_id] + continue + if node_id == donor_node.id: + expected = donor_population + elif node_id == attach.FILTER_NODE: + expected = boundary.expected + elif node_id == attach.MASK_NODE: + expected = population_ops.patch( + current[version], node, attach._mask_result(boundary.expected.frame) + ) + elif node_id == attach.ATTACH_NODE: + expected = population_ops.patch(current[version], node, result) + else: + expected = current[version] + physical.replay.same_replayed_population(expected, observed[node_id]) + current[version] = expected + return current[attach.FILTER_NODE] + + +def run_survey_puf55( + financial_run, + *, + donor_sources, + seed=578, + n_estimators=100, + zero_atol=1e-8, + fixture_definition=None, + resume="auto", +): + """Run one extension over an actual issued run; no pre-run or donor injection. + + ``fixture_definition`` is the canonical owner's explicit invented-source + route. It cannot claim packaged-source admission. A separate call with + ``resume='require'`` must reproduce every source/producer and all values. + """ + require(resume in ("auto", "require"), "RESUME") + boundary = _construct( + financial_run, + donor_sources, + fixture_definition=fixture_definition, + seed=seed, + n_estimators=n_estimators, + zero_atol=zero_atol, + ) + observed, stamps = {}, {} + + def observe(node_id, population): + require(node_id not in observed, "OBSERVER_DUPLICATE") + observed[node_id] = population + stamps[node_id] = physical._population_stamp(population) + + manifest = run_graph( + boundary.compiled, + sources=dict(boundary.paths), + store=boundary.store, + kernels=boundary.kernels, + resume=resume, + _population_observer=observe, + ) + require(tuple(observed) == boundary.compiled.order, "OBSERVER_ROSTER") + if resume == "require": + require( + all(manifest.node(n).hit for n in boundary.compiled.order), "REQUIRED_HITS" + ) + # Complete source-key/implementation/type ancestry before loading any + # trusted model or independently invoking the replay numerical verifier. + boundary.borrow() + loaded = financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + # The mixed-manifest physical baseline includes donor and survey versions + # and is captured once. Cross-store comparison uses the still-issued + # upstream manifest as expected, retaining its original lifetime seals. + full_seals = _heterogeneous_manifest_seals(manifest, boundary.compiled) + boundary.pure() + _check_replayed_survey_manifest( + financial_run.manifest, manifest, financial_run.compiled + ) + boundary.pure() + values = _loaded_values(boundary, manifest, loaded, boundary.nodes[2]) + boundary.projections(values) + if boundary.computed is None: + result = boundary.finalize(values) + else: + boundary.pure() + result = boundary.computed[0] + require( + loaded[attach.ATTACH_NODE, "finalization"] == result.artifacts["finalization"], + "FINALIZATION_ARTIFACT", + ) + expected = _reconstruct(boundary, observed, values, result) + population = observed[attach.ATTACH_NODE] + physical.replay.same_replayed_population(expected, population) + receipt = codec.encode_json( + { + "protocol": attach.PROTOCOL, + "node_count": len(boundary.compiled.order), + "profiles": [r.profile.value for r in boundary.routes], + "financial_run_sha256": codec.sha(boundary.entry[1]), + "manifest_key": manifest.key, + "one_extension_execution": True, + "upstream_already_issued": True, + "complete_population_compared": True, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + output = SurveyPuf55Run( + financial_run, + population, + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + boundary.paths, + receipt, + ) + manifest_bytes = manifest.to_json() + expected_stamp = physical._population_stamp(expected) + artifact_hashes = tuple( + sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items()) + ) + # Last reads include actual source owners/code, manifest store identities, + # and every returned Population. Nothing below the fence performs I/O. + boundary.borrow() + fresh = financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + boundary.borrow() + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in fresh.items())) + == artifact_hashes, + "LATE_ARTIFACT_CHANGED", + ) + boundary.pure() + require( + output.financial_run is financial_run + and output.population is population + and output.manifest is manifest + and output.compiled is boundary.compiled + and output.store is boundary.store + and output.kernels is boundary.kernels + and output.sources == boundary.paths + and output.receipt == receipt + and manifest.to_json() == manifest_bytes + and _heterogeneous_manifest_seals(manifest, boundary.compiled) == full_seals + and physical._population_stamp(expected) == expected_stamp + and all(physical._population_stamp(observed[n]) == stamps[n] for n in observed), + "FINAL_OUTPUT_CHANGED", + ) + physical.replay.same_replayed_population(expected, population) + state = _RunState( + boundary, + population, + stamps[attach.ATTACH_NODE], + expected, + expected_stamp, + manifest, + manifest_bytes, + full_seals, + receipt, + artifact_hashes, + ) + # Baselines were captured before the preceding last-I/O fence. The entry + # retains the final output, not all 245 executor-observed snapshots. + try: + entry = _issue_run(output, state) + _pure_run(output, entry) + except BaseException: + candidate = _ISSUED_RUNS.get(id(output)) + if candidate is not None and candidate[0]() is output: + _forget_run(output, candidate) + raise + return output diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_us_survey_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_us_survey_enrichment.py new file mode 100644 index 000000000..c9c3ae590 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_us_survey_enrichment.py @@ -0,0 +1,1043 @@ +"""Fixed US post-PUF host for source-qualified survey enrichment fragments. + +The owning boundary retains a real checked PUF run and authenticated source +projection. Models use separate original-design donor branches; attachment +adds the declared amount and coverage families to the existing receiving frame. +""" + +from __future__ import annotations + +import sys +import weakref +from dataclasses import dataclass, replace +from types import FunctionType, SimpleNamespace + +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input, qrf, qrf_target +from microcosm.fit.graph_legacy_apply_matrix import ( + MATRIX_APPLY_STATE_TYPE, + decode_matrix_apply_state, +) +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + StructuralDelta, + codecs, + compile_graph, + run_graph, + source_hash, +) +from microcosm.graph import population as population_ops +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys +from microcosm.graph.serialize import graph_to_json + +from . import current_survey_amounts as values +from . import graph_current_survey_health as health_graph +from . import graph_current_survey_predictors as predictor_graph + +parent = values.parent_host +physical = values.physical +require = values.require +PROJECTION_NODE = "survey_amounts.source_projection" +ATTACH_NODE = "survey_amounts.attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_amount_projection", 1) +ATTACHMENT_TYPE = ArtifactType("microcosm.us.current_survey_amount_attachment", 1) +_ISSUED = {} + + +def _live(): + result = [] + for module in ( + sys.modules[__name__], + values, + values.unemployment, + health_graph, + health_graph.health, + health_graph.source, + ): + for name, item in vars(module).items(): + if type(item) is FunctionType: + result.append( + ( + module.__name__, + name, + values.predictors.source._function_seal(item), + ) + ) + elif isinstance(item, type) and item.__module__ == module.__name__: + result.append((module.__name__, name, item)) + for member, function in vars(item).items(): + if isinstance(function, (classmethod, staticmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if type(function) is FunctionType: + result.append( + ( + module.__name__, + name, + member, + values.predictors.source._function_seal(function), + ) + ) + result.append( + ( + "amount_configuration", + values.SEED, + tuple((g.key, g.fields, g.targets) for g in values.GROUPS), + values.UC_REPORT_COLUMNS, + values.unemployment.PROTOCOL, + values.unemployment.READ_COLUMNS, + codec.encode_json(values.unemployment.DICTIONARY), + ) + ) + result.append( + ( + "health_configuration", + health_graph.health.PROTOCOL, + tuple( + (f.output, f.asec, f.acs, f.acs_gap) for f in health_graph.health.FIELDS + ), + health_graph.health.RAW_COLUMNS, + health_graph.health.SOURCE_PREFIX, + health_graph.source.ASEC_COLUMNS, + health_graph.source.ACS_COLUMNS, + ) + ) + return tuple(result) + + +def _amount_edge(): + return ArtifactInput( + "amount_attachment", ATTACH_NODE, "attachment", ATTACHMENT_TYPE + ) + + +def _upstream_edge(): + return ArtifactInput( + "puf_finalization", + parent.attach.ATTACH_NODE, + "finalization", + parent.attach.FINALIZATION_TYPE, + ) + + +def _projection_edge(): + return ArtifactInput("projection", PROJECTION_NODE, "projection", PROJECTION_TYPE) + + +def _ids(group): + base = "survey_amounts." + group.spec.key + return base + ".donor", base + ".columns", base + ".fit", base + ".apply" + + +def amount_nodes(qualified, receiving, *, parent_digest, n_estimators): + require( + type(n_estimators) is int and n_estimators > 0 and codec._hash(parent_digest), + "NODE_PARAMETERS", + ) + params = { + "projection_sha256": codec.sha(qualified.projection), + "parent_sha256": parent_digest, + "n_estimators": n_estimators, + "groups": tuple(g.spec.key for g in qualified.groups), + "protocol": values.PROTOCOL, + } + nodes = [ + Node( + PROJECTION_NODE, + CurrentSurveyAmountProjectionKernel.ref, + population=parent.attach.FILTER_NODE, + inputs=predictor_graph._inputs(receiving), + params=params, + artifact_inputs=(_upstream_edge(),), + artifact_outputs=( + ArtifactOutput("projection", PROJECTION_TYPE), + *( + ArtifactOutput( + g.spec.key + "_matrix", model_input.RECIPIENT_MATRIX_TYPE + ) + for g in qualified.groups + ), + ), + description="Qualify original current survey amount and reporting-universe evidence; preserve source unknowns.", + ) + ] + attach_edges = [_projection_edge(), _upstream_edge()] + for group in qualified.groups: + donor, columns, fit_prefix, apply_prefix = _ids(group) + group_params = {**params, "group": group.spec.key} + matrix_edge = ArtifactInput( + group.spec.key + "_matrix", + PROJECTION_NODE, + group.spec.key + "_matrix", + model_input.RECIPIENT_MATRIX_TYPE, + ) + nodes.extend( + ( + Node( + donor, + CurrentSurveyAmountDonorKernel.ref, + base=values.predictors.host.survey_graph.CREATE_NODE, + structural=StructuralDelta.FILTER, + mass="free", + inputs=predictor_graph._inputs(qualified.source_frame), + params=group_params, + artifact_inputs=(_projection_edge(),), + description="Select source ASEC persons with jointly known targets; retain original design weights before allocation and clones.", + ), + Node( + columns, + CurrentSurveyAmountColumnsKernel.ref, + population=donor, + inputs=predictor_graph._inputs(group.donor_frame), + params=group_params, + outputs=tuple( + Owned("person", c, "float64") + for c in (*qualified.features, *group.spec.targets) + ), + artifact_inputs=(_projection_edge(),), + description="Map qualified current amounts and source predictors without numeric or reporting-unknown fills.", + ), + ) + ) + fits = legacy_qrf_train_nodes( + fit_prefix, + population=donor, + entity="person", + predictors=qualified.features, + targets=group.spec.targets, + seed=values.SEED, + n_estimators=n_estimators, + zero_atol=0, + phase="current_survey_amounts." + group.spec.key, + ) + applies = legacy_qrf_apply_matrix_nodes( + apply_prefix, + population=parent.attach.FILTER_NODE, + fit_nodes=fits, + matrix_producer=PROJECTION_NODE, + matrix_artifact=group.spec.key + "_matrix", + seed=values.SEED, + phase="current_survey_amounts." + group.spec.key, + ) + nodes.extend((*fits, *applies)) + attach_edges.append(matrix_edge) + for i, node in enumerate(applies): + attach_edges.extend( + ( + ArtifactInput( + f"{group.spec.key}_raw_{i}", + node.id, + "raw_draw", + codec.RAW_TARGET_TYPE, + ), + ArtifactInput( + f"{group.spec.key}_state_{i}", + node.id, + "apply_state", + MATRIX_APPLY_STATE_TYPE, + ), + ) + ) + columns = pd.concat([qualified.native, qualified.reports], axis=1) + nodes.append( + Node( + ATTACH_NODE, + CurrentSurveyAmountAttachKernel.ref, + population=parent.attach.FILTER_NODE, + inputs=predictor_graph._inputs(receiving), + params=params, + outputs=tuple(Owned("person", c, str(columns[c].dtype)) for c in columns), + artifact_inputs=tuple(attach_edges), + artifact_outputs=(ArtifactOutput("attachment", ATTACHMENT_TYPE),), + description="Join source observations and ACS conditional draws to both support clones; retain all PUF and source columns, geography, weights and ledger.", + ) + ) + return tuple(nodes) + + +class Boundary: + """Internal retained-value seam; a detached projection cannot construct it.""" + + def __init__(self, run, *, groups, n_estimators): + self.run = run + self.parent_view = parent.check_survey_puf55_run(run) + self.parent_entry = parent._run_entry(run) + self.qualified = values.qualify_current_survey_amounts(run, groups=groups) + self.qualified_stamp = values.seal(self.qualified) + self.parent_stamp = physical._population_stamp(run.population) + self.preparation = run.financial_run.prefix.preparation + self.health = health_graph.qualify_health_coverage(self.preparation) + self.health_stamp = health_graph.health_coverage_seal(self.health) + self.n_estimators = n_estimators + self.amount_nodes = amount_nodes( + self.qualified, + run.population.frame, + parent_digest=self.parent_view.digest, + n_estimators=n_estimators, + ) + self.health_nodes = health_graph.health_coverage_nodes( + self.health, + receiving_version=parent.attach.FILTER_NODE, + after=_amount_edge(), + ) + self.nodes = (*self.amount_nodes, *self.health_nodes) + self.declaration = tuple(self.nodes) + self.live = _live() + self.compiled = self.kernels = self.store = None + self.paths = self.source_keys = self.keys = self.implementations = None + self.parent_objects = ( + run.population, + run.compiled, + run.manifest, + run.store, + run.kernels, + run.sources, + ) + + def pure(self): + parent._pure_run(self.run, self.parent_entry) + require( + parent._run_entry(self.run) is self.parent_entry + and all( + a is b + for a, b in zip( + self.parent_objects, + ( + self.run.population, + self.run.compiled, + self.run.manifest, + self.run.store, + self.run.kernels, + self.run.sources, + ), + strict=True, + ) + ) + and physical._population_stamp(self.run.population) == self.parent_stamp + and values.seal(self.qualified) == self.qualified_stamp + and self.run.financial_run.prefix.preparation is self.preparation + and health_graph.health_coverage_seal(self.health) == self.health_stamp + and self.nodes == self.declaration + and _live() == self.live, + "BOUNDARY_CHANGED", + ) + require( + self.amount_nodes + == amount_nodes( + self.qualified, + self.run.population.frame, + parent_digest=self.parent_view.digest, + n_estimators=self.n_estimators, + ) + and self.health_nodes + == health_graph.health_coverage_nodes( + self.health, + receiving_version=parent.attach.FILTER_NODE, + after=_amount_edge(), + ) + and self.nodes == (*self.amount_nodes, *self.health_nodes), + "BOUNDARY_DECLARATIONS", + ) + if self.compiled is not None: + require( + all( + a is b + for a, b in zip( + self.bound_objects, + (self.compiled, self.store, self.kernels, self.store.codecs), + strict=True, + ) + ) + and tuple(self.kernels.as_mapping().items()) == self.kernel_items + and tuple(self.store.codecs.as_mapping().items()) == self.codec_items + and tuple(self.store.codecs.as_bytes_mapping().items()) + == self.bytes_codec_items, + "BOUND_REGISTRY_CHANGED", + ) + require( + self.compiled == compile_graph(self.compiled.graph) + and graph_to_json(self.compiled.graph) == self.graph_json, + "COMPILED_CHANGED", + ) + + def borrow(self): + require( + parent.check_survey_puf55_run(self.run).payload == self.parent_view.payload, + "PARENT_IDENTITY", + ) + if self.compiled is not None: + paths, source_keys = _source_paths_and_keys( + self.compiled, dict(self.paths), self.store + ) + keys, implementations = _all_node_keys( + self.compiled, self.kernels, source_keys + ) + require( + tuple(sorted(paths.items())) == self.paths + and tuple(sorted(source_keys.items())) == self.source_keys + and tuple(sorted(keys.items())) == self.keys + and tuple(sorted(implementations.items())) == self.implementations, + "SOURCE_OR_IMPLEMENTATION_CHANGED", + ) + self.pure() + + def context(self, context): + # Entry/final host fences check the complete PUF owner. These ordinary + # fragment calls consume retained, source-qualified values and check + # their pure seals; they do not repeatedly reread the full PUF pipeline. + self.pure() + require( + not context.sources + and context.node in self.nodes + and set(context.artifacts) + == {e.name for e in context.node.artifact_inputs}, + "CONTEXT_DECLARATION", + ) + for edge in context.node.artifact_inputs: + value = values.predictors.host.shared.artifact( + context, edge.name, edge.type + ) + require( + value.producer_key == dict(self.keys)[edge.producer], + "ARTIFACT_PRODUCER_KEY", + ) + if edge.name == "projection": + require(value.payload == self.qualified.projection, "PROJECTION_BYTES") + if edge == _upstream_edge(): + record = self.run.manifest.node(edge.producer) + payload = self.run.store.load_bytes( + record.opaque_artifacts[edge.artifact] + ) + require( + value.producer_key == record.key + and value.key == record.opaque_artifacts[edge.artifact] + and value.payload == payload, + "PARENT_ARTIFACT", + ) + self.pure() + return self.qualified + + def requalify(self): + """Reconstruct owned source transformations at the host's final I/O fence.""" + fresh = values.qualify_current_survey_amounts( + self.run, groups=tuple(g.spec.key for g in self.qualified.groups) + ) + require( + values.seal(fresh) == self.qualified_stamp, "SOURCE_REQUALIFICATION_CHANGED" + ) + fresh_health = health_graph.qualify_health_coverage(self.preparation) + require( + health_graph.health_coverage_seal(fresh_health) == self.health_stamp, + "HEALTH_SOURCE_REQUALIFICATION_CHANGED", + ) + self.pure() + + +class _Kernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=predictor_graph._Kernel.capabilities.dependencies, + ) + + def __init__(self, boundary): + self.boundary = boundary + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + values, + values.unemployment, + health_graph, + parent, + physical, + predictor_graph, + values.predictors, + values.unemployment.coverage, + values.unemployment.source_csv_builtin, + population_ops, + model_input, + qrf, + qrf_target, + dependencies=self.capabilities.dependencies, + ) + + +class CurrentSurveyAmountProjectionKernel(_Kernel): + ref = "us.survey_amounts.source_projection@1" + + def run(self, context): + qualified = self.boundary.context(context) + values.predictors.host._current_context_frame( + context, self.boundary.run.population.frame + ) + return KernelResult( + artifacts={ + "projection": qualified.projection, + **{g.spec.key + "_matrix": g.matrix for g in qualified.groups}, + }, + receipt=qualified.evidence, + ) + + +class CurrentSurveyAmountDonorKernel(_Kernel): + ref = "us.survey_amounts.source_donor@1" + capabilities = replace(_Kernel.capabilities, structural=StructuralDelta.FILTER) + + def run(self, context): + qualified = self.boundary.context(context) + values.predictors.host._current_context_frame(context, qualified.source_frame) + group = next( + g for g in qualified.groups if g.spec.key == context.params["group"] + ) + return KernelResult( + keep=pd.Series( + group.keep.copy(), + index=pd.Index( + qualified.source_frame.person.person_id.to_numpy(), name="person_id" + ), + ), + receipt={ + "group": group.spec.key, + "donor_persons": int(group.keep.sum()), + "selection": "joint_known_current_ASEC_person_targets", + "weight_kind": "design", + }, + ) + + +class CurrentSurveyAmountColumnsKernel(_Kernel): + ref = "us.survey_amounts.source_columns@1" + + def run(self, context): + qualified = self.boundary.context(context) + group = next( + g for g in qualified.groups if g.spec.key == context.params["group"] + ) + values.predictors.host._current_context_frame(context, group.donor_frame) + return KernelResult( + columns={ + ("person", c): group.donor_columns[c] for c in group.donor_columns + }, + receipt={ + "group": group.spec.key, + "projection_sha256": codec.sha(qualified.projection), + }, + ) + + +def read_draws(qualified, artifacts): + """Check target history, exact matrix producer and immutable draw bytes.""" + draws = {} + for group in qualified.groups: + name = group.spec.key + matrix_value = artifacts[name + "_matrix"] + require(matrix_value.payload == group.matrix, "MATRIX_BYTES") + matrix = model_input.decode_recipient_matrix(group.matrix) + require(tuple(matrix.features) == qualified.features, "MATRIX_FEATURES") + result = pd.DataFrame(index=matrix.features.index) + models, raw_history = [], [] + for i, target in enumerate(group.spec.targets): + raw = artifacts[f"{name}_raw_{i}"].payload + state_value = artifacts[f"{name}_state_{i}"] + require( + state_value.producer_key == artifacts[f"{name}_raw_{i}"].producer_key, + "RAW_STATE_SIBLINGS", + ) + packet = decode_matrix_apply_state(state_value.payload) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + raw_history.append({"target": target, "sha256": codec.sha(raw)}) + require( + packet["matrix_sha256"] == codec.sha(group.matrix) + and packet["matrix_producer_key"] == matrix_value.producer_key + and chain.entity == "person" + and tuple(chain.predictors) == qualified.features + and tuple(chain.targets) == group.spec.targets + and tuple(chain.completed_targets) == group.spec.targets[: i + 1] + and chain.recipient_index == qrf._index_identity(matrix.features.index) + and application["seed"] == values.SEED + and application["raw_targets"] == raw_history + and application["models"][:i] == models + and len(application["models"]) == i + 1, + "DRAW_CHAIN", + ) + models = application["models"] + result[target] = codec.read_raw_target( + raw, target=target, index=matrix.features.index + ) + draws[name] = result + return draws + + +def _attachment_result(boundary, artifacts): + qualified = boundary.qualified + require( + artifacts["projection"].payload == qualified.projection, "ATTACHMENT_PROJECTION" + ) + draws = read_draws(qualified, artifacts) + columns = values.attach_columns(qualified, boundary.run.population.frame, draws) + receipt = { + "protocol": values.PROTOCOL, + "parent_sha256": boundary.parent_view.digest, + "projection_sha256": codec.sha(qualified.projection), + "draw_sha256": { + n: codec.sha(v.payload) for n, v in artifacts.items() if "_raw_" in n + }, + "source_unknowns_preserved": True, + "weights_changed": False, + "prior_wages_consumed": False, + "release_eligible": False, + } + return KernelResult( + columns=columns, + artifacts={"attachment": codec.encode_json(receipt)}, + receipt=receipt, + ) + + +class CurrentSurveyAmountAttachKernel(_Kernel): + ref = "us.survey_amounts.attach@1" + + def run(self, context): + self.boundary.context(context) + values.predictors.host._current_context_frame( + context, self.boundary.run.population.frame + ) + result = _attachment_result(self.boundary, context.artifacts) + self.boundary.pure() + return result + + +def _construct(run, *, groups, n_estimators): + boundary = Boundary(run, groups=groups, n_estimators=n_estimators) + compiled = compile_graph( + replace(run.compiled.graph, nodes=(*run.compiled.graph.nodes, *boundary.nodes)) + ) + kernels = KernelRegistry() + for kernel in run.kernels.as_mapping().values(): + kernels.register(kernel) + for cls in ( + CurrentSurveyAmountProjectionKernel, + CurrentSurveyAmountDonorKernel, + CurrentSurveyAmountColumnsKernel, + CurrentSurveyAmountAttachKernel, + ): + require(cls.ref not in kernels.refs(), "KERNEL_COLLISION") + kernels.register(cls(boundary)) + for kernel in health_graph.health_coverage_kernels( + boundary.health, + receiving_version=parent.attach.FILTER_NODE, + after=_amount_edge(), + require_current=boundary.pure, + ): + require(kernel.ref not in kernels.refs(), "HEALTH_KERNEL_COLLISION") + kernels.register(kernel) + registry = parent._registry(run.store.codecs, codecs.SourceCodecRegistry()) + store = ContentStore(run.store.root, codecs=registry) + paths, source_keys = _source_paths_and_keys(compiled, dict(run.sources), store) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + require( + all( + keys[n] == run.manifest.node(n).key + and implementations[n] == run.manifest.node(n).kernel_impl_hash + and compiled.graph.node(n) == run.compiled.graph.node(n) + for n in run.compiled.order + ), + "PREFIX_DECLARATION_OR_KEY", + ) + boundary.compiled, boundary.kernels, boundary.store = compiled, kernels, store + boundary.paths = tuple(sorted(paths.items())) + boundary.source_keys = tuple(sorted(source_keys.items())) + boundary.keys = tuple(sorted(keys.items())) + boundary.implementations = tuple(sorted(implementations.items())) + boundary.graph_json = graph_to_json(compiled.graph) + boundary.bound_objects = (compiled, store, kernels, store.codecs) + boundary.kernel_items = tuple(kernels.as_mapping().items()) + boundary.codec_items = tuple(store.codecs.as_mapping().items()) + boundary.bytes_codec_items = tuple(store.codecs.as_bytes_mapping().items()) + boundary.borrow() + return boundary + + +def _verify_models(boundary, loaded, donors): + """Verify persisted models against actual donor values after artifact admission.""" + for group in boundary.qualified.groups: + donor_id, _, fit_prefix, apply_prefix = _ids(group) + donor = donors[donor_id] + first = boundary.compiled.graph.node(fit_prefix + ".000") + model_frame = codec.model_frame( + SimpleNamespace(weights={"person": donor.frame.resolve_weights("person")}), + first.inputs[0], + donor.frame.person, + ) + model = qrf.RegimeGatedQRF( + seed=values.SEED, + n_estimators=boundary.n_estimators, + zero_atol=0, + max_samples_leaf=None, + ) + before = qrf_target.LegacyQRFTrainingState.from_chain( + model.start_chain( + model_frame, + list(boundary.qualified.features), + list(group.spec.targets), + weights="design", + ) + ) + history = [] + for i, target in enumerate(group.spec.targets): + node = f"{fit_prefix}.{i:03d}" + payload = loaded[node, "model"] + packet, after = codec.read_training(loaded[node, "training_state"]) + fitted = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=codec.sha(payload) + ) + require( + fitted.training_state == before + and fitted.next_training_state == after + and fitted.donor_sha256 + == qrf_target._consumed_values_sha256( + model_frame.person, + (*boundary.qualified.features, *group.spec.targets[:i], target), + ), + "TRAINING_DONOR", + ) + history.append( + { + "target": target, + "sha256": codec.sha(payload), + "training_id": fitted.training_id, + } + ) + require( + packet["models"] == history + and decode_matrix_apply_state( + loaded[f"{apply_prefix}.{i:03d}", "apply_state"] + )["application"]["models"] + == history, + "TRAINING_APPLY_HISTORY", + ) + before = after + + +@dataclass(frozen=True) +class SurveyEnrichmentRun: + parent_run: parent.SurveyPuf55Run + population: population_ops.Population + manifest: object + compiled: object + store: ContentStore + kernels: KernelRegistry + sources: tuple + receipt: bytes + + def checked_view(self): + return check_survey_enrichment_run(self) + + +@dataclass(frozen=True) +class CheckedSurveyEnrichmentRun: + """Descriptive values; authority remains in the original retained host.""" + + payload: bytes + digest: str + population: population_ops.Population + + +def _run_seal(run): + return ( + run.receipt, + run.manifest.to_json(), + graph_to_json(run.compiled.graph), + physical._population_stamp(run.population), + parent._heterogeneous_manifest_seals(run.manifest, run.compiled), + tuple(sorted(run.manifest.populations)), + tuple(sorted(run.manifest.mass_ledgers)), + ) + + +def check_survey_enrichment_run(run): + entry = _ISSUED.get(id(run)) + require( + type(run) is SurveyEnrichmentRun and entry is not None and entry[0]() is run, + "UNISSUED_RUN", + ) + _, boundary, stamp, objects, artifacts = entry + try: + boundary.borrow() + require( + all( + a is b + for a, b in zip( + objects, + ( + run.parent_run, + run.population, + run.manifest, + run.compiled, + run.store, + run.kernels, + run.sources, + ), + strict=True, + ) + ), + "RUN_OBJECTS", + ) + loaded = parent.financial._artifacts( + run.manifest, + run.compiled, + run.store, + run.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items())) + == artifacts, + "RUN_ARTIFACTS", + ) + boundary.borrow() + require(_run_seal(run) == stamp, "RUN_CHANGED") + boundary.pure() + except BaseException: + if _ISSUED.get(id(run)) is entry: + del _ISSUED[id(run)] + raise + return CheckedSurveyEnrichmentRun( + run.receipt, codec.sha(run.receipt), run.population + ) + + +def run_us_survey_enrichment( + run, *, groups=("unemployment", "health_costs"), n_estimators=100, resume="auto" +): + """Execute one extension and verify its complete observed parent and output.""" + require(resume in ("auto", "require"), "RESUME") + boundary = _construct(run, groups=groups, n_estimators=n_estimators) + observed, stamps = {}, {} + + def observe(node_id, population): + require(node_id not in observed, "OBSERVER_DUPLICATE") + observed[node_id] = population + stamps[node_id] = physical._population_stamp(population) + + manifest = run_graph( + boundary.compiled, + sources=dict(boundary.paths), + store=boundary.store, + kernels=boundary.kernels, + resume=resume, + _population_observer=observe, + ) + require(tuple(observed) == boundary.compiled.order, "OBSERVER_ROSTER") + if resume == "require": + require(all(record.hit for record in manifest.nodes.values()), "REQUIRED_HITS") + boundary.borrow() + loaded = parent.financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + for node in run.compiled.graph.nodes: + old = run.manifest.node(node.id) + for name, key in old.opaque_artifacts.items(): + require( + loaded[node.id, name] == run.store.load_bytes(key), "PREFIX_ARTIFACT" + ) + physical.replay.same_replayed_population( + run.population, observed[parent.attach.ATTACH_NODE] + ) + require( + loaded[PROJECTION_NODE, "projection"] == boundary.qualified.projection, + "SOURCE_PROJECTION", + ) + attach = boundary.compiled.graph.node(ATTACH_NODE) + artifacts = parent._loaded_values(boundary, manifest, loaded, attach) + result = _attachment_result(boundary, artifacts) + require( + loaded[ATTACH_NODE, "attachment"] == result.artifacts["attachment"], + "ATTACHMENT_ARTIFACT", + ) + # Compare every unchanged prefix terminal Frame and ledger against the + # retained parent manifest. Full execution owners/design anchors on the + # receiving population are checked separately above against the original. + prefix_terminal = {} + for node_id in run.compiled.order: + prefix_terminal[run.compiled.versions[node_id]] = observed[node_id] + for version, population in prefix_terminal.items(): + physical.replay.same_replayed_population( + population_ops.Population.from_frame( + population.frame, version, mass_ledger=population.mass_ledger + ), + population_ops.Population.from_frame( + run.manifest.population(version), + version, + mass_ledger=run.manifest.mass_ledger(version), + ), + ) + current, donors = {}, {} + health_ids = {n.id for n in boundary.health_nodes} + group_nodes = {_ids(g)[0]: g for g in boundary.qualified.groups} + column_nodes = {_ids(g)[1]: g for g in boundary.qualified.groups} + original = population_ops.Population.from_frame( + boundary.qualified.source_frame, values.predictors.host.survey_graph.CREATE_NODE + ) + for node_id in boundary.compiled.order: + node = boundary.compiled.graph.node(node_id) + version = boundary.compiled.versions[node_id] + if node_id in run.compiled.order: + current[version] = observed[node_id] + continue + if node_id in health_ids: + health_artifacts = parent._loaded_values(boundary, manifest, loaded, node) + expected = health_graph.expected_health_population( + node_id, + current.get(version), + qualified=boundary.health, + node=node, + artifacts=health_artifacts, + ) + # Bind every persisted source/recode/attachment artifact to its + # independent domain result, including attachment metadata. + expected_result = health_graph._result( + boundary.health, + node, + None if current.get(version) is None else current[version].frame.person, + ) + require( + all( + loaded[node_id, name] == payload + for name, payload in expected_result.artifacts.items() + ), + "HEALTH_RESULT_ARTIFACT", + ) + elif node_id in group_nodes: + expected = population_ops.patch( + original, node, KernelResult(frame=group_nodes[node_id].donor_frame) + ) + elif node_id in column_nodes: + group = column_nodes[node_id] + expected = population_ops.patch( + current[version], + node, + KernelResult( + columns={ + ("person", c): group.donor_columns[c] + for c in group.donor_columns + } + ), + ) + donors[_ids(group)[0]] = expected + elif node_id == ATTACH_NODE: + expected = population_ops.patch(current[version], node, result) + else: + expected = current[version] + physical.replay.same_replayed_population(expected, observed[node_id]) + current[version] = expected + _verify_models(boundary, loaded, donors) + for version, population in current.items(): + physical.replay.same_replayed_population( + population_ops.Population.from_frame( + population.frame, version, mass_ledger=population.mass_ledger + ), + population_ops.Population.from_frame( + manifest.population(version), + version, + mass_ledger=manifest.mass_ledger(version), + ), + ) + receipt = codec.encode_json( + { + "protocol": values.PROTOCOL, + "parent_sha256": boundary.parent_view.digest, + "manifest_key": manifest.key, + "node_count": len(boundary.compiled.order), + "groups": list(groups), + "complete_population_compared": True, + "projection_sha256": codec.sha(boundary.qualified.projection), + "attachment_sha256": codec.sha(result.artifacts["attachment"]), + "health_projection_sha256": codec.sha(boundary.health.projection), + "health_attachment_sha256": codec.sha( + loaded[health_graph.ATTACH_NODE, "attachment"] + ), + "health_fields": [f.output for f in health_graph.health.FIELDS], + "release_eligible": False, + } + ) + output = SurveyEnrichmentRun( + run, + observed[health_graph.ATTACH_NODE], + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + boundary.paths, + receipt, + ) + objects = ( + output.parent_run, + output.population, + output.manifest, + output.compiled, + output.store, + output.kernels, + output.sources, + ) + stamp = _run_seal(output) + artifact_hashes = tuple( + sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items()) + ) + boundary.requalify() + boundary.borrow() + fresh = parent.financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + boundary.borrow() + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in fresh.items())) + == artifact_hashes + and _run_seal(output) == stamp + and all(physical._population_stamp(observed[n]) == stamps[n] for n in observed), + "FINAL_OUTPUT", + ) + boundary.pure() + require(id(output) not in _ISSUED, "RUN_ALREADY_ISSUED") + ident = id(output) + + def forget(ref): + old = _ISSUED.get(ident) + if old is not None and old[0] is ref: + del _ISSUED[ident] + + entry = (weakref.ref(output, forget), boundary, stamp, objects, artifact_hashes) + _ISSUED[ident] = entry + return output diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py b/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py index df08aa857..528957104 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py @@ -59,7 +59,7 @@ SourceRuntimeError, run_source_stage, ) -from microcosm.frame import Frame +from microcosm.frame import Frame, Weights from microcosm.frame.units import US_SCHEMA __all__ = [ @@ -70,10 +70,13 @@ "US_HOURS_WORKED_REQUIRED_SOURCE_COLUMNS", "US_HOURS_WORKED_STAGE_NAME", "derive_us_hours_worked_from_manifest", + "us_hours_worked_gate_from_summary", + "us_hours_worked_person_summary", "us_hours_worked_signal_gate", "us_hours_worked_summary", "us_hours_worked_stage_spec", "with_us_hours_worked_inputs", + "with_us_hours_worked_person", ] US_HOURS_WORKED_STAGE_NAME = "hours_worked" @@ -233,8 +236,36 @@ def with_us_hours_worked_inputs(frame: Frame, *, seed: int, time_period: int) -> if have_all and _weekly_hours_carry_signal(person): return frame + produced = with_us_hours_worked_person( + person, + weights=frame.resolve_weights("person"), + seed=seed, + time_period=time_period, + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"] = produced + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def with_us_hours_worked_person( + person: pd.DataFrame, *, weights: Weights, seed: int, time_period: int +) -> pd.DataFrame: + """Run the same manifest handler on a declared person view and typed weights.""" + if len(weights.values) != len(person): + raise ValueError("US hours-worked person weights are not row-aligned.") + if all(column in person for column in US_HOURS_WORKED_OUTPUT_COLUMNS) and ( + _weekly_hours_carry_signal(person) + ): + return person stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = weights.values output = run_source_stage( us_hours_worked_stage_spec(), tables={"person": stage_person}, @@ -251,24 +282,27 @@ def with_us_hours_worked_inputs(frame: Frame, *, seed: int, time_period: int) -> f"{column!r}." ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} + produced = person.copy() for column in US_HOURS_WORKED_OUTPUT_COLUMNS: - tables["person"][column] = aligned[column].to_numpy(dtype=np.float64) - return Frame( - tables, - frame.schema, - {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, - frame.strata, - mass_log=frame.mass_log, - metadata=frame.metadata, - ) + produced[column] = aligned[column].to_numpy(dtype=np.float64) + return produced def us_hours_worked_summary(frame: Frame) -> dict[str, object]: """Weighted hours-distribution summary for gates and release manifests.""" - person = frame.table("person") - weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + return us_hours_worked_person_summary( + frame.table("person"), weights=frame.resolve_weights("person") + ) + + +def us_hours_worked_person_summary( + person: pd.DataFrame, *, weights: Weights +) -> dict[str, object]: + """Observe the actual person view with canonically resolved, aligned weights.""" + if len(weights.values) != len(person): + raise ValueError("US hours-worked person weights are not row-aligned.") + weights = np.asarray(weights.values, dtype=np.float64) weekly = pd.to_numeric( person["weekly_hours_worked_before_lsr"], errors="coerce" ).fillna(0.0) @@ -306,7 +340,6 @@ def us_hours_worked_signal_gate(frame: Frame) -> GateResult: """ person = frame.table("person") - failures: list[str] = [] missing = [ column for column in US_HOURS_WORKED_OUTPUT_COLUMNS @@ -321,7 +354,40 @@ def us_hours_worked_signal_gate(frame: Frame) -> GateResult: ) summary = us_hours_worked_summary(frame) - for column, count in summary["unique_counts"].items(): + return us_hours_worked_gate_from_summary(summary) + + +def us_hours_worked_gate_from_summary(summary: dict[str, object]) -> GateResult: + """Apply the original signal checks to actual typed producer observations.""" + if ( + not isinstance(summary, dict) + or set(summary) + != { + "worked_share", + "mean_weekly_hours_workers", + "worked_share_band", + "mean_weekly_hours_band", + "unique_counts", + } + or summary["worked_share_band"] != list(_WORKED_SHARE_BAND) + or summary["mean_weekly_hours_band"] != list(_MEAN_WEEKLY_HOURS_BAND) + or not isinstance(summary["unique_counts"], dict) + or set(summary["unique_counts"]) != set(US_HOURS_WORKED_OUTPUT_COLUMNS) + or any( + type(value) is not int or value < 0 + for value in summary["unique_counts"].values() + ) + or any( + type(summary[name]) not in (int, float) or not np.isfinite(summary[name]) + for name in ("worked_share", "mean_weekly_hours_workers") + ) + ): + raise ValueError("US hours-worked gate requires the complete original summary.") + failures: list[str] = [] + # Canonical artifact JSON sorts mappings; failure order remains the + # original declared output order, including after a cache roundtrip. + for column in US_HOURS_WORKED_OUTPUT_COLUMNS: + count = summary["unique_counts"][column] if count < 2: failures.append( f"{column}: constant column (one observed value) — the hours " diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py index 1f83bd82c..f6f1dbbba 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py @@ -24,8 +24,11 @@ import hashlib import warnings +from collections.abc import Mapping +from dataclasses import dataclass from importlib.resources import files from pathlib import Path +from types import MappingProxyType from typing import Any import numpy as np @@ -63,6 +66,11 @@ "US_HOUSING_REQUIRED_PERSON_SOURCE_COLUMNS", "US_HOUSING_SPM_UNIT_OUTPUT_COLUMNS", "derive_us_housing_inputs", + "AcsRentDonorPreparation", + "AcsRentRecipientPreparation", + "prepare_acs_rent_donor", + "prepare_acs_rent_recipient", + "finalize_acs_rent", "impute_us_pre_subsidy_rent", "impute_us_housing_assistance_to_puf_support", "load_acs_2022_rent_donor", @@ -465,14 +473,17 @@ def _constant_source_by_unit( return values.to_numpy(dtype=np.float64) -def derive_us_housing_inputs(frame: Frame) -> Frame: - """Carry the three exact ASEC housing/tenure inputs onto their entities.""" +def derive_us_housing_tables( + person: pd.DataFrame, household: pd.DataFrame, spm_unit: pd.DataFrame +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Return copied household/SPM tables with the exact measured housing leaves. + + Inputs are the actual linked tables (or declared views) at the existing + housing derivation boundary. SPM raw values align through person_spm_unit_id; + returned group row order, indices, other columns and input tables are kept. + This helper neither reconstructs other entities nor changes raw code meaning. + """ - if frame.schema != US_SCHEMA: - raise ValueError("US housing inputs require the US schema.") - person = frame.table("person") - household = frame.table("household") - spm_unit = frame.table("spm_unit") _required_columns( person, ("SPM_CAPHOUSESUB", "SPM_TENMORTSTATUS", "person_spm_unit_id"), @@ -521,15 +532,30 @@ def derive_us_housing_inputs(frame: Frame) -> Frame: f"{unknown_spm_codes}." ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} - tables["household"]["tenure_type"] = np.asarray( + household_result = household.copy() + spm_result = spm_unit.copy() + household_result["tenure_type"] = np.asarray( [_HOUSEHOLD_TENURE_MAP[code] for code in raw_household_codes], dtype=object ) - tables["spm_unit"]["receives_housing_assistance"] = subsidy > 0.0 - tables["spm_unit"]["takes_up_housing_assistance_if_eligible"] = subsidy > 0.0 - tables["spm_unit"]["spm_unit_tenure_type"] = np.asarray( + spm_result["receives_housing_assistance"] = subsidy > 0.0 + spm_result["takes_up_housing_assistance_if_eligible"] = subsidy > 0.0 + spm_result["spm_unit_tenure_type"] = np.asarray( [_SPM_TENURE_MAP[code] for code in raw_spm_codes], dtype=object ) + return household_result, spm_result + + +def derive_us_housing_inputs(frame: Frame) -> Frame: + """Carry the three exact ASEC housing/tenure inputs onto their entities.""" + + if frame.schema != US_SCHEMA: + raise ValueError("US housing inputs require the US schema.") + household, spm_unit = derive_us_housing_tables( + frame.table("person"), frame.table("household"), frame.table("spm_unit") + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["household"] = household + tables["spm_unit"] = spm_unit return Frame( tables, frame.schema, @@ -703,38 +729,203 @@ def _encode_acs_predictors( ) -> tuple[pd.DataFrame, pd.DataFrame, tuple[str, ...]]: """Dummy-encode the retired QRF's string predictors without ordinality.""" + context = _rent_encoding_context(training) + return ( + _encode_rent_predictors(training, context=context), + _encode_rent_predictors(prediction, context=context), + context["predictors"], + ) + + +def _rent_encoding_context(training: pd.DataFrame) -> Mapping[str, object]: numeric_predictors = tuple( predictor for predictor in ACS_RENT_PREDICTORS if predictor not in _ACS_CATEGORICAL_PREDICTORS ) - encoded_training = training.loc[:, numeric_predictors].copy() - encoded_prediction = prediction.loc[:, numeric_predictors].copy() encoded_columns = list(numeric_predictors) + category_levels = {} for column in _ACS_CATEGORICAL_PREDICTORS: training_values = training[column].astype(str) - prediction_values = prediction[column].astype(str) levels = tuple(sorted(training_values.unique())) - unknown = sorted(set(prediction_values.unique()) - set(levels)) - if unknown: - raise ValueError( - f"ACS rent recipient {column!r} has donor-unsupported value(s): " - f"{unknown}." - ) if len(levels) < 2: raise ValueError( f"ACS rent donor categorical predictor {column!r} is constant." ) + category_levels[column] = levels + encoded_columns.extend(f"{column}__{level}" for level in levels[1:]) + return MappingProxyType( + { + "schema": "microcosm.us.acs_rent_encoding.v1", + "predictors": tuple(encoded_columns), + "category_levels": MappingProxyType(category_levels), + } + ) + + +def _validated_rent_encoding_context( + context: Mapping[str, object], +) -> Mapping[str, object]: + if ( + not isinstance(context, Mapping) + or set(context) != {"schema", "predictors", "category_levels"} + or context.get("schema") != "microcosm.us.acs_rent_encoding.v1" + ): + raise ValueError("Unsupported ACS rent donor encoding context.") + levels_by_column = context["category_levels"] + if not isinstance(levels_by_column, Mapping) or set(levels_by_column) != set( + _ACS_CATEGORICAL_PREDICTORS + ): + raise ValueError( + "ACS rent donor encoding must name its exact categorical columns." + ) + expected = [ + name for name in ACS_RENT_PREDICTORS if name not in _ACS_CATEGORICAL_PREDICTORS + ] + frozen_levels = {} + for column in _ACS_CATEGORICAL_PREDICTORS: + levels = levels_by_column[column] + if ( + not isinstance(levels, (tuple, list)) + or len(levels) < 2 + or any(not isinstance(value, str) for value in levels) + or list(levels) != sorted(set(levels)) + ): + raise ValueError( + "ACS rent donor encoding levels must be sorted unique strings." + ) + frozen_levels[column] = tuple(levels) + expected.extend(f"{column}__{level}" for level in levels[1:]) + if not isinstance(context["predictors"], (tuple, list)) or tuple( + context["predictors"] + ) != tuple(expected): + raise ValueError( + "ACS rent donor encoding predictor order differs from its levels." + ) + return MappingProxyType( + { + "schema": context["schema"], + "predictors": tuple(expected), + "category_levels": MappingProxyType(frozen_levels), + } + ) + + +def _encode_rent_predictors( + table: pd.DataFrame, *, context: Mapping[str, object] +) -> pd.DataFrame: + numeric = tuple( + name for name in ACS_RENT_PREDICTORS if name not in _ACS_CATEGORICAL_PREDICTORS + ) + encoded = table.loc[:, numeric].copy() + for column in _ACS_CATEGORICAL_PREDICTORS: + values = table[column].astype(str) + levels = context["category_levels"][column] + unknown = sorted(set(values.unique()) - set(levels)) + if unknown: + raise ValueError( + f"ACS rent recipient {column!r} has donor-unsupported value(s): {unknown}." + ) for level in levels[1:]: - dummy = f"{column}__{level}" - encoded_training[dummy] = training_values.eq(level).to_numpy( - dtype=np.float64 + encoded[f"{column}__{level}"] = values.eq(level).to_numpy(dtype=np.float64) + return encoded + + +@dataclass(frozen=True) +class AcsRentDonorPreparation: + """Actual sampled donor training rows and recipient-independent encoding.""" + + training: pd.DataFrame + predictors: tuple[str, ...] + context: Mapping[str, object] + + def __post_init__(self) -> None: + context = _validated_rent_encoding_context(self.context) + if ( + self.predictors != context["predictors"] + or not isinstance(self.training, pd.DataFrame) + or tuple(self.training.columns) + != (*self.predictors, "rent", _DONOR_WEIGHT_COLUMN) + ): + raise ValueError( + "ACS rent donor training columns differ from their encoding." ) - encoded_prediction[dummy] = prediction_values.eq(level).to_numpy( - dtype=np.float64 + object.__setattr__(self, "context", context) + + +@dataclass(frozen=True) +class AcsRentRecipientPreparation: + """Predictor rows and their household heads' original person positions. + + ``head_positions[i]`` belongs to household ``predictors.index[i]``; it + need not be sorted when person and household table orders differ. + """ + + predictors: pd.DataFrame + head_positions: np.ndarray + person_index: pd.Index + + def __post_init__(self) -> None: + positions = np.asarray(self.head_positions) + if ( + not isinstance(self.predictors, pd.DataFrame) + or not isinstance(self.person_index, pd.Index) + or positions.ndim != 1 + or not np.issubdtype(positions.dtype, np.integer) + or len(positions) != len(self.predictors) + or (positions < 0).any() + or (positions >= len(self.person_index)).any() + or len(np.unique(positions)) != len(positions) + ): + raise ValueError( + "ACS rent head positions must align to the original person rows." ) - encoded_columns.append(dummy) - return encoded_training, encoded_prediction, tuple(encoded_columns) + positions = positions.astype(np.int64, copy=True) + positions.flags.writeable = False + object.__setattr__(self, "head_positions", positions) + object.__setattr__(self, "person_index", self.person_index.copy(deep=True)) + + +def prepare_acs_rent_recipient( + frame: Frame, *, donor_context: Mapping[str, object] +) -> AcsRentRecipientPreparation: + """Encode actual head predictors against donor-owned categorical levels.""" + context = _validated_rent_encoding_context(donor_context) + features, head_mask = _recipient_head_features(frame) + head_positions = np.flatnonzero(head_mask) + head_households = pd.Index( + frame.person["person_household_id"].to_numpy()[head_positions] + ) + household_order = head_households.get_indexer(features.index) + if (household_order < 0).any(): + raise ValueError("ACS rent predictor household has no selected person head.") + return AcsRentRecipientPreparation( + _encode_rent_predictors(features, context=context), + head_positions[household_order], + frame.person.index, + ) + + +def finalize_acs_rent( + prepared: AcsRentRecipientPreparation, raw_rent: pd.Series | pd.DataFrame +) -> np.ndarray: + """Clip raw draws and scatter each household's value to its own head.""" + if not isinstance(prepared, AcsRentRecipientPreparation): + raise TypeError("ACS rent finalization requires recipient preparation.") + if isinstance(raw_rent, pd.DataFrame) and tuple(raw_rent.columns) == ("rent",): + raw_rent = raw_rent["rent"] + if not isinstance(raw_rent, pd.Series) or raw_rent.name != "rent": + raise ValueError("ACS rent finalization requires the named raw rent Series.") + if not raw_rent.index.equals(prepared.predictors.index): + raise ValueError( + "ACS rent raw prediction index differs from prepared household order." + ) + predicted = pd.to_numeric(raw_rent, errors="coerce").to_numpy(dtype=np.float64) + if not np.isfinite(predicted).all(): + raise ValueError("ACS rent QRF produced nonfinite predictions.") + person_rent = np.zeros(len(prepared.person_index), dtype=np.float64) + person_rent[prepared.head_positions] = np.maximum(predicted, 0.0) + return person_rent def _stable_string_hash(value: str) -> np.uint64: @@ -822,14 +1013,12 @@ def _archived_joint_training_sample( return sampled, sampled_masks -def impute_us_pre_subsidy_rent( - frame: Frame, - donor: pd.DataFrame, - *, - seed: int, - n_estimators: int = _DEFAULT_N_ESTIMATORS, -) -> np.ndarray: - """Draw annual ACS rent once per CPS household and place it on the head.""" +def prepare_acs_rent_donor(donor: pd.DataFrame) -> AcsRentDonorPreparation: + """Select and encode actual archived ACS rent support without recipients. + + The joint rent/real-estate-tax sampling, allocation masks, reset-index + order and household design weights retain their existing semantics. + """ required = ( *ACS_RENT_PREDICTORS, @@ -869,35 +1058,39 @@ def impute_us_pre_subsidy_rent( if float(fit_frame[_DONOR_WEIGHT_COLUMN].sum()) <= 0.0: raise ValueError("ACS rent donor sampled weights sum to zero.") + context = _rent_encoding_context(fit_frame) + encoded_training = _encode_rent_predictors(fit_frame, context=context) + encoded_training["rent"] = fit_frame["rent"].to_numpy(dtype=np.float64) + encoded_training[_DONOR_WEIGHT_COLUMN] = fit_frame[_DONOR_WEIGHT_COLUMN].to_numpy( + dtype=np.float64 + ) + return AcsRentDonorPreparation(encoded_training, context["predictors"], context) + + +def impute_us_pre_subsidy_rent( + frame: Frame, + donor: pd.DataFrame, + *, + seed: int, + n_estimators: int = _DEFAULT_N_ESTIMATORS, +) -> np.ndarray: + """Draw annual ACS rent once per CPS household and place it on the head.""" + + prepared_donor = prepare_acs_rent_donor(donor) + global QRF if QRF is None: from importlib import import_module QRF = import_module("microcosm.fit").QRF - features, head_mask = _recipient_head_features(frame) - encoded_training, encoded_features, encoded_predictors = _encode_acs_predictors( - fit_frame, - features, - ) - encoded_training["rent"] = fit_frame["rent"].to_numpy(dtype=np.float64) - encoded_training[_DONOR_WEIGHT_COLUMN] = fit_frame[_DONOR_WEIGHT_COLUMN].to_numpy( - dtype=np.float64 - ) + recipient = prepare_acs_rent_recipient(frame, donor_context=prepared_donor.context) fitted = QRF(n_estimators=int(n_estimators), seed=int(seed)).fit( - encoded_training, - predictors=list(encoded_predictors), + prepared_donor.training, + predictors=list(prepared_donor.predictors), targets=["rent"], weights=_DONOR_WEIGHT_COLUMN, ) - predicted = pd.to_numeric( - fitted.predict(encoded_features)["rent"], errors="coerce" - ).to_numpy(dtype=np.float64) - if not np.isfinite(predicted).all(): - raise ValueError("ACS rent QRF produced nonfinite predictions.") - predicted = np.maximum(predicted, 0.0) - person_rent = np.zeros(frame.n("person"), dtype=np.float64) - person_rent[head_mask] = predicted - return person_rent + return finalize_acs_rent(recipient, fitted.predict(recipient.predictors)) def _person_puf_predictors(frame: Frame) -> pd.DataFrame: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py b/packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py new file mode 100644 index 000000000..21f7ad81f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py @@ -0,0 +1,202 @@ +"""Explicit input-name profiles; neither a release gate nor source admission. + +The historical release gate retains its existing 163-input default. This +lightweight declaration adds the national/CD projection without importing its +engine-dependent anti-rot helpers or reading package resources. Its exact source +manifest pin documents the extraction, not authority over a receiving Frame. +Grain/applicability are deliberately not guessed from engine variable names. +""" + +from enum import StrEnum + + +class USInputProfile(StrEnum): + HISTORICAL = "us_release_163_v1" + NATIONAL_CD = "us_national_cd_161_v1" + + +MANIFEST_SHA256 = "d96d0e98078906a9b47d47662e173553b1937851c7a8dd576010b50705ca1b1d" +HISTORICAL_REQUIRED_INPUTS = ( + "age", + "alimony_expense", + "alimony_income", + "attends_eligible_educational_institution_for_american_opportunity_credit", + "auto_loan_balance", + "auto_loan_interest", + "bank_account_assets", + "block_geoid", + "bond_assets", + "business_is_sstb", + "casualty_loss", + "charitable_cash_donations", + "charitable_non_cash_donations", + "child_support_expense", + "child_support_received", + "congressional_district_geoid", + "county_fips", + "cps_race", + "detailed_occupation_recode", + "disability_benefits", + "domestic_production_ald", + "educational_assistance", + "educator_expense", + "employment_income_before_lsr", + "estate_income", + "estate_income_would_be_qualified", + "farm_income", + "farm_operations_income", + "farm_operations_income_would_be_qualified", + "farm_rent_income", + "farm_rent_income_would_be_qualified", + "first_home_mortgage_balance", + "first_home_mortgage_interest", + "first_home_mortgage_origination_year", + "fsla_overtime_premium", + "has_american_opportunity_credit_1098_t_or_exception", + "has_american_opportunity_credit_institution_ein", + "has_champva_health_coverage_at_interview", + "has_esi", + "has_indian_health_service_coverage_at_interview", + "has_marketplace_health_coverage_at_interview", + "has_medicaid_health_coverage_at_interview", + "has_never_worked", + "has_non_marketplace_direct_purchase_health_coverage_at_interview", + "has_other_means_tested_health_coverage_at_interview", + "has_tricare_health_coverage_at_interview", + "has_va_health_coverage_at_interview", + "health_insurance_premiums", + "health_insurance_premiums_without_medicare_part_b", + "health_savings_account_ald", + "home_mortgage_interest", + "hourly_wage", + "hours_worked_last_week", + "household_vehicles_owned", + "household_vehicles_value", + "household_weight", + "immigration_status_str", + "investment_income_elected_form_4952", + "investment_interest_expense", + "is_blind", + "is_computer_scientist", + "is_disabled", + "is_enrolled_at_least_half_time_for_american_opportunity_credit", + "is_executive_administrative_professional", + "is_farmer_fisher", + "is_female", + "is_full_time_college_student", + "is_hispanic", + "is_household_head", + "is_incapable_of_self_care", + "is_military", + "is_paid_hourly", + "is_pregnant", + "is_pursuing_credential_for_american_opportunity_credit", + "is_self_employed", + "is_separated", + "is_surviving_spouse", + "is_union_member_or_covered", + "keogh_distributions", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "meets_ssi_disability_criteria", + "miscellaneous_income", + "net_worth", + "non_qualified_dividend_income", + "non_sch_d_capital_gains", + "other_health_insurance_premiums", + "other_medical_expenses", + "over_the_counter_health_expenses", + "own_children_in_household", + "partnership_s_corp_income_would_be_qualified", + "pre_subsidy_care_expenses", + "pre_subsidy_rent", + "previous_year_income_available", + "qualified_bdc_income", + "qualified_dividend_income", + "qualified_passenger_vehicle_loan_interest", + "qualified_reit_and_ptp_income", + "qualified_tuition_expenses", + "real_estate_taxes", + "receives_housing_assistance", + "rental_income", + "rental_income_would_be_qualified", + "roth_401k_contributions_desired", + "roth_ira_contributions_desired", + "salt_refund_income", + "schedule_d_capital_gain_distributions", + "selected_marketplace_plan_benchmark_ratio", + "self_employed_pension_contributions_desired", + "self_employment_income_before_lsr", + "self_employment_income_last_year", + "self_employment_income_would_be_qualified", + "short_term_capital_gains", + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + "spm_unit_energy_subsidy", + "spm_unit_pre_subsidy_childcare_expenses", + "spm_unit_tenure_type", + "ssn_card_type", + "sstb_self_employment_income_before_lsr", + "sstb_self_employment_income_would_be_qualified", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "state_fips", + "stock_assets", + "student_loan_interest", + "takes_up_aca_if_eligible", + "takes_up_eitc", + "takes_up_head_start_if_eligible", + "takes_up_housing_assistance_if_eligible", + "takes_up_medicaid_if_eligible", + "takes_up_medicare_if_eligible", + "takes_up_snap_if_eligible", + "takes_up_ssi_if_eligible", + "takes_up_tanf_if_eligible", + "takes_up_wic_if_eligible", + "tax_exempt_interest_income", + "tax_exempt_ira_distributions", + "tax_exempt_private_pension_income", + "taxable_401k_distributions", + "taxable_403b_distributions", + "taxable_interest_income", + "taxable_ira_distributions", + "taxable_private_pension_income", + "taxable_sep_distributions", + "tenure_type", + "tip_income", + "tract_geoid", + "traditional_401k_contributions_desired", + "traditional_ira_contributions_desired", + "treasury_tipped_occupation_code", + "unadjusted_basis_qualified_property", + "unemployment_compensation", + "unrecaptured_section_1250_gain", + "unreimbursed_business_employee_expenses", + "veterans_benefits", + "w2_wages_from_qualified_business", + "weekly_hours_worked_before_lsr", + "weeks_unemployed", + "workers_compensation", + "would_file_taxes_voluntarily", +) +NATIONAL_CD_ENGINE_OMISSIONS = ("block_geoid", "tract_geoid") +NATIONAL_CD_REQUIRED_INPUTS = tuple( + name + for name in HISTORICAL_REQUIRED_INPUTS + if name not in NATIONAL_CD_ENGINE_OMISSIONS +) +# Independent build state, even when not an engine input in the selected profile. +ASSIGNED_BLOCK_COLUMN = "census_block_geoid" + + +def required_us_inputs( + profile: USInputProfile = USInputProfile.HISTORICAL, +) -> tuple[str, ...]: + """Return a closed profile; no custom name list or silent string fallback.""" + if type(profile) is not USInputProfile: + raise TypeError("US_INPUT_PROFILE_TYPE") + if profile is USInputProfile.HISTORICAL: + return HISTORICAL_REQUIRED_INPUTS + return NATIONAL_CD_REQUIRED_INPUTS diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py index 8f2ea06b9..1f4a86047 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py @@ -15,7 +15,7 @@ import hashlib import json -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field from functools import lru_cache from importlib.metadata import version @@ -38,12 +38,18 @@ from microcosm.build.us_runtime.adult_care import with_us_adult_care_inputs from microcosm.build.us_runtime.child_support import with_us_child_support_inputs from microcosm.build.us_runtime.childcare import with_us_childcare_inputs -from microcosm.build.us_runtime.cps_carried import derive_us_cps_carried_inputs +from microcosm.build.us_runtime.cps_carried import ( + CpsCarriedTables, + derive_us_cps_carried_inputs, + derive_us_cps_carried_tables, +) from microcosm.build.us_runtime.disability_benefits import ( with_us_disability_benefits, ) from microcosm.build.us_runtime.education_inputs import with_us_education_inputs from microcosm.build.us_runtime.eligibility_inputs import ( + prepare_us_eligibility_person, + us_eligibility_inputs_person_carries_signal, with_us_eligibility_inputs, ) from microcosm.build.us_runtime.energy_subsidy import ( @@ -51,8 +57,11 @@ ) from microcosm.build.us_runtime.hours_worked import ( US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS, + us_hours_worked_gate_from_summary, + us_hours_worked_person_summary, us_hours_worked_signal_gate, with_us_hours_worked_inputs, + with_us_hours_worked_person, ) from microcosm.build.us_runtime.housing_inputs import ( US_HOUSING_ASSISTANCE_PUF_MAX_TRAIN_SAMPLES, @@ -70,6 +79,7 @@ ) from microcosm.build.us_runtime.pregnancy import with_us_pregnancy_inputs from microcosm.build.us_runtime.prior_year_income import ( + prepare_us_prior_year_person, with_us_prior_year_income_inputs, ) from microcosm.build.us_runtime.puf_qrf_chain import PRIMARY_QRF_TARGET_ORDER @@ -84,6 +94,8 @@ with_us_qbi_input_reconciliation, ) from microcosm.build.us_runtime.relationship_inputs import ( + prepare_us_relationship_person, + us_relationship_inputs_person_carries_signal, with_us_relationship_inputs, ) from microcosm.build.us_runtime.retirement_contributions import ( @@ -92,10 +104,6 @@ from microcosm.build.us_runtime.retirement_distributions import ( with_us_retirement_distribution_inputs, ) -from microcosm.build.us_runtime.spine_agreement import ( - default_spine_agreement_registry, - spine_agreement_gate, -) from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.build.us_runtime.support_provenance import ( SPINE_ASSEMBLY_MANIFEST_KEY, @@ -121,7 +129,7 @@ from microcosm.build.us_runtime.workers_compensation import ( with_us_workers_compensation, ) -from microcosm.frame import US_SCHEMA, Frame +from microcosm.frame import US_SCHEMA, Frame, Weights from microcosm.frame.adapters.policyengine_us import ( PolicyEngineUSVariableMetadataIndex, VariableDependencyClosure, @@ -163,6 +171,18 @@ "finalize_multispine_source_inputs", "materialize_multispine_agreement_outputs", "materialize_pool_deferred_transfer_inputs", + "multispine_hours_worked_boundary_columns", + "multispine_housing_boundary_columns", + "multispine_housing_implementation_contract", + "multispine_housing_output_family", + "multispine_housing_source_selection", + "assert_multispine_source_merge_labels_supported", + "merge_multispine_housing_source_outputs", + "validate_multispine_housing_person_boundary", + "observe_multispine_hours_person", + "observe_multispine_hours_worked_inputs", + "validate_multispine_hours_person_boundary", + "validate_multispine_hours_worked_boundary", "pool_input_surface", "pool_engine_input_projection_receipt", "pool_remaining_stage_input_manifest", @@ -861,9 +881,8 @@ def _resolve_take_up_program_bindings( for program in load_take_up_contract().programs ) for index, binding in enumerate(bindings): - if ( - len(binding) != 3 - or not all(isinstance(value, str) and value for value in binding) + if len(binding) != 3 or not all( + isinstance(value, str) and value for value in binding ): raise ValueError( "Take-up manifest program binding must contain three non-empty " @@ -1333,9 +1352,7 @@ def surface_provision(variable: str) -> str: variable, execution_scope="whole_pool", provision=provision, - available_by=( - "transferred" if variable in transfer_owned else "seeded" - ), + available_by=("transferred" if variable in transfer_owned else "seeded"), fallback=fallback, ) @@ -1715,6 +1732,14 @@ def materialize_pool_deferred_transfer_inputs(frame: Frame) -> PoolStageOutput: return PoolStageOutput(result, {"inputs": receipts}) +# The default agreement registry validates the engine ABI, whose fresh +# manifest reads the pool functions above. Define that surface before importing +# the registry so either module can be the first import in a fresh process. +from microcosm.build.us_runtime.spine_agreement import ( # noqa: E402 + default_spine_agreement_registry, + spine_agreement_gate, +) + POOL_SPINE_AGREEMENT_REGISTRY = default_spine_agreement_registry( pool_transfer_target_families() ) @@ -1792,32 +1817,872 @@ def prepare_multispine_source_inputs_for_clone( def _with_gated_us_hours_worked_inputs(frame: Frame) -> PoolStageOutput: """Run the shared hours kernel, then keep only pool-owned input leaves.""" + observed = _observe_us_hours_worked_inputs(frame) + gate = observed.receipt["hours_worked_signal_gate"] + if not gate["passed"]: + raise ValueError( + "Pool pre-clone hours-worked signal gate failed:\n " + + "\n ".join(gate["failures"]) + ) + return observed + + +def _observe_us_hours_worked_inputs(frame: Frame) -> PoolStageOutput: + """Compute actual values and gate evidence without certifying the surface.""" + produced = with_us_hours_worked_inputs( frame, seed=POOL_RANDOM_SEED, time_period=POOL_TIME_PERIOD, ) gate = us_hours_worked_signal_gate(produced) - if not gate.passed: - raise ValueError( - "Pool pre-clone hours-worked signal gate failed:\n " - + "\n ".join(gate.failures) - ) pool_surface, removed = _drop_source_output_columns( produced, {"person": US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS}, ) return PoolStageOutput( pool_surface, + _hours_gate_receipt(gate, removed), + ) + + +def _hours_gate_receipt( + gate: GateResult, removed: Mapping[str, list[str]] +) -> dict[str, object]: + return { + "hours_worked_signal_gate": { + "name": gate.name, + "passed": gate.passed, + "failures": list(gate.failures), + "details": dict(gate.details), + }, + "pool_excluded_outputs_removed": dict(removed), + } + + +def multispine_hours_worked_boundary_columns() -> tuple[str, ...]: + """Declare source-boundary evidence without exposing role dispatch downstream.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def multispine_cps_carried_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone CPS evidence to its graph boundary.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_cps_carried_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Retain source-role validation in the existing registered owner.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_cps_carried_implementation_contract() -> dict[str, object]: + """Bind live provider projections actually consumed by this source operator.""" + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[ + "cps_carried" + ].items() + }, + "transient_outputs": _transient_source_outputs( + ("derive_us_cps_carried_inputs",), PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + ), + } + + +def observe_multispine_cps_carried_tables( + person: pd.DataFrame, + spm_unit: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[CpsCarriedTables, dict[str, object]]: + """Use actual person/SPM views for the original source projection and merge.""" + boundary = validate_multispine_cps_carried_person_boundary( + person, metadata=metadata + ) + if ( + set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or pool_rows["spm_unit"] != len(spm_unit) + or len(weights) != len(person) + ): + raise ValueError("CPS-carried population counts or weights are misaligned.") + for group in US_SCHEMA.group_entities: + if pool_rows[group] != int( + person[US_SCHEMA.membership_column(group)].nunique() + ): + raise ValueError("CPS-carried population counts differ from memberships.") + if spm_unit.spm_unit_id.duplicated().any() or set(spm_unit.spm_unit_id) != set( + person.person_spm_unit_id + ): + raise ValueError("CPS-carried SPM rows differ from actual memberships.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected_person = without_support_role_metadata(person.loc[mask], entity="person") + selected_spm = without_support_role_metadata( + spm_unit.loc[ + spm_unit.spm_unit_id.isin(selected_person.person_spm_unit_id) + ].reset_index(drop=True), + entity="spm_unit", + ) + # Match selected Frame weight validation without constructing a partial + # Frame carrying whole-population metadata or unobserved entity tables. + weights.with_values(weights.values[mask], kind=weights.kind) + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["cps_carried"] + selected = { + "person": selected_person.drop( + columns=_unavailable_output_columns(selected_person, outputs["person"]) + ), + "spm_unit": selected_spm.drop( + columns=_unavailable_output_columns(selected_spm, outputs["spm_unit"]) + ), + } + produced = derive_us_cps_carried_tables(selected["person"], selected["spm_unit"]) + merged = {} + merged_rows = {} + for entity, outcome in ( + ("person", produced.person), + ("spm_unit", produced.spm_unit), + ): + identity = US_SCHEMA.entity_id_column(entity) + _assert_source_table_identity( + selected[entity], + outcome, + entity_id=identity, + operator_name="derive_us_cps_carried_inputs", + ) + target = person if entity == "person" else spm_unit + merged[entity], merged_rows[entity] = _merge_source_person_or_group_outputs( + target.copy(), + outcome, + outputs[entity], + entity=entity, + entity_id=identity, + operator_name="derive_us_cps_carried_inputs", + ) + selected_rows = { + "person": len(selected_person), + **{ + group: int(selected_person[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + }, + } + receipt = _source_operator_receipt( + order_index=0, + operator_name="derive_us_cps_carried_inputs", + family="cps_carried", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["derive_us_cps_carried_inputs"], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows=merged_rows, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return CpsCarriedTables( + merged["person"], merged["spm_unit"] + ), _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("derive_us_cps_carried_inputs",), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_prior_year_boundary_columns() -> tuple[str, ...]: + """Declare the existing pre-clone evidence at its registered source owner.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_prior_year_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Reject support populations before the source-only prior-year join.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_prior_year_implementation_contract() -> dict[str, object]: + """Bind the live output and transient projections consumed by this operator.""" + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[ + "prior_year_income" + ].items() + }, + "transient_outputs": _transient_source_outputs( + ("with_us_prior_year_income_inputs",), + PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ), + "seed": POOL_RANDOM_SEED, + "time_period": POOL_TIME_PERIOD, + } + + +def observe_multispine_prior_year_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the real pre-clone join on its declared source person projection. + + Canonical person weights are selected in the original person order. Actual + memberships determine selected group counts; authenticated full counts + describe the unchanged pool. No incomplete Frame is constructed. Support + role fields are removed only from the validated ephemeral source view, so + the helper's support-QRF path cannot run at this boundary. + """ + boundary = validate_multispine_prior_year_person_boundary(person, metadata=metadata) + if ( + not isinstance(weights, Weights) + or set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or len(weights) != len(person) + ): + raise ValueError("Prior-year population counts or weights are misaligned.") + for group in US_SCHEMA.group_entities: + if pool_rows[group] != int( + person[US_SCHEMA.membership_column(group)].nunique() + ): + raise ValueError("Prior-year population counts differ from memberships.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected = without_support_role_metadata(person.loc[mask], entity="person") + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["prior_year_income"] + selected = selected.drop( + columns=_unavailable_output_columns(selected, outputs["person"]) + ) + selected_weights = weights.with_values(weights.values[mask], kind=weights.kind) + produced = prepare_us_prior_year_person( + selected, + weights=selected_weights, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ) + _assert_source_table_identity( + selected, + produced, + entity_id="person_id", + operator_name="with_us_prior_year_income_inputs", + ) + merged, count = _merge_source_person_or_group_outputs( + person.copy(), + produced, + outputs["person"], + entity="person", + entity_id="person_id", + operator_name="with_us_prior_year_income_inputs", + ) + selected_rows = { + "person": len(selected), + **{ + group: int(selected[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + }, + } + receipt = _source_operator_receipt( + order_index=0, + operator_name="with_us_prior_year_income_inputs", + family="prior_year_income", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["with_us_prior_year_income_inputs"], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows={"person": count}, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return merged, _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_prior_year_income_inputs",), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_relationship_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone evidence for relationship preparation.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def multispine_eligibility_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone evidence for eligibility preparation.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_relationship_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Reject support populations before the source-only relationship derivation.""" + return _validate_multispine_person_preparation_boundary(person, metadata=metadata) + + +def validate_multispine_eligibility_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Reject support populations before the source-only eligibility derivation.""" + return _validate_multispine_person_preparation_boundary(person, metadata=metadata) + + +def _validate_multispine_person_preparation_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Validate actual person evidence at the registered source boundary.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_relationship_implementation_contract() -> dict[str, object]: + """Bind the live projections and options this source operator executes.""" + return _multispine_person_preparation_contract( + "with_us_relationship_inputs", "relationship_inputs" + ) + + +def multispine_eligibility_implementation_contract() -> dict[str, object]: + """Bind the live projections and options this source operator executes.""" + return _multispine_person_preparation_contract( + "with_us_eligibility_inputs", "eligibility_inputs" + ) + + +def _multispine_person_preparation_contract( + operator_name: str, family: str +) -> dict[str, object]: + """Project the live output/transient rosters and the executed options. + + These values move through imported registries without an edit here, so a + graph identity that omits them would not describe what actually ran. + """ + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[family].items() + }, + "transient_outputs": _transient_source_outputs( + (operator_name,), PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + ), + "seed": POOL_RANDOM_SEED, + "time_period": POOL_TIME_PERIOD, + } + + +def observe_multispine_relationship_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the real pre-clone relationship derivation on its person projection. + + Canonical person weights are selected in the original person order and the + authenticated full counts describe the unchanged pool. No incomplete Frame + is constructed. Support role fields are removed only from the validated + ephemeral source view. The public pass-through predicate decides whether + the always-recomputing helper runs, so the wrapper's order — signal check + before weight validation and before any raw-column requirement — holds. + """ + return _observe_multispine_person_preparation( + person, + weights=weights, + metadata=metadata, + pool_rows=pool_rows, + operator_name="with_us_relationship_inputs", + family="relationship_inputs", + label="Relationship-input", + carries_signal=lambda table: us_relationship_inputs_person_carries_signal( + table + ), + prepare=lambda table, values: prepare_us_relationship_person( + table, + values, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ), + ) + + +def observe_multispine_eligibility_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the real pre-clone eligibility derivation on its person projection. + + The same registered boundary, canonical weight selection and full-pool ID + merge as the relationship observer. The legacy partial-null pass-through + rule — all five outputs present and observed disability varying — is the + public predicate's, not this helper's, so the graph cannot strengthen or + weaken it. + """ + return _observe_multispine_person_preparation( + person, + weights=weights, + metadata=metadata, + pool_rows=pool_rows, + operator_name="with_us_eligibility_inputs", + family="eligibility_inputs", + label="Eligibility-input", + carries_signal=lambda table: us_eligibility_inputs_person_carries_signal(table), + prepare=lambda table, values: prepare_us_eligibility_person( + table, + values, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ), + ) + + +def _observe_multispine_person_preparation( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], + operator_name: str, + family: str, + label: str, + carries_signal: Callable[[pd.DataFrame], bool], + prepare: Callable[[pd.DataFrame, np.ndarray], pd.DataFrame], +) -> tuple[pd.DataFrame, dict[str, object]]: + """The shared person-preparation source projection, run and full-pool merge.""" + boundary = _validate_multispine_person_preparation_boundary( + person, metadata=metadata + ) + if ( + not isinstance(weights, Weights) + or set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or len(weights) != len(person) + ): + raise ValueError(f"{label} population counts or weights are misaligned.") + for group in US_SCHEMA.group_entities: + if pool_rows[group] != int( + person[US_SCHEMA.membership_column(group)].nunique() + ): + raise ValueError(f"{label} population counts differ from memberships.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected = without_support_role_metadata(person.loc[mask], entity="person") + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[family] + selected = selected.drop( + columns=_unavailable_output_columns(selected, outputs["person"]) + ) + selected_weights = weights.with_values(weights.values[mask], kind=weights.kind) + # The wrapper returns its frame untouched on a signal-carrying surface, + # reading neither weights nor raw source columns. Preserve that order. + produced = ( + selected + if carries_signal(selected) + else prepare(selected, selected_weights.values) + ) + _assert_source_table_identity( + selected, + produced, + entity_id="person_id", + operator_name=operator_name, + ) + merged, count = _merge_source_person_or_group_outputs( + person.copy(), + produced, + outputs["person"], + entity="person", + entity_id="person_id", + operator_name=operator_name, + ) + selected_rows = { + "person": len(selected), + **{ + group: int(selected[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + }, + } + receipt = _source_operator_receipt( + order_index=0, + operator_name=operator_name, + family=family, + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS[operator_name], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows={"person": count}, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return merged, _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=(operator_name,), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_hours_implementation_contract() -> dict[str, object]: + """Bind live provider rosters actually consumed by the hours projection. + + These values can change through imported registries without an edit to + this module. Keep their actual projection in graph implementation identity. + """ + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[ + "hours_worked" + ].items() + }, + "transient_outputs": _transient_source_outputs( + ("with_us_hours_worked_inputs",), + PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ), + } + + +def validate_multispine_hours_worked_boundary(frame: Frame) -> dict[str, object]: + """Validate the existing pre-clone boundary before graph hours derivation.""" + return validate_multispine_hours_person_boundary( + frame.person, metadata=frame.metadata + ) + + +def validate_multispine_hours_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Validate actual person evidence at the registered source boundary.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + available = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(available.sum())} + + +def observe_multispine_hours_worked_inputs(frame: Frame) -> PoolStageOutput: + """Run one registered source operator and retain its uncertified gate evidence. + + The graph's mandatory gate must accept this evidence before any downstream + context is released. Projection, structure checks and ID merge are the + exact same source-boundary implementation used by the direct path. + """ + return _run_source_operator_chain( + frame, + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_hours_worked_inputs",), + operators={"with_us_hours_worked_inputs": _observe_us_hours_worked_inputs}, + ) + + +def observe_multispine_hours_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the registered hours projection using only actual person columns. + + ``weights`` must be the pool's canonically resolved person vector. Full + entity counts come from the authenticated population descriptor; selected + group counts use actual membership IDs, as Frame.select prunes each group + to the referenced IDs. No partial Frame carries whole-pool metadata. + """ + boundary = validate_multispine_hours_person_boundary(person, metadata=metadata) + if ( + set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or len(weights) != len(person) + ): + raise ValueError("Measured hours population counts or weights are misaligned.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected = without_support_role_metadata(person.loc[mask], entity="person") + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["hours_worked"] + selected = selected.drop( + columns=_unavailable_output_columns(selected, outputs["person"]) + ) + selected_weights = weights.with_values(weights.values[mask], kind=weights.kind) + selected_rows = {"person": len(selected)} + selected_rows.update( { - "hours_worked_signal_gate": { - "name": gate.name, - "passed": True, - "failures": [], - "details": dict(gate.details), - }, - "pool_excluded_outputs_removed": removed, + group: int(selected[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + } + ) + if any(selected_rows[entity] > pool_rows[entity] for entity in pool_rows): + raise ValueError("Measured hours membership counts exceed the population.") + produced = with_us_hours_worked_person( + selected, + weights=selected_weights, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ) + gate = us_hours_worked_gate_from_summary( + us_hours_worked_person_summary(produced, weights=selected_weights) + ) + removed = sorted(US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS & set(produced.columns)) + produced = produced.drop(columns=removed) + _assert_source_table_identity( + selected, + produced, + entity_id="person_id", + operator_name="with_us_hours_worked_inputs", + ) + merged, count = _merge_source_person_or_group_outputs( + person.copy(), + produced, + outputs["person"], + entity="person", + entity_id="person_id", + operator_name="with_us_hours_worked_inputs", + ) + receipt = _source_operator_receipt( + order_index=0, + operator_name="with_us_hours_worked_inputs", + family="hours_worked", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["with_us_hours_worked_inputs"], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows={"person": count}, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt=_hours_gate_receipt( + gate, {"person": removed} if removed else {} + ), + overlap_ownership=None, + ) + return merged, _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_hours_worked_inputs",), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_housing_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone housing evidence to its graph boundary.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_housing_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Retain source-role validation in the existing registered owner.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_housing_source_selection( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> tuple[pd.Series, dict[str, object]]: + """Return the registered CPS person mask and its boundary receipt. + + The mask is the same one the direct source-operator chain applies, so a + graph FILTER built from it selects exactly the historical projection rows. + """ + boundary = validate_multispine_housing_person_boundary(person, metadata=metadata) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return mask, boundary + + +def multispine_housing_output_family() -> Mapping[str, frozenset[str]]: + """Return the live registered housing output roster.""" + return PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["housing_inputs"] + + +def multispine_housing_implementation_contract() -> dict[str, object]: + """Bind live provider rosters actually consumed by this source operator.""" + contract = POOL_OPERATOR_CONTRACTS["with_us_housing_inputs"] + return { + "outputs": { + entity: sorted(columns) + for entity, columns in multispine_housing_output_family().items() + }, + "transient_outputs": _transient_source_outputs( + ("with_us_housing_inputs",), PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + ), + "formula_owned_outputs": { + entity: sorted( + set(columns) & set(_FORMULA_OWNED_SOURCE_OUTPUTS.get(entity, ())) + ) + for entity, columns in multispine_housing_output_family().items() }, + "execution_scope": contract.execution_scope, + "phases": list(contract.phases), + "family": contract.family, + "seed": POOL_RANDOM_SEED, + "time_period": POOL_TIME_PERIOD, + } + + +def assert_multispine_source_merge_labels_supported( + target: pd.DataFrame, + columns: Iterable[str], + *, + entity: str, + operator_name: str, + source: pd.DataFrame | None = None, +) -> None: + """Refuse the inherited label-based merge seam instead of working around it. + + :func:`_merge_source_person_or_group_outputs` writes a non-boolean output + through ``target.loc[target.index[positions]]``. With repeated pandas index + labels that assignment reaches every row sharing a selected label, so the + combination of duplicated labels and that write is refused at this explicit + scope boundary. Fixing the shared merge is a separate approved scope; + unique labels retain exact original parity. + + The merge picks its write path from the **produced** values, so pass + ``source`` to decide from the same series it does. Without ``source`` this + falls back to the incumbent rather than the actual write dtype: it can + over-refuse a boolean-producing column whose incumbent is not boolean, and + it would under-refuse a non-boolean-producing column whose incumbent is. + """ + if not target.index.has_duplicates: + return + + def writes_by_label(column: str) -> bool: + decided = ( + target[column] + if source is None or column not in source + else (source[column]) + ) + return not _is_physical_boolean_series(decided) + + affected = sorted( + column for column in columns if column in target and writes_by_label(column) + ) + if affected: + raise ValueError( + f"Multispine source operator {operator_name!r} cannot merge " + f"{entity!r} output(s) {affected} onto repeated index labels; the " + "shared label-based merge would write every row sharing a label. " + "Fixing that inherited seam is a separate reviewed scope." + ) + + +def merge_multispine_housing_source_outputs( + tables: Mapping[str, pd.DataFrame], + produced: Mapping[str, pd.DataFrame], + *, + pool_rows: Mapping[str, int], + selected_rows: Mapping[str, int], + evidence_rows: int, +) -> tuple[dict[str, pd.DataFrame], dict[str, int], dict[str, object]]: + """Merge the five declared housing leaves by entity ID and receipt the order. + + ``tables`` are isolated full-pool copies and ``produced`` the projected + source rows carrying every declared output. Column values, the ID merge and + the ordered receipt come from the existing registered helpers; this wrapper + adds no new formula and changes no selection semantics. + """ + outputs = multispine_housing_output_family() + if set(produced) != set(outputs) or set(outputs) - set(tables): + raise ValueError( + "US housing source merge requires its three declared output tables." + ) + merged: dict[str, pd.DataFrame] = { + entity: table for entity, table in tables.items() + } + merged_rows: dict[str, int] = {} + for entity in sorted(outputs): + columns = outputs[entity] + identity = US_SCHEMA.entity_id_column(entity) + assert_multispine_source_merge_labels_supported( + merged[entity], + columns, + entity=entity, + operator_name="with_us_housing_inputs", + source=produced[entity], + ) + merged[entity], merged_rows[entity] = _merge_source_person_or_group_outputs( + merged[entity], + produced[entity], + columns, + entity=entity, + entity_id=identity, + operator_name="with_us_housing_inputs", + ) + receipt = _source_operator_receipt( + order_index=0, + operator_name="with_us_housing_inputs", + family="housing_inputs", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["with_us_housing_inputs"], + before_rows=dict(pool_rows), + available_rows=dict(selected_rows), + output_rows=dict(selected_rows), + merged_rows=merged_rows, + declared_outputs=dict(outputs), + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return ( + merged, + merged_rows, + _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_housing_inputs",), + evidence_rows=evidence_rows, + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ), ) @@ -2265,43 +3130,21 @@ def _run_source_operator_chain( f"{operator_name!r}: input={before_rows}, output={after_rows}." ) receipts.append( - { - "order_index": order_index, - "operator": operator_name, - "family": family, - "phase": phase, - "execution_scope": contract.execution_scope, - "pool_input_rows": before_rows, - "operator_input_rows": available_rows, - "cps_available_rows": ( - available_rows - if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE - else None - ), - "operator_output_rows": output_rows, - "merged_rows": merged_rows, - "operator_projection": { - "selection": ( - _CPS_SOURCE_EVIDENCE_COLUMN - if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE - else _WHOLE_POOL_EXECUTION_SCOPE - ), - "lineage_state_persisted": ( - contract.execution_scope == _WHOLE_POOL_EXECUTION_SCOPE - ), - "support_role_metadata_exposed": phase == _POST_CLONE_PHASE, - }, - "output_columns": { - entity: sorted(columns) - for entity, columns in declared_outputs.items() - if columns - }, - "formula_owned_outputs_removed": formula_owned_removed, - "kernel_receipt": dict(kernel_receipt), - "overlap_ownership": ( - dict(overlap_ownership) if overlap_ownership is not None else None - ), - } + _source_operator_receipt( + order_index=order_index, + operator_name=operator_name, + family=family, + phase=phase, + contract=contract, + before_rows=before_rows, + available_rows=available_rows, + output_rows=output_rows, + merged_rows=merged_rows, + declared_outputs=declared_outputs, + formula_owned_removed=formula_owned_removed, + kernel_receipt=kernel_receipt, + overlap_ownership=overlap_ownership, + ) ) uses_cps_source = any( POOL_OPERATOR_CONTRACTS[name].execution_scope == _CPS_SOURCE_EXECUTION_SCOPE @@ -2309,38 +3152,126 @@ def _run_source_operator_chain( ) return PoolStageOutput( current, - { - "phase": phase, - "operator_order": list(operator_names), - "cps_source_evidence": ( - { - "column": _CPS_SOURCE_EVIDENCE_COLUMN, - "person_rows": int( - _cps_source_evidence_mask(frame, phase=phase).sum() - ), - } + _source_chain_receipt( + phase=phase, + operator_names=operator_names, + evidence_rows=( + int(_cps_source_evidence_mask(frame, phase=phase).sum()) if uses_cps_source else None ), - "transient_outputs_carried_through_clone": ( - _transient_source_outputs(operator_names, output_families) - if phase == _PRE_CLONE_PHASE - else {} + receipts=receipts, + output_families=output_families, + ), + ) + + +def _source_chain_receipt( + *, + phase: str, + operator_names: tuple[str, ...], + evidence_rows: int | None, + receipts: list[dict[str, object]], + output_families: Mapping[str, Mapping[str, frozenset[str]]], +) -> dict[str, object]: + return { + "phase": phase, + "operator_order": list(operator_names), + "cps_source_evidence": ( + { + "column": _CPS_SOURCE_EVIDENCE_COLUMN, + "person_rows": evidence_rows, + } + if evidence_rows is not None + else None + ), + "transient_outputs_carried_through_clone": ( + _transient_source_outputs(operator_names, output_families) + if phase == _PRE_CLONE_PHASE + else {} + ), + "suboperators": receipts, + } + + +def _source_operator_receipt( + *, + order_index: int, + operator_name: str, + family: str, + phase: str, + contract: SourceOperatorContract, + before_rows: Mapping[str, int], + available_rows: Mapping[str, int], + output_rows: Mapping[str, int], + merged_rows: Mapping[str, int], + declared_outputs: Mapping[str, frozenset[str]], + formula_owned_removed: Mapping[str, list[str]], + kernel_receipt: Mapping[str, object], + overlap_ownership: Mapping[str, object] | None, +) -> dict[str, object]: + return { + "order_index": order_index, + "operator": operator_name, + "family": family, + "phase": phase, + "execution_scope": contract.execution_scope, + "pool_input_rows": before_rows, + "operator_input_rows": available_rows, + "cps_available_rows": ( + available_rows + if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE + else None + ), + "operator_output_rows": output_rows, + "merged_rows": merged_rows, + "operator_projection": { + "selection": ( + _CPS_SOURCE_EVIDENCE_COLUMN + if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE + else _WHOLE_POOL_EXECUTION_SCOPE + ), + "lineage_state_persisted": ( + contract.execution_scope == _WHOLE_POOL_EXECUTION_SCOPE ), - "suboperators": receipts, + "support_role_metadata_exposed": phase == _POST_CLONE_PHASE, }, - ) + "output_columns": { + entity: sorted(columns) + for entity, columns in declared_outputs.items() + if columns + }, + "formula_owned_outputs_removed": formula_owned_removed, + "kernel_receipt": dict(kernel_receipt), + "overlap_ownership": ( + dict(overlap_ownership) if overlap_ownership is not None else None + ), + } def _assert_source_operator_boundary(frame: Frame, *, phase: str) -> None: - manifest = frame.metadata.get(SPINE_ASSEMBLY_MANIFEST_KEY) + _assert_source_person_boundary( + frame.table(frame.schema.person_entity), + metadata=frame.metadata, + person_entity=frame.schema.person_entity, + phase=phase, + ) + + +def _assert_source_person_boundary( + person: pd.DataFrame, + *, + metadata: Mapping[str, object], + person_entity: str, + phase: str, +) -> None: + manifest = metadata.get(SPINE_ASSEMBLY_MANIFEST_KEY) if not isinstance(manifest, Mapping): raise ValueError( "Multispine source operators require the immutable spine assembly " "manifest before any source derivation." ) - person = frame.table(frame.schema.person_entity) - clone_column = support_clone_index_column(frame.schema.person_entity) + clone_column = support_clone_index_column(person_entity) if clone_column not in person: raise ValueError( "Multispine source operators require post-assembly clone provenance; " @@ -2370,7 +3301,19 @@ def _assert_source_operator_boundary(frame: Frame, *, phase: str) -> None: def _cps_source_evidence_mask(frame: Frame, *, phase: str) -> pd.Series: """Select CPS lineage only from a raw column unavailable on ACS.""" - person = frame.table(frame.schema.person_entity) + return _cps_person_evidence_mask( + frame.table(frame.schema.person_entity), + person_entity=frame.schema.person_entity, + phase=phase, + ) + + +def _cps_person_evidence_mask( + person: pd.DataFrame, + *, + person_entity: str, + phase: str, +) -> pd.Series: if _CPS_SOURCE_EVIDENCE_COLUMN not in person: raise ValueError( "Multispine source operators require raw CPS evidence column " @@ -2385,7 +3328,7 @@ def _cps_source_evidence_mask(frame: Frame, *, phase: str) -> pd.Series: "Multispine source operators found no CPS-evidenced person rows in " f"{_CPS_SOURCE_EVIDENCE_COLUMN!r}." ) - clone_column = support_clone_index_column(frame.schema.person_entity) + clone_column = support_clone_index_column(person_entity) clone_index = pd.to_numeric(person[clone_column], errors="coerce") evidenced_clones = set(clone_index.loc[available].astype(int).tolist()) invalid_evidence = ( @@ -2679,15 +3622,39 @@ def _assert_source_operator_structure( ) for entity in before.entities: entity_id = before.schema.entity_id_column(entity) - before_ids = before.table(entity)[entity_id] - after_ids = after.table(entity)[entity_id] - if after_ids.duplicated().any() or set(after_ids.tolist()) != set( - before_ids.tolist() - ): - raise ValueError( - f"Multispine source operator {operator_name!r} changed structural " - f"{entity_id!r} values." - ) + _assert_source_table_identity( + before.table(entity), + after.table(entity), + entity_id=entity_id, + operator_name=operator_name, + ) + + +def _assert_source_table_identity( + before: pd.DataFrame, + after: pd.DataFrame, + *, + entity_id: str, + operator_name: str, +) -> None: + before_ids = before[entity_id] + after_ids = after[entity_id] + if after_ids.duplicated().any() or set(after_ids.tolist()) != set( + before_ids.tolist() + ): + raise ValueError( + f"Multispine source operator {operator_name!r} changed structural " + f"{entity_id!r} values." + ) + + +def _unavailable_output_columns( + table: pd.DataFrame, + columns: frozenset[str], +) -> list[str]: + return [ + column for column in columns if column in table and table[column].isna().all() + ] def _without_unavailable_output_columns( @@ -2701,11 +3668,7 @@ def _without_unavailable_output_columns( for entity, columns in outputs.items(): if entity not in tables: continue - unavailable = [ - column - for column in columns - if column in tables[entity] and tables[entity][column].isna().all() - ] + unavailable = _unavailable_output_columns(tables[entity], columns) if unavailable: tables[entity] = tables[entity].drop(columns=unavailable) dropped = True @@ -2743,83 +3706,14 @@ def _merge_source_operator_outputs( target = tables[entity] source = operated.table(entity) entity_id = pool.schema.entity_id_column(entity) - if entity_id not in target or entity_id not in source: - raise ValueError( - f"Multispine source operator {operator_name!r} cannot align " - f"{entity!r} without {entity_id!r}." - ) - if source[entity_id].duplicated().any(): - raise ValueError( - f"Multispine source operator {operator_name!r} returned duplicate " - f"{entity_id!r} values." - ) - missing_outputs = sorted(set(columns) - set(source.columns)) - if missing_outputs: - raise ValueError( - f"Multispine source operator {operator_name!r} did not emit its " - f"declared {entity!r} output(s): {missing_outputs}." - ) - source_by_id = source.set_index(entity_id) - target_ids = target[entity_id] - eligible = target_ids.isin(source_by_id.index) - if int(eligible.sum()) != len(source): - raise ValueError( - f"Multispine source operator {operator_name!r} output IDs do not " - f"align one-to-one with the {entity!r} pool." - ) - for column in sorted(columns): - source_values = source_by_id[column] - aligned = source_values.reindex(target_ids) - source_is_boolean = _is_physical_boolean_series(source_values) - if source_is_boolean: - positions = np.flatnonzero(eligible.to_numpy()) - aligned_boolean = pd.Series( - pd.array(aligned, dtype="boolean"), - index=target.index, - name=column, - ) - if column not in target: - target[column] = aligned_boolean - continue - incumbent = target[column] - invalid_incumbent = incumbent.dropna().map( - lambda value: not isinstance(value, (bool, np.bool_)) - ) - if invalid_incumbent.any(): - offending_types = sorted( - { - f"{type(value).__module__}.{type(value).__qualname__}" - for value in incumbent.dropna().loc[invalid_incumbent] - } - ) - raise TypeError( - f"Multispine source operator {operator_name!r} emitted " - f"physical booleans for {entity}.{column}, but the pool " - "materialized observed non-boolean values with " - f"dtype {incumbent.dtype!s}: {offending_types}." - ) - merged_boolean = pd.Series( - pd.array(incumbent, dtype="boolean"), - index=target.index, - name=column, - ) - merged_boolean.iloc[positions] = aligned_boolean.iloc[positions].array - target[column] = merged_boolean - continue - if column in target and pd.api.types.is_bool_dtype(target[column].dtype): - raise TypeError( - f"Multispine source operator {operator_name!r} emitted " - f"non-boolean values for boolean-materialized " - f"{entity}.{column}; source dtype={source_values.dtype!s}." - ) - if column not in target: - target[column] = aligned.to_numpy() - else: - positions = np.flatnonzero(eligible.to_numpy()) - target.loc[target.index[positions], column] = aligned.iloc[ - positions - ].to_numpy() - merged_rows[entity] = int(eligible.sum()) + tables[entity], merged_rows[entity] = _merge_source_person_or_group_outputs( + target, + source, + columns, + entity=entity, + entity_id=entity_id, + operator_name=operator_name, + ) merged = Frame( tables, @@ -2832,6 +3726,95 @@ def _merge_source_operator_outputs( return merged, merged_rows +def _merge_source_person_or_group_outputs( + target: pd.DataFrame, + source: pd.DataFrame, + columns: frozenset[str], + *, + entity: str, + entity_id: str, + operator_name: str, +) -> tuple[pd.DataFrame, int]: + """The shared source-boundary ID merge; callers supply an isolated target.""" + if entity_id not in target or entity_id not in source: + raise ValueError( + f"Multispine source operator {operator_name!r} cannot align " + f"{entity!r} without {entity_id!r}." + ) + if source[entity_id].duplicated().any(): + raise ValueError( + f"Multispine source operator {operator_name!r} returned duplicate " + f"{entity_id!r} values." + ) + missing_outputs = sorted(set(columns) - set(source.columns)) + if missing_outputs: + raise ValueError( + f"Multispine source operator {operator_name!r} did not emit its " + f"declared {entity!r} output(s): {missing_outputs}." + ) + source_by_id = source.set_index(entity_id) + target_ids = target[entity_id] + eligible = target_ids.isin(source_by_id.index) + if int(eligible.sum()) != len(source): + raise ValueError( + f"Multispine source operator {operator_name!r} output IDs do not " + f"align one-to-one with the {entity!r} pool." + ) + for column in sorted(columns): + source_values = source_by_id[column] + aligned = source_values.reindex(target_ids) + source_is_boolean = _is_physical_boolean_series(source_values) + if source_is_boolean: + positions = np.flatnonzero(eligible.to_numpy()) + aligned_boolean = pd.Series( + pd.array(aligned, dtype="boolean"), + index=target.index, + name=column, + ) + if column not in target: + target[column] = aligned_boolean + continue + incumbent = target[column] + invalid_incumbent = incumbent.dropna().map( + lambda value: not isinstance(value, (bool, np.bool_)) + ) + if invalid_incumbent.any(): + offending_types = sorted( + { + f"{type(value).__module__}.{type(value).__qualname__}" + for value in incumbent.dropna().loc[invalid_incumbent] + } + ) + raise TypeError( + f"Multispine source operator {operator_name!r} emitted " + f"physical booleans for {entity}.{column}, but the pool " + "materialized observed non-boolean values with " + f"dtype {incumbent.dtype!s}: {offending_types}." + ) + merged_boolean = pd.Series( + pd.array(incumbent, dtype="boolean"), + index=target.index, + name=column, + ) + merged_boolean.iloc[positions] = aligned_boolean.iloc[positions].array + target[column] = merged_boolean + continue + if column in target and pd.api.types.is_bool_dtype(target[column].dtype): + raise TypeError( + f"Multispine source operator {operator_name!r} emitted " + f"non-boolean values for boolean-materialized " + f"{entity}.{column}; source dtype={source_values.dtype!s}." + ) + if column not in target: + target[column] = aligned.to_numpy() + else: + positions = np.flatnonzero(eligible.to_numpy()) + target.loc[target.index[positions], column] = aligned.iloc[ + positions + ].to_numpy() + return target, int(eligible.sum()) + + def _is_physical_boolean_series(values: pd.Series) -> bool: """Recognize boolean values without treating numeric 0/1 as booleans.""" diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py new file mode 100644 index 000000000..bf1a03535 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py @@ -0,0 +1,721 @@ +"""Explicit activation of the direct-national ACS 2024 published age counts. + +A captured reference bundle is evidence, not permission. The accepted +`national-derived-inventory.json` says so about itself: `target_activation` is +false, `calibration_target_registry` is `not_created`, and every table carries +`targets_allowed: false`. This module never reads that flag as authority and +never edits the bundle. Authority comes from +:data:`NATIONAL_AGE_ACTIVATION` — an immutable declaration, checked into this +repository and reviewed as code, that names the source digest, the exact +publisher cells, the target role, the population universe, the target period +and the age convention. :func:`activate_national_age_targets` refuses unless +the pinned bytes reproduce that declaration, and it then builds a **new** +:class:`~microcosm.calibrate.registry.TargetRegistry` artifact. + +What is authenticated here: the inventory bytes against the declared digest; +each selected raw Census response against the digest recorded for it *and* +against its request descriptor; the publisher's own labels, group, concept and +integer predicate for every selected cell; the single national row's +`GEO_ID`/`us`/`NAME` identity; and a from-bytes re-derivation of every selected +cell through the same :func:`classify_acs_value` precedence the district +inventory uses. Cells are parsed from exactly the bytes that were hashed. + +What is *not* established here: that the resulting calibration is nationally or +congressional-district valid, that any modelled person was observed in 2024, or +that this registry may be released. B19001 and B25003 are reserved same-source +holdouts and are refused outright — this module cannot activate them, and it +cannot mark them fresh or unseen. + +Published margins of error are converted to standard errors with the Census +90 percent factor and travel on the specs. The current Adam loss does not +consume them. A cell whose margin is a controlled-estimate sentinel carries a +**null** standard error, never zero: "no published variance" is not "no +variance". +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from dataclasses import asdict, dataclass +from pathlib import Path + +from microcosm.calibrate.hierarchy import ( + CalibrationHierarchy, + HierarchyCategory, + HierarchyDimension, + HierarchyGeography, + HierarchyNode, +) +from microcosm.calibrate.provider_labels import calibration_provider_label +from microcosm.calibrate.registry import TargetRegistry, TargetSpec +from microcosm.calibrate.variable_labels import calibration_variable_label + +from ..cd_benchmark.protocol import RESERVED_FAMILIES +from .cd_reference import AUTHORITIES, classify_acs_value +from .cd_reference_sources import strict_json + +__all__ = [ + "ACTIVATION_SCHEMA_VERSION", + "ActivationError", + "AgeBand", + "NationalAgeActivation", + "NATIONAL_AGE_ACTIVATION", + "activate_national_age_targets", + "activation_digest", + "band_columns", +] + +#: Revision of the activation declaration's canonical form. Bump it when the +#: meaning of a field changes; the digest already moves when a value changes. +ACTIVATION_SCHEMA_VERSION = 1 + +#: Census publishes ACS margins of error at 90 percent confidence; this is the +#: documented divisor that turns one into a standard error. The authority is +#: the same sample-size-and-data-quality note the district inventory cites. +_MOE_90_TO_SE = 1.645 +_MOE_AUTHORITY = AUTHORITIES["acs_confidence"] + +#: The published label prefix of the disjoint AGE distribution in S0101. Cells +#: under `SELECTED AGE CATEGORIES` overlap it and are refused by construction. +_AGE_LABEL_PREFIX = "Estimate!!Total!!Total population!!AGE!!" + +_HEX64 = re.compile(r"[0-9a-f]{64}") + + +class ActivationError(ValueError): + """The pinned evidence does not reproduce the activation declaration.""" + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ActivationError(reason) + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +@dataclass(frozen=True, slots=True) +class AgeBand: + """One activated publisher cell and the household column that answers it. + + Attributes: + variable: The publisher cell, e.g. ``"S0101_C01_002"``. + label: The publisher's exact label for that cell's estimate. Binding + it here means an edited cell list cannot silently pull a + differently-defined published quantity. + low: Inclusive lower bound in completed years. + high: Inclusive upper bound, or ``None`` for the open top band. + column: The household count column the population operator owns. + """ + + variable: str + label: str + low: int + high: int | None + column: str + + def __post_init__(self) -> None: + _require( + isinstance(self.variable, str) + and isinstance(self.label, str) + and isinstance(self.column, str) + and self.variable + and self.label + and self.column, + "AGE_BAND_TEXT", + ) + _require( + type(self.low) is int + and self.low >= 0 + and ( + self.high is None or (type(self.high) is int and self.high >= self.low) + ), + f"AGE_BAND_BOUNDS:{self.variable}", + ) + + +#: Diagnostics schema 8 hierarchy vocabulary for activated national age cells. +#: The provider and category labels come from the reviewed US label catalogs; +#: the geography is the activation's own national publisher row; the single +#: dimension is the publisher cell whose published label the band binds. +HIERARCHY_PROVIDER = "census_acs" +HIERARCHY_DIMENSION_ID = "publisher_cell" +HIERARCHY_DIMENSION_LABEL = "Publisher cell" +NATIONAL_GEOGRAPHY_ID = "0100000US" +NATIONAL_GEOGRAPHY_LABEL = "United States" +NATIONAL_GEOGRAPHY_LEVEL = "country" + + +def _hierarchy_category_id(table: str) -> str: + if table == "S0101": + return "population_by_age" + if table == "B01001": + return "population_by_sex_and_age" + raise ActivationError(f"UNSUPPORTED_TABLE:{table}") + + +def expected_demographic_hierarchy( + table: str, *, variable: str, label: str, geography: str +) -> CalibrationHierarchy: + """The only hierarchy a national demographic count cell may carry. + + Everything but the published cell label is fixed by the table and the + national geography, so a registry validator can rebuild it from a spec and + refuse any other provider, category, geography, dimension, or target. + """ + _require(geography == NATIONAL_GEOGRAPHY_ID, f"UNSUPPORTED_GEOGRAPHY:{geography}") + category_id = _hierarchy_category_id(table) + provider_label = calibration_provider_label("us", HIERARCHY_PROVIDER) + category_label = calibration_variable_label("us", HIERARCHY_PROVIDER, category_id) + _require(bool(provider_label) and bool(category_label), "HIERARCHY_LABELS") + return CalibrationHierarchy( + provider=HierarchyNode(HIERARCHY_PROVIDER, provider_label), + category=HierarchyCategory( + f"{HIERARCHY_PROVIDER}.{category_id}", category_label, HIERARCHY_PROVIDER + ), + geography=HierarchyGeography( + NATIONAL_GEOGRAPHY_ID, NATIONAL_GEOGRAPHY_LABEL, NATIONAL_GEOGRAPHY_LEVEL + ), + dimensions=( + HierarchyDimension( + HIERARCHY_DIMENSION_ID, HIERARCHY_DIMENSION_LABEL, variable, label + ), + ), + target=HierarchyNode(variable, label), + ) + + +def demographic_target_hierarchy( + table: str, band: AgeBand, *, geography: str +) -> CalibrationHierarchy: + """The complete schema 8 hierarchy for one activated band's target spec.""" + _require(type(band) is AgeBand, "AGE_BAND_TYPE") + return expected_demographic_hierarchy( + table, variable=band.variable, label=band.label, geography=geography + ) + + +def _bands() -> tuple[AgeBand, ...]: + quinary = [(low, low + 4) for low in range(0, 85, 5)] + bands = [] + for index, (low, high) in enumerate(quinary): + if low == 0: + label = "Under 5 years" + else: + label = f"{low} to {high} years" + bands.append( + AgeBand( + variable=f"S0101_C01_{index + 2:03d}", + label=_AGE_LABEL_PREFIX + label, + low=low, + high=high, + column=f"people_age_{low}_{high}", + ) + ) + bands.append( + AgeBand( + variable="S0101_C01_019", + label=_AGE_LABEL_PREFIX + "85 years and over", + low=85, + high=None, + column="people_age_85_plus", + ) + ) + return tuple(bands) + + +@dataclass(frozen=True, slots=True) +class NationalAgeActivation: + """The immutable authority for one calibration connection. + + Every field is normative: :func:`activation_digest` covers all of them, and + the digest travels into each produced :class:`TargetSpec`, so a changed + declaration produces a different registry version and a different node key. + + Attributes: + inventory_sha256: Digest of the accepted derived inventory's bytes. + table: The activated publisher table. + dataset: The Census API dataset the selected responses come from. + acs_year: The publisher's data year. + acs_release: The publisher's release. + geography: The direct national geography id. + geography_query: The publisher query that produced that single row. + published_name: The publisher's own name for that row. + confidence_level: The confidence level of the published margins. + role: What the activated cells are for. + universe: The population universe the cells count, stated in full. + entity: The entity whose weights the targets constrain. + period: The target period tag. + age_convention_id: The closed token naming the convention. The + population operator implements exactly this one and refuses any + other, so the declaration and the operator cannot drift apart. + age_convention: How model ages are read against the published bands. + allowed_reservation_status: The only inventory reservation status this + activation may act on. The same-source holdouts carry a different + one and can never satisfy it. + bands: The activated cells, in publisher order. + """ + + inventory_sha256: str + table: str + dataset: str + acs_year: int + acs_release: str + geography: str + geography_query: str + published_name: str + confidence_level: float + role: str + universe: str + entity: str + period: str + age_convention_id: str + age_convention: str + allowed_reservation_status: str + bands: tuple[AgeBand, ...] + + def __post_init__(self) -> None: + _require( + _HEX64.fullmatch(self.inventory_sha256) is not None, + "ACTIVATION_INVENTORY_DIGEST", + ) + _require(self.table not in RESERVED_FAMILIES, f"RESERVED_TABLE:{self.table}") + _require(self.table == "S0101", f"UNSUPPORTED_TABLE:{self.table}") + _require(self.role == "calibration", f"UNSUPPORTED_ROLE:{self.role}") + _require(self.universe.startswith("population"), "UNSUPPORTED_UNIVERSE") + _require(self.entity == "household", f"UNSUPPORTED_ENTITY:{self.entity}") + _require(self.period == "2024", f"UNSUPPORTED_PERIOD:{self.period}") + _require( + self.age_convention_id == "observed_interview_age_completed_years", + f"UNSUPPORTED_AGE_CONVENTION:{self.age_convention_id}", + ) + _require(self.acs_year == 2024, "UNSUPPORTED_ACS_YEAR") + _require(self.geography == "0100000US", "UNSUPPORTED_GEOGRAPHY") + _require( + self.allowed_reservation_status == "inventory_only", + "UNSUPPORTED_RESERVATION_STATUS", + ) + _require( + isinstance(self.bands, tuple) + and all(isinstance(band, AgeBand) for band in self.bands), + "ACTIVATION_BANDS_TYPE", + ) + _require(len(self.bands) == 18, f"ACTIVATION_BAND_COUNT:{len(self.bands)}") + _require( + len({band.variable for band in self.bands}) == len(self.bands) + and len({band.column for band in self.bands}) == len(self.bands), + "ACTIVATION_BANDS_NOT_DISTINCT", + ) + expected_low = 0 + for index, band in enumerate(self.bands): + _require(band.low == expected_low, f"ACTIVATION_BANDS_GAP:{band.variable}") + _require( + band.label.startswith(_AGE_LABEL_PREFIX), + f"ACTIVATION_BAND_NOT_AGE_DISTRIBUTION:{band.variable}", + ) + last = index == len(self.bands) - 1 + _require( + (band.high is None) is last, + f"ACTIVATION_BAND_OPEN_INTERVAL:{band.variable}", + ) + if band.high is not None: + expected_low = band.high + 1 + # Publisher cell order is the band order; nothing may be reordered or + # substituted without changing the digest. + _require( + [band.variable for band in self.bands] + == [f"S0101_C01_{index:03d}" for index in range(2, 20)], + "ACTIVATION_CELL_SELECTOR", + ) + + +#: The reviewed authority. Root's first engineering rung: observed interview +#: ages from pooled source cohorts, calibrated to the 2024 published national +#: distribution. No birth dates, no invented aging, and no claim that any +#: modelled person was interviewed in 2024. +NATIONAL_AGE_ACTIVATION = NationalAgeActivation( + inventory_sha256=( + "750b624e89557944c442600befb5557847dd75be6ce439c20daa9dfea5a0baf7" + ), + table="S0101", + dataset="acs/acs1/subject", + acs_year=2024, + acs_release="1-year", + geography="0100000US", + geography_query="us:*", + published_name="United States", + confidence_level=0.9, + role="calibration", + universe=( + "population: every resident person the publisher counts, household " + "and group-quarters alike; no household-population subsetting" + ), + entity="household", + period="2024", + age_convention_id="observed_interview_age_completed_years", + age_convention=( + "age in completed years as observed at each person's own source " + "interview, used verbatim; source cohorts are pooled and their " + "observed age distribution is calibrated to the 2024 published " + "counts; no birth dates, no aging, and no assertion that any person " + "was observed in 2024" + ), + allowed_reservation_status="inventory_only", + bands=_bands(), +) + + +def activation_digest(declaration: NationalAgeActivation) -> str: + """The content digest of a declaration's canonical form.""" + + # asdict recurses into the frozen AgeBand rows, so every bound, label and + # column name is inside the digest. + return hashlib.sha256( + _canonical({"schema_version": ACTIVATION_SCHEMA_VERSION, **asdict(declaration)}) + ).hexdigest() + + +def band_columns( + declaration: NationalAgeActivation = NATIONAL_AGE_ACTIVATION, +) -> tuple[str, ...]: + """The household count columns the activation expects, in band order.""" + + return tuple(band.column for band in declaration.bands) + + +def _read_pinned(path: Path, digest: str, *, what: str) -> tuple[bytes, object]: + """Hash the bytes, then parse *those* bytes. Never re-read from disk.""" + + _require(_HEX64.fullmatch(digest) is not None, f"PINNED_DIGEST_SHAPE:{what}") + _require(path.is_file(), f"PINNED_FILE_MISSING:{what}") + payload = path.read_bytes() + actual = hashlib.sha256(payload).hexdigest() + _require(actual == digest, f"PINNED_DIGEST_MISMATCH:{what}:{actual}") + return payload, strict_json(payload) + + +def _selected_source(inventory: dict, declaration: NationalAgeActivation, kind: str): + sources = inventory.get("sources") + _require(isinstance(sources, dict), "INVENTORY_SOURCES") + census = sources.get("census") + _require(isinstance(census, list), "INVENTORY_CENSUS_SOURCES") + entries = [ + entry + for entry in census + if isinstance(entry, dict) + and entry.get("table") == declaration.table + and entry.get("kind") == kind + ] + _require(len(entries) == 1, f"SOURCE_ENTRY_NOT_UNIQUE:{kind}:{len(entries)}") + entry = entries[0] + digest = entry.get("sha256") + _require( + isinstance(digest, str) and _HEX64.fullmatch(digest) is not None, + f"SOURCE_DIGEST:{kind}", + ) + _require(entry.get("dataset") == declaration.dataset, f"SOURCE_DATASET:{kind}") + _require(entry.get("year") == declaration.acs_year, f"SOURCE_YEAR:{kind}") + _require( + entry.get("geography") == declaration.geography_query, + f"SOURCE_GEOGRAPHY:{kind}", + ) + url = entry.get("url") + _require(isinstance(url, str) and url.startswith("https://"), f"SOURCE_URL:{kind}") + _require( + f"/{declaration.acs_year}/{declaration.dataset}" in url, + f"SOURCE_URL_SCOPE:{kind}", + ) + _require( + entry.get("path") == f"raw/{entry.get('sha256')}.json", f"SOURCE_PATH:{kind}" + ) + return entry + + +def _authenticated_response( + root: Path, entry: dict, *, kind: str +) -> tuple[bytes, object]: + """Authenticate the request descriptor and the response it names.""" + + descriptor_path = ( + root / "requests" / f"{hashlib.sha256(entry['url'].encode()).hexdigest()}.json" + ) + _require(descriptor_path.is_file(), f"REQUEST_DESCRIPTOR_MISSING:{kind}") + descriptor = strict_json(descriptor_path.read_bytes()) + _require(descriptor == entry, f"REQUEST_DESCRIPTOR_MISMATCH:{kind}") + payload, document = _read_pinned( + root / entry["path"], entry["sha256"], what=f"response:{kind}" + ) + _require(len(payload) == entry.get("size_bytes"), f"RESPONSE_SIZE:{kind}") + return payload, document + + +def _national_row(document: object, declaration: NationalAgeActivation) -> dict: + _require( + isinstance(document, list) and len(document) == 2, + "RESPONSE_NOT_A_SINGLE_NATIONAL_ROW", + ) + header, values = document + _require( + isinstance(header, list) and all(isinstance(name, str) for name in header), + "RESPONSE_HEADER_SHAPE", + ) + _require(len(set(header)) == len(header), "RESPONSE_DUPLICATE_HEADER") + _require(isinstance(values, list) and len(values) == len(header), "RESPONSE_WIDTH") + row = dict(zip(header, values, strict=True)) + _require(row.get("GEO_ID") == declaration.geography, "RESPONSE_GEO_ID") + _require(row.get("us") == "1", "RESPONSE_NOT_THE_NATION") + _require(row.get("NAME") == declaration.published_name, "RESPONSE_PUBLISHED_NAME") + return row + + +def _published_variable(metadata: object, name: str, declaration) -> dict: + _require(isinstance(metadata, dict), "METADATA_SHAPE") + variables = metadata.get("variables") + _require(isinstance(variables, dict), "METADATA_VARIABLES") + entry = variables.get(name) + _require(isinstance(entry, dict), f"METADATA_VARIABLE_MISSING:{name}") + _require(entry.get("group") == declaration.table, f"METADATA_GROUP:{name}") + _require(entry.get("concept") == "Age and Sex", f"METADATA_CONCEPT:{name}") + return entry + + +def _count_estimate(cell: dict, variable: str) -> int: + estimate = cell["estimate"] + _require( + estimate["class"] == "numeric" and not estimate["annotation_conflict"], + f"NON_COUNT_ESTIMATE:{variable}:{estimate['class']}", + ) + value = estimate["numeric_value"] + _require( + value is not None + and math.isfinite(value) + and value >= 0 + and float(value).is_integer(), + f"ESTIMATE_NOT_A_NONNEGATIVE_COUNT:{variable}", + ) + return int(value) + + +def _standard_error( + cell: dict, variable: str, *, confidence_level: float +) -> float | None: + """A published margin becomes a standard error; a sentinel becomes null.""" + + moe = cell["moe"] + _require(not moe["annotation_conflict"], f"MOE_ANNOTATION_CONFLICT:{variable}") + if moe["class"] != "numeric": + # Controlled, suppressed and open-interval margins carry no published + # variance. Null says "unknown"; zero would claim certainty. + return None + _require(confidence_level == 0.9, "UNSUPPORTED_CONFIDENCE_LEVEL") + value = moe["numeric_value"] + _require( + value is not None and math.isfinite(value) and value >= 0, + f"MOE_NOT_NONNEGATIVE:{variable}", + ) + if value == 0: + # A zero margin is a statement about a controlled cell, not a claim + # that the count is certain. Null again, never a zero standard error. + return None + return float(value) / _MOE_90_TO_SE + + +def activate_national_age_targets( + bundle_dir: str | Path, + *, + declaration: NationalAgeActivation = NATIONAL_AGE_ACTIVATION, +) -> TargetRegistry: + """Authenticate the pinned bundle and mint a new age-count registry. + + Args: + bundle_dir: The captured reference bundle root, holding + ``national-derived-inventory.json``, ``raw/`` and ``requests/``. + declaration: The activation authority. The default is the reviewed + one; a caller-supplied declaration is validated identically and + has no privileges of its own. + + Returns: + A new :class:`TargetRegistry` of 18 disjoint national age counts. + + Raises: + ActivationError: On any digest, identity, universe, label, schema, + reservation, selector or value violation. The bundle is never + written to, and a reserved same-source holdout can never be + activated. + """ + + root = Path(bundle_dir) + _, inventory = _read_pinned( + root / "national-derived-inventory.json", + declaration.inventory_sha256, + what="inventory", + ) + _require(isinstance(inventory, dict), "INVENTORY_SHAPE") + _require( + inventory.get("kind") == "us_national_acs_reference_inventory", + "INVENTORY_KIND", + ) + # The bundle states it is not an activation. Assert that rather than + # letting an edited descriptive field pass for authority. + _require(inventory.get("target_activation") is False, "INVENTORY_CLAIMS_ACTIVATION") + _require( + inventory.get("calibration_target_registry") == "not_created", + "INVENTORY_CLAIMS_A_REGISTRY", + ) + scope = inventory.get("scope") + _require(isinstance(scope, dict), "INVENTORY_SCOPE") + _require(scope.get("country") == "us", "INVENTORY_COUNTRY") + _require(scope.get("acs_year") == declaration.acs_year, "INVENTORY_ACS_YEAR") + _require(scope.get("acs_release") == declaration.acs_release, "INVENTORY_RELEASE") + _require( + scope.get("geography") == f"nation (for={declaration.geography_query})", + "INVENTORY_GEOGRAPHY", + ) + + tables = inventory.get("acs") + _require(isinstance(tables, dict), "INVENTORY_TABLES") + for reserved in sorted(RESERVED_FAMILIES): + held = tables.get(reserved) + _require(isinstance(held, dict), f"RESERVED_TABLE_ABSENT:{reserved}") + held_reservation = held.get("reservation") + _require( + isinstance(held_reservation, dict), + f"RESERVED_RESERVATION_SHAPE:{reserved}", + ) + _require( + held_reservation.get("status") == "reserved_same_source_holdout", + f"RESERVED_TABLE_RELABELLED:{reserved}", + ) + + entry = tables.get(declaration.table) + _require(isinstance(entry, dict), f"TABLE_ABSENT:{declaration.table}") + _require(entry.get("lineage") == "published", "TABLE_LINEAGE") + _require(entry.get("district_derived") is False, "TABLE_IS_DISTRICT_DERIVED") + _require(entry.get("geography_level") == "nation", "TABLE_GEOGRAPHY_LEVEL") + _require(entry.get("geography_id") == declaration.geography, "TABLE_GEOGRAPHY_ID") + _require( + entry.get("source_geography_query") == declaration.geography_query, + "TABLE_GEOGRAPHY_QUERY", + ) + _require(entry.get("published_name") == declaration.published_name, "TABLE_NAME") + _require(entry.get("year") == declaration.acs_year, "TABLE_YEAR") + _require( + entry.get("confidence_level") == declaration.confidence_level, + "TABLE_CONFIDENCE_LEVEL", + ) + _require( + entry.get("grouping") == "subject_age_and_sex_published_groups", + "TABLE_GROUPING", + ) + reservation = entry.get("reservation") + _require(isinstance(reservation, dict), "TABLE_RESERVATION") + _require( + reservation.get("status") == declaration.allowed_reservation_status, + f"TABLE_RESERVATION_STATUS:{reservation.get('status')}", + ) + # The bundle's own permission flags stay false and stay unread as + # authority; an inventory edited to claim permission is refused, and the + # digest above would already have rejected it. + _require(reservation.get("targets_allowed") is False, "INVENTORY_GRANTS_TARGETS") + _require(reservation.get("tuning_allowed") is False, "INVENTORY_GRANTS_TUNING") + + metadata_entry = _selected_source(inventory, declaration, "metadata") + data_entry = _selected_source(inventory, declaration, "data") + _, metadata = _authenticated_response(root, metadata_entry, kind="metadata") + _, data = _authenticated_response(root, data_entry, kind="data") + row = _national_row(data, declaration) + + inventory_cells = entry.get("cells") + _require(isinstance(inventory_cells, list), "TABLE_CELLS") + by_variable: dict[str, dict] = {} + for cell in inventory_cells: + _require(isinstance(cell, dict), "CELL_SHAPE") + variable = cell.get("variable") + _require(isinstance(variable, str), "CELL_VARIABLE") + _require(variable not in by_variable, f"DUPLICATE_CELL:{variable}") + by_variable[variable] = cell + + def derived_cell(variable: str) -> dict: + """Re-derive one cell from the authenticated bytes and cross-check it. + + The same annotation-precedence reader the district inventory uses runs + again over exactly the response that was hashed; the accepted inventory + must then agree with it cell for cell. + """ + + for suffix in ("E", "M", "EA", "MA"): + _require( + f"{variable}{suffix}" in row, + f"RESPONSE_CELL_MISSING:{variable}{suffix}", + ) + derived = { + "variable": variable, + "estimate": classify_acs_value(row[f"{variable}E"], row[f"{variable}EA"]), + "moe": classify_acs_value(row[f"{variable}M"], row[f"{variable}MA"]), + } + recorded = by_variable.get(variable) + _require(recorded is not None, f"CELL_ABSENT_FROM_INVENTORY:{variable}") + _require(derived == recorded, f"CELL_DISAGREES_WITH_INVENTORY:{variable}") + return derived + + specs: list[TargetSpec] = [] + digest = activation_digest(declaration) + for band in declaration.bands: + variable = band.variable + published = _published_variable(metadata, f"{variable}E", declaration) + _require(published.get("label") == band.label, f"PUBLISHED_LABEL:{variable}") + _require( + published.get("predicateType") == "int", f"PUBLISHED_NOT_A_COUNT:{variable}" + ) + derived = derived_cell(variable) + value = _count_estimate(derived, variable) + specs.append( + TargetSpec( + name=variable, + entity=declaration.entity, + measure=band.column, + value=float(value), + period=declaration.period, + se=_standard_error( + derived, variable, confidence_level=declaration.confidence_level + ), + source=f"{data_entry['url']} ({declaration.table} {variable}E)", + family=f"acs.{declaration.table}", + hierarchy=demographic_target_hierarchy( + declaration.table, band, geography=declaration.geography + ), + notes=( + f"activation={digest}; band={band.low}-" + f"{'+' if band.high is None else band.high}; " + f"label={band.label}; " + f"age_convention={declaration.age_convention}; " + f"uncertainty=published 90% MOE converted at " + f"{_MOE_90_TO_SE} ({_MOE_AUTHORITY}), not consumed " + f"by the current loss" + ), + metadata={ + "table": declaration.table, + "reference_sha256": declaration.inventory_sha256, + "geography": declaration.geography, + "universe": "population", + "role": declaration.role, + "evidence_scope": "source_documented", + }, + ) + ) + + # Coherence with the publisher's own total, checked and then discarded: + # the total is deliberately not activated as a nineteenth loss term. + total_variable = f"{declaration.table}_C01_001" + total = _count_estimate(derived_cell(total_variable), total_variable) + _require( + sum(int(spec.value) for spec in specs) == total, + "ACTIVATED_BANDS_DO_NOT_PARTITION_THE_PUBLISHED_TOTAL", + ) + return TargetRegistry(specs, country="us") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py b/packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py new file mode 100644 index 000000000..72f27cfd7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py @@ -0,0 +1,666 @@ +"""Private native household origins, authenticated from original source members. + +ACS uses the existing archive codec. ASEC requires a separately reviewed original +HOUSEHOLD CSV member registry: prepared household HDF and PERSON-member pins are +not substitutes. The three registered members have an independently reviewed +archive/member audit and a complete accepted-population identifier bridge. + +This module owns source lineage, not population treatments or benchmark scoring. +Source capsules can be issued only by a reader. Graph transport is distinct from +source authentication; its consumer must retain the actual producer artifact edge. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import InitVar, asdict, dataclass, field +from pathlib import Path + +import numpy as np + +from microcosm.build.cd_benchmark.origin import ( + AcsHouseholdOrigin, + AsecHouseholdOrigin, + SourceMember, + origin_key, +) +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.canonical import canonical_json + +from . import acs_housing_universe_source as acs +from .asec_student_controls import _snapshot +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES +from .support_provenance import ( + support_channel_column, + support_clone_index_column, + support_source_id_column, +) + +_TOKEN = object() +_BOUND_TOKEN = object() +SOURCE_SCHEMA = "microcosm.us.native-household-origin-source.v1" +BINDING_SCHEMA = "microcosm.us.population-household-origins.v1" +MAX_SOURCE_BYTES = 512 * 1024**2 +_SHA = re.compile(r"[0-9a-f]{64}") + + +class NativeOriginError(ValueError): + """Bounded refusal code, without a private identifier or source token.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise NativeOriginError(reason) + + +def _sha(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _integer(value: object) -> int: + _require( + not isinstance(value, (bool, np.bool_)) + and isinstance(value, (int, np.integer)), + "NATIVE_INTEGER", + ) + return int(value) + + +def _carried_native_integer(value: object) -> int: + # Union assembly may store a source-only integer column as canonical text + # beside the other arm's missing values. Accept only its lossless canonical + # decimal representation; source membership is still established against + # the independently authenticated integer H_SEQ roster, never this cast. + if isinstance(value, str): + _require( + re.fullmatch(r"[1-9][0-9]{0,17}", value) is not None, + "CARRIED_NATIVE_INTEGER", + ) + return int(value) + return _integer(value) + + +@dataclass(frozen=True) +class AsecNativeMemberPin: + """A reviewed original household member, not authority merely by construction. + + A root source audit must establish membership in the already-pinned Census + archive before adding this declaration to the closed registry below. + """ + + income_year: int + survey_year: int + canonical_member_id: str + member_name: str + archive_sha256: str + member_sha256: str + size_bytes: int + rows: int + + def __post_init__(self) -> None: + _require( + type(self.income_year) is int + and self.income_year in ASEC_EDUCATION_ASSISTANCE_ARCHIVES + and self.survey_year == self.income_year + 1, + "ASEC_MEMBER_COHORT", + ) + _require( + isinstance(self.canonical_member_id, str) + and bool(self.canonical_member_id) + and self.member_name == f"hhpub{str(self.survey_year)[-2:]}.csv", + "ASEC_MEMBER_IDENTITY", + ) + _require( + isinstance(self.member_sha256, str) + and _SHA.fullmatch(self.member_sha256) is not None + and self.archive_sha256 + == ASEC_EDUCATION_ASSISTANCE_ARCHIVES[self.income_year].zip_sha256, + "ASEC_MEMBER_ARCHIVE_BINDING", + ) + _require( + type(self.size_bytes) is int + and 0 < self.size_bytes <= MAX_SOURCE_BYTES + and type(self.rows) is int + and 0 < self.rows <= 1_000_000, + "ASEC_MEMBER_BOUNDS", + ) + + +# Root acceptance: us-asec-native-household-root-acceptance.json, SHA-256 +# f997bbdc3573ed346d0e84d29f90c7b7ded9a12720c036d2789a7d561a04d602. +# The original archive/member bytes and all accepted attachment/selected native +# household identifiers were independently verified. Tests use invented pins. +_ASEC_MEMBER_PINS: tuple[AsecNativeMemberPin, ...] = ( + AsecNativeMemberPin( + income_year=2022, + survey_year=2023, + canonical_member_id="census/cps/asec/2023/household/hhpub23.csv", + member_name="hhpub23.csv", + archive_sha256="d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11", + member_sha256="c88192cb3c963a90ea98022606112a01ade0eb8e2c17df4149052970a5d5246f", + size_bytes=30_259_450, + rows=88_978, + ), + AsecNativeMemberPin( + income_year=2023, + survey_year=2024, + canonical_member_id="census/cps/asec/2024/household/hhpub24.csv", + member_name="hhpub24.csv", + archive_sha256="cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + member_sha256="b12c1078e02766a4033ba1a031abdd6b9ef9df546aee5886dc8439a35abaa2ac", + size_bytes=30_448_089, + rows=89_473, + ), + AsecNativeMemberPin( + income_year=2024, + survey_year=2025, + canonical_member_id="census/cps/asec/2025/household/hhpub25.csv", + member_name="hhpub25.csv", + archive_sha256="318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + member_sha256="b5b7351d5d4e5d79ff189f1d90096b16b2d4749671c14328ec6474cfe83ce116", + size_bytes=33_298_479, + rows=88_932, + ), +) + + +def asec_member_registry() -> list[dict[str, object]]: + """Return the exact live registry for the separately scoped graph identity.""" + return [asdict(pin) for pin in _ASEC_MEMBER_PINS] + + +@dataclass(frozen=True) +class AuthenticatedNativeOriginSource: + """Owned source projection. Its constructor is not a public admission path.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token: object) -> None: + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + object.__setattr__(self, "_issued_sha256", _sha(self.payload)) + + @property + def document(self) -> dict: + _require(_sha(self.payload) == self._issued_sha256, "SOURCE_MUTATION") + return _source_document(self.payload) + + +def _source_document(payload: bytes) -> dict: + _require(type(payload) is bytes and len(payload) <= MAX_SOURCE_BYTES, "SOURCE_SIZE") + try: + value = json.loads(payload) + _require(canonical_json(value) == payload, "SOURCE_CANONICAL") + _require( + set(value) + == { + "schema", + "arm", + "members", + "records", + "source_receipt_sha256", + "release_eligible", + } + and value["schema"] == SOURCE_SCHEMA + and value["arm"] in ("acs", "asec") + and value["release_eligible"] is False + and _SHA.fullmatch(value["source_receipt_sha256"]) is not None, + "SOURCE_SHAPE", + ) + members = [SourceMember(**member) for member in value["members"]] + _require(bool(members) and len(set(members)) == len(members), "SOURCE_MEMBERS") + seen = set() + for period, native, member_index, key in value["records"]: + _require( + type(member_index) is int and 0 <= member_index < len(members), + "SOURCE_MEMBER_INDEX", + ) + origin = ( + AcsHouseholdOrigin(members[member_index], period, native) + if value["arm"] == "acs" + else AsecHouseholdOrigin(members[member_index], period, native) + ) + _require(origin_key(origin) == key, "SOURCE_ORIGIN_KEY") + identity = (period, native) + _require(identity not in seen, "SOURCE_NATIVE_KEY_AMBIGUOUS") + seen.add(identity) + _require(bool(seen), "SOURCE_EMPTY") + return value + except NativeOriginError: + raise + except (ValueError, TypeError, KeyError, IndexError, OverflowError): + raise NativeOriginError("SOURCE_SHAPE") from None + + +def _issue(arm, members, records, receipt_sha): + payload = canonical_json( + { + "schema": SOURCE_SCHEMA, + "arm": arm, + "members": members, + "records": records, + "source_receipt_sha256": receipt_sha, + "release_eligible": False, + } + ) + _source_document(payload) + return AuthenticatedNativeOriginSource(payload, _token=_TOKEN) + + +def produce_acs_native_origins(source_dir, *, snapshot_root, output_dir): + """Reuse the real ACS source reader; never accept caller-provided source rows.""" + source = acs.produce_acs_housing_source( + source_dir, + snapshot_root=snapshot_root, + output_dir=output_dir, + ) + receipt = json.loads(source.receipt_json) + table = source.households + members = [ + SourceMember( + # Original member identity is stable across staging aliases and + # recompression, and its original member bytes hash is explicit. + f"census/acs/pums/2024/household/{member['name']}", + member["sha256"], + ) + for member in receipt["members"]["household"] + if member["is_data"] + ] + names = [ + member["name"] + for member in receipt["members"]["household"] + if member["is_data"] + ] + indices = {name: i for i, name in enumerate(names)} + records = [] + for native, name in zip(table.SERIALNO, table.source_member, strict=True): + index = indices[name] + key = origin_key(AcsHouseholdOrigin(members[index], 2024, native)) + records.append([2024, native, index, key]) + return _issue( + "acs", + [member.as_payload() for member in members], + records, + _sha(source.receipt_json), + ) + + +def produce_asec_native_origins(member_paths: Mapping[int, Path]): + """Hash and parse registered original HOUSEHOLD CSV members only. + + No member name or digest is inferred from the person-side restoration. + Unregistered genuine sources refuse before any path is opened. + """ + pins = tuple(_ASEC_MEMBER_PINS) + _require(bool(pins), "ASEC_MEMBER_REGISTRY_UNAVAILABLE") + _require( + len({pin.income_year for pin in pins}) == len(pins) + and set(member_paths) == {pin.income_year for pin in pins}, + "ASEC_MEMBER_ROSTER", + ) + records, members = [], [] + with tempfile.TemporaryDirectory(prefix="native-asec-origin-") as tmp: + for index, pin in enumerate(pins): + # Revalidate the entire declared registry, including archive/cohort. + AsecNativeMemberPin(**asdict(pin)) + capture = Path(tmp) / f"{index}.csv" + _require( + _snapshot(member_paths[pin.income_year], capture, size=pin.size_bytes) + == pin.member_sha256, + "ASEC_MEMBER_SHA256", + ) + member = SourceMember(pin.canonical_member_id, pin.member_sha256) + members.append(member.as_payload()) + with capture.open("r", encoding="utf-8", newline="") as handle: + rows = csv.reader(handle, strict=True) + header = next(rows, []) + _require( + bool(header) + and len(header) == len(set(header)) + and "H_SEQ" in header, + "ASEC_MEMBER_HEADER", + ) + position = header.index("H_SEQ") + count = 0 + for row in rows: + _require(len(row) == len(header), "ASEC_MEMBER_ROW_WIDTH") + token = row[position] + _require( + re.fullmatch(r"[0-9]{1,18}", token) is not None + and int(token) > 0, + "ASEC_MEMBER_NATIVE_KEY", + ) + native = int(token) + records.append( + [ + pin.income_year, + native, + index, + origin_key( + AsecHouseholdOrigin(member, pin.income_year, native) + ), + ] + ) + count += 1 + _require(count <= pin.rows, "ASEC_MEMBER_ROWS") + _require(count == pin.rows, "ASEC_MEMBER_ROWS") + _require( + _sha(capture.read_bytes()) == pin.member_sha256, "ASEC_CAPTURE_CHANGED" + ) + return _issue( + "asec", members, records, _sha(canonical_json(asec_member_registry())) + ) + + +def _population_content(frame: Frame) -> str: + # Defined over all cells/axes, effective weights and strata. Graph context + # deliberately has no Frame.metadata, graph ledger or original design anchors. + try: + normalized = Frame( + { + entity: frame.table(entity).loc[:, sorted(frame.table(entity))] + for entity in frame.entities + }, + frame.schema, + {entity: frame.resolve_weights(entity) for entity in frame.entities}, + frame.strata, + ) + return acs.frame_content_sha256(normalized) + except (ValueError, TypeError, KeyError, OverflowError): + raise NativeOriginError("POPULATION_CONTENT") from None + + +@dataclass(frozen=True) +class PopulationOriginBinding: + """Private population-grain evidence; its summary carries no native keys.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token): + _require(_token is _BOUND_TOKEN, "POPULATION_BINDING_CONSTRUCTOR") + object.__setattr__(self, "_issued_sha256", _sha(self.payload)) + + @property + def document(self): + _require(_sha(self.payload) == self._issued_sha256, "BINDING_MUTATION") + return json.loads(self.payload) + + @property + def summary(self): + return self.document["summary"] + + +def verify_population_origin_binding(frame: Frame, binding: PopulationOriginBinding): + """Reject evidence from any other full population content/ordered axes.""" + _require(type(binding) is PopulationOriginBinding, "POPULATION_BINDING_TYPE") + _require( + binding.document["population_content_sha256"] == _population_content(frame), + "POPULATION_CONTENT", + ) + + +def _entity_rows(frame, household_origins): + person = frame.person + household_ids = person.person_household_id.to_numpy() + rows = {} + for entity in frame.entities: + table = frame.table(entity) + ids = table[US_SCHEMA.entity_id_column(entity)].to_numpy() + if entity == "person": + membership = dict(zip(ids.tolist(), household_ids.tolist(), strict=True)) + elif entity == "household": + membership = dict(zip(ids.tolist(), ids.tolist(), strict=True)) + else: + pairs = person[ + [US_SCHEMA.membership_column(entity), "person_household_id"] + ].drop_duplicates() + _require( + not pairs.iloc[:, 0].duplicated().any(), "GROUP_CROSSES_HOUSEHOLDS" + ) + membership = dict(pairs.itertuples(index=False, name=None)) + _require(set(ids) == set(membership), "ENTITY_HOUSEHOLD_COVERAGE") + records = [] + effective_weights = frame.resolve_weights(entity).values + for i, identity in enumerate(ids): + hh = household_origins[_integer(membership[identity])] + channel = table[support_channel_column(entity)].iloc[i] + source_id = _integer(table[support_source_id_column(entity)].iloc[i]) + role = _integer(table[support_clone_index_column(entity)].iloc[i]) + _require( + channel == hh["channel"] and role == hh["clone_index"], + "ENTITY_HOUSEHOLD_LINEAGE", + ) + records.append( + { + "entity_id": _integer(identity), + "entity_source_id": source_id, + "clone_index": role, + "channel": channel, + "household_source_id": hh["entity_source_id"], + "origin_key": hh["origin_key"], + "weight_hex": float(effective_weights[i]).hex(), + } + ) + _require( + len({(r["entity_source_id"], r["clone_index"]) for r in records}) + == len(records), + "ENTITY_LINEAGE_DUPLICATE", + ) + rows[entity] = records + groups = { + entity: {row["entity_id"]: row for row in rows[entity]} + for entity in US_SCHEMA.group_entities + } + for position, row in enumerate(rows["person"]): + memberships = {} + for entity, lookup in groups.items(): + member_id = _integer( + person[US_SCHEMA.membership_column(entity)].iloc[position] + ) + member = lookup[member_id] + _require( + all( + member[k] == row[k] + for k in ( + "clone_index", + "channel", + "household_source_id", + "origin_key", + ) + ), + "PERSON_GROUP_LINEAGE", + ) + memberships[entity] = member["entity_source_id"] + row["membership_source_ids"] = memberships + return rows + + +def _bind_population_origins( + frame: Frame, + *, + sources: Sequence[AuthenticatedNativeOriginSource], + parent: PopulationOriginBinding | None = None, +): + """Resolve every household and entity row, including all zero-weight records. + + Native columns identify original records; structural source IDs bind the + selection/clone ancestry and cannot substitute for that native identity. + """ + _require(frame.schema == US_SCHEMA and len(sources) == 2, "SOURCE_ARMS") + documents = [] + for source in sources: + _require(type(source) is AuthenticatedNativeOriginSource, "SOURCE_TYPE") + documents.append(source.document) + _require({d["arm"] for d in documents} == {"acs", "asec"}, "SOURCE_ARMS") + lookups = { + d["arm"]: {(p, n): key for p, n, _member, key in d["records"]} + for d in documents + } + household = frame.table("household") + person = frame.person + cohort_rows = person[["person_household_id", "source_year"]].drop_duplicates() + _require( + not cohort_rows.person_household_id.duplicated().any(), + "HOUSEHOLD_COHORT_AMBIGUOUS", + ) + cohorts = dict(cohort_rows.itertuples(index=False, name=None)) + weights = frame.resolve_weights("household").values + rows = [] + for i, hh in household.iterrows(): + identity = _integer(hh.household_id) + channel = hh[support_channel_column("household")] + _require(channel in lookups and identity in cohorts, "NATIVE_KEY_UNRESOLVED") + raw_period = cohorts[identity] + _require( + isinstance(raw_period, (str, int, np.integer)) + and re.fullmatch(r"[0-9]{4}", str(raw_period)) is not None, + "HOUSEHOLD_COHORT", + ) + period = int(raw_period) + native = ( + hh.SERIALNO if channel == "acs" else _carried_native_integer(hh.asec_H_SEQ) + ) + key = lookups[channel].get((period, native)) + _require(key is not None, "NATIVE_KEY_UNRESOLVED") + role = _integer(hh[support_clone_index_column("household")]) + _require(role in (0, 1), "CLONE_ROLE") + position = household.index.get_loc(i) + rows.append( + { + "household_id": identity, + "entity_source_id": _integer(hh[support_source_id_column("household")]), + "clone_index": role, + "channel": channel, + "origin_key": key, + "weight_hex": float(weights[position]).hex(), + } + ) + _require(len({r["household_id"] for r in rows}) == len(rows), "HOUSEHOLD_IDS") + entity_rows = _entity_rows(frame, {r["household_id"]: r for r in rows}) + cloned = any(r["clone_index"] for r in rows) + _require(not cloned or parent is not None, "CLONE_PARENT_REQUIRED") + pair_conserved = None + if parent is not None: + _require(type(parent) is PopulationOriginBinding, "PARENT_TYPE") + prior = parent.document + _require( + prior["source_payload_sha256"] == sorted(_sha(s.payload) for s in sources), + "PARENT_SOURCE_MISMATCH", + ) + _require( + not any(r["clone_index"] for r in prior["households"]), + "PARENT_MUST_BE_NATIVE", + ) + for entity, current in entity_rows.items(): + old = {r["entity_source_id"]: r for r in prior["entities"][entity]} + for row in current: + before = old.get(row["entity_source_id"]) + _require( + before is not None + and all( + row[k] == before[k] + for k in ("channel", "household_source_id", "origin_key") + ), + "PARENT_MEMBERSHIP_LINEAGE", + ) + if entity == "person": + _require( + row["membership_source_ids"] == before["membership_source_ids"], + "PARENT_MEMBERSHIP_LINEAGE", + ) + before_weight = float.fromhex(before["weight_hex"]) + if cloned: + half = before_weight * 0.5 + _require( + row["weight_hex"] == half.hex() and half * 2.0 == before_weight, + "CLONE_ENTITY_PAIR_WEIGHT", + ) + else: + _require( + row["weight_hex"] == before["weight_hex"], + "PARENT_ENTITY_WEIGHT", + ) + if cloned: + _require( + {(r["entity_source_id"], r["clone_index"]) for r in current} + == {(source_id, role) for source_id in old for role in (0, 1)}, + "CLONE_PAIR_COVERAGE", + ) + else: + retained_households = {r["entity_source_id"] for r in rows} + _require( + {r["entity_source_id"] for r in current} + == { + r["entity_source_id"] + for r in prior["entities"][entity] + if r["household_source_id"] in retained_households + }, + "PARENT_HOUSEHOLD_SELECTION", + ) + if cloned: + old = { + r["entity_source_id"]: float.fromhex(r["weight_hex"]) + for r in prior["households"] + } + _require( + all( + row["weight_hex"] == (old[row["entity_source_id"]] * 0.5).hex() + and (old[row["entity_source_id"]] * 0.5) * 2.0 + == old[row["entity_source_id"]] + for row in rows + ), + "CLONE_PAIR_WEIGHT", + ) + pair_conserved = True + content = _population_content(frame) + summary = { + "households": len(rows), + "original_household_origins": len({r["origin_key"] for r in rows}), + "zero_weight_households_with_lineage": sum( + float.fromhex(r["weight_hex"]) == 0 for r in rows + ), + "entity_rows": {entity: len(values) for entity, values in entity_rows.items()}, + "population_content_sha256": content, + "parent_pair_mass_conserved": pair_conserved, + "design_anchors": "not_in_frame_context_not_evaluated", + "release_eligible": False, + } + payload = canonical_json( + { + "schema": BINDING_SCHEMA, + "population_content_sha256": content, + "ordered_entity_ids_sha256": { + e: _sha(canonical_json([r["entity_id"] for r in records])) + for e, records in entity_rows.items() + }, + "source_payload_sha256": sorted(_sha(s.payload) for s in sources), + "parent_binding_sha256": None if parent is None else _sha(parent.payload), + "households": rows, + "entities": entity_rows, + "summary": summary, + } + ) + return PopulationOriginBinding(payload, _token=_BOUND_TOKEN) + + +def bind_population_origins( + frame: Frame, + *, + sources: Sequence[AuthenticatedNativeOriginSource], + parent: PopulationOriginBinding | None = None, +): + """Bind every current row; malformed inputs refuse without leaking native IDs.""" + try: + return _bind_population_origins(frame, sources=sources, parent=parent) + except NativeOriginError: + raise + except (ValueError, TypeError, KeyError, IndexError, OverflowError): + raise NativeOriginError("POPULATION_BINDING_CONTRACT") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/native_origin_graph_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/native_origin_graph_inventory.json new file mode 100644 index 000000000..7232f1347 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/native_origin_graph_inventory.json @@ -0,0 +1,89 @@ +{ + "base_stages": [ + "composed_asec_binding_v1", + "acs_housing_universe_2024" + ], + "contracts": { + "microcosm.build/cd_benchmark/canonical.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/cd_benchmark/origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.canonical" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_native_household_origin.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_native_origin_implementation", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.artifact_edges", + "microcosm.graph.canonical", + "microcosm.graph.keys" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "6420544565b4a2b11dc7238e06eef099a429982afa3dfbcec3dbdfbba7a27bd1" + }, + "microcosm.build/us_runtime/graph_native_origin_implementation.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_implementation" + ], + "resource_accesses_sha256": "ecc7b6bd345950216623b7de94c9aeb392504af36e398094481f04ad203914f9", + "unbound_uses_sha256": "51778c9d32f80dfeba9741570ff5fa19ef83b464d7b207bf2f438cb008ae3ae5" + }, + "microcosm.build/us_runtime/native_household_origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "740699f0e8648d061a7057844a4aabf04bb5ce2df7118fa4256dc2d215820c93", + "unbound_uses_sha256": "d4cc434fc5fa5a17c6fa46592dbeb8019d061927ff218c618036288221e76503" + } + }, + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "extra_modules": [ + "microcosm.build/cd_benchmark/canonical.py", + "microcosm.build/cd_benchmark/origin.py", + "microcosm.build/us_runtime/native_household_origin.py", + "microcosm.build/us_runtime/graph_native_household_origin.py", + "microcosm.build/us_runtime/graph_native_origin_implementation.py" + ], + "import_classifications": { + "microcosm.build.cd_benchmark.canonical": "Frozen v4 canonical origin encoding helpers; extra module bytes attested.", + "microcosm.build.cd_benchmark.origin": "Frozen v4 typed original-household keys; extra module bytes attested, protocol unchanged.", + "microcosm.build.us_runtime": "Relative native source/ACS reader imports; exact imported-symbol use fences plus declared module closures.", + "microcosm.build.us_runtime.asec_student_controls": "Existing bounded immutable member snapshot helper, attested by the composition closure.", + "microcosm.build.us_runtime.education_assistance_source": "Existing original ASEC archive and cohort pins, attested by the composition closure.", + "microcosm.build.us_runtime.graph_implementation": "Unchanged upstream identity service and AST/resource dependency classifier.", + "microcosm.build.us_runtime.graph_native_origin_implementation": "This isolated extension identity; its own module bytes and inventory are attested.", + "microcosm.build.us_runtime.support_provenance": "Existing descendant source-ID, channel and role column names; no population treatment.", + "microcosm.frame": "Frame schema, linkage, effective typed weights and population content; existing complete Frame closure.", + "microcosm.graph": "Graph declarations, typed artifact outputs and kernel contracts; existing complete graph closure.", + "microcosm.graph.artifact_edges": "Actual graph typed-descriptor validation at full materialization boundary; existing graph closure.", + "microcosm.graph.canonical": "Graph canonical JSON encoding; existing complete graph closure.", + "microcosm.graph.keys": "Actual opaque producer/output artifact-key derivation; existing complete graph closure.", + "numpy": "Installed NumPy distribution version plus platform numeric scope; strict native integer handling." + }, + "schema": "microcosm.us.native-origin-extension-inventory.v1", + "stage": "native_household_origin_v1" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py index e3e18961e..310b9953c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py @@ -14,7 +14,6 @@ import pandas as pd -from microcosm.build.us_runtime.acs_transfer import ACS_DERIVED_TRANSFER_INPUTS from microcosm.build.us_runtime.adult_care import US_ADULT_CARE_OUTPUT_COLUMNS from microcosm.build.us_runtime.child_support import ( US_CHILD_SUPPORT_OUTPUT_COLUMNS, @@ -59,20 +58,6 @@ US_PRIOR_YEAR_INCOME_FORMULA_OWNED_OUTPUT_COLUMNS, US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS, ) -from microcosm.build.us_runtime.puf_capital_gains_tail import ( - PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN, - PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN, -) -from microcosm.build.us_runtime.puf_support import ( - PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, - PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, -) from microcosm.build.us_runtime.qbi_inputs import US_QBI_RECONCILED_PERSON_COLUMNS from microcosm.build.us_runtime.relationship_inputs import ( US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS, @@ -96,6 +81,20 @@ ) from microcosm.frame import Frame +from .operator_column_contracts import ( + ACS_DERIVED_TRANSFER_INPUTS, + PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN, + PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, + PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, + PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN, + PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, + PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, +) + __all__ = [ "FORMULA_OWNED_SOURCE_COLUMNS", "PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py new file mode 100644 index 000000000..d5117016b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py @@ -0,0 +1,157 @@ +"""Source-visible operator column declarations, without operation imports. + +The operation modules re-export these same objects. Source boundary checks can +inspect ownership without loading transfer code or its reference tables. +""" + +from __future__ import annotations + +# Declared by qbi_inputs.py; operations retain their historical imports. + +_GENERAL_QUALIFICATION_FLAGS: tuple[str, ...] = ( + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", +) + +_SSTB_QUALIFICATION_FLAG = "sstb_self_employment_income_would_be_qualified" + +US_QBI_BOOLEAN_OUTPUT_COLUMNS: tuple[str, ...] = ( + *_GENERAL_QUALIFICATION_FLAGS, + _SSTB_QUALIFICATION_FLAG, + # Keep the classifier last so the chained QRF can condition the SSTB draw + # on the qualification flags it must agree with. + "business_is_sstb", +) + +US_QBI_NONNEGATIVE_OUTPUT_COLUMNS: tuple[str, ...] = ( + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", +) + +US_QBI_OUTPUT_COLUMNS: tuple[str, ...] = ( + *US_QBI_BOOLEAN_OUTPUT_COLUMNS, + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_self_employment_income_before_lsr", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", +) + + +# Declared by acs_transfer.py; operations retain their historical imports. + +#: Person columns the default transfer DERIVES deterministically after the +#: QRF fits (never fitted themselves). Coverage checks require them on the +#: recipient exactly like declared plan targets. +ACS_DERIVED_TRANSFER_INPUTS: tuple[str, ...] = ( + "schedule_d_capital_gain_distributions", +) + + +# Declared by puf_support.py; operations retain their historical imports. + +# Shared structural limit; importing it must not load the PUF donor operations. +PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID = 10**15 - 1 + +US_PUF_SUPPORT_STAGE_NAME = "puf_support_channel" + +PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS = ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "tax_exempt_interest_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", + "taxable_private_pension_income", + "taxable_ira_distributions", + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", + "alimony_income", + "alimony_expense", + "salt_refund_income", + "charitable_cash_donations", + "charitable_non_cash_donations", + "real_estate_taxes", + "home_mortgage_interest", + "investment_interest_expense", + "investment_income_elected_form_4952", + "student_loan_interest", + "educator_expense", + "qualified_tuition_expenses", + "casualty_loss", + "unreimbursed_business_employee_expenses", + # The engine owns the realized contribution amounts through the + # IRA-limit scale and self-employment caps; the persistable leaves are + # the desired contributions, equal to the PUF's observed deductions at + # baseline (issue #278). + "traditional_ira_contributions_desired", + "self_employed_pension_contributions_desired", + "rental_income", + "estate_income", + "farm_income", + "farm_operations_income", + "farm_rent_income", + "miscellaneous_income", + "partnership_income", + "s_corp_income", + "partnership_self_employment_net_earnings", + *US_QBI_OUTPUT_COLUMNS, +) + +PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS: tuple[str, ...] = ( + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "first_home_mortgage_balance", + "second_home_mortgage_balance", + "first_home_mortgage_interest", + "second_home_mortgage_interest", + "first_home_mortgage_origination_year", + "second_home_mortgage_origination_year", + "health_savings_account_ald", +) + + +# Declared by puf_capital_gains_tail.py; operations retain their historical imports. + +PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS = ( + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", +) + +PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS = ("unrecaptured_section_1250_gain",) + +PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN = "puf_capital_gains_tail_transfer_applied" + +PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN = "puf_capital_gains_tail_donor_source_id" + +PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN = ( + "puf_capital_gains_tail_donor_is_synthetic" +) + +PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN = ( + "puf_capital_gains_tail_donor_filing_status_code" +) + +PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN = ( + "puf_capital_gains_tail_donor_agi_band_index" +) + +PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN = "puf_capital_gains_tail_transfer_weight" diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py new file mode 100644 index 000000000..e4a0911c4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py @@ -0,0 +1,536 @@ +"""Describe required inputs on one actual graph Population and attached manifest. + +This is a diagnostic, never an issuer, source qualifier, applicability model, +release gate or claim of non-default statistical signal. Call inside an actual +host's checked lifetime after its source/producer checks. Population and manifest +constructors are public: even coherent supplied objects cannot prove original +source ancestry. There is deliberately no supplied expected hash/list shortcut. + +The roster is a versioned coverage contract, not a claim that every name is +scientifically independent or consumed by every engine route. The current +source manifest does not declare grain or applicability. Present +columns report their actual grains; missing names retain unresolved grain. +Every row's applicability remains unresolved, including declared ABSENT cells. +Column ownership identifies the last writer, not the measurement source of all +rows: writer-mask counts and carried counts stay separate. Source names and +typed artifact ancestry below describe declarations/receipts, not fresh reads. + +Only the current group-linked survey schema is supported. Experimental link +tables refuse, rather than being dropped by the maintained replay comparator. +The single possible borrowed I/O is the attached manifest population lookup. +All detached output is built afterward, with final pure input seals. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import asdict, dataclass + +import numpy as np +import pandas as pd + +from microcosm.graph import CompiledGraph, Population, RunManifest, compile_graph +from microcosm.graph.availability import execution_state +from microcosm.graph.canonical import canonical_json +from microcosm.graph.decl import Ownership, StructuralDelta +from microcosm.graph.population import _storage_parts, owned_ids +from microcosm.graph.serialize import graph_to_json +from microcosm.graph.store import ( + _axis_name_payload, + _encode_frame_metadata, + _encode_object_scalar, +) + +from .input_coverage_profile import ( + ASSIGNED_BLOCK_COLUMN, + MANIFEST_SHA256, + USInputProfile, + required_us_inputs, +) +from .support_provenance import ( + spine_source_id_column, + support_channel_column, + support_clone_index_column, +) +from .survey_population_replay import same_replayed_frame + + +@dataclass(frozen=True) +class CoverageGroup: + entity: str + origin: str + clone: int + id_dtype: str + ordered_ids: tuple[int | str, ...] + source_id_dtype: str + ordered_source_ids: tuple[int | str, ...] + + +@dataclass(frozen=True) +class CoverageCounts: + group: int + known: int + unknown: int + invalid: int + writer_rows: int + carried_rows: int + applicability_unresolved: int + # Neither nulls nor Ownership.ABSENT are evidence of non-applicability. + not_applicable: int = 0 + + +@dataclass(frozen=True) +class InputCoverage: + name: str + entity: str | None + dtype: str | None + owner: str | None + declaration: str + writer_mask: str | None + counts: tuple[CoverageCounts, ...] + + +@dataclass(frozen=True) +class CoverageProducer: + node: str + key: str + kernel_ref: str + implementation: str + declared_sources: tuple[tuple[str, str], ...] + predecessors: tuple[tuple[str, str], ...] + typed_artifacts_json: str + + +@dataclass(frozen=True) +class PopulationInputCoverage: + profile: USInputProfile + version: str + graph_sha256: str + manifest_key: str + population_storage_sha256: str + groups: tuple[CoverageGroup, ...] + inputs: tuple[InputCoverage, ...] + producers: tuple[CoverageProducer, ...] + missing_inputs: tuple[str, ...] + ambiguous_grains: tuple[str, ...] + assigned_block: InputCoverage + block_storage_issues: tuple[str, ...] + protocol: str = "microcosm.us.input_coverage_diagnostic.v1" + profile_reference_sha256: str = MANIFEST_SHA256 + profile_authority: str = "tracked_input_coverage_contract_v1" + source_ancestry_verified: bool = False + applicability_complete: bool = False + statistical_signal_verified: bool = False + release_eligible: bool = False + + def to_bytes(self) -> bytes: + return canonical_json(asdict(self)) + + +def _require(condition, reason): + if not condition: + raise ValueError("US_INPUT_COVERAGE_" + reason) + + +def _series_stamp(series): + digest = hashlib.sha256() + digest.update(canonical_json((str(series.dtype), _axis_name_payload(series.name)))) + if pd.api.types.is_object_dtype(series.dtype): + # Typed scalar encoding, never process-specific PyObject addresses. + for value in series: + part = _encode_object_scalar(value) + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + else: + # Whole series: the slice selects every row in order, byte for byte. + for part in _storage_parts(series, slice(None)): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + return digest.hexdigest() + + +def _axis_stamp(axis): + _require(not isinstance(axis, pd.MultiIndex), "MULTIINDEX") + return (type(axis).__name__, _series_stamp(pd.Series(axis.array, name=axis.name))) + + +def _frame_stamp(frame): + _require(frame.links == (), "LINK_TABLES_UNSUPPORTED") + tables = [] + for entity in frame.entities: + table = frame.table(entity) + tables.append( + ( + entity, + _axis_stamp(table.index), + _axis_stamp(table.columns), + table.flags.allows_duplicate_labels, + tuple((column, _series_stamp(table[column])) for column in table), + ) + ) + return hashlib.sha256( + canonical_json( + { + "schema": asdict(frame.schema), + "tables": tables, + "strata": ( + _axis_stamp(frame.strata.index), + _series_stamp(frame.strata), + ), + "weights": tuple( + ( + entity, + frame.weights_for(entity).kind.value, + frame.weights_for(entity).values.dtype.str, + frame.weights_for(entity).values.tobytes().hex(), + ) + for entity in frame.weighted_entities + ), + "metadata": _encode_frame_metadata(frame.metadata), + "mass_log": tuple(asdict(record) for record in frame.mass_log), + } + ) + ).hexdigest() + + +def _population_stamp(population): + return hashlib.sha256( + canonical_json( + { + "frame": _frame_stamp(population.frame), + "version": population.version, + "owners": sorted(population.owners.items()), + "kinds": sorted( + (k, v.value) for k, v in population.weight_kind.items() + ), + "ledger": tuple(asdict(record) for record in population.mass_ledger), + "design": tuple( + (k, v.dtype.str, v.shape, v.tobytes().hex()) + for k, v in sorted(population.design_weights.items()) + ), + } + ) + ).hexdigest() + + +def _ids(series): + _require(not series.isna().any(), "UNKNOWN_ID") + if pd.api.types.is_integer_dtype(series.dtype) and not pd.api.types.is_bool_dtype( + series.dtype + ): + return tuple(int(value) for value in series) + _require(isinstance(series.dtype, pd.StringDtype), "ID_DTYPE") + return tuple(str(value) for value in series) + + +def _groups(frame): + groups, selections, roles = [], [], {} + for entity in frame.entities: + table = frame.table(entity) + names = ( + support_channel_column(entity), + support_clone_index_column(entity), + spine_source_id_column(entity), + ) + _require(all(name in table for name in names), "ORIGIN_COLUMNS:" + entity) + channel, clone, native = (table[name] for name in names) + _require( + not channel.isna().any() and set(channel) <= {"acs", "asec"}, + "ORIGIN:" + entity, + ) + _require( + pd.api.types.is_integer_dtype(clone.dtype) + and not pd.api.types.is_bool_dtype(clone.dtype) + and not clone.isna().any() + and set(clone) <= {0, 1}, + "CLONE:" + entity, + ) + ids = _ids(table[frame.schema.entity_id_column(entity)]) + native_ids = _ids(native) + _require(len(set(ids)) == len(ids), "DUPLICATE_ID:" + entity) + row_roles = tuple((str(a), int(b)) for a, b in zip(channel, clone, strict=True)) + _require( + len(set(zip(row_roles, native_ids, strict=True))) == len(ids), + "DUPLICATE_SOURCE_ROLE:" + entity, + ) + roles[entity] = dict(zip(ids, row_roles, strict=True)) + for origin, role in sorted(set(row_roles)): + selected = np.array( + [v == (origin, role) for v in row_roles], dtype=np.bool_ + ) + groups.append( + CoverageGroup( + entity, + origin, + role, + str(table[frame.schema.entity_id_column(entity)].dtype), + tuple(v for v, use in zip(ids, selected, strict=True) if use), + str(native.dtype), + tuple( + v for v, use in zip(native_ids, selected, strict=True) if use + ), + ) + ) + selections.append(selected) + person = frame.table(frame.schema.person_entity) + person_ids = _ids(person[frame.schema.person_id_column]) + for entity in frame.schema.group_entities: + members = _ids(person[frame.schema.membership_column(entity)]) + _require( + all( + member in roles[entity] + and roles[entity][member] == roles[frame.schema.person_entity][pid] + for pid, member in zip(person_ids, members, strict=True) + ), + "MEMBERSHIP_ORIGIN_CLONE:" + entity, + ) + return tuple(groups), tuple(selections) + + +def _writer(population, compiled, entity, name): + owner = population.owners[entity, name] + node = compiled.graph.node(owner) + outputs = [o for o in node.outputs if (o.entity, o.column) == (entity, name)] + if not outputs: + _require(node.structural is not StructuralDelta.NONE, "OWNER_DECLARATION") + return ( + owner, + "structural_carrier", + None, + np.zeros(population.frame.n(entity), dtype=np.bool_), + ) + _require(len(outputs) == 1, "OWNER_DECLARATION") + output = outputs[0] + table = population.frame.table(entity) + selected = ( + table[population.frame.schema.entity_id_column(entity)] + .isin(owned_ids(population, output)) + .to_numpy(dtype=np.bool_) + ) + if output.ownership is Ownership.ABSENT: + _require(table[name].loc[selected].isna().all(), "ABSENT_VALUE") + return owner, output.ownership.value, output.rows, selected + + +def _input(population, compiled, groups, selections, name, entity): + if entity is None: + return InputCoverage( + name, None, None, None, "missing_unresolved_grain", None, () + ) + frame, table = population.frame, population.frame.table(entity) + if name == "household_weight": + series = pd.Series(frame.weights_for(entity).values) + owner, declaration, mask = population.version, "typed_weight_carrier", None + writer = np.zeros(len(series), dtype=np.bool_) + else: + series = table[name] + owner, declaration, mask, writer = _writer(population, compiled, entity, name) + unknown = series.isna().to_numpy(dtype=np.bool_) + invalid = np.zeros(len(series), dtype=np.bool_) + if pd.api.types.is_numeric_dtype(series.dtype): + numeric = series.to_numpy(dtype=np.float64, na_value=np.nan) + invalid = ~unknown & ~np.isfinite(numeric) + known = ~unknown & ~invalid + counts = tuple( + CoverageCounts( + i, + int(known[selected].sum()), + int(unknown[selected].sum()), + int(invalid[selected].sum()), + int(writer[selected].sum()), + int((~writer[selected]).sum()), + int(selected.sum()), + ) + for i, (group, selected) in enumerate(zip(groups, selections, strict=True)) + if group.entity == entity + ) + return InputCoverage( + name, entity, str(series.dtype), owner, declaration, mask, counts + ) + + +def diagnose_us_input_coverage( + population: Population, + *, + compiled: CompiledGraph, + manifest: RunManifest, + profile: USInputProfile = USInputProfile.NATIONAL_CD, +) -> PopulationInputCoverage: + """Bind descriptive coverage to actual tables and current graph ownership. + + This verifies supplied graph consistency, not actual-run/source issuance. + The final host must retain/requalify the real upstream owners separately. + No applicability exemption is accepted from caller metadata or a hash. + """ + names = required_us_inputs(profile) + _require( + type(population) is Population + and type(compiled) is CompiledGraph + and type(manifest) is RunManifest, + "TYPES", + ) + graph_json = graph_to_json(compiled.graph) + fresh = compile_graph(compiled.graph) + _require(compiled == fresh, "COMPILED_GRAPH") + _require( + manifest.country == compiled.graph.country == "us" + and tuple(manifest.nodes) == compiled.order, + "MANIFEST_ROSTER", + ) + _require(not manifest.known_failures, "MANIFEST_FAILURE") + for node_id in compiled.order: + record = manifest.node(node_id) + _require( + record.kernel_ref == compiled.graph.node(node_id).kernel + and execution_state(record.receipt) is None, + "PRODUCER_RECEIPT", + ) + frame = population.frame + frame.revalidate() + _require( + set(population.owners) + == { + (entity, str(column)) + for entity in frame.entities + for column in frame.table(entity) + }, + "PHYSICAL_OWNERS", + ) + _require( + compiled.versions.get(population.version) == population.version, + "POPULATION_VERSION", + ) + # CompiledGraph.owners contains explicit claims, not carried columns. + # Every structural step reowns its complete carrier; only writers in the + # current version replace those owners. Derive the roster from declarations + # along the base chain, never by accepting arbitrary physical columns. + expected_owners = {} + version = population.version + while True: + for (declared_version, entity, column), owner in compiled.owners.items(): + if declared_version == version: + expected_owners.setdefault( + (entity, column), + owner if version == population.version else population.version, + ) + holder = compiled.graph.node(version) + if holder.structural is StructuralDelta.CREATE: + break + version = holder.base + structural = { + (entity, frame.schema.entity_id_column(entity)) for entity in frame.entities + } + structural.update( + (frame.schema.person_entity, frame.schema.membership_column(entity)) + for entity in frame.schema.group_entities + ) + expected_owners.update(dict.fromkeys(structural, population.version)) + _require( + expected_owners and dict(population.owners) == expected_owners, "CURRENT_OWNERS" + ) + _require( + population.weight_kind + == { + entity: frame.weights_for(entity).kind for entity in frame.weighted_entities + }, + "WEIGHT_KIND", + ) + before, manifest_json = _population_stamp(population), manifest.to_json() + # A lazy manifest can perform store I/O here. Never construct detached output + # before this final borrow. Replay normalization is restricted to this edge. + materialized = manifest.population(population.version) + ledger = manifest.mass_ledger(population.version) + same_replayed_frame(frame, materialized) + _require( + canonical_json(tuple(asdict(r) for r in ledger)) + == canonical_json(tuple(asdict(r) for r in population.mass_ledger)), + "MASS_LEDGER", + ) + _require( + _population_stamp(population) == before and manifest.to_json() == manifest_json, + "BORROW_MUTATION", + ) + materialized_stamp = _frame_stamp(materialized) + groups, selections = _groups(frame) + inputs, missing, ambiguous = [], [], [] + for name in names: + entities = ( + ["household"] + if name == "household_weight" and "household" in frame.weighted_entities + else [entity for entity in frame.entities if name in frame.table(entity)] + ) + if name == "household_weight" and entities != ["household"]: + # Never accept a redundant ordinary column instead of typed weights. + entities = [] + if not entities: + missing.append(name) + inputs.append(_input(population, compiled, groups, selections, name, None)) + else: + if len(entities) > 1: + ambiguous.append(name) + inputs.extend( + _input(population, compiled, groups, selections, name, entity) + for entity in entities + ) + household = frame.table("household") + block = _input( + population, + compiled, + groups, + selections, + ASSIGNED_BLOCK_COLUMN, + "household" if ASSIGNED_BLOCK_COLUMN in household else None, + ) + block_issues = [] + if block.entity is None: + block_issues.append("missing_assigned_block") + else: + values = household[ASSIGNED_BLOCK_COLUMN] + if values.isna().any(): + block_issues.append("unknown_assigned_block") + if any( + not isinstance(v, str) or len(v) != 15 or not v.isascii() or not v.isdigit() + for v in values[values.notna()] + ): + block_issues.append("malformed_assigned_block") + sources = {source.name: source.codec for source in compiled.graph.sources} + producers = tuple( + CoverageProducer( + node_id, + manifest.node(node_id).key, + manifest.node(node_id).kernel_ref, + manifest.node(node_id).kernel_impl_hash, + tuple( + (name, sources[name]) for name in compiled.graph.node(node_id).sources + ), + tuple( + (name, manifest.node(name).key) + for name in compiled.predecessors[node_id] + ), + canonical_json(manifest.node(node_id).typed_artifacts).decode(), + ) + for node_id in compiled.order + ) + # Pure checks after decoding and grouping. No source/store/owner borrow follows. + _require( + _population_stamp(population) == before + and _frame_stamp(materialized) == materialized_stamp + and manifest.to_json() == manifest_json + and graph_to_json(compiled.graph) == graph_json + and compiled == fresh, + "FINAL_MUTATION", + ) + return PopulationInputCoverage( + profile, + population.version, + hashlib.sha256(graph_json.encode()).hexdigest(), + manifest.key, + before, + groups, + tuple(inputs), + producers, + tuple(missing), + tuple(ambiguous), + block, + tuple(block_issues), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py b/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py index ab3e56d6f..7983db5bf 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py @@ -45,7 +45,7 @@ support_role_series, without_support_role_metadata, ) -from microcosm.frame import Frame +from microcosm.frame import Frame, Weights from microcosm.frame.units import US_SCHEMA __all__ = [ @@ -63,6 +63,7 @@ "US_PRIOR_YEAR_INCOME_STAGE_NAME", "derive_us_prior_year_income_from_manifest", "impute_us_prior_year_income_to_puf_support_from_manifest", + "prepare_us_prior_year_person", "us_prior_year_income_signal_gate", "us_prior_year_income_source_reconciliation_gate", "us_prior_year_income_stage_spec", @@ -213,11 +214,7 @@ def _previous_year_availability_match_survival_factor(frame: Frame) -> float: f"spine manifest version {version!r}." ) factor = manifest.get("sample_fraction") - if ( - type(factor) is not float - or not np.isfinite(factor) - or not 0.0 < factor <= 1.0 - ): + if type(factor) is not float or not np.isfinite(factor) or not 0.0 < factor <= 1.0: raise ValueError( "US prior-year-income availability requires a finite production " "stacked sample_fraction in (0, 1]." @@ -695,34 +692,57 @@ def _replace_person_table(frame: Frame, person: pd.DataFrame) -> Frame: ) -def with_us_prior_year_income_inputs( - frame: Frame, +def _prior_year_person_is_complete(person: pd.DataFrame) -> bool: + return ( + not has_support_role_metadata(person, entity="person") + and all(column in person for column in US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS) + and not all( + column in person for column in US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS + ) + ) + + +def prepare_us_prior_year_person( + person: pd.DataFrame, *, + weights: Weights, seed: int, time_period: int, -) -> Frame: - """Materialize adjacent-year earnings and PUF-support replacements.""" + predictors: pd.DataFrame | None = None, +) -> pd.DataFrame: + """Run the actual prior-year stage on a real person table. - if frame.schema != US_SCHEMA: - raise ValueError("US prior-year income requires the US schema.") - person = frame.table("person") - has_support_roles = has_support_role_metadata(person, entity="person") - has_raw_sources = all( - column in person for column in US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS - ) - if ( - not has_support_roles - and all(column in person for column in US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS) - and not has_raw_sources - ): - return frame + ``weights`` must be resolved for this person order by the caller. The + source-only, pre-clone path requires no other entity table. The legacy + post-clone path supplies the actual person/tax-unit predictor projection; + it is never reconstructed from placeholder entities here. Monetary values + retain the existing source-period treatment; this helper does not uprate. + """ + if not isinstance(weights, Weights): + raise TypeError("US prior-year person weights require canonical Weights.") + if len(weights) != len(person): + raise ValueError("US prior-year weights must match the person row order.") + if _prior_year_person_is_complete(person): + if predictors is not None: + raise ValueError("Source-only prior-year inputs require no PUF predictors.") + return person + has_support_roles = has_support_role_metadata(person, entity="person") stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = weights.values if has_support_roles: - predictors = _person_prior_year_income_predictors(frame) + if ( + not isinstance(predictors, pd.DataFrame) + or not predictors.index.equals(person.index) + or tuple(predictors.columns) != _PUF_PREDICTORS + ): + raise ValueError( + "Prior-year support inputs require the actual aligned PUF predictors." + ) for column in _PUF_PREDICTORS: stage_person[_PUF_PREDICTOR_PREFIX + column] = predictors[column].to_numpy() + elif predictors is not None: + raise ValueError("Source-only prior-year inputs require no PUF predictors.") output = run_source_stage( us_prior_year_income_stage_spec(), tables={"person": stage_person}, @@ -742,6 +762,34 @@ def with_us_prior_year_income_inputs( ) if has_support_roles: output = output.drop(columns=[_FORMULA_OWNED_OUTPUT]) + return output + + +def with_us_prior_year_income_inputs( + frame: Frame, + *, + seed: int, + time_period: int, +) -> Frame: + """Materialize adjacent-year earnings and PUF-support replacements.""" + if frame.schema != US_SCHEMA: + raise ValueError("US prior-year income requires the US schema.") + person = frame.table("person") + if _prior_year_person_is_complete(person): + return frame + weights = frame.resolve_weights("person") + predictors = ( + _person_prior_year_income_predictors(frame) + if has_support_role_metadata(person, entity="person") + else None + ) + output = prepare_us_prior_year_person( + person, + weights=weights, + seed=seed, + time_period=time_period, + predictors=predictors, + ) return _replace_person_table(frame, output) @@ -892,12 +940,10 @@ def us_prior_year_income_signal_gate(frame: Frame) -> GateResult: if match_survival_factor != 1.0: authored_lower, authored_upper = summary[availability_band_key] applied_floor = float(authored_lower) * match_survival_factor - availability_band_key = ( - "previous_year_income_available_applied_share_band" + availability_band_key = "previous_year_income_available_applied_share_band" + summary["previous_year_income_available_sampled_match_survival_factor"] = ( + match_survival_factor ) - summary[ - "previous_year_income_available_sampled_match_survival_factor" - ] = match_survival_factor summary["previous_year_income_available_applied_floor"] = applied_floor summary[availability_band_key] = [applied_floor, authored_upper] failures: list[str] = [] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/property_income_constants.py b/packages/microcosm-build/src/microcosm/build/us_runtime/property_income_constants.py new file mode 100644 index 000000000..1b1456ad4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/property_income_constants.py @@ -0,0 +1,10 @@ +"""Canonical columns for the declared ACS/ASEC property-income bridge.""" + +PROPERTY_COMPONENTS = ( + "property_ordinary_interest", + "property_retirement_interest", + "property_dividends", + "property_broad_receipts", +) +PROPERTY_REPORTED_TOTAL = "property_reported_total" +PROPERTY_DRAW_COLUMNS = tuple("draw_" + name for name in PROPERTY_COMPONENTS) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py new file mode 100644 index 000000000..4eae37e78 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py @@ -0,0 +1,141 @@ +"""Project a bound canonical59 artifact into the survey-SS PUF55 donor. + +This in-memory converter grants no source authority. The caller authenticates +the artifact upstream and supplies its expected identity. No file is opened, +person is invented, or recipient Social Security component is produced here. +""" + +from __future__ import annotations + +import hashlib +import re + +import numpy as np +import pandas as pd + +from . import full_puf_enrichment as enrichment +from .puf59_canonical_artifact import decode_canonical_puf59 + +_SS_CARRIER_METHOD = ( + "Component-neutral carrier: total SS in retirement, other component carriers " + "zero; requires recipient-profile SS reconciliation. Zeros are not " + "observations of absent benefits." +) + + +def canonical_puf55_donor_from_artifact( + payload: bytes, + *, + expected_artifact_sha256: str, + expected_growth_scheme: str = "family_observed", + profile=enrichment.PUF55_SURVEY_SS, +): + """Return the ordered donor and descriptive projection metadata. + + The canonical59 v2 producer puts grown E02400 into its retirement carrier + and zeroes the other three carriers. Require that exact modeled convention + before relabeling the total as a predictor. It is transported tax-return + income, not an observed current-year total or a beneficiary component. + + The explicit eight-predictor profile omits this total from conditioning; + both profiles retain the same 55 targets, source rows, weights and capacity. + Both still validate the complete canonical artifact and its carrier recipe. + + The returned table has the source RECID axis. Its mutable values and this + metadata do not constitute an authenticated graph edge or release receipt. + """ + if type(profile) is not enrichment.PufOutputProfile or profile not in ( + enrichment.PUF55_SURVEY_SS, + enrichment.PUF55_SURVEY_SS_NO_TOTAL, + ): + raise ValueError("PUF55_DONOR_PROFILE") + if ( + type(payload) is not bytes + or type(expected_artifact_sha256) is not str + or re.fullmatch(r"[0-9a-f]{64}", expected_artifact_sha256) is None + or hashlib.sha256(payload).hexdigest() != expected_artifact_sha256 + ): + raise ValueError("PUF55_DONOR_ARTIFACT_IDENTITY") + arrays, source_receipt = decode_canonical_puf59( + payload, expected_growth_scheme=expected_growth_scheme + ) + assumptions = source_receipt.get("model_assumptions") + if ( + not isinstance(assumptions, dict) + or assumptions.get("schema") != "microcosm.us.puf59_baseline_models/1" + or assumptions.get("ss_method") != _SS_CARRIER_METHOD + or any( + np.any(arrays[name] != 0) + for name in enrichment.SURVEY_SS_COMPONENTS + if name != "social_security_retirement" + ) + ): + raise ValueError("PUF55_DONOR_SS_CARRIER") + + index = pd.Index(arrays["RECID"], name="tax_unit_id") + tax_unit = pd.DataFrame( + { + "tax_unit_id": arrays["RECID"], + "weight": arrays["weight"], + "filing_status_code": arrays["puf_2015_filing_status_code"], + "puf_person_incidence_capacity": arrays["puf_person_incidence_capacity"], + **{name: arrays[name] for name in profile.targets}, + **{name: arrays[name] for name in enrichment.PUF59.source_predictors}, + **( + { + enrichment.SURVEY_SS_TOTAL_PREDICTOR: arrays[ + "social_security_retirement" + ] + } + if profile is enrichment.PUF55_SURVEY_SS + else {} + ), + }, + index=index, + ) + # Known here means present canonical cells, including modeled values. It + # does not reclassify the inherited source/model provenance as observed. + known = pd.DataFrame( + True, + index=index, + columns=( + "weight", + "filing_status_code", + "puf_person_incidence_capacity", + *profile.targets, + *profile.source_predictors, + ), + ) + donor = enrichment.canonical_full_puf_donor( + None, + tax_unit, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=profile.person_outputs, + profile=profile, + ) + return donor, { + "schema": "microcosm.us.puf55_canonical_donor_projection/1", + "artifact_sha256": expected_artifact_sha256, + "source_receipt_sha256": source_receipt["sha256"], + "source_statistical_year": 2015, + "money_year": 2024, + "rows": len(donor), + "profile": profile.value, + "ordered_columns": list(donor.columns), + "row_axis": "source RECID, tax-return grain", + "social_security_total": { + "predictor": ( + enrichment.SURVEY_SS_TOTAL_PREDICTOR + if profile is enrichment.PUF55_SURVEY_SS + else None + ), + "raw_field": "E02400", + "origin": "modeled_transport", + "growth_scheme": expected_growth_scheme, + "growth_recipe_sha256": source_receipt["growth"]["recipe_sha256"], + "component_interpretation": "No observed component allocation", + }, + "source_authority_granted": False, + "release_eligible": False, + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py new file mode 100644 index 000000000..8cfd66785 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py @@ -0,0 +1,518 @@ +"""Validate and merge the two PUF55 raw chains before one finalization. + +This is a values-only boundary. The host must freshly qualify the recipient +matrices against its actual financial run, authenticate every typed graph edge, +and verify donor/model ownership before using the result. Caller bytes and this +receipt issue no source, Population, donor, fit or release authority. The public +raw merger loads no model. The private finalization bridge below requires an +upstream-trusted final model for each nonempty route and returns a candidate +Frame. It fits no model and issues no Population attachment. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.fit import model_input + +from . import full_puf_enrichment as full +from . import graph_full_puf_enrichment as physical +from . import support_provenance as provenance + +PROTOCOL = "microcosm.us.puf55-route-raw-merge.v1" +FINALIZATION_PROTOCOL = "microcosm.us.puf55-route-finalization.v2" +PROFILES = (full.PUF55_SURVEY_SS, full.PUF55_SURVEY_SS_NO_TOTAL) + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_ROUTE_" + reason) + + +@dataclass(frozen=True) +class Puf55RouteDraws: + """Descriptive typed-chain bytes, never an authenticated producer capsule.""" + + profile: full.PufOutputProfile + matrix: bytes + matrix_producer_key: str + raw_draws: tuple[tuple[str, bytes], ...] + apply_state: bytes + training_state: bytes + + +def _route_snapshot(route): + _require(type(route) is Puf55RouteDraws, "INPUT_TYPE") + # Frozen dataclasses can still be changed with object.__setattr__. Read each + # field once and use only these immutable retained values during decoding. + result = ( + route.profile, + route.matrix, + route.matrix_producer_key, + route.raw_draws, + route.apply_state, + route.training_state, + ) + profile, matrix, key, raw, apply_state, training_state = result + _require(type(profile) is full.PufOutputProfile and profile in PROFILES, "PROFILE") + _require( + all(type(value) is bytes for value in (matrix, apply_state, training_state)) + and type(key) is str + and full.codec._hash(key), + "ARTIFACT_TYPES", + ) + _require( + type(raw) is tuple + and all( + type(pair) is tuple + and len(pair) == 2 + and type(pair[0]) is str + and type(pair[1]) is bytes + for pair in raw + ) + and tuple(pair[0] for pair in raw) == profile.targets, + "RAW_ROSTER", + ) + return result + + +def _receiving_axis(frame): + table = frame.table("tax_unit") + column = provenance.support_clone_index_column("tax_unit") + _require( + table.columns.is_unique + and table.index.is_unique + and column in table + and table[column].dtype == np.dtype("int64") + and bool(table[column].isin((0, 1)).all()), + "CLONE_AXIS", + ) + all_ids = full._ids(table.tax_unit_id, "route_receiving") + _require(len(set(all_ids.tolist())) == len(all_ids), "RECEIVING_IDS") + mask = table[column].eq(1).to_numpy() + ids = all_ids[mask] + _require(len(ids) > 0, "NO_RECIPIENTS") + return ids, table.index[mask].copy() + + +def merge_puf55_route_draws(frame, *, recipient_matrices, route_draws, seed): + """Return all 55 raw targets in complete receiving pandas row order. + + ``recipient_matrices`` is the exact immutable matrix roster freshly derived + by the host's authenticated recipient qualifier. Empty routes are omitted + in nine/eight order. This function checks values, not that upstream duty. + Finalize the complete returned table once, after checking both donor/model + bindings; never finalize subsets or attach a route over another route. + """ + _require(type(seed) is int and seed >= 0, "SEED") + _require( + type(recipient_matrices) is tuple + and all( + type(pair) is tuple + and len(pair) == 2 + and type(pair[0]) is str + and type(pair[1]) is bytes + for pair in recipient_matrices + ), + "MATRIX_ROSTER", + ) + _require(type(route_draws) is tuple and 1 <= len(route_draws) <= 2, "ROUTE_ROSTER") + retained = tuple(_route_snapshot(route) for route in route_draws) + names = tuple(values[0].value for values in retained) + _require( + names == tuple(p.value for p in PROFILES if p.value in names) + and tuple(name for name, _ in recipient_matrices) == names, + "ROUTE_ORDER", + ) + ids, row_index = _receiving_axis(frame) + tables, seen, evidence, raw_seals = [], set(), [], [] + for values, (_, expected_matrix) in zip(retained, recipient_matrices, strict=True): + profile, matrix, key, raw, apply_state, training_state = values + _require(matrix == expected_matrix, "RECIPIENT_MATRIX_CHANGED") + prepared = model_input.decode_recipient_matrix(matrix) + route_ids = tuple(prepared.entity_ids.tolist()) + _require( + prepared.entity == "tax_unit" + and tuple(prepared.features.columns) == profile.predictors + and prepared.features.index.name == "tax_unit_id", + "MATRIX_PROFILE", + ) + _require(not seen.intersection(route_ids), "OVERLAPPING_IDS") + seen.update(route_ids) + decoded = full.decode_full_puf_draws( + matrix=matrix, + matrix_producer_key=key, + raw_draws=dict(raw), + apply_state=apply_state, + training_state=training_state, + seed=seed, + profile=profile, + ) + # Check the detached decoder result against the immutable input bytes; + # comparing two mutable returned tables would not anchor its values. + _require( + type(decoded) is pd.DataFrame + and tuple(decoded.columns) == profile.targets + and decoded.index.equals(pd.Index(route_ids, name="tax_unit_id")) + and all(dtype == np.dtype("float64") for dtype in decoded.dtypes) + and all( + full.codec.encode_raw_target( + decoded[target].to_numpy(), target=target, index=decoded.index + ) + == payload + for target, payload in raw + ), + "DECODED_RAW_CHANGED", + ) + tables.append(decoded.copy(deep=True)) + raw_seals.append((route_ids, raw)) + evidence.append( + { + "profile": profile.value, + "rows": len(route_ids), + "matrix_sha256": full.codec.sha(matrix), + "matrix_producer_key": key, + "apply_state_sha256": full.codec.sha(apply_state), + "training_state_sha256": full.codec.sha(training_state), + "raw_target_sha256": {target: full.codec.sha(p) for target, p in raw}, + } + ) + _require(seen == set(ids.tolist()), "INCOMPLETE_RECIPIENT_UNION") + combined = ( + pd.concat(tables, axis=0).loc[ids.tolist(), list(PROFILES[0].targets)].copy() + ) + # The IDs above establish the one explicit bridge to the existing pandas + # row index required by the maintained whole-cohort finalizer. + combined.index = row_index.copy() + fresh_ids, fresh_index = _receiving_axis(frame) + _require( + np.array_equal(ids, fresh_ids) and row_index.identical(fresh_index), + "RECEIVING_AXIS_CHANGED", + ) + receipt = full.codec.encode_json( + { + "protocol": PROTOCOL, + "routes": evidence, + "recipient_rows": len(ids), + "target_order": list(PROFILES[0].targets), + "ordered_recipient_ids_sha256": full.codec.sha(ids.tobytes()), + "merge": "disjoint complete clone-one union in receiving order", + "finalization_performed": False, + "donor_model_binding_verified_here": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + _require(len(receipt) <= 128 * 1024, "RECEIPT_SIZE") + # A later route decoder could have changed an earlier detached table. Check + # every final route slice against its original bytes after all decoding. + positions = {value: position for position, value in enumerate(ids.tolist())} + _require( + tuple(combined.columns) == PROFILES[0].targets + and combined.index.identical(row_index) + and all(dtype == np.dtype("float64") for dtype in combined.dtypes) + and all( + full.codec.encode_raw_target( + combined[target].to_numpy()[[positions[value] for value in route_ids]], + target=target, + index=pd.Index(route_ids, name="tax_unit_id"), + ) + == payload + for route_ids, raw in raw_seals + for target, payload in raw + ), + "MERGED_RAW_CHANGED", + ) + return combined, receipt + + +@dataclass(frozen=True) +class Puf55RouteFinalizationInput: + """Numerical inputs only, never evidence that model pickle is trustworthy. + + The graph host must authenticate the actual canonical donor construction, + receiving projection, complete typed fit/apply ancestry and phase before + passing these bytes to the private finalizer. A public dataclass, hash or + this module's descriptive receipt does not satisfy that precondition. + """ + + draws: Puf55RouteDraws + donor: pd.DataFrame + last_model: bytes + phase: str + + +def _donor_identity(donor, columns): + """Exact numerical projection seal, excluding descriptive DataFrame attrs.""" + ids = full._ids(donor.index, "route_donor_RECID") + _require( + donor.index.name == "tax_unit_id" and donor.index.is_unique, "DONOR_RECID_AXIS" + ) + for name in columns: + full._numeric(donor[name], label="route_donor." + name) + return full.codec.sha( + full.codec.encode_json( + { + "columns": list(columns), + "index_name": donor.index.name, + "dtypes": [str(donor[name].dtype) for name in columns], + "recid_sha256": full.codec.sha(ids.tobytes()), + "values_sha256": full.qrf_target._consumed_values_sha256( + donor, tuple(columns) + ), + } + ) + ) + + +def _model_donor_frame(model_donor): + """Convert already validated PUF55 donor values without recipient access. + + The caller first checks the closed profile and selected donor domains. This + fence preserves source RECID values under the canonical tax_unit_id pandas + axis, with separate maintained technical model IDs and exact design weights. + It issues no donor/source or Population authority. + """ + _require( + type(model_donor) is pd.DataFrame + and tuple(model_donor.columns)[-1:] == ("weight",), + "MODEL_DONOR_INPUT", + ) + seal = _donor_identity(model_donor, tuple(model_donor.columns)) + frame = full.support._tax_unit_model_frame(model_donor) + _require(type(frame) is full.Frame, "MODEL_DONOR_FRAME_TYPE") + table = frame.table("tax_unit") + ids = np.arange(1, len(model_donor) + 1, dtype=np.int64) + _require( + _donor_identity(model_donor, tuple(model_donor.columns)) == seal + and tuple(table.columns) == ("tax_unit_id", *model_donor.columns[:-1]) + and table.index.identical(model_donor.index) + and table.drop(columns="tax_unit_id").equals(model_donor.drop(columns="weight")) + and table.tax_unit_id.dtype == np.dtype("int64") + and np.array_equal(table.tax_unit_id.to_numpy(), ids) + and tuple(frame.person.columns) == ("person_id", "person_tax_unit_id") + and frame.person.index.identical(pd.RangeIndex(len(ids))) + and frame.person.person_id.dtype == np.dtype("int64") + and frame.person.person_tax_unit_id.dtype == np.dtype("int64") + and np.array_equal(frame.person.person_id.to_numpy(), ids) + and np.array_equal(frame.person.person_tax_unit_id.to_numpy(), ids) + and frame.weights_for("tax_unit").kind.value == "design" + and frame.weights_for("tax_unit").values.dtype == np.dtype("float64") + and frame.weights_for("tax_unit").values.tobytes() + == model_donor.weight.to_numpy(dtype=np.float64).tobytes(), + "MODEL_DONOR_CONVERSION", + ) + return frame + + +def _candidate_frame_sha256(frame): + """Complete physical candidate seal, not Population or source issuance.""" + _require(type(frame) is full.Frame, "FINALIZATION_RESULT_TYPE") + return physical._population_stamp( + physical.Population.from_frame(frame, "puf55_route_numerical_candidate") + ) + + +def _finalize_puf55_routes(frame, *, recipient_matrices, routes, seed): + """Validate two trusted raw/model chains, then finalize the whole cohort once. + + Mandatory caller precondition: authenticate donor source/construction and + every typed producer edge BEFORE this function can deserialize final-model + bytes. ``phase`` is checked as a value here; only the host can bind it to the + real graph node. Fresh source/recipient qualification and complete upstream + and materialized Population checks remain host duties after this function's + last I/O. This private numerical result grants no source or owner admission. + + Supply only nonempty routes, in nine/eight order. Each donor keeps its full + canonical RECID order, including zero design weights and incidence capacity. + No route-specific Frame/clone selection or full-nine matrix is constructed. + An entirely empty receiving clone-one cohort is refused, consistently with + the maintained finalizer's nonempty PUF detail requirement. + """ + _require(type(frame) is full.Frame, "FINALIZATION_FRAME") + _require(type(seed) is int and seed >= 0, "SEED") + _require(type(routes) is tuple and 1 <= len(routes) <= 2, "FINALIZATION_ROUTES") + _receiving_axis(frame) # Refuse an empty cohort before any trusted decoder. + retained, copies = [], [] + common_columns = ( + *PROFILES[1].predictors, + *PROFILES[1].targets, + "weight", + *PROFILES[1].donor_auxiliary_columns, + ) + for item in routes: + _require(type(item) is Puf55RouteFinalizationInput, "FINALIZATION_INPUT_TYPE") + draws, donor, last_model, phase = ( + item.draws, + item.donor, + item.last_model, + item.phase, + ) + values = _route_snapshot(draws) + profile = full.require_puf_output_profile(values[0]) + _require(type(last_model) is bytes and bool(last_model), "FINAL_MODEL_BYTES") + _require(type(phase) is str and phase == profile.phase, "PHASE") + _require(type(donor) is pd.DataFrame, "DONOR_TYPE") + model_donor = full._validated_model_donor(donor, profile=profile) + donor_seal = _donor_identity(donor, tuple(donor.columns)) + common_seal = _donor_identity(donor, common_columns) + _, training = full.codec.read_training(values[-1]) + _require(training.to_dict()["model_config"]["seed"] == seed, "MODEL_SEED") + # The maintained conversion retains the pandas RECID axis and inserts + # separate model entity IDs. It performs no source admission or fillna. + donor_frame = _model_donor_frame(model_donor) + model_seal = _donor_identity(model_donor, tuple(model_donor.columns)) + retained.append( + ( + item, + draws, + values, + donor, + donor_seal, + model_donor, + model_seal, + donor_frame, + last_model, + phase, + common_seal, + ) + ) + copies.append(Puf55RouteDraws(*values)) + _require(len({row[-1] for row in retained}) == 1, "DONOR_PARITY") + # The original merger validates every profile/matrix/raw predecessor before + # any trusted model decode and performs the single ID-to-pandas relabeling. + combined, merge_receipt = merge_puf55_route_draws( + frame, + recipient_matrices=recipient_matrices, + route_draws=tuple(copies), + seed=seed, + ) + raw_seal = full.qrf_target._consumed_values_sha256(combined, PROFILES[0].targets) + receiving_ids, receiving_rows = _receiving_axis(frame) + for row in retained: + _, _, values, _, _, _, _, donor_frame, last_model, _, _ = row + full._check_fitted_model_donor( + donor_frame, + training_state=values[-1], + last_model=last_model, + profile=values[0], + ) + + def check_inputs(): + for row in retained: + ( + item, + draws, + values, + donor, + donor_seal, + model_donor, + model_seal, + donor_frame, + last_model, + phase, + _, + ) = row + _require( + item.draws is draws + and item.donor is donor + and type(item.last_model) is bytes + and item.last_model == last_model + and type(item.phase) is str + and item.phase == phase + and _route_snapshot(draws) == values + and _donor_identity(donor, tuple(donor.columns)) == donor_seal + and _donor_identity(model_donor, tuple(model_donor.columns)) + == model_seal, + "FINALIZATION_INPUT_CHANGED", + ) + resolved = full.qrf._resolve_qrf_fit_input( + donor_frame, + list(values[0].predictors), + list(values[0].targets), + "design", + ) + _require( + resolved.table.index.identical(model_donor.index) + and resolved.table.loc[:, list(model_donor.columns[:-1])].equals( + model_donor.drop(columns="weight") + ) + and np.array_equal(resolved.weights, model_donor.weight.to_numpy()), + "MODEL_DONOR_FRAME_CHANGED", + ) + fresh_ids, fresh_rows = _receiving_axis(frame) + _require( + np.array_equal(fresh_ids, receiving_ids) + and fresh_rows.identical(receiving_rows) + and combined.index.identical(receiving_rows) + and tuple(combined.columns) == PROFILES[0].targets + and all(dtype == np.dtype("float64") for dtype in combined.dtypes) + and full.qrf_target._consumed_values_sha256(combined, PROFILES[0].targets) + == raw_seal, + "FINALIZATION_RAW_CHANGED", + ) + + check_inputs() + caps = [] + # Both projections have the same target/weight surface. The first nonempty + # route supplies it; its additional predictors are ignored by this finalizer. + candidate = full.support.finalize_us_puf_tax_detail_predictions( + frame, + retained[0][5], + combined.copy(deep=True), + person_outputs=PROFILES[0].person_outputs, + tax_unit_outputs=PROFILES[0].tax_unit_outputs, + tail_bound_diagnostics=caps, + absent_cells=full.support.PUF_ABSENT_CELLS_PRESERVE_NULLS, + ) + check_inputs() + _require(type(candidate) is full.Frame, "FINALIZATION_RESULT_TYPE") + # Construct detached descriptive output only after the last trusted-model + # and finalizer I/O plus the pure retained-input checks. + evidence = [ + { + "profile": row[2][0].value, + "declared_phase": row[9], + "donor_rows": len(row[3]), + "donor_numerical_sha256": row[4], + "common_donor_numerical_sha256": row[10], + "last_model_sha256": full.codec.sha(row[8]), + } + for row in retained + ] + # This immutable digest crosses the function-return boundary with the + # candidate. A caller must compare it before capturing its own baseline. + candidate_sha256 = _candidate_frame_sha256(candidate) + receipt = full.codec.encode_json( + { + "protocol": FINALIZATION_PROTOCOL, + "candidate_frame_sha256": candidate_sha256, + "routes": evidence, + "raw_merge_receipt_sha256": full.codec.sha(merge_receipt), + "common_donor_columns": list(common_columns), + "recipient_rows": len(receiving_ids), + "ordered_recipient_ids_sha256": full.codec.sha(receiving_ids.tobytes()), + "target_order": list(PROFILES[0].targets), + "tail_bounds": caps, + "finalizer_calls": 1, + "absent_cells": full.support.PUF_ABSENT_CELLS_PRESERVE_NULLS, + "model_consumed_donor_values_checked": True, + "source_and_typed_producer_authentication": "mandatory upstream host precondition", + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + _require(type(receipt) is bytes and len(receipt) <= 128 * 1024, "RECEIPT_SIZE") + # Receipt encoding is a final callback boundary. Keep every prior retained + # input check and reject a changed candidate before handing it to the host. + check_inputs() + _require( + _candidate_frame_sha256(candidate) == candidate_sha256, + "FINALIZATION_CANDIDATE_CHANGED", + ) + return candidate, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py new file mode 100644 index 000000000..264a57a5f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py @@ -0,0 +1,574 @@ +"""Disjoint PUF55 recipient values from an actual, authenticated financial run. + +Preparation only: no donor, fit, attachment or Population admission. Retained +modeled return roles are not observed filers or a current-money reconstruction. +The six existing money features retain their all-member aggregation contract. +Source SS reports/components/masks are detached projections on both clone arms; +the receiving Frame and its canonical benefit cells are never modified. +""" + +from __future__ import annotations + +import csv +import hashlib +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.fit import model_input +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES + +from . import graph_atomic_survey_financial as financial +from . import puf55_survey_ss_measurement as ss +from . import puf_full_source as puf_source +from . import support_provenance as provenance + +source, full, support = ss.source, ss.full, ss.full.support +PROTOCOL = "microcosm.us.puf55-survey-recipients.v1" +PROFILES = (full.PUF55_SURVEY_SS, full.PUF55_SURVEY_SS_NO_TOTAL) +MONEY_PREDICTORS = full.PUF59.predictors[2:] +MONEY_COLUMNS = ( + ("employment_income_before_lsr",), + ("self_employment_income_before_lsr",), + ("taxable_interest_income",), + ("non_qualified_dividend_income", "qualified_dividend_income"), + ("short_term_capital_gains",), + ("long_term_capital_gains_before_response",), +) +_STATUS = { + "SINGLE": 1, + "JOINT": 2, + "SEPARATE": 3, + "HEAD_OF_HOUSEHOLD": 4, + "SURVIVING_SPOUSE": 5, +} +MAX_RECEIPT_BYTES = 128 * 1024 + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_SURVEY_RECIPIENTS_" + reason) + + +def _table_digest(table): + """Exact axes, storage domains, values and unknowns for our detached tables.""" + _require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE") + digest = hashlib.sha256() + digest.update( + source._encode( + [ + list(table.columns), + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + [ + [ + str(d), + getattr(d, "storage", None), + str(getattr(d, "na_value", "")), + ] + for d in table.dtypes + ], + ] + ) + ) + for row in table.itertuples(index=True, name=None): + for value in row: + payload = source._frame_cell_encode(value) + digest.update(len(payload).to_bytes(8, "big")) + digest.update(payload) + for series in (pd.Series(table.index), *(table[c] for c in table)): + if series.dtype == np.dtype("object"): + # These projections use object storage only for non-null strings. + # Their bytes were encoded above; ndarray.tobytes() here would hash + # process-local Python pointers and make equivalent replay unstable. + _require( + series.map(lambda value: type(value) is str).all(), + "OBJECT_STRING_STORAGE", + ) + continue + # Whole series: the slice selects every row in order, byte for byte. + for part in financial.reconstruction._storage_parts(series, slice(None)): + digest.update(len(part).to_bytes(8, "big")) + digest.update(part) + return digest.hexdigest() + + +def _literal_snapshot(state): + """Capture only the selected literal rows against the actual closed pin. + + Immutable strings come from the independently hashed original CSV stream, + not from a detached qualifier result or carried reason/allocation columns. + The existing coverage capture owns file handling; this adds no source issuer. + """ + owner = ss.reports.coverage + native = state.native[1] + entry = source.asec_native._ISSUED.get(id(native)) + _require( + entry is not None and entry[0]() is native and entry[1] == native.payload, + "LITERAL_NATIVE_OWNER", + ) + pins = tuple(pin for pin in owner._MEMBER_PINS if pin[0] == 2024) + _require(len(pins) == 1, "LITERAL_PIN") + year, member, archive, digest, rows, size = pins[0] + retained = [ + row + for row in entry[2].coverage.receipt["sources"] + if row["source_year"] == year + ] + _require( + len(retained) == 1 + and all( + retained[0][key] == value + for key, value in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "LITERAL_NATIVE_PIN", + ) + people = state.frame.person + keys = set( + people.loc[ + people[provenance.support_channel_column("person")].eq("asec"), "PERIDNUM" + ] + ) + reader = ss.reports.source_csv_builtin.capture_csv_reader(csv) + _require(reader is not None, "LITERAL_CSV_READER") + captured_rows, count, total = {}, 0, 0 + hashed = hashlib.sha256() + with tempfile.TemporaryDirectory(prefix="microcosm-puf55-ss-literals-") as tmp: + path = Path(tmp) / member + owner._capture( + state.root / "asec" / member, + path, + size=size, + digest=digest, + budget=[owner._BODY_MAX], + ) + with path.open("rb") as stream: + + def lines(): + nonlocal total + while chunk := stream.readline(owner._ROW_MAX + 1): + _require( + len(chunk) <= owner._ROW_MAX and total + len(chunk) <= size, + "LITERAL_BYTES", + ) + first = total == 0 + total += len(chunk) + hashed.update(chunk) + yield chunk.decode("utf-8-sig" if first else "utf-8") + + records = reader(lines(), strict=True) + header = next(records, []) + _require( + len(header) == len(set(header)) + and set(ss.reports.READ_COLUMNS) <= set(header), + "LITERAL_HEADER", + ) + positions = [header.index(c) for c in ss.reports.READ_COLUMNS] + for record in records: + count += 1 + _require(count <= rows and len(record) == len(header), "LITERAL_ROWS") + selected = tuple(record[i] for i in positions) + if selected[0] in keys: + _require(selected[0] not in captured_rows, "LITERAL_DUPLICATE") + captured_rows[selected[0]] = selected + _require( + count == rows + and total == size + and hashed.hexdigest() == digest + and set(captured_rows) == keys, + "LITERAL_SOURCE_BYTES", + ) + return pins[0], tuple(captured_rows[key] for key in sorted(captured_rows)) + + +def _source_report(state, snapshot): + """Pure full projection from pinned literals and retained native source cells.""" + pin, records = snapshot + _require( + tuple(p for p in ss.reports.coverage._MEMBER_PINS if p[0] == 2024) == (pin,), + "LITERAL_PIN_CHANGED", + ) + people = state.frame.person + channel = people[provenance.support_channel_column("person")] + result = pd.DataFrame(index=pd.Index(people.person_id, name="person_id")) + result["native_person_id"] = people[ + provenance.spine_source_id_column("person") + ].to_numpy() + result["source"] = channel.to_numpy() + result["social_security_source_total"] = np.nan + result["source_reporting_universe"] = False + result["source_reporting_unit"] = "person_report_record" + components = ss.reports.basis_owner.COMPONENTS + for column in components: + result[column], result["allowed_" + column] = np.nan, True + result["basis_origin"] = "unresolved" + result["allocation_origin"] = "unresolved_allocation_provenance" + asec = people.loc[channel.eq("asec")] + literals = ( + pd.DataFrame(records, columns=ss.reports.READ_COLUMNS) + .set_index("PERIDNUM", drop=False) + .loc[asec.PERIDNUM] + ) + _require( + np.array_equal(literals.A_AGE.astype("int64"), asec.A_AGE) + and np.array_equal( + ss.reports._codes(literals.SS_VAL, range(100000)), + ss.reports._numeric(asec.SS_VAL), + ), + "LITERAL_SOURCE_IDENTITY", + ) + amount, basis, allowed, labels = ss.reports.basis_owner.asec_reporting_basis( + ss.reports._numeric(asec.SS_VAL), + ss.reports._numeric(asec.A_AGE), + ss.reports._codes(literals.SS_YN, {0, 1, 2}), + ss.reports._codes(literals.RESNSS1, range(9)), + ss.reports._codes(literals.RESNSS2, range(9)), + ) + ids = asec.person_id.to_numpy() + result.loc[ids, "social_security_source_total"] = amount + result.loc[ids, "source_reporting_universe"] = ss.reports._numeric(asec.A_AGE) >= 15 + result.loc[ids, "source_reporting_unit"] = ( + "person_report_may_combine_family_payments" + ) + for j, column in enumerate(components): + result.loc[ids, column], result.loc[ids, "allowed_" + column] = ( + basis[:, j], + allowed[:, j], + ) + result.loc[ids, "basis_origin"] = labels + result.loc[ids, "allocation_origin"] = ss.reports._allocation_labels( + literals + ).allocation_origin.to_numpy() + acs = people.loc[channel.eq("acs")] + eligible = ss.reports._numeric(acs.AGEP) >= 15 + total = ss.reports._numeric(acs.acs_social_security_income) + result.loc[acs.person_id, "social_security_source_total"] = total + result.loc[acs.person_id, "source_reporting_universe"] = eligible + zero = acs.person_id.to_numpy()[eligible & (total == 0)] + result.loc[zero, list(components)] = 0.0 + result.loc[zero, ["allowed_" + c for c in components]] = False + result.loc[zero, "basis_origin"] = "known_total_zero" + result.loc[acs.person_id.to_numpy()[eligible & (total > 0)], "basis_origin"] = ( + "acs_combined_positive_requires_model" + ) + result.loc[acs.person_id.to_numpy()[~eligible], "basis_origin"] = ( + "acs_below15_outside_reporting_universe" + ) + return result + + +def _aligned(source_frame, frame, entity): + original, receiving = source_frame.table(entity), frame.table(entity) + identity = entity + "_id" + key, arm = ( + provenance.support_source_id_column(entity), + provenance.support_clone_index_column(entity), + ) + for column in ( + original[identity], + receiving[identity], + receiving[key], + receiving[arm], + ): + ss._axis(column) + _require(original[identity].is_unique and receiving[identity].is_unique, "AXIS") + pairs = receiving[[key, arm]] + _require( + receiving[arm].isin((0, 1)).all() + and not pairs.duplicated().any() + and set(receiving[key]) == set(original[identity]) + and pairs.groupby(key, sort=False).size().eq(2).all(), + "CLONE_PAIRS", + ) + aligned = original.set_index(identity).loc[receiving[key]].copy(deep=True) + for column in ( + provenance.spine_source_id_column(entity), + provenance.support_channel_column(entity), + ): + _require( + np.array_equal(aligned[column].to_numpy(), receiving[column].to_numpy()), + "CLONE_SOURCE_IDENTITY", + ) + aligned.index = pd.Index(receiving[identity].to_numpy(copy=True), name=identity) + return aligned + + +def _current_money_surface(frame, recipient_mask): + """Aggregate already-qualified financial leaves; issue no source authority. + + The public caller rechecks the actual financial run before this arithmetic. + Its source owner already verifies ACS adjustment and age-universe zeros. + A raw stacked-spine universe gate would incorrectly require ACS columns on + an ASEC-only surface and is not the input boundary for these produced leaves. + """ + person, units = frame.person, frame.table("tax_unit") + plans = {} + for predictor, columns in zip(MONEY_PREDICTORS, MONEY_COLUMNS, strict=True): + plan = support._strict_predictor_source_plan( + predictor, tax_unit=units, person=person + ) + _require( + plan.entity == "person" and plan.columns == columns, + "CURRENT_MONEY_SOURCE_PLAN", + ) + plans[predictor] = plan + support._require_complete_recipient_predictor_sources( + frame, recipient_mask, MONEY_PREDICTORS, source_plans=plans + ) + features = support._tax_unit_feature_frame( + frame, MONEY_PREDICTORS, preserve_nulls=True, source_plans=plans + ) + support._require_complete_recipient_predictors( + features, recipient_mask, MONEY_PREDICTORS + ) + evidence = { + "input_boundary": "current_financial_person_leaves_after_upstream_qualification", + "aggregation": "sum_all_modeled_tax_unit_members", + "predictor_source_mapping": { + predictor: {"entity": plan.entity, "columns": list(plan.columns)} + for predictor, plan in plans.items() + }, + "recipient_tax_unit_rows": int(recipient_mask.sum()), + "recipient_feature_values_sha256": support._predictor_feature_values_sha256( + features.loc[recipient_mask, list(MONEY_PREDICTORS)], + units.loc[recipient_mask, "tax_unit_id"], + ), + "raw_acs_universe_requalified_here": False, + "source_admission_issued": False, + } + evidence["sha256"] = support._receipt_sha256(evidence) + return features, evidence + + +def _project(source_frame, frame, report, measured): + """Pure projection after live ownership checks; never an authority issuer.""" + people, units = frame.person, frame.table("tax_unit") + original_people = _aligned(source_frame, frame, "person") + original_units = _aligned(source_frame, frame, "tax_unit") + unit_source = units.set_index("tax_unit_id")[ + provenance.support_source_id_column("tax_unit") + ] + _require( + np.array_equal( + people.tax_unit_role_input.to_numpy(), + original_people.tax_unit_role_input.to_numpy(), + ) + and np.array_equal( + units.filing_status_input.to_numpy(), + original_units.filing_status_input.to_numpy(), + ) + and np.array_equal( + people.person_tax_unit_id.map(unit_source).to_numpy(), + original_people.person_tax_unit_id.to_numpy(), + ), + "MODELED_ROLE_INHERITANCE", + ) + person_ids = people[provenance.support_source_id_column("person")].to_numpy() + person_report = report.loc[person_ids].copy(deep=True) + person_report.index = pd.Index( + people.person_id.to_numpy(copy=True), name="person_id" + ) + # Recheck every receiving role structure, and independently reproduce the + # source measurement on both arms. Invalid roles never select route eight. + received_measurement, _ = ss._measure(people, units, person_report) + expected = measured.loc[ + units[provenance.support_source_id_column("tax_unit")] + ].copy(deep=True) + expected.index = received_measurement.index + _require( + ss._projection_digest(received_measurement) == ss._projection_digest(expected), + "CLONE_MEASUREMENT", + ) + dependent_count = ( + people.tax_unit_role_input.eq("DEPENDENT") + .groupby(people.person_tax_unit_id) + .sum() + ) + first = puf_source.puf_2015_receiver_size_measurement( + units.filing_status_input.map(_STATUS).to_numpy(dtype=np.int64), + dependent_count.reindex(units.tax_unit_id).to_numpy(dtype=np.int64), + ) + for name, values in first.items(): + received_measurement[name] = values + recipients = ( + units[provenance.support_clone_index_column("tax_unit")].eq(1).to_numpy() + ) + features, money_evidence = _current_money_surface(frame, recipients) + for name in MONEY_PREDICTORS: + received_measurement[name] = features[name].to_numpy( + dtype=np.float64, copy=True + ) + matrices, selected = [], [] + for profile in PROFILES: + mask = recipients & received_measurement[ss.ROUTE].eq(profile.value).to_numpy() + ids = units.loc[mask, "tax_unit_id"].to_numpy(copy=True) + selected.extend(ids.tolist()) + if not len(ids): + continue + _require( + len(ids) * (1 + len(profile.predictors)) * 8 <= RAW_BYTES_MAX_BYTES, + "MATRIX_SIZE", + ) + table = received_measurement.loc[ids, list(profile.predictors)].astype( + "float64" + ) + payload = model_input.encode_recipient_matrix( + table, entity="tax_unit", entity_ids=ids + ) + _require(len(payload) <= RAW_BYTES_MAX_BYTES, "MATRIX_SIZE") + matrices.append((profile.value, payload)) + _require( + len(selected) == len(set(selected)) + and set(selected) == set(units.loc[recipients, "tax_unit_id"]), + "RECIPIENT_PARTITION", + ) + return person_report, received_measurement, tuple(matrices), money_evidence + + +@dataclass(frozen=True) +class Puf55SurveyRecipients: + """Detached outputs with their actual upstream run; public copies are values.""" + + financial_run: financial.AtomicSurveyFinancialRunValues + person: pd.DataFrame + tax_unit: pd.DataFrame + matrices: tuple[tuple[str, bytes], ...] + receipt: bytes + + +def _result_stamp(result): + _require( + type(result) is Puf55SurveyRecipients + and type(result.receipt) is bytes + and type(result.matrices) is tuple + and all( + type(row) is tuple + and len(row) == 2 + and type(row[0]) is str + and type(row[1]) is bytes + for row in result.matrices + ), + "RESULT_TYPE", + ) + return ( + _table_digest(result.person), + _table_digest(result.tax_unit), + result.matrices, + result.receipt, + ) + + +def qualify_puf55_survey_recipients(financial_run): + """Require the actual financial run and requalify source observations. + + This admits numerical preparation under retained modeled roles only. It + cannot establish current-money tax-unit reconstruction or release validity. + Source receipt objects and caller-supplied role tables are not accepted. + """ + financial.check_atomic_survey_financial_run(financial_run) + financial.require_complete_property_taxes(financial_run) + entry = financial._run_entry(financial_run) + state = entry[2] + preparation = state.prefix.preparation + source_frame = state.preparation_entry[2].frame + frame = state.financial_population.frame + snapshot = _literal_snapshot(state.preparation_entry[2]) + report = ss.reports.qualify_current_social_security(preparation) + ss._source_identity(source_frame.person, report) + authoritative_report = _source_report(state.preparation_entry[2], snapshot) + _require( + _table_digest(report.person) == _table_digest(authoritative_report), + "COMPLETE_SOURCE_REPORT", + ) + expected_measurement, measurement_counts = ss._measure( + source_frame.person, source_frame.table("tax_unit"), authoritative_report + ) + person, units, matrices, money_evidence = _project( + source_frame, frame, authoritative_report, expected_measurement + ) + receipt = source._encode( + { + "protocol": PROTOCOL, + "financial_run_sha256": financial.codec.sha(entry[1]), + "preparation_sha256": financial.codec.sha(state.preparation_entry[1]), + "person_projection_sha256": _table_digest(person), + "tax_unit_projection_sha256": _table_digest(units), + "source_report_projection_sha256": _table_digest(authoritative_report), + "asec_current_literal_member_sha256": snapshot[0][3], + "asec_selected_literal_rows": len(snapshot[1]), + "asec_income_year": 2024, + "asec_interview_year": 2025, + "acs_income_window": "rolling prior12 months at2024 interview; ADJINC price basis", + "reporting_grain": "person report; family/child aggregation and repayment/Railroad alignment unresolved", + "ss_measurement": ss.MEASUREMENT, + "native_measurement_counts": measurement_counts, + "measurement_knownness": "all included HEAD/actual JOINT SPOUSE reports available; no dependent, component or partial-sum fallback", + "money_predictor_evidence": money_evidence, + "routes": [ + { + "profile": name, + "matrix_sha256": financial.codec.sha(payload), + "rows": len(model_input.decode_recipient_matrix(payload).features), + "predictors": list(full.PufOutputProfile(name).predictors), + "outputs": list(full.PufOutputProfile(name).targets), + } + for name, payload in matrices + ], + "recipient_tax_units": sum( + len(model_input.decode_recipient_matrix(p).features) + for _, p in matrices + ), + "receiving_tax_units": len(units), + "receiving_persons": len(person), + "partition": "all clone-one tax units exactly once; empty routes omitted in nine/eight order", + "role_boundary": "retained modeled HEAD/actual JOINT SPOUSE roles; not observed filer status", + "money_contract": "existing six features sum all modeled tax-unit members; current financial leaves only", + "current_money_tax_units_reconstructed_here": False, + "person_social_security_changed": False, + "population_changed": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + }, + maximum=MAX_RECEIPT_BYTES, + ) + result = Puf55SurveyRecipients(financial_run, person, units, matrices, receipt) + stamp = _result_stamp(result) + financial.check_atomic_survey_financial_run(financial_run) + financial._pure_run(financial_run, entry) + fresh = _source_report(state.preparation_entry[2], snapshot) + fresh_measurement, _ = ss._measure( + source_frame.person, source_frame.table("tax_unit"), fresh + ) + final_person, final_units, final_matrices, final_money = _project( + source_frame, frame, fresh, fresh_measurement + ) + _require( + financial._run_entry(financial_run) is entry + and result.financial_run is financial_run + and _result_stamp(result) == stamp + and _table_digest(fresh) == _table_digest(authoritative_report) + and ( + _table_digest(final_person), + _table_digest(final_units), + final_matrices, + receipt, + ) + == stamp + and final_money == money_evidence, + "FINAL_PROJECTION", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py new file mode 100644 index 000000000..d5abe6afb --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py @@ -0,0 +1,320 @@ +"""Filer/joint-spouse report sums for conditioning, never beneficiary amounts. + +The public entry point requires a live authenticated survey preparation. Its +retained modeled return roles are inputs, not roles inferred from the coarsened +PUF filing-class predictor. The private arithmetic is not a source issuer. +Graph hosts must retain/requalify the preparation and authenticate clone +inheritance, routing and typed model edges independently. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from . import current_social_security_source as reports +from . import full_puf_enrichment as full +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.puf55-survey-ss-measurement.v1" +MEASUREMENT = "social_security_filer_joint_spouse_report_sum_proxy" +TOTAL = full.SURVEY_SS_TOTAL_PREDICTOR +KNOWN = "puf_conditioning_social_security_total_known" +ROUTE = "puf55_conditioning_profile" +COLUMNS = (TOTAL, KNOWN, ROUTE) +_PERSON_COLUMNS = ("person_id", "person_tax_unit_id", "tax_unit_role_input") +_UNIT_COLUMNS = ("tax_unit_id", "filing_status_input") +_REPORT_COLUMNS = ( + "social_security_source_total", + "source_reporting_universe", +) +_STATUSES = {"SINGLE", "JOINT", "SEPARATE", "HEAD_OF_HOUSEHOLD", "SURVIVING_SPOUSE"} + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_SURVEY_SS_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _axis(values): + _require( + values.dtype == np.dtype("int64") + and not values.isna().any() + and values.ge(0).all(), + "AXIS", + ) + + +def _measure(person, tax_unit, report): + """Pure numerical checks on already-qualified values; no admission issued.""" + for table, columns in ( + (person, _PERSON_COLUMNS), + (tax_unit, _UNIT_COLUMNS), + (report, _REPORT_COLUMNS), + ): + _require( + type(table) is pd.DataFrame + and table.columns.is_unique + and set(columns) <= set(table), + "COLUMNS", + ) + for column in (person.person_id, person.person_tax_unit_id, tax_unit.tax_unit_id): + _axis(column) + _require( + len(tax_unit) > 0 + and person.person_id.is_unique + and tax_unit.tax_unit_id.is_unique + and set(person.person_tax_unit_id) == set(tax_unit.tax_unit_id) + and report.index.name == "person_id" + and report.index.dtype == np.dtype("int64") + and report.index.is_unique + and set(report.index) == set(person.person_id), + "MEMBERSHIP", + ) + roles, statuses = person.tax_unit_role_input, tax_unit.filing_status_input + _require( + not roles.isna().any() + and roles.map(lambda v: type(v) is str).all() + and roles.isin(("HEAD", "SPOUSE", "DEPENDENT")).all() + and not statuses.isna().any() + and statuses.map(lambda v: type(v) is str).all() + and statuses.isin(_STATUSES).all(), + "ROLE_VALUES", + ) + # Validate every return's roles before deciding any missing-report route. + # SURVIVING_SPOUSE maps to PUF class 2 elsewhere but has no actual spouse. + groups = person.groupby("person_tax_unit_id", sort=False) + included = [] + for tid, status in tax_unit[list(_UNIT_COLUMNS)].itertuples(index=False, name=None): + members = groups.get_group(tid) + head = members.tax_unit_role_input.eq("HEAD") + spouse = members.tax_unit_role_input.eq("SPOUSE") + _require( + head.sum() == 1 and spouse.sum() == int(status == "JOINT"), + "ROLE_STRUCTURE", + ) + included.append(members.loc[head | spouse, "person_id"].to_numpy(copy=True)) + amount, universe = report[_REPORT_COLUMNS[0]], report[_REPORT_COLUMNS[1]] + _require( + pd.api.types.is_numeric_dtype(amount.dtype) + and not pd.api.types.is_bool_dtype(amount.dtype) + and not pd.api.types.is_complex_dtype(amount.dtype), + "REPORT_PHYSICAL_TYPE", + ) + _require(universe.dtype == np.dtype("bool"), "UNIVERSE_PHYSICAL_TYPE") + if pd.api.types.is_integer_dtype(amount.dtype): + _require(amount.dropna().between(-(2**53), 2**53).all(), "REPORT_PRECISION") + numeric = amount.to_numpy(dtype=np.float64, na_value=np.nan, copy=True) + available = np.isfinite(numeric) + _require( + (np.isnan(numeric) | available).all() and (numeric[available] >= 0).all(), + "REPORT_DOMAIN", + ) + _require( + np.isnan(numeric[~universe.to_numpy()]).all(), "OUTSIDE_UNIVERSE_OBSERVATION" + ) + values, known, routes = [], [], [] + missing_people, included_people = 0, 0 + for ids in included: + selected = report.loc[ids] + observed = selected[_REPORT_COLUMNS[0]].to_numpy( + dtype=np.float64, na_value=np.nan, copy=True + ) + present = selected[_REPORT_COLUMNS[1]].to_numpy() & np.isfinite(observed) + complete = bool(present.all()) + included_people += len(ids) + missing_people += int((~present).sum()) + # No pandas skipna, empty sum, dependent zero, component sum or partial + # report sum can manufacture a known measurement. + total = float(np.sum(observed)) if complete else np.nan + _require(not complete or np.isfinite(total), "SUM_OVERFLOW") + values.append(total) + known.append(complete) + routes.append( + (full.PUF55_SURVEY_SS if complete else full.PUF55_SURVEY_SS_NO_TOTAL).value + ) + result = pd.DataFrame( + { + TOTAL: np.asarray(values, dtype=np.float64), + KNOWN: np.asarray(known, dtype=bool), + ROUTE: pd.array(routes, dtype="string"), + }, + index=pd.Index(tax_unit.tax_unit_id.to_numpy(copy=True), name="tax_unit_id"), + ) + counts = { + "returns": len(result), + "included_reporters": included_people, + "excluded_dependents": len(person) - included_people, + "unavailable_included_reports": missing_people, + "nine_predictor_returns": int(result[KNOWN].sum()), + "eight_predictor_returns": int((~result[KNOWN]).sum()), + } + return result, counts + + +def _projection_digest(table): + _require( + type(table) is pd.DataFrame + and tuple(table) == COLUMNS + and table.index.name == "tax_unit_id" + and table.index.dtype == np.dtype("int64") + and table.index.is_unique + and table[TOTAL].dtype == np.dtype("float64") + and table[KNOWN].dtype == np.dtype("bool"), + "PROJECTION_STORAGE", + ) + digest = hashlib.sha256() + for tid, total, known, route in table.itertuples(index=True, name=None): + payload = source._encode( + [ + int(tid), + None if np.isnan(total) else float(total).hex(), + bool(known), + route, + ] + ) + digest.update(len(payload).to_bytes(4, "big")) + digest.update(payload) + return digest.hexdigest() + + +def _source_identity(people, projection): + """Check detached source output against the exact retained source cells. + + SS qualification already authenticates original current ASEC literals and + ACS SSP adjustment. This additional comparison prevents a return callback + from changing its detached amount/universe without changing the live owner. + """ + _require(type(projection) is reports.CurrentSocialSecurityProjection, "REPORT_TYPE") + report = projection.person + _require( + report.index.is_unique and set(report.index) == set(people.person_id), + "SOURCE_AXIS", + ) + report = report.reindex(people.person_id) + channels = people[support_channel_column("person")] + _require( + np.array_equal(report.source.to_numpy(), channels.to_numpy()) + and np.array_equal( + report.native_person_id.to_numpy(), + people[spine_source_id_column("person")].to_numpy(), + ), + "SOURCE_ORIGIN", + ) + for channel, age_column, amount_column in ( + ("asec", "A_AGE", "SS_VAL"), + ("acs", "AGEP", "acs_social_security_income"), + ): + mask = channels.eq(channel).to_numpy() + selected = people.loc[mask] + age = reports._numeric(selected[age_column]) + expected = reports._numeric(selected[amount_column]) + eligible = age >= 15 + expected[~eligible] = np.nan + observed = report.iloc[np.flatnonzero(mask)] + _require( + np.array_equal(observed.source_reporting_universe.to_numpy(), eligible) + and np.array_equal( + reports._numeric(observed.social_security_source_total), + expected, + equal_nan=True, + ), + "SOURCE_REPORT_IDENTITY", + ) + + +@dataclass(frozen=True) +class Puf55SurveySSMeasurement: + """Detached values and live upstream reference, not a reusable certificate.""" + + preparation: source.AuthenticatedSurveyPopulationPreparation + tax_unit: pd.DataFrame + evidence: bytes + + +def qualify_puf55_survey_ss_measurement(preparation): + """Project native stacked returns; clone inheritance remains the host's duty. + + Roles are the modeled roles retained by the exact preparation, not a claim + that this helper reran current-money tax-unit construction. Receipt bytes, + caller-made tables, and this result cannot replace the live upstream owner. + """ + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + projection = reports.qualify_current_social_security(preparation) + _source_identity(state.frame.person, projection) + report_evidence = source._encode(projection.evidence) + _require( + projection.evidence["preparation_sha256"] == _sha(entry[1]), + "REPORT_PREPARATION", + ) + values, counts = _measure( + state.frame.person, state.frame.table("tax_unit"), projection.person + ) + stamp = _projection_digest(values) + evidence = source._encode( + { + "protocol": PROTOCOL, + "measurement": MEASUREMENT, + "preparation_sha256": _sha(entry[1]), + "source_report_evidence_sha256": _sha(report_evidence), + "projection_sha256": stamp, + "counts": counts, + "columns": list(COLUMNS), + "membership": "HEAD plus SPOUSE only for literal JOINT; no dependents", + "role_boundary": "exact modeled return roles retained by live preparation", + "current_money_tax_units_reconstructed_here": False, + "knownness": "all included source reports available; not benefit ownership", + "fallback": full.PUF55_SURVEY_SS_NO_TOTAL.value, + "family_and_child_reports": "retained without allocation or deduplication", + "source_periods": { + "asec": "2024 calendar income; 2025 interview", + "acs": "rolling prior 12 months at 2024 interview; ADJINC price basis", + }, + "repayment_and_railroad_alignment": "unresolved; no adjustment applied", + "donor_E02400_editing": "unresolved; separate donor transport convention", + "person_social_security_changed": False, + "individual_beneficiary_assignment_claim": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + }, + maximum=64 * 1024, + ) + result = Puf55SurveySSMeasurement(preparation, values, evidence) + # Finish live owner I/O, then only compare retained owners and computed + # values. The returned projection is never itself treated as authority. + final_entry = preparation._checked() + _require( + final_entry is entry + and source._ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1], + "FINAL_ISSUANCE", + ) + source._pure_final(state) + _source_identity(state.frame.person, projection) + fresh, fresh_counts = _measure( + state.frame.person, state.frame.table("tax_unit"), projection.person + ) + _require( + source._encode(projection.evidence) == report_evidence + and fresh_counts == counts + and _projection_digest(fresh) == stamp + and result.preparation is preparation + and result.evidence == evidence + and _projection_digest(result.tax_unit) == stamp, + "FINAL_PROJECTION", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical.py new file mode 100644 index 000000000..af1290c9f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical.py @@ -0,0 +1,307 @@ +"""Return-grain canonical PUF59 with explicit model/growth lineage. + +No tax engine, invented donor people, loan balances or loan vintages. The +observed statistical2015 source remains immutable beside modeled outputs. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +from . import puf_full_source as source_owner +from . import puf_qbi_model as qbi +from . import puf_target2024_growth as growth + +VERSION = "microcosm.us.puf59_canonical_return/2" +INTEREST_ASSET_SHA256 = ( + "c3356ae216487f365cb0e0a7ab1ba46843a6c52950b75eae1bfab9b0b80a735a" +) + +INTEREST_BAND_FACTS_SHA256 = ( + "3ea894c474302dd818f347f5bfde3b331bf80468b0ac928ffe76ed1e6de62cbc" +) +PREFIX_NAMES = ( + "RECID", + "weight", + "puf_person_incidence_capacity", + "puf_2015_filing_status_code", + "puf_2015_capped_return_size", +) + + +def prefix_values_digest(arrays): + """Bind identities, source weights and source-specific features in row order.""" + digest = hashlib.sha256() + for name in PREFIX_NAMES: + digest.update(name.encode("ascii") + b"\0") + digest.update( + np.asarray( + arrays[name], dtype="= -(2**53)) & (a <= 2**53)).all()), + "PUF59_MODEL_INTEGER_PRECISION:" + name, + ) + a = a.astype(np.float64) + _require(bool(np.isfinite(a).all()), "PUF59_MODEL_NONFINITE:" + name) + return a + + +def baseline_modeled_columns(auxiliary, *, n, interest_bands, interest_asset_sha256): + """Eleven explicit baseline outputs absent as exact raw canonical leaves.""" + _require( + interest_asset_sha256 == INTEREST_ASSET_SHA256, "PUF59_INTEREST_ASSET_IDENTITY" + ) + _require(isinstance(auxiliary, Mapping), "PUF59_MODEL_AUXILIARY_MAPPING") + names = ( + "raw_adjusted_gross_income", + "raw_total_interest_deduction", + "raw_total_social_security", + "raw_realized_ira_deduction", + "raw_realized_keogh_deduction", + "raw_tuition_fees_deduction", + "raw_lifetime_learning_qualified_expenses", + "raw_miscellaneous_itemized_deductions", + "raw_partnership_nonpassive_net", + ) + _require(all(k in auxiliary for k in names), "PUF59_MODEL_MISSING_AUXILIARY") + a = {k: _money(auxiliary[k], k, n) for k in names} + for k in names: + if k not in ("raw_adjusted_gross_income", "raw_partnership_nonpassive_net"): + _require(bool((a[k] >= 0).all()), "PUF59_MODEL_NEGATIVE_SOURCE:" + k) + bands = tuple(interest_bands) + _require(len(bands) == 22, "PUF59_INTEREST_BAND_COUNT") + _require( + bands[0].lower_bound is None and bands[-1].upper_bound is None, + "PUF59_INTEREST_FULL_AGI", + ) + for left, right in zip(bands[:-1], bands[1:], strict=True): + _require(left.upper_bound == right.lower_bound, "PUF59_INTEREST_CONTIGUOUS") + total = a["raw_total_interest_deduction"] + agi = a["raw_adjusted_gross_income"] + investment = np.zeros(n) + covered = np.zeros(n, dtype=np.uint8) + band_facts = [] + fields = ( + "home_mortgage_interest_amount", + "deductible_points_amount", + "qualified_mortgage_insurance_premiums_amount", + "investment_interest_amount", + ) + for i, band in enumerate(bands): + _require(band.source_row == 11 + i, "PUF59_INTEREST_SOURCE_ROW") + lo, hi = band.lower_bound, band.upper_bound + _require( + (lo is None or type(lo) in (int, float) and np.isfinite(lo)) + and (hi is None or type(hi) in (int, float) and np.isfinite(hi)) + and (lo is None or hi is None or lo < hi), + "PUF59_INTEREST_BAND_BOUNDS", + ) + amounts = np.asarray([getattr(band, k) for k in fields], dtype=np.float64) + _require( + bool(np.isfinite(amounts).all() and (amounts >= 0).all()) + and amounts.sum() > 0, + "PUF59_INTEREST_COMPONENT_DOMAIN", + ) + _require( + type(band.total_interest_paid_amount) is int + and abs( + band.total_interest_paid_amount - sum(getattr(band, k) for k in fields) + ) + <= 1, + "PUF59_INTEREST_SOURCE_ROUNDING", + ) + mask = np.ones(n, dtype=bool) + if lo is not None: + mask &= agi >= lo + if hi is not None: + mask &= agi < hi + investment[mask] = total[mask] * (amounts[3] / amounts.sum()) + covered[mask] += 1 + band_facts.append( + { + "source_row": band.source_row, + "lower_bound": lo, + "upper_bound": hi, + "amounts_thousands_usd": amounts.tolist(), + } + ) + band_facts_sha256 = hashlib.sha256( + json.dumps(band_facts, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + _require( + band_facts_sha256 == INTEREST_BAND_FACTS_SHA256, + "PUF59_INTEREST_BAND_FACTS_IDENTITY", + ) + _require(bool((covered == 1).all()), "PUF59_INTEREST_PARTITION") + home = total - investment + investment = total - home + _require(np.array_equal(home + investment, total), "PUF59_INTEREST_CONSERVATION") + columns = { + "home_mortgage_interest": home, + "investment_interest_expense": investment, + "social_security_retirement": a["raw_total_social_security"].copy(), + "social_security_disability": np.zeros(n), + "social_security_dependents": np.zeros(n), + "social_security_survivors": np.zeros(n), + "traditional_ira_contributions_desired": a["raw_realized_ira_deduction"].copy(), + "self_employed_pension_contributions_desired": a[ + "raw_realized_keogh_deduction" + ].copy(), + "qualified_tuition_expenses": np.maximum( + a["raw_tuition_fees_deduction"], + a["raw_lifetime_learning_qualified_expenses"], + ), + "unreimbursed_business_employee_expenses": a[ + "raw_miscellaneous_itemized_deductions" + ].copy(), + "partnership_self_employment_net_earnings": a[ + "raw_partnership_nonpassive_net" + ].copy(), + } + return columns, { + "schema": "microcosm.us.puf59_baseline_models/1", + "input_money_year": 2015, + "interest_asset_sha256": interest_asset_sha256, + "interest_band_facts_sha256": band_facts_sha256, + "interest_method": "Normalize all four published component amounts. Map mortgage+points+insurance to the available home interest leaf and investment only to investment expense; complementary rounding conserves E19200 exactly.", + "ss_method": "Component-neutral carrier: total SS in retirement, other component carriers zero; requires recipient-profile SS reconciliation. Zeros are not observations of absent benefits.", + "desired_deductions": "Realized IRA/Keogh deductions proxy for desired amounts; uncensored demand is not observed.", + "tuition": "Maximum of realized tuition deduction and Lifetime Learning qualified expenses; incomplete source expense proxy.", + "employee_expense": "Total miscellaneous itemized deductions proxy; broader than employee business expenses.", + "partnership_earnings": "Nonpassive partnership net earnings proxy; not reconstructed from capped ScheduleSE amounts.", + "mortgage_structure_generated": False, + "origin": "modeled_not_observed", + } + + +@dataclass(frozen=True) +class CanonicalPuf59Result: + columns: Mapping[str, np.ndarray] + source_predictors: Mapping[str, np.ndarray] + recids: np.ndarray + design_weight: np.ndarray + person_incidence_capacity: np.ndarray + money_year: int + qbi_calibration: qbi.QbiEmploymentCalibration + receipt: Mapping[str, object] + + +def construct_canonical_puf59( + source, + *, + interest_bands, + interest_asset_sha256, + selected_recids=None, + qbi_employment_calibration=None, + growth_scheme="family_observed", + seed=0, +): + """Construct canonical outputs from an owner-decoded ordinary source cohort. + + The caller supplies source-authenticated interest bands. Selection is owned + by the source decoder. No data source is opened or sampled in this function. + """ + observed, aux, status = source_owner.observed_and_derived_return_columns( + source, selected_recids=selected_recids + ) + n = len(status["RECID"]) + _require(n > 0, "PUF59_EMPTY_COHORT") + modeled, model_receipt = baseline_modeled_columns( + aux, + n=n, + interest_bands=interest_bands, + interest_asset_sha256=interest_asset_sha256, + ) + _require(not set(observed) & set(modeled), "PUF59_CANONICAL_COLLISION") + canonical = {**observed, **modeled} + known = {k: np.ones(n, dtype=bool) for k in canonical} + qresult = qbi.model_full_puf_qbi( + canonical, + status["RECID"], + known=known, + input_money_year=2015, + seed=seed, + employment_calibration=qbi_employment_calibration, + ) + canonical.update(qresult.columns) + _require(set(canonical) == set(growth.OUTPUTS), "PUF59_CANONICAL_COMPLETE_ROSTER") + known = {k: np.ones(n, dtype=bool) for k in canonical} + result = growth.grow_puf_2015_to_2024( + canonical, known=known, input_money_year=2015, scheme=growth_scheme + ) + weight = status["S006"].astype(np.float64) / 100 + _require( + bool((weight >= 0).all() and np.isfinite(weight).all() and weight.sum() > 0), + "PUF59_WEIGHT_DOMAIN", + ) + predictors = { + k: np.frombuffer(np.asarray(aux[k], dtype=np.int64).tobytes(), dtype=np.int64) + for k in ("puf_2015_filing_status_code", "puf_2015_capped_return_size") + } + prefix_arrays = { + "RECID": status["RECID"], + "weight": weight, + "puf_person_incidence_capacity": np.ones(n, dtype=np.int64), + **predictors, + } + receipt = { + "prefix_values_sha256": prefix_values_digest(prefix_arrays), + "schema": VERSION, + "source_definition_sha256": source.definition_sha256, + "source_sha256": dict(source.source_sha256), + "source_statistical_year": 2015, + "input_money_year": 2015, + "output_money_year": 2024, + "rows": n, + "model_assumptions": model_receipt, + "qbi": dict(qresult.receipt), + "growth": dict(result.receipt), + "return_size": "Source PUF2015 coarsened filing-status plus capped dependent count; no physical person claim.", + "person_incidence_capacity": "Constant1 for zero/one modeled return incidence; excluded from conditioning predictors.", + "weight": "Source S006 exact hundredths divided once by100; no weight growth.", + "recipient_conditioning_self_employment": "Must sum base and SSTB ScheduleC outcomes for the total source predictor.", + "source_observation_status": "33 direct/derived baseline columns; 11 explicit baseline models; 15 added QBI leaves and one replacement; all monetary transport is modeled.", + "mortgage_structure_generated": False, + "release_eligible": False, + } + receipt["sha256"] = hashlib.sha256( + json.dumps( + receipt, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + + def readonly(a): + return np.frombuffer(a.tobytes(), dtype=a.dtype) + + return CanonicalPuf59Result( + result.columns, + MappingProxyType(predictors), + readonly(status["RECID"].astype(np.int64)), + readonly(weight), + readonly(np.ones(n, dtype=np.int64)), + 2024, + qresult.calibration, + MappingProxyType(receipt), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical_artifact.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical_artifact.py new file mode 100644 index 000000000..d0b5566c8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical_artifact.py @@ -0,0 +1,237 @@ +"""Bounded deterministic private canonical59 array envelope; no source admission.""" + +from __future__ import annotations + +import hashlib +import json +import struct +from collections.abc import Mapping + +import numpy as np + +from .puf59_canonical import PREFIX_NAMES, VERSION, prefix_values_digest +from .puf_target2024_growth import INCIDENCE_FIELDS, OUTPUTS, RECIPE_SHA256, _digest + +MAGIC = b"MCPUF59\x02" +MAX_HEADER = 128 * 1024 +MAX_BODY = 128 * 1024 * 1024 +PREFIX = PREFIX_NAMES +NAMES = (*PREFIX, *OUTPUTS) +INTEGERS = set(PREFIX) - {"weight"} | set(INCIDENCE_FIELDS) + + +def _require(value, code): + if not value: + raise ValueError(code) + + +def _receipt(receipt, expected_growth_scheme): + _require( + expected_growth_scheme in ("family_observed", "cpi_only"), + "PUF59_ARTIFACT_EXPECTED_SCHEME", + ) + _require(isinstance(receipt, Mapping), "PUF59_ARTIFACT_RECEIPT_TYPE") + r = dict(receipt) + _require( + r.get("schema") == VERSION + and type(r.get("input_money_year")) is int + and r["input_money_year"] == 2015 + and type(r.get("output_money_year")) is int + and r["output_money_year"] == 2024 + and type(r.get("rows")) is int + and 0 < r["rows"] <= MAX_BODY // (len(NAMES) * 8) + and r.get("release_eligible") is False, + "PUF59_ARTIFACT_RECEIPT_CONTRACT", + ) + gr = r.get("growth") + _require( + isinstance(gr, Mapping) and gr.get("scheme") == expected_growth_scheme, + "PUF59_ARTIFACT_GROWTH_SCHEME", + ) + _require(gr.get("recipe_sha256") == RECIPE_SHA256, "PUF59_ARTIFACT_GROWTH_RECIPE") + h = r.pop("sha256", None) + _require( + h + == hashlib.sha256( + json.dumps( + r, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest(), + "PUF59_ARTIFACT_RECEIPT_HASH", + ) + r["sha256"] = h + return r + + +def _validate(arrays, n): + _require( + type(n) is int and 0 < n <= MAX_BODY // (len(NAMES) * 8), "PUF59_ARTIFACT_ROWS" + ) + _require( + isinstance(arrays, Mapping) and set(arrays) == set(NAMES), + "PUF59_ARTIFACT_COLUMNS", + ) + for name in NAMES: + a = np.asarray(arrays[name]) + kind = "iu" if name in INTEGERS else "fiu" + _require( + a.shape == (n,) and a.dtype.kind in kind, + "PUF59_ARTIFACT_COLUMN_TYPE:" + name, + ) + _require(bool(np.isfinite(a).all()), "PUF59_ARTIFACT_NONFINITE:" + name) + if a.dtype.kind in "iu": + _require( + bool(((a >= -(2**53)) & (a <= 2**53)).all()), + "PUF59_ARTIFACT_INTEGER_RANGE:" + name, + ) + ids = arrays["RECID"] + w = arrays["weight"] + _require( + bool((ids > 0).all()) + and len(np.unique(ids)) == n + and not np.isin(ids, [999996, 999997, 999998, 999999]).any(), + "PUF59_ARTIFACT_ID_DOMAIN", + ) + _require( + bool((w >= 0).all()) and np.isfinite(w.sum()) and w.sum() > 0, + "PUF59_ARTIFACT_WEIGHT", + ) + _require( + bool((arrays["puf_person_incidence_capacity"] == 1).all()), + "PUF59_ARTIFACT_CAPACITY", + ) + _require( + bool(np.isin(arrays["puf_2015_filing_status_code"], [1, 2, 3, 4]).all()), + "PUF59_ARTIFACT_STATUS", + ) + size = arrays["puf_2015_capped_return_size"] + _require(bool(((size >= 1) & (size <= 5)).all()), "PUF59_ARTIFACT_RETURN_SIZE") + for name in INCIDENCE_FIELDS: + _require(bool(np.isin(arrays[name], [0, 1]).all()), "PUF59_ARTIFACT_INCIDENCE") + + +def _bindings(arrays, receipt): + _require( + prefix_values_digest(arrays) == receipt.get("prefix_values_sha256"), + "PUF59_ARTIFACT_PREFIX_BINDING", + ) + _require( + _digest(arrays) == receipt["growth"].get("output_values_sha256"), + "PUF59_ARTIFACT_GROWTH_BINDING", + ) + + +def _encode(arrays, receipt, expected_growth_scheme): + _require( + isinstance(arrays, Mapping) and set(arrays) == set(NAMES), + "PUF59_ARTIFACT_COLUMNS", + ) + n = len(arrays["RECID"]) + _validate(arrays, n) + r = _receipt(receipt, expected_growth_scheme) + _bindings(arrays, r) + _require(r["rows"] == n, "PUF59_ARTIFACT_RECEIPT_ROWS") + blocks = [ + np.asarray(arrays[name], dtype=" len(MAGIC) + 4 + length, + "PUF59_ARTIFACT_HEADER_SIZE", + ) + start = len(MAGIC) + 4 + + def pairs(items): + d = {} + for k, v in items: + _require(k not in d, "PUF59_ARTIFACT_DUPLICATE_JSON_KEY") + d[k] = v + return d + + h = json.loads( + payload[start : start + length], + object_pairs_hook=pairs, + parse_constant=lambda _: (_ for _ in ()).throw( + ValueError("PUF59_ARTIFACT_JSON_NONFINITE") + ), + ) + _require( + type(h) is dict + and set(h) == {"schema", "rows", "names", "dtypes", "body_sha256", "receipt"}, + "PUF59_ARTIFACT_HEADER_CONTRACT", + ) + _require( + json.dumps(h, sort_keys=True, separators=(",", ":"), allow_nan=False).encode( + "utf-8" + ) + == payload[start : start + length], + "PUF59_ARTIFACT_HEADER_CANONICAL", + ) + _require( + h["schema"] == "microcosm.us.puf59_canonical_artifact/2" + and h["names"] == list(NAMES) + and h["dtypes"] == [" None: + """Refuse using a stable reason, without reporting source row values.""" + if not condition: + raise ValueError(reason) + + +def fixture_sources(definition_text: str, projection_text: str): + """Validate actual fixture definitions; packaged source pins cannot enter.""" + # Source documents and fit packets use distinct JSON canonical conventions. Validate each document against its own producer's exact bytes. + definition = raw.fixture_definition(json.loads(definition_text)) + projection = agi.fixture_agi_projection_document( + json.loads(projection_text), definition + ) + require( + definition.params_text == definition_text + and projection.params_text == projection_text, + "FIXTURE_SOURCE_CANONICAL", + ) + require(tuple(sorted(projection.fields)) == MONEY_FIELDS, "FIXTURE_MONEY_FIELDS") + require( + all(c.fit_admission == SOURCE_ADMISSION for c in projection.columns), + "SOURCE_ADMISSION_CHANGED", + ) + return definition, projection + + +def decode_price_arrays( + payload: bytes, *, expected: Mapping, expected_sha256: str, max_rows=MAX_ROWS +): + """Read the exact reviewed local helper envelope, with external identity pins. + + This parser does not create a GrownPufTable, repeat growth, or admit a fit. + The returned arrays are read-only views of the authenticated body bytes. + """ + require(type(max_rows) is int and 0 < max_rows <= 250_000, "PRICE_ROW_BOUND") + require( + isinstance(payload, bytes) + and len(payload) <= MAX_BYTES + and payload.startswith(MAGIC), + "PRICE_ENVELOPE", + ) + require( + codec._hash(expected_sha256) and codec.sha(payload) == expected_sha256, + "PRICE_PAYLOAD_SHA", + ) + offset = len(MAGIC) + require(len(payload) >= offset + 8, "PRICE_HEADER_LENGTH") + size = int.from_bytes(payload[offset : offset + 8], "big") + start = offset + 8 + require(0 < size <= 65536 and len(payload) >= start + size, "PRICE_HEADER_BOUND") + header = codec.decode_json(payload[start : start + size]) + require( + set(expected) == HEADER_FIELDS and set(header) == HEADER_FIELDS | {"columns"}, + "PRICE_HEADER_FIELDS", + ) + require({k: header[k] for k in HEADER_FIELDS} == dict(expected), "PRICE_IDENTITY") + require( + header["schema"] == "local.puf.price_restatement.arrays.v1" + and header["release_eligible"] is False, + "PRICE_SCOPE", + ) + for field in ("input_binding_sha256", "contract_sha256"): + require(codec._hash(header[field]), "PRICE_HEADER_SHA") + for field in ("source_head", "helper_head"): + require( + isinstance(header[field], str) + and len(header[field]) == 40 + and all(c in "0123456789abcdef" for c in header[field]), + "PRICE_CODE_ID", + ) + columns = header["columns"] + require( + isinstance(columns, dict) and set(columns) == set(ARRAY_DTYPES), + "PRICE_COLUMN_ROSTER", + ) + cursor, rows, result = start + size, None, {} + for name in sorted(ARRAY_DTYPES): + entry = columns[name] + require( + isinstance(entry, dict) and set(entry) == {"dtype", "rows", "sha256"}, + "PRICE_COLUMN_FIELDS", + ) + require( + entry["dtype"] == ARRAY_DTYPES[name] + and type(entry["rows"]) is int + and 0 < entry["rows"] <= max_rows, + "PRICE_COLUMN_TYPE_ROWS", + ) + rows = entry["rows"] if rows is None else rows + require(entry["rows"] == rows, "PRICE_ROW_ALIGNMENT") + end = cursor + rows * np.dtype(entry["dtype"]).itemsize + require(end <= len(payload), "PRICE_BODY_TRUNCATED") + body = payload[cursor:end] + require(codec.sha(body) == entry["sha256"], "PRICE_BODY_SHA") + values = np.frombuffer(body, dtype=entry["dtype"]) + require(np.isfinite(values).all(), "PRICE_NONFINITE") + result[name] = values + cursor = end + require(cursor == len(payload), "PRICE_TRAILING_BYTES") + return result + + +def donor_frame(arrays: Mapping, status, decoded, projection) -> Frame: + """One computational person/return wrapper, with explicit raw S006 DESIGN weights.""" + require(projection.route == "test_fixture", "GENUINE_FIT_NOT_ADMITTED") + return _donor_frame(arrays, status, decoded, projection, scope=SCOPE) + + +def _donor_frame(arrays, status, decoded, projection, *, scope): + """Shared mechanical checks; the entry point must qualify its source/recipe.""" + known = np.asarray(decoded.amount_known, dtype=bool) + require( + np.array_equal(status.typed["RECID"], decoded.typed["RECID"]), + "DONOR_STATUS_JOIN", + ) + require( + np.array_equal(status.typed["disclosure_aggregate"], ~known), "DONOR_UNIVERSE" + ) + ids = status.typed["RECID"][known] + require( + len(ids) == len(np.unique(ids)) + and np.array_equal(ids, arrays["RECID"]) + and np.array_equal(ids, arrays[growth.PROVENANCE_RECID_COLUMN]), + "DONOR_RECID_ORDER", + ) + aggregate_ids = status.typed["RECID"][~known] + require( + len(aggregate_ids) == 4 + and len(np.unique(aggregate_ids)) == 4 + and set(aggregate_ids) == set(raw.PUF_AGGREGATE_RECIDS), + "DONOR_AGGREGATE_ROSTER", + ) + for field, delivered in ( + ("S006", "S006_delivered_int64"), + ("FLPDYR", "FLPDYR_delivered_int16"), + ("demographic_status", "demographic_status_delivered_int8"), + ): + original = status.typed[field][known] + require( + np.array_equal(original, arrays[delivered]) + and np.array_equal(original, arrays[field]), + "DONOR_CARRIED_STATUS", + ) + require( + np.array_equal( + decoded.typed["E00100"][known], arrays[growth.PROVENANCE_SOURCE_AGI_COLUMN] + ), + "DONOR_SOURCE_AGI", + ) + weights = arrays["S006_delivered_int64"].astype("float64") + require( + np.all(arrays["S006_delivered_int64"] >= 0) + and np.all(arrays["S006_delivered_int64"] <= 2**53) + and np.array_equal(weights.astype("int64"), arrays["S006_delivered_int64"]) + and weights.sum() > 0, + "DONOR_DESIGN_WEIGHTS", + ) + wage = arrays["E00200"] + require(np.isfinite(wage).all() and np.all(wage >= 0), "DONOR_WAGE_FEATURE") + mars = status.typed["MARS"][known] + require(np.isin(mars, (1, 2, 3, 4)).all(), "DONOR_MARS") + table = pd.DataFrame( + { + "tax_unit_id": ids.copy(), + FEATURES[0]: wage.copy(), + **{f"mars_{i}": (mars == i).astype("float64") for i in range(1, 5)}, + TARGET: arrays[TARGET].copy(), + } + ) + return Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + {"person_id": ids.copy(), "person_tax_unit_id": ids.copy()} + ), + }, + EntitySchema(group_entities=("tax_unit",)), + {"tax_unit": Weights(weights, WeightKind.DESIGN)}, + metadata={ + "scope": scope, + "source_projection_sha256": projection.sha256, + "source_projection_document": projection.canonical.decode("utf-8"), + "carrier": "one technical person per return; not a taxpayer count", + "weight_units": "delivered_S006_integer_hundredths", + }, + ) + + +def population_content(frame: Frame) -> str: + """Full materialized cells/axes/effective weights/strata; no metadata shortcuts.""" + return origin._population_content(frame) + + +def recipient_matrix(frame: Frame, *, role_column=None, person_wages=None): + """Use every detail unit, including zero-weight rows, and actual filing roles.""" + require(frame.schema == US_SCHEMA, "HOST_SCHEMA") + # Build a manifest from the current native roster only for validation. The + # separate fixture source edge binds that roster's authority and full bytes. + tables = {e: frame.table(e) for e in frame.entities} + for entity, table in tables.items(): + role = table[provenance.support_clone_index_column(entity)] + require( + role.dtype == np.dtype("int64") and role.isin((0, 1)).all(), + "HOST_CLONE_TYPE", + ) + manifest = provenance.spine_assembly_manifest( + { + e: t.loc[t[provenance.support_clone_index_column(e)].eq(0)] + for e, t in tables.items() + }, + channels=("acs", "asec"), + ) + checked = Frame( + tables, + frame.schema, + {e: frame.weights_for(e) for e in frame.weighted_entities}, + frame.strata, + metadata=manifest, + ) + provenance.validate_assembly_provenance(checked, boundary="invented_puf_detail") + person, units = tables["person"], tables["tax_unit"] + if person_wages is not None: + require( + type(person_wages) is pd.Series + and person_wages.dtype == np.dtype("float64") + and person_wages.index.dtype == np.dtype("int64") + and person_wages.index.is_unique + and np.array_equal(person_wages.index.to_numpy(), person.person_id), + "HOST_WAGE_ROW_AXIS", + ) + clone_role = person[provenance.support_clone_index_column("person")] + wage_values = person_wages.to_numpy() + require( + np.array_equal( + wage_values[clone_role.eq(0)].view("uint64"), + wage_values[clone_role.eq(1)].view("uint64"), + ), + "HOST_WAGE_PAIR_BYTES", + ) + pairs = {} + for entity, table in tables.items(): + src, role = ( + provenance.support_source_id_column(entity), + provenance.support_clone_index_column(entity), + ) + require(table[src].dtype == np.dtype("int64"), "HOST_SOURCE_ID_TYPE") + require(not table.duplicated([src, role]).any(), "HOST_ROLE_PAIR_DUPLICATE") + native, detail = table.loc[table[role].eq(0)], table.loc[table[role].eq(1)] + require( + len(native) == len(detail) and np.array_equal(native[src], detail[src]), + "HOST_ROLE_PAIR_COVERAGE", + ) + structural = {US_SCHEMA.entity_id_column(entity), role} + if entity == "person": + structural.update( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + for name in table.columns: + if name not in structural: + require( + native[name] + .reset_index(drop=True) + .equals(detail[name].reset_index(drop=True)), + "HOST_COPIED_SOURCE_CELL", + ) + weights = frame.resolve_weights(entity).values + require( + np.array_equal( + weights[table[role].eq(0)].view("uint64"), + weights[table[role].eq(1)].view("uint64"), + ), + "HOST_PAIR_WEIGHTS", + ) + pairs[entity] = dict( + zip( + table[US_SCHEMA.entity_id_column(entity)], + zip(table[src], table[role], strict=True), + strict=True, + ) + ) + for group in US_SCHEMA.group_entities: + member = US_SCHEMA.membership_column(group) + for pid, gid in zip(person.person_id, person[member], strict=True): + require(pairs["person"][pid][1] == pairs[group][gid][1], "HOST_MEMBER_ROLE") + by_source = {} + for pid, gid in zip(person.person_id, person[member], strict=True): + source, role = pairs["person"][pid] + group_source = pairs[group][gid][0] + if role == 0: + by_source[source] = group_source + else: + require(by_source.get(source) == group_source, "HOST_CLONE_MEMBERSHIP") + require( + set(units[provenance.support_channel_column("tax_unit")]) == {"acs", "asec"}, + "HOST_BOTH_ARMS", + ) + mask = units[provenance.support_clone_index_column("tax_unit")].eq(1).to_numpy() + selected = units.loc[mask] + values = [] + for row in selected.itertuples(index=False): + members = person.loc[person.person_tax_unit_id.eq(row.tax_unit_id)] + status = row.filing_status_input + require(status in MARS, "HOST_FILING_STATUS") + if role_column is None: + for name in ( + "is_tax_unit_head", + "is_tax_unit_spouse", + "is_tax_unit_dependent", + ): + require( + name in members + and members[name].dtype == np.dtype("bool") + and not members[name].isna().any(), + "HOST_FILING_ROLE", + ) + head, spouse, dependent = ( + members[name].to_numpy() + for name in ( + "is_tax_unit_head", + "is_tax_unit_spouse", + "is_tax_unit_dependent", + ) + ) + else: + require( + role_column == "tax_unit_role_input" and role_column in members, + "HOST_ROLE_INPUT_COLUMN", + ) + roles = members[role_column] + require( + pd.api.types.is_string_dtype(roles.dtype) + and not roles.isna().any() + and roles.isin(("HEAD", "SPOUSE", "DEPENDENT")).all(), + "HOST_ROLE_INPUT_VALUES", + ) + head, spouse, dependent = ( + roles.eq(name).to_numpy(dtype=bool) + for name in ("HEAD", "SPOUSE", "DEPENDENT") + ) + require( + np.all(head.astype(int) + spouse.astype(int) + dependent.astype(int) == 1) + and head.sum() == 1 + and spouse.sum() == int(status == "JOINT"), + "HOST_FILING_STRUCTURE", + ) + channel = getattr(row, provenance.support_channel_column("tax_unit")) + name = ( + "asec_reported_wage_income_2024_price" + if channel == "asec" + else "employment_income_before_lsr" + ) + earners = head | spouse + if person_wages is None: + require(name in members, "HOST_MISSING_FEATURE") + wage = members.loc[earners, name] + else: + wage = person_wages.loc[members.loc[earners, "person_id"]] + require( + pd.api.types.is_numeric_dtype(wage) + and not pd.api.types.is_bool_dtype(wage) + and not pd.api.types.is_complex_dtype(wage) + and not wage.isna().any(), + "HOST_MISSING_FEATURE", + ) + a = wage.to_numpy(dtype="float64") + require(np.isfinite(a).all() and (a >= 0).all(), "HOST_NONFINITE_FEATURE") + amount = float(a.sum()) + require(np.isfinite(amount), "HOST_FEATURE_OVERFLOW") + values.append([amount, *(float(MARS[status] == i) for i in range(1, 5))]) + ids = selected.tax_unit_id.to_numpy(copy=True) + features = pd.DataFrame( + values, index=pd.Index(ids), columns=FEATURES, dtype="float64" + ) + return encode_recipient_matrix(features, entity="tax_unit", entity_ids=ids), mask diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py new file mode 100644 index 000000000..79b34e697 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py @@ -0,0 +1,847 @@ +"""Qualified transport for one development-only PUF conditional diagnostic. + +Callers supply independently reviewed content pins. Receipt labels or fixture +definitions cannot replace those pins. No function opens a file or fits a model. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input + +from . import asec_current_money as current_money +from . import graph_asec_income as asec_income +from . import native_household_origin as origin +from . import puf_detail_transfer as detail +from . import puf_monetary_agi_projection as agi +from . import puf_raw_source as raw +from . import support_provenance as provenance +from .graph_context import _row_identity + +require = detail.require +RECIPE_ID = "us.puf.schedule_c_wage_mars_diagnostic.v1" +SCOPE = "development_conditional_support_not_tax_inputs" +SOURCE_HEAD = "c5c34a048fc38c01840b2da5454385e0a51d1c49" +HELPER_HEAD = "dea036278cc3f7c29c99da67bcadd968822c1d7a" +MAX_DONORS = 250_000 +ROLES = ( + "acceptance", + "proposal", + "config", + "completion", + "cold", + "required", + "definition", + "projection_definition", + "status", + "projection", + "arrays", +) + + +def recipe_document(): + """A diagnostic modeling choice, never a modification of source admission.""" + return { + "id": RECIPE_ID, + "scope": SCOPE, + "target": detail.TARGET, + "features": list(detail.FEATURES), + "output": detail.OUTPUT, + "trees": 2, + "seed": 578, + "zero_atol": 0.0, + "donor_weight": "delivered_S006_integer_hundredths_DESIGN", + "wage_proxy": "head_and_actual_JOINT_spouse_only_dependents_excluded", + "host_role_source": "microunit_tax_unit_role_input", + "filing_mapping": detail.MARS, + "missing_wage": "refuse_selected_earner_no_age_only_zero", + "prices": "2015_file_CPI_U_to_2024_development_prices", + "source_period": "unresolved_raw_FLPDYR", + "source_fit_admission": detail.SOURCE_ADMISSION, + "population_meaning": "potential_filing_unit_support_not_observed_filers", + "tax_input_admission": False, + "scientific_launch_admission": False, + "release_eligible": False, + } + + +def parse_document(payload): + """Strict JSON, allowing the reviewed helper's formatted receipt encoding.""" + require( + type(payload) is bytes and 0 < len(payload) <= 16_000_000, + "DIAGNOSTIC_DOCUMENT_BOUND", + ) + + def pairs(items): + result = {} + for key, value in items: + require(key not in result, "DIAGNOSTIC_DUPLICATE_KEY") + result[key] = value + return result + + def bad(_): + raise ValueError("DIAGNOSTIC_NONFINITE_JSON") + + value = json.loads(payload, object_pairs_hook=pairs, parse_constant=bad) + require(isinstance(value, dict), "DIAGNOSTIC_DOCUMENT_TYPE") + return value + + +def content_pin(payload): + return {"bytes": len(payload), "sha256": codec.sha(payload)} + + +def check_pin(payload, pin, reason): + require( + isinstance(pin, Mapping) + and type(pin.get("bytes")) is int + and codec._hash(pin.get("sha256")) + and content_pin(payload) == {k: pin[k] for k in ("bytes", "sha256")}, + reason, + ) + + +def qualify_price_transport(payloads, expected_pins, *, route, recipe): + """Validate DEA completion → config/source → grown-array transport. + + Expected pins are execution inputs established outside this parser. The + production route accepts only the packaged original source definitions; + tests use a visibly different acceptance/mode with actual fixture codecs. + """ + require( + codec.encode_json(recipe) == codec.encode_json(recipe_document()), + "DIAGNOSTIC_RECIPE", + ) + require(route in ("packaged", "test_fixture"), "DIAGNOSTIC_SOURCE_ROUTE") + require(set(payloads) == set(expected_pins) == set(ROLES), "PRICE_TRANSPORT_ROSTER") + for name in ROLES: + check_pin(payloads[name], expected_pins[name], "PRICE_TRANSPORT_PIN:" + name) + documents = {n: parse_document(payloads[n]) for n in ROLES[:6]} + acceptance, proposal, config, completed, cold, warm = ( + documents[n] for n in ROLES[:6] + ) + genuine = route == "packaged" + require( + acceptance["status"] + == ( + "ACCEPTED_GENUINE_DEVELOPMENTAL_PRICE_RESTATEMENT" + if genuine + else "ACCEPTED_INVENTED_DEVELOPMENTAL_PRICE_RESTATEMENT" + ) + and acceptance["release_eligible"] is False + and acceptance["fit_admission"] == detail.SOURCE_ADMISSION + and config["mode"] == ("genuine" if genuine else "bound_invented") + and completed["genuine_input_bound"] is genuine + and completed["status"] + == ( + "GENUINE_DEVELOPMENTAL_PRICE_RESTATEMENT_COMPLETED" + if genuine + else "INVENTED_PRICE_RESTATEMENT_PREPARATION_PASSED" + ), + "PRICE_TRANSPORT_AUTHORITY_SCOPE", + ) + for name in ( + "config", + "completion", + "cold", + "required", + "arrays", + "status", + "projection", + "definition", + "projection_definition", + ): + require( + any( + content_pin(payloads[name]) == {k: p[k] for k in ("bytes", "sha256")} + for p in acceptance["saved_file_pins_checked"] + ), + "PRICE_ACCEPTANCE_PIN:" + name, + ) + check_pin( + payloads["proposal"], acceptance["reviewed_proposal"], "PRICE_PROPOSAL_PIN" + ) + for name, receipt in (("cold", cold), ("required", warm)): + require( + completed[name]["sha256"] == codec.sha(payloads[name]), "PRICE_CHILD_PIN" + ) + require(receipt["phase"] == name, "PRICE_CHILD_PHASE") + check_pin(payloads["config"], completed["config_pin"], "PRICE_CONFIG_PIN") + require( + {k: v for k, v in config.items() if k != "execution"} == proposal["config"] + and proposal["price_execution_authorized"] is False + and proposal["release_eligible"] is False, + "PRICE_REVIEWED_CONFIGURATION", + ) + require( + config["inputs"] == cold["input_binding"] == warm["input_binding"], + "PRICE_SOURCE_INPUTS", + ) + require( + set(config["inputs"]) + == {"definition", "projection_definition", "status", "projection"}, + "PRICE_SOURCE_ROSTER", + ) + for name, pin in config["inputs"].items(): + check_pin(payloads[name], pin, "PRICE_SOURCE_PIN:" + name) + source = config["bound_source"] + require( + source["route"] == route and source["invented"] is (not genuine), + "PRICE_BOUND_ROUTE", + ) + source_sha = codec.sha(codec.encode_json(source)) + require( + source_sha + == config["binding_sha256"] + == cold["bound_source_identity"] + == warm["bound_source_identity"] + == completed["bound_source_identity"], + "PRICE_BOUND_SOURCE_SHA", + ) + runtime_keys = ( + "source_head", + "helper_head", + "python", + "executable", + "executable_sha256", + "versions", + ) + runtime = {k: cold["runtime"][k] for k in runtime_keys} + require( + all( + {k: item["runtime"][k] for k in runtime_keys} == runtime + for item in (warm, completed) + ), + "PRICE_RUNTIME_BINDING", + ) + require( + runtime["source_head"] == SOURCE_HEAD + and runtime["helper_head"] == HELPER_HEAD + and runtime["executable_sha256"] + == "bf4ea26231212e330b8e9b6f24969a30f894fec88665bf42754070d114ac7fc6" + and runtime["versions"] == {"numpy": "2.4.6", "pandas": "3.0.3"}, + "PRICE_REVIEWED_RUNTIME", + ) + for item in (cold, warm, completed): + isolation = item["runtime"]["bytecode_isolation"] + require( + isolation["isolated"] is True + and isolation["dont_write_bytecode"] is True + and isolation["absent_absolute_prefix"].startswith("/"), + "PRICE_BYTECODE_ISOLATION", + ) + require( + cold["runtime"]["bytecode_isolation"]["absent_absolute_prefix"] + != warm["runtime"]["bytecode_isolation"]["absent_absolute_prefix"], + "PRICE_FRESH_PROCESS_PREFIX", + ) + require( + cold["pid"] != warm["pid"] + and warm["all_hits"] + and warm["kernel_calls"] == [] + and warm["raising_replay_stubs"] is True + and all(v == 0 for v in warm["transform_calls"].values()), + "PRICE_REQUIRED_PROOF", + ) + for name in ( + "contract", + "array_identity", + "graph_artifacts", + "grown_artifact", + "excluded_lexical_artifact", + ): + require(cold[name] == warm[name], "PRICE_COLD_WARM_IDENTITY:" + name) + contract = cold["contract"] + require(source["contract"] == contract, "PRICE_SOURCE_CONTRACT") + require( + source["runtime"]["executable_sha256"] == runtime["executable_sha256"] + and source["runtime"]["package_versions"] == runtime["versions"], + "PRICE_SOURCE_RUNTIME", + ) + require( + contract["ratio"] == [313689, 237017] + and contract["ratio_bits"] == "0aa9e810012df53f" + and contract["release_eligible"] is False + and contract["per_record_preliminary_inflation"] is False + and contract["nominal_income_growth_claim"] is False, + "PRICE_BASIS", + ) + require( + acceptance["source_head"] == SOURCE_HEAD + and acceptance["helper_head"] == HELPER_HEAD + and config["helper_head"] == HELPER_HEAD, + "PRICE_REVIEWED_CODE", + ) + if genuine: + definition, projection = ( + raw.packaged_definition(), + agi.packaged_agi_projection(), + ) + else: + definition = raw.fixture_definition(parse_document(payloads["definition"])) + projection = agi.fixture_agi_projection_document( + parse_document(payloads["projection_definition"]), definition + ) + # Source files may use formatted JSON; their exact input pins were checked above. + require( + parse_document(payloads["definition"]) + == raw.definition_document_json(definition), + "PRICE_DEFINITION", + ) + require( + parse_document(payloads["projection_definition"]) + == json.loads(projection.canonical), + "PRICE_PROJECTION_DEFINITION", + ) + require( + cold["source_definition_sha256"] == definition.sha256 + and cold["projection_definition_sha256"] == projection.sha256, + "PRICE_DEFINITION_IDENTITY", + ) + require( + source["source"]["head"] == SOURCE_HEAD + and source["source"]["definition_canonical_sha256"] == definition.sha256 + and source["source"]["projection_canonical_sha256"] == projection.sha256, + "PRICE_DECLARED_SOURCE_IDENTITY", + ) + metadata = source["source_column_metadata"] + require( + set(metadata) == set(detail.MONEY_FIELDS) + and codec.sha(codec.encode_json(metadata)) == source["source_basis_sha256"] + and cold["source_column_metadata_preserved"] + == warm["source_column_metadata_preserved"] + == metadata, + "PRICE_SOURCE_METADATA", + ) + for value in metadata.values(): + require( + value["fit_admission"] == detail.SOURCE_ADMISSION + and value["period_semantics"] == "unresolved_raw_FLPDYR" + and value["amount_units"] == "whole_usd_source_year" + and value["grain"] == "source_return" + and value["projection_sha256"] == projection.sha256, + "PRICE_SOURCE_MEANING", + ) + rows = config["ordinary_rows"] + require( + type(rows) is int + and 0 < rows <= MAX_DONORS + and rows + == source["ordinary_rows"] + == acceptance["ordinary_rows"] + == completed["ordinary_rows"] + and acceptance["excluded_aggregate_rows"] == completed["aggregate_rows"] == 4, + "PRICE_ORDINARY_ROSTER", + ) + check_pin(payloads["arrays"], cold["grown_artifact"], "PRICE_EXPORT_PIN") + header = { + "schema": "local.puf.price_restatement.arrays.v1", + "input_binding_sha256": codec.sha(codec.encode_json(config["inputs"])), + "contract_sha256": contract["sha256"], + "source_head": SOURCE_HEAD, + "helper_head": HELPER_HEAD, + "release_eligible": False, + } + arrays = detail.decode_price_arrays( + payloads["arrays"], + expected=header, + expected_sha256=codec.sha(payloads["arrays"]), + max_rows=rows, + ) + require(len(arrays["RECID"]) == rows, "PRICE_EXACT_ROWS") + for name, values in arrays.items(): + require( + cold["array_identity"][name] + == { + "dtype": values.dtype.str, + "rows": len(values), + "sha256": codec.sha(values.tobytes()), + }, + "PRICE_ARRAY_RECEIPT", + ) + status = raw.decode_return_status(payloads["status"], definition) + decoded = agi.decode_agi_projection( + payloads["projection"], + projection, + expected_status_artifact_sha256=codec.sha(payloads["status"]), + ) + require( + metadata == agi._thawed(decoded.column_metadata), + "PRICE_ORIGINAL_METADATA_BINDING", + ) + frame = detail._donor_frame(arrays, status, decoded, projection, scope=SCOPE) + binding = { + "schema": "microcosm.us.puf_diagnostic_price_transport.v1", + "route": route, + "recipe": recipe, + "input_pins": {n: content_pin(payloads[n]) for n in ROLES}, + "source_column_metadata": metadata, + "source_projection_document": projection.canonical.decode(), + "original_source_admission": detail.SOURCE_ADMISSION, + "contract": contract, + "source_binding_sha256": source_sha, + "ordinary_rows": rows, + "donor_model_input_sha256": donor_model_input_sha256(frame), + "release_eligible": False, + } + return frame, codec.encode_json(binding) + + +def donor_model_input_sha256(frame): + """Exact donor IDs/features/target and DESIGN resampling weights.""" + table = frame.table("tax_unit").loc[:, [*detail.FEATURES, detail.TARGET]].copy() + table.index = pd.Index( + frame.table("tax_unit").tax_unit_id.to_numpy(dtype="int64"), name="tax_unit_id" + ) + payload = model_input.encode_recipient_matrix( + table.loc[:, [*detail.FEATURES, detail.TARGET]], + entity="tax_unit", + entity_ids=table.index.to_numpy(dtype="int64"), + ) + weight = frame.weights_for("tax_unit") + return codec.sha( + codec.encode_json( + { + "matrix_sha256": codec.sha(payload), + "weight_kind": weight.kind.value, + "weight_sha256": codec.sha(weight.values.astype("= 0) & (age <= 99) & (age == np.floor(age))).all() + and np.array_equal(age, _numeric(person.age, "ACS_WAGE_AGE")), + "ACS_WAGE_AGE", + ) + require(person.source_year.astype(str).eq("2024").all(), "ACS_WAGE_COHORT") + wage = _numeric(person.WAGP, "ACS_WAGE_RAW_TYPE", nullable=True) + adjustment = _numeric(person.ADJINC, "ACS_WAGE_ADJINC_TYPE", nullable=True) + carried = _numeric( + person.employment_income_before_lsr, "ACS_WAGE_CARRIED_TYPE", nullable=True + ) + observed = ~np.isnan(wage) + require( + np.isfinite(wage[observed]).all() + and (wage[observed] >= 0).all() + and np.isfinite(adjustment[observed]).all() + and (adjustment[observed] > 0).all(), + "ACS_WAGE_DOMAIN", + ) + require(not (observed & (age < 15)).any(), "ACS_WAGE_OUTSIDE_UNIVERSE_OBSERVED") + expected = wage * (adjustment / 1_000_000.0) + require( + np.array_equal(np.isnan(carried), ~observed) + and np.array_equal( + expected[observed].view("uint64"), carried[observed].view("uint64") + ), + "ACS_WAGE_ADJUSTED_BINDING", + ) + return { + "ids": person.person_id.to_numpy(dtype="int64").tolist(), + "age_sha256": codec.sha(age.astype("= 15).tolist(), + "missing_treatment": "preserved_no_zero_substitution", + "arithmetic": "WAGP*(ADJINC/1000000)", + "source_period_equivalence_claim": False, + } + + +def asec_wage_evidence(person, values): + """Reconstruct the accepted income artifact, then join its native person axis. + + ``values`` are executor-checked typed artifacts. H and its T1 descendant + have legitimate differing source ancestry; the accepted readers bind the + prepared receipt, current-money buffers, cohort and exact income payload. + """ + from . import graph_composed_asec_binding as asec_binding + from . import graph_composed_asec_measures as measures + + binding = measures._binding(values) + prepared = measures._prepared(values["prepared_receipt"]) + document = measures._document(values["frame_context"]) + measures._bound_population(document, binding) + selected = measures._selected_money(binding, prepared, values) + rows = asec_binding.bind_composed_asec_arm_rows( + values["arm_rows"].payload, binding_document=binding + ) + original = rows.array("person", "original_ids") + + def integer(name): + return asec_binding._int64_view(person[name], reason="ASEC_HOST:" + name) + + require( + np.array_equal(integer(provenance.spine_source_id_column("person")), original), + "ASEC_HOST_NATIVE_PERSON_ROSTER", + ) + composed = integer(provenance.support_source_id_column("person")) + require( + _row_identity(pd.DataFrame({"person_id": composed}), "person")[ + "ordered_ids_sha256" + ] + == binding["arm"]["person"]["composed_ordered_ids_sha256"], + "ASEC_HOST_COMPOSED_ROSTER", + ) + text_year = person.source_year.astype(str) + require(text_year.isin(("2022", "2023", "2024")).all(), "ASEC_HOST_COHORT") + years = text_year.to_numpy().astype("int64") + income = asec_income.bind_income_observations( + values["income_observations"].payload, prepared_receipt=prepared + ) + result = asec_income.read_reported_income( + values["reported_income"].payload, + selected, + income, + person_ids=original, + income_years=years, + person_positions_sha256=binding["arm"]["person"]["source_positions_sha256"], + prepared_receipt_sha256=codec.sha(values["prepared_receipt"].payload), + selection_sha256=codec.sha(values["asec_binding"].payload), + selected_money_sha256=codec.sha(values["selected_current_money"].payload), + income_payload_sha256=codec.sha(values["income_observations"].payload), + source_producer_key=values["prepared_receipt"].producer_key, + selection_producer_key=values["asec_binding"].producer_key, + frame_context_sha256=codec.sha(values["frame_context"].payload), + ) + require( + np.array_equal(integer("A_AGE"), result.array("A_AGE")) + and np.array_equal( + _numeric(person.age, "ASEC_HOST_AGE"), result.array("A_AGE") + ), + "ASEC_HOST_NATIVE_AGE", + ) + name = "asec_reported_wage_income_2024_price" + actual = _numeric(person[name], "ASEC_HOST_WAGE_TYPE") + require( + np.isfinite(actual).all() + and (actual >= 0).all() + and np.array_equal(actual.view("uint64"), result.array(name).view("uint64")), + "ASEC_HOST_WAGE_BINDING", + ) + require((result.array("WSAL_VAL.validity") == 1).all(), "ASEC_HOST_WAGE_VALIDITY") + return { + "ids": person.person_id.to_numpy(dtype="int64").tolist(), + "source_native_person_sha256": codec.sha(original.tobytes()), + "income_year_sha256": codec.sha(years.tobytes()), + "reported_income_sha256": codec.sha(values["reported_income"].payload), + "wage_sha256": codec.sha(actual.tobytes()), + "evidence_axes": { + n: result.array("WSAL_VAL." + n).tolist() + for n in ("status", "validity", "zero_origin") + }, + "monetary_basis": json.loads(result.header)["monetary_basis"], + "missing_treatment": "refuse_no_evidence_upgrade", + "source_period_equivalence_claim": False, + } + + +def recipient_matrix(frame): + """Read the actual microunit role carrier; do not calculate policy roles.""" + return detail.recipient_matrix(frame, role_column="tax_unit_role_input") + + +def qualify_host_population(frame, values): + """Bind the complete untouched clone pool before selecting model features. + + Reconstruct native membership from typed original-source projections and + native-parent binding. Exact content comparison also catches a caller's + omitted declared column; the materialized verifier remains mandatory. + """ + sources = [] + for name in ("acs", "asec"): + value = values[name] + require( + origin._source_document(value.payload)["arm"] == name, "HOST_ORIGIN_ARM" + ) + sources.append( + origin.AuthenticatedNativeOriginSource(value.payload, _token=origin._TOKEN) + ) + parent_doc = codec.decode_json(values["parent_binding"].payload) + require(parent_doc["schema"] == origin.BINDING_SCHEMA, "HOST_PARENT_SCHEMA") + require( + parent_doc["graph_parent_edges"] + == { + n: { + "producer_key": values[n].producer_key, + "artifact_key": values[n].key, + "payload_sha256": codec.sha(values[n].payload), + } + for n in ("acs", "asec") + }, + "HOST_NATIVE_ORIGIN_PRODUCERS", + ) + parent = origin.PopulationOriginBinding( + values["parent_binding"].payload, _token=origin._BOUND_TOKEN + ) + expected = origin.bind_population_origins( + frame, sources=sources, parent=parent + ).document + recorded = codec.decode_json(values["clone_binding"].payload) + require( + { + k: v + for k, v in recorded.items() + if k not in ("graph_parent_edges", "population_node") + } + == expected, + "HOST_FULL_ORIGIN_BINDING", + ) + require( + recorded["graph_parent_edges"] + == { + n: { + "producer_key": values[n].producer_key, + "artifact_key": values[n].key, + "payload_sha256": codec.sha(values[n].payload), + } + for n in ("acs", "asec", "parent_binding") + }, + "HOST_ORIGIN_PRODUCERS", + ) + # Exact pair checks include every carried cell, linked entity and weight, + # including zero-weight rows. This also refuses incomplete selected wages. + matrix, mask = recipient_matrix(frame) + native = frame.person.loc[ + frame.person[provenance.support_clone_index_column("person")].eq(0) + ] + channel = native[provenance.support_channel_column("person")] + acs_person = native.loc[channel.eq("acs")].reset_index(drop=True) + asec_person = native.loc[channel.eq("asec")].reset_index(drop=True) + require(len(acs_person) > 0 and len(asec_person) > 0, "HOST_BOTH_ARMS") + evidence = { + "acs": acs_wage_evidence(acs_person), + "asec": asec_wage_evidence(asec_person, values), + } + return { + "schema": "microcosm.us.puf_diagnostic_host_features.v1", + "recipe": recipe_document(), + "full_population_content_sha256": detail.population_content(frame), + "origin_binding_sha256": codec.sha(values["clone_binding"].payload), + "wage_evidence": evidence, + "matrix_sha256": codec.sha(matrix), + "recipients": int(mask.sum()), + "all_clone_rows_including_zero_weights": True, + "release_eligible": False, + } + + +CURRENT_SURVEY_ROUTE = "authenticated_current_survey_v1" +CURRENT_SURVEY_PROJECTION_PROTOCOL = "microcosm.us.survey-puf-host-projection.v1" +CURRENT_SURVEY_MAX_BYTES = 64 * 1024**2 + + +def current_survey_recipient_matrix(frame, projection): + """Pure typed-transport consumer; live owner verification remains mandatory. + + Both this parser and the existing matrix validate every clone pair. The + projection retains current ASEC evidence bytes. ACS arithmetic/missingness + remains the existing WAGP/ADJINC contract, including under-15 refusal for a + missing selected earner. No serialized field confers source admission. + """ + require( + type(projection) is bytes and 0 < len(projection) <= CURRENT_SURVEY_MAX_BYTES, + "SURVEY_HOST_PROJECTION_BOUND", + ) + document = codec.decode_json(projection) + require( + set(document) + == { + "protocol", + "route", + "preparation_sha256", + "allocation_sha256", + "clone_sha256", + "asec", + "release_eligible", + }, + "SURVEY_HOST_PROJECTION_FIELDS", + ) + require( + document["protocol"] == CURRENT_SURVEY_PROJECTION_PROTOCOL + and document["route"] == CURRENT_SURVEY_ROUTE + and document["release_eligible"] is False + and all( + codec._hash(document[k]) + for k in ("preparation_sha256", "allocation_sha256", "clone_sha256") + ), + "SURVEY_HOST_PROJECTION_CONTRACT", + ) + asec = document["asec"] + require( + type(asec) is dict + and set(asec) + == { + "preparation_sha256", + "asec_native_sha256", + "money_header", + "money_header_sha256", + "domain", + "zero_origin_code", + "columns", + "rows", + }, + "SURVEY_WAGE_FIELDS", + ) + require( + asec["preparation_sha256"] == document["preparation_sha256"] + and codec._hash(asec["asec_native_sha256"]) + and codec.sha(codec.encode_json(asec["money_header"])) + == asec["money_header_sha256"] + and asec["money_header"]["target_year"] == 2024 + and asec["money_header"]["semantic"] == "annual_current_money" + and asec["columns"] + == [ + "stacked_person_id", + "native_person_id", + "income_year", + "amount_f64le_hex", + "status", + "validity", + "zero_origin", + ], + "SURVEY_WAGE_CONTRACT", + ) + people = frame.person + native_role = people[provenance.support_clone_index_column("person")].eq(0) + native_acs = people.loc[ + native_role & people[provenance.support_channel_column("person")].eq("acs") + ] + native_asec = people.loc[ + native_role & people[provenance.support_channel_column("person")].eq("asec") + ] + acs_evidence = acs_wage_evidence(native_acs) + rows = asec["rows"] + require( + type(rows) is list and len(rows) == len(native_asec) > 0, "SURVEY_WAGE_ROSTER" + ) + require( + type(asec["domain"]) is dict + and set(asec["domain"]) + == {"name", "entity", "column", "grain", "minimum", "maximum", "zero_semantics"} + and asec["domain"]["name"] == "WSAL_VAL" + and asec["domain"]["entity"] == "person" + and type(asec["zero_origin_code"]) is int, + "SURVEY_WAGE_DOMAIN", + ) + amounts = {} + for row in rows: + require( + type(row) is list + and len(row) == 7 + and all(type(row[i]) is int for i in (0, 1, 2, 4, 5, 6)) + and row[2] == 2024 + and row[5] == 1 + and all(0 <= row[i] <= 255 for i in (4, 5, 6)) + and type(row[3]) is str + and len(row[3]) == 16 + and set(row[3]) <= set("0123456789abcdef"), + "SURVEY_WAGE_ROW", + ) + require(row[0] not in amounts, "SURVEY_WAGE_DUPLICATE") + amount = np.frombuffer(bytes.fromhex(row[3]), dtype="= 0, "SURVEY_WAGE_AMOUNT") + amounts[row[0]] = (row[1], amount) + # Reuse the maintained money owner's complete buffer/evidence/domain check. + # These value objects do not issue money/source authority. The live verifier + # compares the complete projection, including this source-owned domain. + current_money._validate_field( + current_money.MoneyField( + "WSAL_VAL", + b"".join(bytes.fromhex(row[3]) for row in rows), + bytes(row[4] for row in rows), + bytes(row[5] for row in rows), + bytes(row[6] for row in rows), + ), + current_money.MoneyDomain(**asec["domain"]), + len(rows), + nominal=True, + zero_origin_code=current_money.ZeroOrigin(asec["zero_origin_code"]), + ) + source_id = provenance.support_source_id_column("person") + original_id = provenance.spine_source_id_column("person") + require( + [r[0] for r in rows] == native_asec[source_id].tolist() + and [r[1] for r in rows] == native_asec[original_id].tolist() + and native_asec.source_year.astype(str).eq("2024").all(), + "SURVEY_WAGE_ORIGIN", + ) + wages = [] + columns = [ + provenance.support_channel_column("person"), + source_id, + original_id, + "employment_income_before_lsr", + ] + for channel, stacked, native, value in people[columns].itertuples( + index=False, name=None + ): + if channel == "asec": + require(stacked in amounts, "SURVEY_WAGE_COVERAGE") + original, amount = amounts[stacked] + require(original == native, "SURVEY_WAGE_ORIGIN") + wages.append(amount) + else: + require(channel == "acs", "SURVEY_WAGE_CHANNEL") + wages.append(np.nan if pd.isna(value) else value) + projected = pd.Series( + wages, index=pd.Index(people.person_id.to_numpy()), dtype="float64" + ) + matrix, mask = detail.recipient_matrix( + frame, role_column="tax_unit_role_input", person_wages=projected + ) + require(len(matrix) <= CURRENT_SURVEY_MAX_BYTES, "SURVEY_MATRIX_BOUND") + return ( + matrix, + mask, + { + "scope": SCOPE, + "host_route": CURRENT_SURVEY_ROUTE, + "projection_sha256": codec.sha(projection), + "acs_wage_evidence": acs_evidence, + "asec_money_header_sha256": asec["money_header_sha256"], + "selected_tax_units": int(mask.sum()), + "release_eligible": False, + "source_period_equivalence_claim": False, + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source.py new file mode 100644 index 000000000..b666afa90 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source.py @@ -0,0 +1,519 @@ +"""Additive full-return PUF source projection, before canonical modeling. + +This is not the older 13-column monetary envelope. It preserves every source +return, raw period, integer S006 and missing demographic state. Only ordinary +returns carry numeric amounts; disclosure aggregates retain sparse lexical +tokens and use an explicit int64 sentinel. The unchanged raw decoder owns CSV +layout, status-code and join validation. This owner authenticates both exact +delivery buffers before calling it and binds the subsequent projection to its +complete RECID sequence. No tax engine, growth, fit or donor admission occurs. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import struct +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +from . import puf_raw_source as raw + +FULL_SOURCE_VERSION = "microcosm.us.puf_2015_full_return_source/3" +FULL_SOURCE_MAX_BYTES = 128 * 1024 * 1024 +OUTSIDE_AMOUNT_UNIVERSE = -(2**63) + +# Explicit publisher amount-field roster. No role is inferred from a prefix. +# E30400/E30500 deliberately excluded: capped Schedule SE amounts cannot +# reconstruct uncapped partnership earnings. +MONEY_COLUMNS = ( + "E00100", + "E00200", + "E00300", + "E00400", + "E00600", + "E00650", + "E00700", + "E00800", + "E00900", + "E01100", + "E01200", + "E01400", + "E01700", + "E02100", + "E02400", + "E03150", + "E03210", + "E03220", + "E03230", + "E03240", + "E03290", + "E03300", + "E03500", + "E18500", + "E19200", + "E19800", + "E20100", + "E20400", + "E20500", + "P22250", + "P23250", + "E24515", + "E24518", + "E25850", + "E25860", + "E25940", + "E25980", + "E25920", + "E25960", + "E26110", + "E26170", + "E26190", + "E26160", + "E26180", + "E26270", + "E26100", + "E26390", + "E26400", + "E27200", + "T27800", + "E58990", + "E87530", +) +COUNT_COLUMNS = ("XFPT", "XFST", "XOCAH", "XOCAWH", "XOODEP", "XOPAR", "XTOT") +DEPENDENT_COLUMNS = ("XOCAH", "XOCAWH", "XOODEP", "XOPAR") +PROJECTED_COLUMNS = (*MONEY_COLUMNS, *COUNT_COLUMNS) +_MONEY = re.compile(r"-?[0-9]{1,12}\Z", re.ASCII) +_COUNT = re.compile(r"[0-9]{1,2}\Z", re.ASCII) +# Operational preservation bound for four out-of-universe source rows. +# Empty, scientific notation and literal formatting are retained, not parsed. +AGGREGATE_LEXEME_MAX_CHARACTERS = 64 +_AGGREGATE = re.compile(r"[\x20-\x7e]{0,64}\Z", re.ASCII) + + +def _require(condition, code): + if not condition: + raise ValueError(code) + + +def _validate_count_fields(values, status): + """Publisher disclosure-coded counts; never a household/person roster.""" + ordinary = status["disclosure_aggregate"] == 0 + mars = status["MARS"][ordinary] + caps = np.select([mars == 1, mars == 2, mars == 3, mars == 4], [2, 3, 1, 3], -1) + _require((caps >= 0).all(), "FULL_COUNT_MARS") + for name in ("XFPT", "XFST"): + _require( + np.isin(values[name][ordinary], (0, 1)).all(), "FULL_EXEMPTION_FLAG:" + name + ) + cumulative = np.zeros(len(mars), dtype=np.int64) + for name in DEPENDENT_COLUMNS: + count = values[name][ordinary] + _require(((count >= 0) & (count <= 3)).all(), "FULL_DEPENDENT_DOMAIN:" + name) + cumulative += count + _require((cumulative <= caps).all(), "FULL_DEPENDENT_SEQUENTIAL_CAP:" + name) + xtot = values["XTOT"][ordinary] + _require(((xtot >= 0) & (xtot <= 5)).all(), "FULL_EXEMPTIONS_DOMAIN") + _require( + np.array_equal( + xtot, values["XFPT"][ordinary] + values["XFST"][ordinary] + cumulative + ), + "FULL_EXEMPTIONS_COMPONENT_IDENTITY", + ) + + +def puf_2015_receiver_size_measurement(filing_status_code, dependent_count): + """Match the PUF's coarse status and censored return-size predictor. + + Generic model status codes are single1/joint2/separate3/HOH4/widow5. + PUF code2 combines joint and surviving-spouse returns. Its baseline of two + is a predictor convention for that combined class, not an observed number + of living filers. Physical survey membership must remain unchanged. + """ + status = np.asarray(filing_status_code) + dependents = np.asarray(dependent_count) + _require( + status.ndim == 1 and dependents.shape == status.shape, + "FULL_RECEIVER_COUNT_SHAPE", + ) + _require( + status.dtype.kind in "iu" and dependents.dtype.kind in "iu", + "FULL_RECEIVER_COUNT_TYPE", + ) + _require(np.isin(status, (1, 2, 3, 4, 5)).all(), "FULL_RECEIVER_STATUS_DOMAIN") + _require( + ((dependents >= 0) & (dependents <= 1000)).all(), + "FULL_RECEIVER_DEPENDENT_BOUND", + ) + mars = np.where(status == 5, 2, status).astype(np.int64) + cap = np.select([mars == 1, mars == 2, mars == 3, mars == 4], [2, 3, 1, 3]) + return { + "puf_2015_filing_status_code": mars, + "puf_2015_capped_return_size": 1 + + (mars == 2).astype(np.int64) + + np.minimum(dependents, cap), + } + + +def _frozen_array(values): + values = np.asarray(values) + return np.frombuffer(values.tobytes(), dtype=values.dtype).reshape(values.shape) + + +@dataclass(frozen=True) +class FullPufSource: + """Read-only source values; available is not an admission or truth flag.""" + + definition_sha256: str + source_sha256: Mapping[str, str] + status: Mapping[str, np.ndarray] + values: Mapping[str, np.ndarray] + aggregate_tokens: Mapping[int, Mapping[str, str]] + status_payload: bytes + + @property + def ordinary(self): + return self.status["disclosure_aggregate"] == 0 + + +def decode_full_puf_source(main_bytes, demographic_bytes, definition): + """Authenticate, decode, and align a complete declared delivery in memory.""" + _require(isinstance(definition, raw.PufRawSourceDefinition), "FULL_DEFINITION") + for data, pin in ( + (main_bytes, definition.main), + (demographic_bytes, definition.demographic), + ): + _require(type(data) is bytes and len(data) == pin.bytes, "FULL_SOURCE_SIZE") + _require(hashlib.sha256(data).hexdigest() == pin.sha256, "FULL_SOURCE_SHA256") + status = raw.decode_puf_raw_source(main_bytes, demographic_bytes, definition) + rows = len(status.typed["RECID"]) + _require( + rows * len(PROJECTED_COLUMNS) * 8 <= FULL_SOURCE_MAX_BYTES, + "FULL_PROJECTION_BODY_LIMIT", + ) + profile = definition.document["csv_profile"] + reader = raw._reader(main_bytes, profile) + header = raw._check_header(reader, definition.main, profile=profile) + _require(set(PROJECTED_COLUMNS) <= set(header), "FULL_PROJECTED_HEADER") + positions = {name: header.index(name) for name in PROJECTED_COLUMNS} + recid_position = header.index("RECID") + values = { + name: np.full(rows, OUTSIDE_AMOUNT_UNIVERSE, dtype="= offset + 8, "FULL_HEADER_LENGTH") + size = struct.unpack(" 0, "FULL_HEADER_LIMIT") + offset += 8 + _require(offset + size <= len(payload), "FULL_TRUNCATED_HEADER") + header_bytes = payload[offset : offset + size] + try: + h = json.loads(header_bytes) + except (UnicodeError, ValueError) as error: + raise ValueError("FULL_HEADER_JSON") from error + _require( + type(h) is dict + and set(h) + == { + "definition_sha256", + "source_sha256", + "status_bytes", + "status_sha256", + "rows", + "columns", + "aggregate_tokens", + }, + "FULL_HEADER_KEYS", + ) + _require( + json.dumps( + h, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ).encode("ascii") + == header_bytes, + "FULL_HEADER_CANONICAL", + ) + _require( + h["definition_sha256"] == definition.sha256 + and h["source_sha256"] + == { + "main": definition.main.sha256, + "demographic": definition.demographic.sha256, + }, + "FULL_ARTIFACT_SOURCE_BINDING", + ) + rows = h["rows"] + _require( + type(rows) is int and rows == definition.main.data_records, "FULL_ARTIFACT_ROWS" + ) + _require( + type(h["status_bytes"]) is int + and 0 < h["status_bytes"] <= FULL_SOURCE_MAX_BYTES, + "FULL_STATUS_LENGTH", + ) + offset += size + status_payload = payload[offset : offset + h["status_bytes"]] + _require( + len(status_payload) == h["status_bytes"] + and hashlib.sha256(status_payload).hexdigest() == h["status_sha256"], + "FULL_STATUS_SHA256", + ) + status = raw.decode_return_status(status_payload, definition=definition) + offset += h["status_bytes"] + _require( + type(h["columns"]) is list and len(h["columns"]) == len(PROJECTED_COLUMNS), + "FULL_ARTIFACT_COLUMNS", + ) + values = {} + for name, entry in zip(PROJECTED_COLUMNS, h["columns"], strict=True): + _require( + type(entry) is dict + and set(entry) == {"name", "bytes", "sha256"} + and entry["name"] == name + and type(entry["bytes"]) is int + and entry["bytes"] == rows * 8, + "FULL_COLUMN_DECLARATION", + ) + data = payload[offset : offset + entry["bytes"]] + _require( + len(data) == entry["bytes"] + and hashlib.sha256(data).hexdigest() == entry["sha256"], + "FULL_COLUMN_SHA256", + ) + values[name] = np.frombuffer(data, dtype="= -cap) & (value[~aggregate] <= cap)).all() + and (name not in COUNT_COLUMNS or (value[~aggregate] >= 0).all()), + "FULL_ORDINARY_TYPED_DOMAIN", + ) + _validate_count_fields(values, status.typed) + return FullPufSource( + definition.sha256, + MappingProxyType(h["source_sha256"]), + status.typed, + MappingProxyType(values), + MappingProxyType({int(k): MappingProxyType(v) for k, v in tokens.items()}), + status_payload, + ) + + +DIRECT_MAPPINGS = { + "employment_income_before_lsr": "E00200", + "self_employment_income_before_lsr": "E00900", + "taxable_interest_income": "E00300", + "qualified_dividend_income": "E00650", + "tax_exempt_interest_income": "E00400", + "short_term_capital_gains": "P22250", + "long_term_capital_gains_before_response": "P23250", + "long_term_capital_gains_on_collectibles": "E24518", + "non_sch_d_capital_gains": "E01100", + "taxable_private_pension_income": "E01700", + "taxable_ira_distributions": "E01400", + "alimony_income": "E00800", + "alimony_expense": "E03500", + "salt_refund_income": "E00700", + "charitable_cash_donations": "E19800", + "charitable_non_cash_donations": "E20100", + "real_estate_taxes": "E18500", + "investment_income_elected_form_4952": "E58990", + "student_loan_interest": "E03210", + "educator_expense": "E03220", + "casualty_loss": "E20500", + "farm_income": "T27800", + "farm_operations_income": "E02100", + "farm_rent_income": "E27200", + "miscellaneous_income": "E01200", + "domestic_production_ald": "E03240", + "unrecaptured_section_1250_gain": "E24515", + "health_savings_account_ald": "E03290", +} + + +def observed_and_derived_return_columns(source, *, selected_recids=None): + """Return canonical observed/derived quantities plus explicit model inputs. + + This stage intentionally retains total SS, total interest, realized IRA/ + Keogh deductions and total miscellaneous itemized deductions under honest + auxiliary names. The next declared model owner supplies component/proxy + outputs; their absence is never silently converted to a known zero here. + """ + _require(isinstance(source, FullPufSource), "FULL_SOURCE_TYPE") + mask = source.ordinary.copy() + if selected_recids is not None: + ids = tuple(selected_recids) + _require( + len(ids) > 0 + and all(type(i) is int for i in ids) + and len(set(ids)) == len(ids), + "FULL_SELECTION_IDS", + ) + _require( + set(ids) <= set(source.status["RECID"][mask].tolist()), + "FULL_SELECTION_UNIVERSE", + ) + mask &= np.isin(source.status["RECID"], ids) + a = {name: value[mask] for name, value in source.values.items()} + out = {name: a[field].copy() for name, field in DIRECT_MAPPINGS.items()} + out.update( + { + "non_qualified_dividend_income": a["E00600"] - a["E00650"], + "rental_income": a["E25850"] - a["E25860"], + "estate_income": a["E26390"] - a["E26400"], + "partnership_income": ( + a["E25940"] + a["E25980"] - a["E25920"] - a["E25960"] - a["E26110"] + ), + "s_corp_income": ( + a["E26170"] + a["E26190"] - a["E26160"] - a["E26180"] - a["E26100"] + ), + } + ) + auxiliary = { + "raw_adjusted_gross_income": a["E00100"], + "raw_total_social_security": a["E02400"], + "raw_total_interest_deduction": a["E19200"], + "raw_realized_ira_deduction": a["E03150"], + "raw_realized_keogh_deduction": a["E03300"], + "raw_tuition_fees_deduction": a["E03230"], + "raw_lifetime_learning_qualified_expenses": a["E87530"], + "raw_miscellaneous_itemized_deductions": a["E20400"], + "raw_partnership_nonpassive_net": a["E25980"] - a["E25960"], + "raw_partnership_s_corp_combined": a["E26270"], + "raw_exemptions_count": a["XTOT"], + **{"raw_" + name.lower(): a[name] for name in COUNT_COLUMNS[:-1]}, + "reported_dependent_count": sum(a[name] for name in DEPENDENT_COLUMNS), + "puf_2015_filing_status_code": source.status["MARS"][mask].copy(), + "puf_2015_capped_return_size": ( + 1 + + (source.status["MARS"][mask] == 2).astype(np.int64) + + sum(a[name] for name in DEPENDENT_COLUMNS) + ), + "raw_ordinary_dividend_income": a["E00600"], + } + auxiliary["partnership_component_reconciliation_residual"] = ( + out["partnership_income"] + out["s_corp_income"] - a["E26270"] + ) + # Canonical intermediates stay integer dollars until the explicit model + # boundary. S006 stays integer hundredths here too. + status = {name: value[mask].copy() for name, value in source.status.items()} + return out, auxiliary, status diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source_graph.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source_graph.py new file mode 100644 index 000000000..6f8f13044 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source_graph.py @@ -0,0 +1,122 @@ +"""Graph owner of the additive complete PUF return-source artifact.""" + +from __future__ import annotations + +import hashlib +from importlib import import_module + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import load_source_bytes + +from . import puf_full_source as full +from . import puf_raw_source as raw + +FULL_RETURN_SOURCE_TYPE = ArtifactType("microcosm.us.puf_2015_full_return_source", 3) + + +class FullPufReturnSourceKernel(KernelBase): + ref = "us.puf.full_return_source@3" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, dependencies=("numpy",) + ) + + def __init__(self, *, definition=None, source_codecs=None): + self.definition = ( + raw.packaged_definition() if definition is None else definition + ) + self.source_codecs = ( + raw.puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self): + base = source_hash( + import_module(__name__), + full, + raw, + dependencies=self.capabilities.dependencies, + ) + return hashlib.sha256( + base.encode("ascii") + b"\0" + canonical_json(raw.csv_acceptance_profile()) + ).hexdigest() + + def run(self, context): + d = self.definition + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources != (d.main.source_name, d.demographic.source_name) + or context.node.inputs + or context.node.outputs + or context.node.artifact_inputs + or context.node.artifact_outputs + != (ArtifactOutput("full_return_source", FULL_RETURN_SOURCE_TYPE),) + or dict(context.params) != {"definition": d.params_text} + ): + raise ValueError("FULL_SOURCE_NODE_DECLARATION") + main = load_source_bytes( + d.main.codec, + context.sources[d.main.source_name], + registry=self.source_codecs, + ) + demo = load_source_bytes( + d.demographic.codec, + context.sources[d.demographic.source_name], + registry=self.source_codecs, + ) + decoded = full.decode_full_puf_source(main, demo, d) + payload = full.encode_full_puf_source(decoded) + return KernelResult( + artifacts={"full_return_source": payload}, + receipt={ + "full_puf_return_source": { + "definition_sha256": d.sha256, + "definition_route": d.route, + "source_sha256": dict(decoded.source_sha256), + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "rows": len(decoded.status["RECID"]), + "ordinary_rows": int(decoded.ordinary.sum()), + "projected_monetary_fields": len(full.MONEY_COLUMNS), + "projected_count_fields": len(full.COUNT_COLUMNS), + "amount_units": "source_whole_us_dollars", + "weight_units": "S006_integer_hundredths", + "period": "raw_FLPDYR_preserved", + "canonical_modeling": False, + "target_year_growth": False, + "donor_admission": False, + } + }, + ) + + +def full_puf_return_source_node( + *, population, definition=None, node_id="us_puf_full_source.return_source" +): + if not isinstance(population, str) or not population: + raise ValueError("FULL_SOURCE_POPULATION") + d = raw.packaged_definition() if definition is None else definition + return Node( + node_id, + FullPufReturnSourceKernel.ref, + population=population, + sources=(d.main.source_name, d.demographic.source_name), + params={"definition": d.params_text}, + artifact_outputs=( + ArtifactOutput("full_return_source", FULL_RETURN_SOURCE_TYPE), + ), + description="Authenticate and type 52 monetary fields, seven exemption/dependent count fields, and complete PUF source status; no model or growth.", + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py new file mode 100644 index 000000000..794486e8e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py @@ -0,0 +1,1713 @@ +"""Pure per-field growth contract and transform for a raw-source PUF successor. + +This is the first bounded slice of `microcosm#530 +`_ item 2: *pin the raw +PUF asset and implement PUF aging as a Microcosm stage, under the same uprating +discipline as the other sources*. Nothing here reads a file, acquires an +artifact, imports a country engine, or generates a factor. It compiles an +explicit contract and applies it to a table that the caller already holds. + +What the contract makes explicit, per consumed field +---------------------------------------------------- +Source reference year, target year, role (money / code / identifier / count / +design weight), growth kind (nominal price restatement versus real per-return +aging), sign branch, and the exact factor-series identity. A field with no +declared rule is refused; a field with two rules that could both select the +same row is refused; there is no implicit factor of one and no dispatch by +string substring. Every factor is resolved by naming a **variable row** and two +**year columns** of the factor table explicitly, and is applied to a given row +at most once. + +Why the row/column axes are named rather than iterated +------------------------------------------------------ +The archived producer looped ``for variable in uprating`` over a DataFrame +pivoted to year columns, so it iterated years while believing it iterated +variable names (:data:`ARCHIVED_PRODUCER_COMMIT`, ``datasets/puf/puf.py`` +against ``utils/uprating.py``). Its 2015→2021 step separately listed two of its +three sign-branched fields in a fallback group as well, so for those two a +sign-specific factor was followed by a second growth factor. Both failure modes are structural +here: a series name is looked up in a variable-keyed mapping, a year is looked +up in a year-keyed mapping, and the sign-branch cover is validated to be exactly +disjoint before any array is touched. + +Authority +--------- +A factor table declares its own authority, and only +:data:`RELEASE_ELIGIBLE_AUTHORITIES` — ``reviewed_resource`` alone — can stand +behind a release. ``invented_fixture_nonauthority`` proves the contract and the +transform on invented levels. ``developmental_public_resource`` is for a +bounded experiment on levels that really were read from a public publisher: it +must carry the same receipt a reviewed table owes, plus the digest and +retrieval time of the exact resource bytes, and it must declare its own divisor +and index/total shape. Labelling such a table an invented fixture would be a +misdescription, so it gets its own authority rather than a lie. + +What it does **not** get is a promotion path. Release eligibility is keyed on +``reviewed_resource`` and nothing else, and a developmental table whose digest +appears in :data:`REVIEWED_FACTOR_TABLE_DIGESTS` is refused outright, so an +allowlist entry cannot launder a developmental resource into a release recipe. +``reviewed_resource`` is itself refused until a reviewed table is generated, +pinned, and added to that deliberately empty tuple. No authority can be +promoted by passing a flag. + +Scope +----- +Money growth and design-weight growth are separate entry points, so a money +transform cannot move a weight. Codes, identifiers and counts are declared but +never written, and preservation is verified rather than assumed. The raw +``RECID`` and the source-year AGI used for SOI band assignment are copied into +separate provenance columns before any growth, so a later change of AGI basis +cannot destroy the band input. Nothing here decides whether the pinned +processed artifact is comparable to a rebuilt one; that artifact stays an +opaque legacy input, outside this contract. +""" + +from __future__ import annotations + +import datetime +import json +import struct +from collections.abc import Mapping, Sequence +from dataclasses import InitVar, dataclass +from enum import StrEnum +from hashlib import sha256 +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.build.monetary_targets import MonetaryBasis +from microcosm.build.us_runtime.puf_source_agi import PUF_SOURCE_YEAR + +__all__ = [ + "DESIGN_WEIGHT_FIELD", + "GROWN_MONEY_VALUATION", + "MAX_EXACT_AMOUNT", + "PROVENANCE_COLUMNS", + "PROVENANCE_RECID_COLUMN", + "PROVENANCE_SOURCE_AGI_COLUMN", + "ARCHIVED_PRODUCER_COMMIT", + "DEVELOPMENTAL_PROVENANCE_KEYS", + "DEVELOPMENTAL_PROVENANCE_LITERALS", + "PROPOSED_TY2015_FIELD_ROSTER", + "RELEASE_ELIGIBLE_AUTHORITIES", + "ROSTER_CITATION", + "ROSTER_SOURCE_REFERENCE_YEAR", + "REVIEWED_FACTOR_TABLE_DIGESTS", + "REVIEWED_PROVENANCE_KEYS", + "SOURCE_AGI_FIELD", + "SOURCE_RECID_FIELD", + "CompiledPufGrowth", + "FactorAuthority", + "FactorSeries", + "FieldRole", + "GrowthFactorTable", + "GrowthKind", + "GrowthRecipe", + "GrowthRule", + "GrownPufTable", + "PufGrowthRefusalError", + "ResolvedFactor", + "RosterEntry", + "UNDOCUMENTED_SOURCE_FIELDS", + "SignBranch", + "apply_puf_design_weight_growth", + "apply_puf_growth", + "compile_puf_growth", + "parse_growth_recipe", + "roster_role", + "puf_growth_monetary_metadata", + "require_transformable", +] + +#: Schema version of the recipe identity this module emits. +RECIPE_SCHEMA_VERSION = 1 + +#: Schema version of the factor-table document this module parses. +FACTOR_TABLE_SCHEMA_VERSION = 1 + +#: Raw IRS PUF field carrying the return identifier. +SOURCE_RECID_FIELD = "RECID" + +#: Raw IRS PUF field carrying adjusted gross income. +SOURCE_AGI_FIELD = "E00100" + +#: Raw IRS PUF field carrying the sample weight (stored times 100). +DESIGN_WEIGHT_FIELD = "S006" + +#: Copy of ``RECID``, written from the source before any growth. No rule may +#: target this column or take its name, but the column itself is an ordinary +#: pandas column: what protects it is the contract, and in the graph the fact +#: that no node declares it as a rewrite. +PROVENANCE_RECID_COLUMN = "puf_source_recid" + +#: Copy of source-year AGI on the same terms, kept for the TY2015 SOI band +#: assignment in :mod:`microcosm.build.us_runtime.puf_source_agi`. +PROVENANCE_SOURCE_AGI_COLUMN = "puf_source_year_agi" + +#: The provenance columns this transform adds, in output order. +PROVENANCE_COLUMNS = (PROVENANCE_RECID_COLUMN, PROVENANCE_SOURCE_AGI_COLUMN) + +#: SHA-256 digests of factor tables that a reviewed authority has accepted. +#: Empty: no factor table has been generated or reviewed for this lineage, so +#: ``reviewed_resource`` authority cannot compile. Adding a digest here is the +#: reviewed act; it is not something a caller can supply. +REVIEWED_FACTOR_TABLE_DIGESTS: tuple[str, ...] = () + +#: What a ``reviewed_resource`` factor table must state about itself. Nothing +#: can satisfy this yet, because :data:`REVIEWED_FACTOR_TABLE_DIGESTS` is +#: empty; the set is here so the receipt a reviewed table owes is written down +#: executably rather than only in prose. +REVIEWED_PROVENANCE_KEYS = frozenset( + { + "dependency_versions", + "generated_by", + "generator_code_sha256", + "index_or_total", + "parameter_paths", + "population_divisor", + "rounding", + "source_url", + } +) + +#: What a ``developmental_public_resource`` factor table must state about +#: itself: everything a reviewed table owes, plus the digest of the exact +#: resource bytes its levels were read from and when they were retrieved. A +#: public resource is a moving distribution, so "which bytes, read when" is +#: part of the claim rather than a footnote. +DEVELOPMENTAL_PROVENANCE_KEYS = frozenset( + REVIEWED_PROVENANCE_KEYS | {"resource_sha256", "retrieved_utc"} +) + +#: Provenance values a developmental table must spell out rather than leave to +#: a reader's inference. A price index has no return or population divisor and +#: is an index, not a total; a table that says otherwise is describing a +#: different quantity and is refused here rather than downstream. +DEVELOPMENTAL_PROVENANCE_LITERALS = MappingProxyType( + {"index_or_total": "index", "population_divisor": "none"} +) + +#: Largest integer amount float64 represents exactly. A larger integer money +#: value is refused rather than silently rounded by the working conversion. +MAX_EXACT_AMOUNT = 2**53 + +#: Largest table this transform will accept, so a mistake is a refusal rather +#: than an unbounded allocation. +MAX_ROWS = 2_000_000 + +#: Largest factor document this transform will parse. +MAX_DOCUMENT_BYTES = 512 * 1024 + +_INTEGER_KINDS = frozenset("iu") +_MONEY_KINDS_ALLOWED = frozenset("iuf") +_RESULT_TOKEN = object() + + +class PufGrowthRefusalError(ValueError): + """Sanitized field/reason refusal; never carries row values or identifiers.""" + + def __init__(self, reason: str, field: str = "contract") -> None: + self.reason = reason + self.field = field + super().__init__(f"{field}: {reason}") + + +def _require(condition: bool, reason: str, field: str = "contract") -> None: + if not condition: + raise PufGrowthRefusalError(reason, field) + + +def _sha(value: bytes) -> str: + return sha256(value).hexdigest() + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +def _parse(document: bytes) -> dict: + """Parse a bounded JSON object, refusing duplicate keys and non-finite values.""" + _require( + type(document) is bytes and 0 < len(document) <= MAX_DOCUMENT_BYTES, + "DOCUMENT_SIZE_OR_TYPE", + ) + + def pairs(items): + result: dict = {} + for key, item in items: + _require(key not in result, "DUPLICATE_DOCUMENT_KEY") + result[key] = item + return result + + def constant(_): + raise PufGrowthRefusalError("NONFINITE_DOCUMENT") + + try: + parsed = json.loads(document, object_pairs_hook=pairs, parse_constant=constant) + except PufGrowthRefusalError: + raise + except (ValueError, RecursionError) as error: + raise PufGrowthRefusalError("MALFORMED_DOCUMENT") from error + _require(type(parsed) is dict, "DOCUMENT_OBJECT_REQUIRED") + return parsed + + +def _digest(value: object) -> bool: + return ( + type(value) is str + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _identifier(value: object, reason: str, field: str = "contract") -> str: + _require( + type(value) is str and 0 < len(value) <= 128 and value.strip() == value, + reason, + field, + ) + return value # type: ignore[return-value] + + +def _year(value: object, reason: str, field: str = "contract") -> int: + _require(type(value) is int and 1900 <= value <= 2200, reason, field) + return value # type: ignore[return-value] + + +class FieldRole(StrEnum): + """What a consumed raw field is, which fixes what may happen to it.""" + + MONEY = "money" + CODE = "code" + IDENTIFIER = "identifier" + COUNT = "count" + DESIGN_WEIGHT = "design_weight" + + +class GrowthKind(StrEnum): + """How a declared factor is meant, which is not recoverable from its value.""" + + #: Restates the same nominal amount into another year's purchasing power. + NOMINAL_PRICE_RESTATEMENT = "nominal_price_restatement" + #: Ages an amount per return: an aggregate total ratio divided by a return + #: count ratio, so it carries real change as well as price change. + REAL_PER_RETURN_AGING = "real_per_return_aging" + #: Grows the design weight itself. Never applied to money. + DESIGN_WEIGHT_GROWTH = "design_weight_growth" + #: Declared and deliberately untouched. + NONE = "none" + + +class SignBranch(StrEnum): + """Which rows of a money field a rule may select.""" + + ANY = "any" + POSITIVE = "positive" + NEGATIVE = "negative" + + +class FactorAuthority(StrEnum): + """What a factor table claims about itself.""" + + INVENTED_FIXTURE = "invented_fixture_nonauthority" + #: Levels really read from a named public publisher, for a bounded + #: developmental experiment. Never release-eligible; see + #: :data:`RELEASE_ELIGIBLE_AUTHORITIES`. + DEVELOPMENTAL_PUBLIC_RESOURCE = "developmental_public_resource" + REVIEWED_RESOURCE = "reviewed_resource" + + +#: The only authority a release recipe may stand behind. Written as a set so +#: the invariant is one readable membership test rather than a comparison +#: repeated at each site, and so a test can assert its contents directly. +RELEASE_ELIGIBLE_AUTHORITIES = frozenset({FactorAuthority.REVIEWED_RESOURCE}) + + +_MONEY_KINDS = frozenset( + {GrowthKind.NOMINAL_PRICE_RESTATEMENT, GrowthKind.REAL_PER_RETURN_AGING} +) +_PRESERVED_ROLES = frozenset({FieldRole.CODE, FieldRole.IDENTIFIER, FieldRole.COUNT}) + +#: How a grown money column's valuation is named, per growth kind. The string +#: goes into :class:`~microcosm.build.monetary_targets.MonetaryBasis`, so a +#: later fit-boundary check compares a restated amount against an aged amount +#: as the different things they are. +GROWN_MONEY_VALUATION = MappingProxyType( + { + GrowthKind.NOMINAL_PRICE_RESTATEMENT: "nominal_price_restated_from_{source}", + GrowthKind.REAL_PER_RETURN_AGING: "real_per_return_aged_from_{source}", + } +) + + +@dataclass(frozen=True) +class GrowthRule: + """One rule: which rows of which field grow by which factor series. + + Attributes: + field: The raw source column this rule governs. + role: What the field is. Only :attr:`FieldRole.MONEY` may carry a sign + branch, and only money and the design weight may grow at all. + source_reference_year: The year the observed amount belongs to. + target_year: The year the output amount is stated in. Preserved fields + retain ``source_reference_year``; growing roles may also use that year. + growth_kind: What the factor means. Fixed by the role for every role + except money, which must choose restatement or aging explicitly. + sign_branch: Which rows this rule selects. ``ANY`` must be the field's + only rule; ``POSITIVE`` and ``NEGATIVE`` must both be present. + factor_series: The exact variable-row name in the factor table. Empty + only for a declared no-growth field. + citation: Where the field's meaning was read. Descriptive. + """ + + field: str + role: FieldRole + source_reference_year: int + target_year: int + growth_kind: GrowthKind + sign_branch: SignBranch = SignBranch.ANY + factor_series: str = "" + citation: str = "" + + def __post_init__(self) -> None: + field = _identifier(self.field, "RULE_FIELD_NAME") + _require(type(self.role) is FieldRole, "RULE_ROLE", field) + _require(type(self.growth_kind) is GrowthKind, "RULE_GROWTH_KIND", field) + _require(type(self.sign_branch) is SignBranch, "RULE_SIGN_BRANCH", field) + _require(type(self.citation) is str, "RULE_CITATION", field) + source = _year(self.source_reference_year, "RULE_SOURCE_YEAR", field) + target = _year(self.target_year, "RULE_TARGET_YEAR", field) + if self.role is FieldRole.MONEY: + _require(self.growth_kind in _MONEY_KINDS, "MONEY_GROWTH_KIND", field) + elif self.role is FieldRole.DESIGN_WEIGHT: + _require( + self.growth_kind is GrowthKind.DESIGN_WEIGHT_GROWTH, + "WEIGHT_GROWTH_KIND", + field, + ) + _require(self.sign_branch is SignBranch.ANY, "WEIGHT_SIGN_BRANCH", field) + else: + _require( + self.growth_kind is GrowthKind.NONE, "PRESERVED_GROWTH_KIND", field + ) + _require(self.sign_branch is SignBranch.ANY, "PRESERVED_SIGN_BRANCH", field) + _require(source == target, "PRESERVED_YEAR_SPAN", field) + if self.growth_kind is GrowthKind.NONE: + _require(self.factor_series == "", "PRESERVED_FACTOR_SERIES", field) + else: + _identifier(self.factor_series, "RULE_FACTOR_SERIES", field) + _require(target >= source, "RULE_YEAR_ORDER", field) + + @property + def grows(self) -> bool: + """Whether this rule multiplies anything at all.""" + return self.growth_kind is not GrowthKind.NONE + + def as_document(self) -> dict: + """Canonical, hashable description of this rule.""" + return { + "field": self.field, + "role": str(self.role), + "source_reference_year": self.source_reference_year, + "target_year": self.target_year, + "growth_kind": str(self.growth_kind), + "sign_branch": str(self.sign_branch), + "factor_series": self.factor_series, + } + + +@dataclass(frozen=True) +class GrowthRecipe: + """A closed set of rules covering every consumed field exactly once. + + Coverage is validated here, before any table is seen: a money field is + covered either by one ``ANY`` rule or by exactly one ``POSITIVE`` and one + ``NEGATIVE`` rule, and never by both shapes. That is the structural refusal + of the archived overlap, where a sign-branched field also sat in a fallback + group and could receive a second factor. + """ + + recipe_id: str + rules: tuple[GrowthRule, ...] + + def __post_init__(self) -> None: + _identifier(self.recipe_id, "RECIPE_ID") + _require( + type(self.rules) is tuple + and bool(self.rules) + and all(type(rule) is GrowthRule for rule in self.rules), + "RECIPE_RULES", + ) + seen: set[tuple[str, SignBranch]] = set() + branches: dict[str, set[SignBranch]] = {} + shape: dict[str, tuple[FieldRole, int, int, GrowthKind]] = {} + for rule in self.rules: + key = (rule.field, rule.sign_branch) + _require(key not in seen, "DUPLICATE_FIELD_RULE", rule.field) + seen.add(key) + branches.setdefault(rule.field, set()).add(rule.sign_branch) + declaration = ( + rule.role, + rule.source_reference_year, + rule.target_year, + rule.growth_kind, + ) + if rule.field in shape: + _require( + shape[rule.field] == declaration, + "AMBIGUOUS_FIELD_DECLARATION", + rule.field, + ) + else: + shape[rule.field] = declaration + for name, present in branches.items(): + if present == {SignBranch.ANY}: + continue + _require(SignBranch.ANY not in present, "OVERLAPPING_SIGN_RULES", name) + _require( + present == {SignBranch.POSITIVE, SignBranch.NEGATIVE}, + "INCOMPLETE_SIGN_COVER", + name, + ) + for rule in self.rules: + _require( + rule.field not in PROVENANCE_COLUMNS, + "PROVENANCE_COLUMN_NAME", + rule.field, + ) + money = [rule for rule in self.rules if rule.role is FieldRole.MONEY] + _require(bool(money), "NO_MONEY_FIELD") + identifiers = [ + rule.field for rule in self.rules if rule.role is FieldRole.IDENTIFIER + ] + _require(len(set(identifiers)) == 1, "PROVENANCE_RECID_FIELD") + _require( + any(rule.field == SOURCE_AGI_FIELD for rule in money), + "PROVENANCE_AGI_FIELD", + SOURCE_AGI_FIELD, + ) + money_targets = {rule.target_year for rule in money} + _require(len(money_targets) == 1, "MIXED_TARGET_YEAR") + _require( + len({rule.source_reference_year for rule in money}) == 1, + "MIXED_SOURCE_REFERENCE_YEAR", + ) + weights = [rule for rule in self.rules if rule.role is FieldRole.DESIGN_WEIGHT] + _require(len(weights) <= 1, "MULTIPLE_DESIGN_WEIGHTS") + if weights and money_targets: + _require( + weights[0].target_year == next(iter(money_targets)), + "WEIGHT_TARGET_YEAR", + weights[0].field, + ) + object.__setattr__(self, "rules", tuple(sorted(self.rules, key=_rule_order))) + + @property + def fields(self) -> tuple[str, ...]: + """Every declared field name, once each, in the rules' sorted order. + + ``__post_init__`` sorts the rules by ``(field, sign_branch)``, so two + recipes that declare the same rules in a different order are the same + recipe and hash the same. + """ + ordered: list[str] = [] + for rule in self.rules: + if rule.field not in ordered: + ordered.append(rule.field) + return tuple(ordered) + + def rules_for(self, field: str) -> tuple[GrowthRule, ...]: + """Every rule governing ``field``.""" + return tuple(rule for rule in self.rules if rule.field == field) + + @property + def identity(self) -> bytes: + """Canonical bytes identifying this recipe.""" + return _json( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "recipe_id": self.recipe_id, + "rules": [rule.as_document() for rule in self.rules], + } + ) + + +def parse_growth_recipe(document: bytes) -> GrowthRecipe: + """Rebuild a recipe from its own :attr:`GrowthRecipe.identity` bytes. + + Round-tripping through the identity document is how a recipe travels as a + graph node parameter without becoming an opaque blob: the bytes a node + carries are exactly the bytes the recipe hashes to. + + Args: + document: A recipe identity document. + + Returns: + The recipe those bytes describe. + + Raises: + PufGrowthRefusalError: If the document is malformed, is a different + schema version, or does not round-trip to itself. + """ + data = _parse(document) + _require( + set(data) == {"schema_version", "recipe_id", "rules"}, "RECIPE_DOCUMENT_SCHEMA" + ) + _require(data["schema_version"] == RECIPE_SCHEMA_VERSION, "RECIPE_SCHEMA_VERSION") + _require( + type(data["rules"]) is list and bool(data["rules"]), "RECIPE_DOCUMENT_RULES" + ) + rules = [] + for entry in data["rules"]: + _require( + type(entry) is dict + and set(entry) + == { + "field", + "role", + "source_reference_year", + "target_year", + "growth_kind", + "sign_branch", + "factor_series", + }, + "RECIPE_DOCUMENT_RULES", + ) + for key, values in ( + ("role", FieldRole), + ("growth_kind", GrowthKind), + ("sign_branch", SignBranch), + ): + _require( + entry[key] in tuple(str(value) for value in values), + "RECIPE_DOCUMENT_RULES", + entry["field"] if type(entry["field"]) is str else "contract", + ) + rules.append( + GrowthRule( + entry["field"], + FieldRole(entry["role"]), + entry["source_reference_year"], + entry["target_year"], + GrowthKind(entry["growth_kind"]), + SignBranch(entry["sign_branch"]), + entry["factor_series"], + ) + ) + recipe = GrowthRecipe(data["recipe_id"], tuple(rules)) + _require(recipe.identity == document, "RECIPE_DOCUMENT_ROUNDTRIP") + return recipe + + +def _rule_order(rule: GrowthRule) -> tuple[str, str]: + return (rule.field, str(rule.sign_branch)) + + +@dataclass(frozen=True) +class FactorSeries: + """One variable row of the factor table: a level per year column. + + Levels are index levels, not ratios. A factor is always a ratio of two + explicitly named year columns of one explicitly named row, so a table + cannot be read as if its year columns were variable names. + """ + + name: str + kind: GrowthKind + levels: tuple[tuple[int, str], ...] + + def __post_init__(self) -> None: + name = _identifier(self.name, "SERIES_NAME") + _require(type(self.kind) is GrowthKind, "SERIES_KIND", name) + _require(self.kind is not GrowthKind.NONE, "SERIES_KIND", name) + _require( + type(self.levels) is tuple and bool(self.levels), "SERIES_LEVELS", name + ) + _require( + all(type(pair) in (tuple, list) and len(pair) == 2 for pair in self.levels), + "SERIES_LEVELS", + name, + ) + # Own inner pairs too: frozen dataclasses do not freeze a caller's lists. + object.__setattr__(self, "levels", tuple(tuple(pair) for pair in self.levels)) + years = [year for year, _ in self.levels] + _require(len(set(years)) == len(years), "DUPLICATE_SERIES_YEAR", name) + _require(years == sorted(years), "UNORDERED_SERIES_YEARS", name) + for year, level in self.levels: + _year(year, "SERIES_YEAR", name) + _require(type(level) is str and level.strip() == level, "LEVEL_TEXT", name) + _require(_finite_positive(level), "LEVEL_VALUE", name) + + def level_text(self, year: int) -> str: + """The exact level text in the ``year`` column of this variable row.""" + for candidate, level in self.levels: + if candidate == year: + return level + raise PufGrowthRefusalError("MISSING_FACTOR_YEAR", self.name) + + @property + def years(self) -> tuple[int, ...]: + return tuple(year for year, _ in self.levels) + + +def _finite_positive(text: str) -> bool: + try: + value = float(text) + except ValueError: + return False + return bool(np.isfinite(value)) and value > 0.0 + + +def _freeze_provenance(value: object, depth: int = 0) -> object: + """Own a bounded JSON tree, including nested caller-supplied containers.""" + _require(depth <= 64, "TABLE_PROVENANCE_DEPTH") + if isinstance(value, Mapping): + _require(all(type(key) is str for key in value), "TABLE_PROVENANCE_KEY") + return MappingProxyType( + {key: _freeze_provenance(item, depth + 1) for key, item in value.items()} + ) + if type(value) in (tuple, list): + return tuple(_freeze_provenance(item, depth + 1) for item in value) + _require( + value is None or type(value) in (str, int, float, bool), + "TABLE_PROVENANCE_VALUE", + ) + if type(value) is float: + _require(bool(np.isfinite(value)), "TABLE_PROVENANCE_VALUE") + return value + + +def _provenance_document(value: object) -> object: + """Return detached JSON containers, never references into the owned tree.""" + if isinstance(value, Mapping): + return {key: _provenance_document(item) for key, item in value.items()} + if type(value) is tuple: + return [_provenance_document(item) for item in value] + return value + + +@dataclass(frozen=True) +class GrowthFactorTable: + """A parsed factor document: variable rows by year columns, plus authority. + + The document is retained verbatim so the table's identity is the bytes a + reviewer would read, not a re-serialization of this object's state. + """ + + document: bytes + table_id: str + authority: FactorAuthority + years: tuple[int, ...] + series: tuple[FactorSeries, ...] + provenance: Mapping[str, object] + + def __post_init__(self) -> None: + _require( + type(self.document) is bytes + and 0 < len(self.document) <= MAX_DOCUMENT_BYTES, + "DOCUMENT_SIZE_OR_TYPE", + ) + _identifier(self.table_id, "TABLE_ID") + _require(type(self.authority) is FactorAuthority, "TABLE_AUTHORITY") + _require( + type(self.series) is tuple + and bool(self.series) + and all(type(item) is FactorSeries for item in self.series), + "TABLE_SERIES", + ) + names = [item.name for item in self.series] + _require(len(set(names)) == len(names), "DUPLICATE_SERIES") + _require(type(self.years) is tuple and bool(self.years), "TABLE_YEARS") + for year in self.years: + _year(year, "TABLE_YEARS") + _require(list(self.years) == sorted(set(self.years)), "TABLE_YEARS") + for item in self.series: + _require(item.years == self.years, "SERIES_YEAR_COVERAGE", item.name) + _require(isinstance(self.provenance, Mapping), "TABLE_PROVENANCE") + object.__setattr__(self, "provenance", _freeze_provenance(self.provenance)) + # Re-serializing proves the retained bytes are the canonical document + # for exactly this parsed state, so no field can drift from the bytes. + _require(self.document == _json(self.as_document()), "TABLE_DOCUMENT") + + @classmethod + def from_bytes(cls, document: bytes) -> GrowthFactorTable: + """Parse a factor document, refusing anything it does not fully close.""" + data = _parse(document) + _require( + set(data) + == { + "authority", + "provenance", + "schema_version", + "series", + "table_id", + "years", + }, + "TABLE_SCHEMA", + ) + _require( + data["schema_version"] == FACTOR_TABLE_SCHEMA_VERSION, + "TABLE_SCHEMA_VERSION", + ) + _require( + data["authority"] in tuple(str(value) for value in FactorAuthority), + "TABLE_AUTHORITY", + ) + _require(type(data["years"]) is list and bool(data["years"]), "TABLE_YEARS") + years = tuple(_year(value, "TABLE_YEARS") for value in data["years"]) + _require(type(data["series"]) is dict and bool(data["series"]), "TABLE_SERIES") + _require(type(data["provenance"]) is dict, "TABLE_PROVENANCE") + series = [] + for name in sorted(data["series"]): + entry = data["series"][name] + _require( + type(entry) is dict and set(entry) == {"kind", "levels"}, + "SERIES_SCHEMA", + name, + ) + _require( + entry["kind"] in tuple(str(value) for value in GrowthKind), + "SERIES_KIND", + name, + ) + levels = entry["levels"] + _require(type(levels) is dict and bool(levels), "SERIES_LEVELS", name) + _require( + set(levels) == {str(year) for year in years}, + "SERIES_YEAR_COVERAGE", + name, + ) + series.append( + FactorSeries( + name, + GrowthKind(entry["kind"]), + tuple((year, levels[str(year)]) for year in years), + ) + ) + return cls( + document, + data["table_id"], + FactorAuthority(data["authority"]), + years, + tuple(series), + data["provenance"], + ) + + def as_document(self) -> dict: + """The canonical document for this table's parsed state.""" + return { + "authority": str(self.authority), + "provenance": _provenance_document(self.provenance), + "schema_version": FACTOR_TABLE_SCHEMA_VERSION, + "series": { + item.name: { + "kind": str(item.kind), + "levels": {str(year): level for year, level in item.levels}, + } + for item in self.series + }, + "table_id": self.table_id, + "years": list(self.years), + } + + def series_for(self, name: str) -> FactorSeries: + """The variable row called ``name``; refuses a year column or a miss.""" + for item in self.series: + if item.name == name: + return item + raise PufGrowthRefusalError("UNKNOWN_FACTOR_SERIES", name) + + @property + def sha256(self) -> str: + return _sha(self.document) + + +def _check_developmental_provenance(factors: GrowthFactorTable) -> None: + """Close what a ``developmental_public_resource`` table must state. + + Everything the reviewed shape owes, plus the digest and retrieval time of + the exact resource bytes, plus the two literals that keep a price index + from being read as a per-return total. The reviewed allowlist is checked + negatively here: a developmental digest listed there is a refusal, so the + one act that promotes a reviewed table cannot promote this one. + """ + _require( + set(factors.provenance) == DEVELOPMENTAL_PROVENANCE_KEYS, + "DEVELOPMENTAL_PROVENANCE_SHAPE", + ) + _require( + factors.sha256 not in REVIEWED_FACTOR_TABLE_DIGESTS, + "DEVELOPMENTAL_TABLE_IN_REVIEWED_ALLOWLIST", + ) + for key, expected in DEVELOPMENTAL_PROVENANCE_LITERALS.items(): + _require(factors.provenance[key] == expected, "DEVELOPMENTAL_PROVENANCE_VALUE") + _require( + _digest(factors.provenance["resource_sha256"]), "DEVELOPMENTAL_RESOURCE_DIGEST" + ) + url = factors.provenance["source_url"] + _require( + type(url) is str and url.startswith("https://") and len(url) > len("https://"), + "DEVELOPMENTAL_SOURCE_URL", + ) + retrieved = factors.provenance["retrieved_utc"] + _require(type(retrieved) is str, "DEVELOPMENTAL_RETRIEVED_UTC") + try: + stamp = datetime.datetime.fromisoformat(retrieved) + except ValueError as error: + raise PufGrowthRefusalError("DEVELOPMENTAL_RETRIEVED_UTC") from error + _require( + stamp.tzinfo is not None and stamp.utcoffset() == datetime.timedelta(0), + "DEVELOPMENTAL_RETRIEVED_UTC", + ) + + +@dataclass(frozen=True) +class ResolvedFactor: + """One rule's factor, resolved once from two named year columns.""" + + field: str + sign_branch: SignBranch + series: str + kind: GrowthKind + source_year: int + target_year: int + source_level: str + target_level: str + ratio_bits: str + + def __post_init__(self) -> None: + field = _identifier(self.field, "RESOLVED_FIELD") + _require(type(self.sign_branch) is SignBranch, "RESOLVED_SIGN_BRANCH", field) + _require( + type(self.kind) is GrowthKind and self.kind is not GrowthKind.NONE, + "RESOLVED_KIND", + field, + ) + _identifier(self.series, "RESOLVED_SERIES", field) + _year(self.source_year, "RESOLVED_SOURCE_YEAR", field) + _year(self.target_year, "RESOLVED_TARGET_YEAR", field) + # The stored bits are the factor. Recomputing them here means a + # hand-built ResolvedFactor cannot claim a factor its own levels do not + # produce, so the bits are evidence rather than an assertion. + expected = _factor_ratio_bits(self.source_level, self.target_level, field) + _require(self.ratio_bits == expected, "RESOLVED_RATIO_BITS", field) + + @property + def value(self) -> float: + """The exact float64 this factor is, recovered from its bit pattern.""" + return struct.unpack(" dict: + return { + "field": self.field, + "kind": str(self.kind), + "ratio_bits": self.ratio_bits, + "series": self.series, + "sign_branch": str(self.sign_branch), + "source_level": self.source_level, + "source_year": self.source_year, + "target_level": self.target_level, + "target_year": self.target_year, + } + + +@dataclass(frozen=True) +class CompiledPufGrowth: + """A recipe bound to a factor table, with every factor already resolved.""" + + recipe: GrowthRecipe + factors: GrowthFactorTable + resolved: tuple[ResolvedFactor, ...] + identity: bytes + + def __post_init__(self) -> None: + _require(type(self.recipe) is GrowthRecipe, "RECIPE_TYPE") + _require(type(self.factors) is GrowthFactorTable, "FACTOR_TABLE_TYPE") + # The authority gate lives here, not only in the compile function, so + # that no construction path can produce a release-eligible contract the + # compile chokepoint would have refused. + if self.factors.authority is FactorAuthority.REVIEWED_RESOURCE: + _require( + self.factors.sha256 in REVIEWED_FACTOR_TABLE_DIGESTS, + "UNREVIEWED_FACTOR_TABLE", + ) + _require( + set(self.factors.provenance) == REVIEWED_PROVENANCE_KEYS, + "REVIEWED_PROVENANCE_SHAPE", + ) + elif self.factors.authority is FactorAuthority.DEVELOPMENTAL_PUBLIC_RESOURCE: + _check_developmental_provenance(self.factors) + _require( + type(self.resolved) is tuple + and all(type(item) is ResolvedFactor for item in self.resolved), + "RESOLVED_TYPE", + ) + # Exactly one resolved factor per growing rule, agreeing with it, and + # none left over: a directly constructed contract cannot describe a + # different computation than its own recipe declares. + expected = tuple( + ( + rule.field, + rule.sign_branch, + rule.factor_series, + rule.growth_kind, + rule.source_reference_year, + rule.target_year, + ) + for rule in self.recipe.rules + if rule.grows + ) + actual = tuple( + ( + factor.field, + factor.sign_branch, + factor.series, + factor.kind, + factor.source_year, + factor.target_year, + ) + for factor in self.resolved + ) + _require(actual == expected, "RESOLVED_COVERAGE") + _require( + self.resolved == _resolve_factors(self.recipe, self.factors), + "RESOLVED_FACTOR_BINDING", + ) + _require( + type(self.identity) is bytes + and self.identity + == _contract_identity(self.recipe, self.factors, self.resolved), + "CONTRACT_BINDING", + ) + + @property + def sha256(self) -> str: + return _sha(self.identity) + + @property + def release_eligible(self) -> bool: + """Whether this contract may stand behind a release recipe. + + ``False`` for every invented fixture and every developmental public + resource, and there is no override: the answer is membership of + :data:`RELEASE_ELIGIBLE_AUTHORITIES`, which no caller, flag or + allowlist entry can widen. + """ + return self.factors.authority in RELEASE_ELIGIBLE_AUTHORITIES + + @property + def money_fields(self) -> tuple[str, ...]: + """Declared money fields, once each, in rule order.""" + return tuple( + dict.fromkeys( + rule.field for rule in self.recipe.rules if rule.role is FieldRole.MONEY + ) + ) + + @property + def money_view_fields(self) -> tuple[str, ...]: + """The columns :func:`apply_puf_growth` consumes: everything but weights. + + The design weight is excluded so a money transform neither reads nor + depends on it. In the graph that keeps the money node and the + design-weight node siblings rather than one ordering the other. + """ + weight = self.design_weight_rule + return tuple( + name + for name in self.recipe.fields + if weight is None or name != weight.field + ) + + @property + def preserved_fields(self) -> tuple[str, ...]: + """Declared code/identifier/count fields, which this contract never writes.""" + return tuple( + dict.fromkeys( + rule.field + for rule in self.recipe.rules + if rule.role in _PRESERVED_ROLES + ) + ) + + @property + def design_weight_rule(self) -> GrowthRule | None: + for rule in self.recipe.rules: + if rule.role is FieldRole.DESIGN_WEIGHT: + return rule + return None + + @property + def target_year(self) -> int: + for rule in self.recipe.rules: + if rule.role is FieldRole.MONEY: + return rule.target_year + raise PufGrowthRefusalError("NO_MONEY_FIELD") + + def factor_for(self, field: str, sign_branch: SignBranch) -> ResolvedFactor: + for factor in self.resolved: + if factor.field == field and factor.sign_branch is sign_branch: + return factor + raise PufGrowthRefusalError("NO_RESOLVED_FACTOR", field) + + +def compile_puf_growth( + recipe: GrowthRecipe, factors: GrowthFactorTable +) -> CompiledPufGrowth: + """Bind a recipe to a table using the same resolver as constructor validation. + + Args: + recipe: The closed rule set. Already coverage-validated. + factors: The factor table the rules name. + + Returns: + An immutable compiled contract carrying its own identity bytes. + + Raises: + PufGrowthRefusalError: If the table's authority is not compilable, if a + rule names a series the table does not have, if a named series + means a different growth kind than the rule declares, or if either + declared year is not a column of that series. + """ + _require(type(recipe) is GrowthRecipe, "RECIPE_TYPE") + _require(type(factors) is GrowthFactorTable, "FACTOR_TABLE_TYPE") + resolved = _resolve_factors(recipe, factors) + return CompiledPufGrowth( + recipe, factors, resolved, _contract_identity(recipe, factors, resolved) + ) + + +def _factor_ratio_bits(source_level: str, target_level: str, field: str) -> str: + """One finite positive float64 ratio for compiler and public constructors.""" + _require( + all( + type(level) is str and level.strip() == level and _finite_positive(level) + for level in (source_level, target_level) + ), + "RESOLVED_LEVEL", + field, + ) + ratio = float(target_level) / float(source_level) + _require(bool(np.isfinite(ratio)) and ratio > 0.0, "FACTOR_VALUE", field) + return struct.pack(" tuple[ResolvedFactor, ...]: + """Resolve the complete recipe from the table's immutable named series.""" + resolved = [] + for rule in recipe.rules: + if not rule.grows: + continue + series = factors.series_for(rule.factor_series) + _require(series.kind is rule.growth_kind, "FACTOR_KIND_MISMATCH", rule.field) + source_level = series.level_text(rule.source_reference_year) + target_level = series.level_text(rule.target_year) + resolved.append( + ResolvedFactor( + rule.field, + rule.sign_branch, + series.name, + rule.growth_kind, + rule.source_reference_year, + rule.target_year, + source_level, + target_level, + _factor_ratio_bits(source_level, target_level, rule.field), + ) + ) + return tuple(resolved) + + +def _contract_identity( + recipe: GrowthRecipe, + factors: GrowthFactorTable, + resolved: tuple[ResolvedFactor, ...], +) -> bytes: + """Canonical bytes identifying one recipe bound to one factor table.""" + return _json( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "recipe": json.loads(recipe.identity), + "factor_table_sha256": factors.sha256, + "factor_table_id": factors.table_id, + "factor_authority": str(factors.authority), + "resolved": [factor.as_document() for factor in resolved], + } + ) + + +def puf_growth_monetary_metadata( + compiled: CompiledPufGrowth, +) -> Mapping[str, MonetaryBasis]: + """Declared output basis per money field, for a later fit-boundary check. + + The check itself is not in this slice. What is here is the metadata it + needs: currency, target period, and a valuation string that distinguishes a + nominal restatement from a real per-return aging, so a donor field and a + recipient predictor cannot be silently matched across different bases. + + Args: + compiled: A compiled contract. + + Returns: + Money field name to its declared output + :class:`~microcosm.build.monetary_targets.MonetaryBasis`. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + result: dict[str, MonetaryBasis] = {} + for rule in compiled.recipe.rules: + if rule.role is not FieldRole.MONEY or rule.field in result: + continue + result[rule.field] = MonetaryBasis( + currency="USD", + unit="base_currency", + period=str(rule.target_year), + temporal_basis="annual_flow", + sector="households", + perimeter="irs_individual_income_tax_returns", + valuation=GROWN_MONEY_VALUATION[rule.growth_kind].format( + source=rule.source_reference_year + ), + ) + return MappingProxyType(result) + + +@dataclass(frozen=True, eq=False) +class GrownPufTable: + """A grown table and its receipt; only :func:`apply_puf_growth` builds one.""" + + table: pd.DataFrame + contract_sha256: str + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token: object) -> None: + _require(_token is _RESULT_TOKEN, "RESULT_CONSTRUCTOR_UNAVAILABLE") + + @property + def receipt(self) -> dict: + """The parsed receipt describing exactly what was applied.""" + return _parse(self._receipt) + + @property + def receipt_bytes(self) -> bytes: + return self._receipt + + +def _checked_source_table( + table: pd.DataFrame, compiled: CompiledPufGrowth +) -> pd.DataFrame: + _require(type(table) is pd.DataFrame, "SOURCE_TABLE_TYPE") + rows = len(table) + _require(0 < rows <= MAX_ROWS, "SOURCE_TABLE_ROWS") + _require(all(type(name) is str for name in table.columns), "SOURCE_TABLE_COLUMNS") + _require(len(set(table.columns)) == len(table.columns), "DUPLICATE_SOURCE_COLUMN") + declared = set(compiled.money_view_fields) + present = set(table.columns) + weight = compiled.design_weight_rule + if weight is not None and weight.field in present: + raise PufGrowthRefusalError("DESIGN_WEIGHT_NOT_IN_MONEY_VIEW", weight.field) + missing = sorted(declared - present) + if missing: + raise PufGrowthRefusalError("MISSING_DECLARED_FIELD", missing[0]) + undeclared = sorted(present - declared) + if undeclared: + raise PufGrowthRefusalError("UNDECLARED_SOURCE_COLUMN", undeclared[0]) + return table + + +def _money_values(table: pd.DataFrame, field: str) -> np.ndarray: + """A money column as float64, refusing anything it would have to coerce.""" + series = table[field] + # An explicit kind allow-list rather than ``is_numeric_dtype``, which also + # admits complex, whose imaginary part a float64 cast would drop in silence. + _require( + series.dtype.kind in _MONEY_KINDS_ALLOWED + or ( + pd.api.types.is_extension_array_dtype(series.dtype) + and str(series.dtype).lower().startswith(("int", "uint", "float")) + ), + "FIELD_DTYPE", + field, + ) + _require(not series.isna().any(), "FIELD_MISSING_VALUE", field) + integral = series.dtype.kind in _INTEGER_KINDS or ( + pd.api.types.is_extension_array_dtype(series.dtype) + and str(series.dtype).lower().startswith(("int", "uint")) + ) + if integral: + # float64 is the working type, so an integer amount it cannot represent + # exactly is refused rather than silently rounded. + _require( + bool((np.abs(series.to_numpy(dtype=object)) <= MAX_EXACT_AMOUNT).all()), + "MONEY_MAGNITUDE", + field, + ) + values = series.to_numpy(dtype=np.float64, copy=True) + _require(bool(np.isfinite(values).all()), "FIELD_NONFINITE", field) + # A negative zero would pass every arithmetic guard untouched and then + # compare equal to zero while carrying a different bit pattern. + _require(not bool(np.signbit(values[values == 0.0]).any()), "NEGATIVE_ZERO", field) + return values + + +def _integral_values(table: pd.DataFrame, field: str, reason: str) -> np.ndarray: + series = table[field] + _require( + series.dtype.kind in _INTEGER_KINDS + or ( + pd.api.types.is_extension_array_dtype(series.dtype) + and str(series.dtype).lower().startswith(("int", "uint")) + ), + reason, + field, + ) + _require(not series.isna().any(), "FIELD_MISSING_VALUE", field) + return series.to_numpy(copy=True) + + +def _selection(values: np.ndarray, branch: SignBranch) -> np.ndarray: + """Rows this branch selects. Zero is never selected, so it never grows.""" + if branch is SignBranch.POSITIVE: + return values > 0.0 + if branch is SignBranch.NEGATIVE: + return values < 0.0 + return values != 0.0 + + +def require_transformable(compiled: CompiledPufGrowth) -> None: + """Check every precondition :func:`apply_puf_growth` needs, without a table. + + A caller that is about to declare work — a graph node factory, say — uses + this so a contract that could never run is refused before anything is + loaded, rather than at the moment the money transform reaches for a field + that is not there. + + Args: + compiled: The contract to check. + + Raises: + PufGrowthRefusalError: If the contract declares no money field, no + single identifier, or no source-year AGI field. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + _ = compiled.target_year + _source_reference_year(compiled) + _recid_field(compiled) + _agi_field(compiled) + + +def apply_puf_growth(table: pd.DataFrame, compiled: CompiledPufGrowth) -> GrownPufTable: + """Grow every declared money field once, preserving everything else. + + The input is never mutated: every refusal happens before any array is + written, and the output is a new frame built from copies. + + Args: + table: One row per raw source record, whose columns are exactly + :attr:`CompiledPufGrowth.money_view_fields` — every declared field + except the design weight. + compiled: The compiled contract. + + Row correspondence is positional, not by pandas label: the output carries a + fresh ``RangeIndex`` and row *i* of the output is row *i* of the input. A + caller that needs the input's labels back must reattach them. + + The input is never mutated and no partial result is ever returned. Most + refusals happen before any arithmetic; the product guards + (``GROWN_NONFINITE``, ``SIGN_CHANGED``, ``FACTOR_APPLIED_TWICE``) + necessarily run after a candidate array has been computed in local memory. + + Returns: + A :class:`GrownPufTable` whose frame carries the grown money columns, + the byte-identical code/identifier/count columns, and the two + provenance columns copied from the source before any growth. The design + weight is not part of this view at all; it grows only through + :func:`apply_puf_design_weight_growth`. + + Raises: + PufGrowthRefusalError: On any contract, dtype, value, or coverage + violation. Nothing is written when this is raised. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + source = _checked_source_table(table, compiled) + rows = len(source) + + # Validate every column and resolve every factor before writing anything. + grown: dict[str, np.ndarray] = {} + applications: dict[str, int] = {} + for name in compiled.money_view_fields: + rules = compiled.recipe.rules_for(name) + role = rules[0].role + if role is FieldRole.MONEY: + values = _money_values(source, name) + applied = np.zeros(rows, dtype=np.int64) + result = values.copy() + for rule in rules: + factor = compiled.factor_for(name, rule.sign_branch) + selected = _selection(values, rule.sign_branch) + applied += selected.astype(np.int64) + # The product is checked below; a warning here would only + # duplicate a refusal that is already explicit. + with np.errstate(over="ignore", under="ignore"): + result[selected] = values[selected] * factor.value + _require(bool((applied <= 1).all()), "FACTOR_APPLIED_TWICE", name) + _require(bool(np.isfinite(result).all()), "GROWN_NONFINITE", name) + _require( + bool((np.sign(result) == np.sign(values)).all()), "SIGN_CHANGED", name + ) + grown[name] = result + applications[name] = int(applied.sum()) + else: + _integral_values(source, name, "PRESERVED_FIELD_DTYPE") + + recid_field = _recid_field(compiled) + agi_field = _agi_field(compiled) + provenance_recid = _integral_values(source, recid_field, "PROVENANCE_RECID_DTYPE") + _require(len(np.unique(provenance_recid)) == rows, "DUPLICATE_RECID", recid_field) + provenance_agi = _money_values(source, agi_field) + + output = pd.DataFrame(index=pd.RangeIndex(rows)) + for name in source.columns: + output[name] = ( + pd.Series(grown[name], index=output.index, dtype="float64") + if name in grown + else source[name].reset_index(drop=True).copy(deep=True) + ) + output[PROVENANCE_RECID_COLUMN] = pd.Series( + provenance_recid, index=output.index, dtype=source[recid_field].dtype + ) + output[PROVENANCE_SOURCE_AGI_COLUMN] = pd.Series( + provenance_agi, index=output.index, dtype="float64" + ) + + receipt = _json( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "artifact_kind": "microcosm.us_puf_growth", + "contract_sha256": _sha(compiled.identity), + "factor_authority": str(compiled.factors.authority), + "factor_table_sha256": compiled.factors.sha256, + "release_eligible": compiled.release_eligible, + "rows": rows, + "source_reference_year": _source_reference_year(compiled), + "target_year": compiled.target_year, + "grown_money_fields": sorted(grown), + "grown_row_counts": {name: applications[name] for name in sorted(grown)}, + "preserved_fields": sorted(compiled.preserved_fields), + "design_weight_field": ( + compiled.design_weight_rule.field + if compiled.design_weight_rule is not None + else None + ), + "design_weight_in_money_view": False, + "provenance_columns": list(PROVENANCE_COLUMNS), + "provenance_recid_source": recid_field, + "provenance_agi_source": agi_field, + } + ) + return GrownPufTable(output, _sha(compiled.identity), receipt, _token=_RESULT_TOKEN) + + +def apply_puf_design_weight_growth( + weights: Sequence[float] | np.ndarray, compiled: CompiledPufGrowth +) -> np.ndarray: + """Grow design weights by their own declared factor, and nothing else. + + Kept apart from :func:`apply_puf_growth` on purpose: money growth and + weight growth are different operations with different factor series, and a + single call that did both could silently move population with prices. + + Args: + weights: The source design weights, one per record. + compiled: The compiled contract, which must declare a design weight. + + Returns: + A new float64 array of grown weights. + + Raises: + PufGrowthRefusalError: If no design-weight rule is declared, or the + weights are not finite and positive. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + rule = compiled.design_weight_rule + _require(rule is not None, "NO_DESIGN_WEIGHT_RULE") + values = np.asarray(weights, dtype=np.float64) + _require(values.ndim == 1 and 0 < values.size <= MAX_ROWS, "WEIGHT_SHAPE") + _require(bool(np.isfinite(values).all()), "WEIGHT_NONFINITE") + _require(bool((values > 0.0).all()), "NONPOSITIVE_DESIGN_WEIGHT") + factor = compiled.factor_for(rule.field, SignBranch.ANY) + _require( + factor.kind is GrowthKind.DESIGN_WEIGHT_GROWTH, "WEIGHT_FACTOR_KIND", rule.field + ) + with np.errstate(over="ignore", under="ignore"): + grown = values * factor.value + _require(bool(np.isfinite(grown).all()), "GROWN_WEIGHT_NONFINITE", rule.field) + _require(bool((grown > 0.0).all()), "GROWN_WEIGHT_NONPOSITIVE", rule.field) + return grown + + +def _source_reference_year(compiled: CompiledPufGrowth) -> int: + years = { + rule.source_reference_year + for rule in compiled.recipe.rules + if rule.role is FieldRole.MONEY + } + _require(len(years) == 1, "MIXED_SOURCE_REFERENCE_YEAR") + return years.pop() + + +def _recid_field(compiled: CompiledPufGrowth) -> str: + """The one declared identifier. :class:`GrowthRecipe` already required it.""" + candidates = [ + rule.field + for rule in compiled.recipe.rules + if rule.role is FieldRole.IDENTIFIER + ] + _require(len(candidates) == 1, "PROVENANCE_RECID_FIELD") + return candidates[0] + + +def _agi_field(compiled: CompiledPufGrowth) -> str: + _require( + SOURCE_AGI_FIELD in compiled.recipe.fields, + "PROVENANCE_AGI_FIELD", + SOURCE_AGI_FIELD, + ) + return SOURCE_AGI_FIELD + + +@dataclass(frozen=True) +class RosterEntry: + """One raw TY2015 PUF field this lineage already names, and what it is. + + A roster entry states a **role** and a documented **meaning**, never a + factor. Which series a field should grow by is a producer decision that + belongs to a reviewed recipe, not to this file. + """ + + field: str + role: FieldRole + meaning: str + citation: str = "" + + def __post_init__(self) -> None: + _identifier(self.field, "ROSTER_FIELD") + _require(type(self.role) is FieldRole, "ROSTER_ROLE", self.field) + _require(bool(self.meaning), "ROSTER_MEANING", self.field) + + +#: The archived producer commit every historical claim in this module is read +#: from. The retired US data package is named by commit rather than by package +#: name because this repository's launch contract forbids naming it in the live +#: tree (``test_no_incumbent_data_package_references_in_live_tree``); paths in +#: the citations below are relative to that repository's Python package +#: directory. Read a blob with ``git show :/``. +ARCHIVED_PRODUCER_COMMIT = "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe" + +#: Where the roster's field meanings come from. The archived file is a comment +#: block that itself cites an IRS booklet; that booklet was not read here, so a +#: meaning below is "the label the archived producer attached to this code", +#: not an independently authenticated IRS definition. +ROSTER_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/aggregate_record_totals.yaml" + " (its own cited source: 2014 PUF General Description Booklet, pp. 33-36," + " June 2020)" +) + +_STRUCTURAL_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/puf.py preprocess_puf" +) + +#: Field code to the meaning the archived codebook records for it. Money only. +_DOCUMENTED_MONEY_MEANINGS: tuple[tuple[str, str], ...] = ( + ("E00100", "AGI"), + ("E00200", "Wages and salaries"), + ("E00300", "Taxable interest"), + ("E00400", "Tax-exempt interest"), + ("E00600", "Ordinary dividends"), + ("E00650", "Qualified dividends"), + ("E00700", "State/local tax refund"), + ("E00800", "Alimony received"), + ("E00900", "Business income/loss"), + ("E01000", "Net capital gain/loss"), + ("E01100", "Capital gain distributions"), + ("E01200", "Other gains/losses"), + ("E01400", "Taxable IRA distributions"), + ("E01500", "Total pensions and annuities"), + ("E01700", "Taxable pensions and annuities"), + ("E02100", "Farm income/loss (Sch F)"), + ("E02300", "Unemployment compensation"), + ("E02400", "Social security benefits"), + ("E03150", "IRA deduction"), + ("E03210", "Student loan interest"), + ("E03220", "Educator expenses"), + ("E03230", "Tuition and fees"), + ("E03240", "Domestic production deduction"), + ("E03270", "Self-employed health insurance"), + ("E03290", "HSA deduction"), + ("E03300", "Self-employed SEP/SIMPLE"), + ("E03400", "Early withdrawal penalty"), + ("E03500", "Alimony paid"), + ("E07240", "Saver's credit"), + ("E07260", "Residential energy credit"), + ("E07300", "Foreign tax credit"), + ("E07400", "General business credit"), + ("E07600", "Prior year min tax credit"), + ("E09600", "AMT"), + ("E09700", "Recapture of investment credit"), + ("E09800", "Unreported SE tax"), + ("E09900", "Penalty on early withdrawal"), + ("E11200", "Excess FICA withheld"), + ("E17500", "Medical expenses"), + ("E18400", "State/local income tax"), + ("E18500", "Real estate taxes"), + ("E19200", "Interest paid"), + ("E19800", "Cash contributions"), + ("E20100", "Non-cash contributions"), + ("E20400", "Misc itemized deductions"), + ("E20500", "Casualty/theft loss"), + ("E22250", "Short-term capital gain/loss"), + ("E24515", "Unrecaptured Sec. 1250 gain"), + ("E24518", "Collectibles gain/loss"), + ("E25850", "Rental income (Sch E gross)"), + ("E25860", "Rental loss (Sch E gross)"), + ("E25940", "Partnership income (gross)"), + ("E25960", "Partnership loss (gross)"), + ("E25980", "Partnership net income"), + ("E26180", "S-corp loss (gross)"), + ("E26190", "S-corp net income"), + ("E26270", "Partnership/S-corp net"), + ("E26390", "Estate/trust income (gross)"), + ("E26400", "Estate/trust loss (gross)"), + ("E27200", "Farm rental income (Sch E)"), + ("E30400", "SE income (taxpayer)"), + ("E30500", "SE income (spouse)"), + ("E32800", "Child care credit expenses"), + ("E58990", "Investment interest (4952)"), + ("E62900", "AMT foreign tax credit"), + ("E87521", "American Opportunity Credit"), + ("P08000", "Other credits"), + ("P23250", "Long-term capital gain/loss"), + ("T27800", "Farm income (Sch J)"), +) + +_SOI_MEASURE_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:utils/soi.py puf_measures" +) + +_SOI_RENAME_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/uprate_puf.py SOI_TO_PUF_STRAIGHT_RENAMES" +) + +_SCREENED_FIELD_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/aggregate_record_utils.py SCREENED_FIELDS" +) + +#: The raw TY2015 PUF fields this lineage names, with the role each one plays. +#: A **proposal**, not an authenticated raw-source manifest: every money code +#: the archived codebook describes, plus the four structural codes, the one +#: screened code, and five money codes the codebook omits that only the +#: archived SOI rename/measure tables name -- each cited to where its meaning +#: was actually read. It is not every code this repository reads, and not every +#: code the delivered file contains. It declares no factor for any field. +#: Adding or removing a field is a reviewed change, not something a recipe may +#: do implicitly. +PROPOSED_TY2015_FIELD_ROSTER: tuple[RosterEntry, ...] = ( + RosterEntry( + SOURCE_RECID_FIELD, + FieldRole.IDENTIFIER, + "Record id (the archived producer copies it to household_id)", + _STRUCTURAL_CITATION, + ), + RosterEntry( + DESIGN_WEIGHT_FIELD, + FieldRole.DESIGN_WEIGHT, + "Sample weight, stored times 100 (the archived producer divides by 100)", + _STRUCTURAL_CITATION, + ), + RosterEntry( + "MARS", + FieldRole.CODE, + "Filing status: 0 marks four aggregate disclosure records; " + "1 single, 2 joint, 3 separate, 4 head of household", + _STRUCTURAL_CITATION, + ), + RosterEntry( + "XTOT", + FieldRole.COUNT, + "Exemptions count", + _STRUCTURAL_CITATION, + ), + RosterEntry( + "P22250", + FieldRole.MONEY, + "Short-term capital gains", + _SCREENED_FIELD_CITATION, + ), + RosterEntry( + "E02500", + FieldRole.MONEY, + "Taxable social security (the archived SOI rename binds the SOI series" + " taxable_social_security to this code; the codebook has no entry)", + _SOI_RENAME_CITATION, + ), + # Four money codes the codebook omits and only the archived SOI measure + # table names. E06500 is bound to two different SOI series there, which is + # itself an open question a successor must settle rather than inherit. + RosterEntry("E04800", FieldRole.MONEY, "Taxable income", _SOI_MEASURE_CITATION), + RosterEntry( + "E06500", + FieldRole.MONEY, + "Income tax; the archived measure table binds it to both" + " total_income_tax and income_tax_before_credits", + _SOI_MEASURE_CITATION, + ), + RosterEntry( + "E08800", FieldRole.MONEY, "Income tax after credits", _SOI_MEASURE_CITATION + ), + RosterEntry( + "E19700", + FieldRole.MONEY, + "Charitable contributions deduction", + _SOI_MEASURE_CITATION, + ), + *( + RosterEntry(field, FieldRole.MONEY, meaning, ROSTER_CITATION) + for field, meaning in _DOCUMENTED_MONEY_MEANINGS + ), +) + +#: Codes the archived producer names in executable code at +#: :data:`ARCHIVED_PRODUCER_COMMIT` while nothing there says what they mean: +#: ``DSI`` and ``EIC`` appear only in a predictor list, ``E25920`` only as a +#: subtracted term, and ``E87530`` only inside two helpers whose names imply a +#: Lifetime-Learning qualified-tuition meaning that no comment states. They are +#: deliberately absent from the roster: a successor must resolve them against +#: real source documentation rather than inherit a guess. This is **not** a +#: complete inventory of undocumented codes -- it is what this slice tripped +#: over, and closing it is a documented remaining obligation. ``E22250`` is in +#: the roster because the codebook documents it, but no archived code path +#: reads it -- every path uses ``P22250`` -- so whether both codes exist in the +#: delivered file is open. +UNDOCUMENTED_SOURCE_FIELDS: tuple[str, ...] = ("DSI", "E25920", "E87530", "EIC") + +#: The source tax year those roster fields are observed in. +ROSTER_SOURCE_REFERENCE_YEAR = PUF_SOURCE_YEAR + + +def roster_role(field: str) -> FieldRole: + """The proposed role of ``field``; refuses a field the roster does not name.""" + for entry in PROPOSED_TY2015_FIELD_ROSTER: + if entry.field == field: + return entry.role + raise PufGrowthRefusalError("FIELD_NOT_IN_ROSTER", field) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth_graph.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth_graph.py new file mode 100644 index 000000000..0c727a5aa --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth_graph.py @@ -0,0 +1,375 @@ +"""Graph nodes that run the pure PUF growth transform inside a real population. + +Three nodes, in the shape the US build already uses for a source operation: + +``.boundary`` + A ``FILTER`` node that keeps every person. It exists because a column a + ``CREATE`` node loaded can only be rewritten once a structural node has + opened a new version. Each kernel reconstructs the declared contract from + its parameters before its computation. The boundary's receipt states the + contract digest, the factor authority, and whether that authority could + ever stand behind a release. + +``.money`` + Owns each declared money column as a rewrite, plus the two provenance + columns. Its whole computation is + :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth`. + +``.design_weight`` + Owns only the declared design-weight column, through + :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_design_weight_growth`. + A separate node so the graph itself records that money growth and weight + growth are different operations reading different factor series. + +The contract travels as node parameters — the recipe identity document and the +factor-table document, verbatim — so a changed contract changes the node key +and nothing is served from the store across a contract change. Codes, +identifiers and counts are read but never owned. Two executor mechanisms are +what actually keep them byte-identical: a kernel's returned coordinates must be +exactly the node's declared outputs, so it cannot write them at all; and the +projected input view is frozen and its digest compared before and after the +kernel runs, so it cannot mutate them in place either. + +The design weight this stage grows is the **raw source column**. Promoting it +into a frame's typed design weights is a separate, later obligation with its +own mass policy; nothing here touches ``Frame`` weights. The weight column is +owned as ``float64``, as are the money rewrites; a base version carrying any +of those columns as an integer would need a +declared dtype change, which is a reviewed decision rather than a cast here. + +**The closed read set is the transform's, not the frame's.** +:func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth` refuses a table +carrying a column no rule declares. These nodes cannot make that promise about +the *population*: a ``CREATE`` node upstream may have loaded raw money columns +this contract never mentions, and they pass through the stage ungrown, because a +node only sees the slices it declares. Closing that gap needs a source-stage +contract that declares the full raw column inventory and checks the population +against it; that is a documented remaining obligation, not something this stage +silently covers. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from importlib import import_module + +import pandas as pd + +from microcosm.build.us_runtime.puf_growth import ( + PROVENANCE_RECID_COLUMN, + PROVENANCE_SOURCE_AGI_COLUMN, + CompiledPufGrowth, + GrowthFactorTable, + PufGrowthRefusalError, + apply_puf_design_weight_growth, + apply_puf_growth, + compile_puf_growth, + parse_growth_recipe, + require_transformable, +) +from microcosm.graph import ( + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) + +__all__ = [ + "PUF_GROWTH_BOUNDARY_NODE", + "PUF_GROWTH_DESIGN_WEIGHT_NODE", + "PUF_GROWTH_MONEY_NODE", + "PUF_GROWTH_PROVENANCE_RECID_DTYPE", + "PUF_GROWTH_STAGE", + "PufGrowthBoundaryKernel", + "PufGrowthDesignWeightKernel", + "PufGrowthMoneyKernel", + "register_us_puf_growth_kernels", + "us_puf_growth_nodes", +] + +#: Stage name, and the node ids derived from it. +PUF_GROWTH_STAGE = "us_puf_growth" +PUF_GROWTH_BOUNDARY_NODE = f"{PUF_GROWTH_STAGE}.boundary" +PUF_GROWTH_MONEY_NODE = f"{PUF_GROWTH_STAGE}.money" +PUF_GROWTH_DESIGN_WEIGHT_NODE = f"{PUF_GROWTH_STAGE}.design_weight" + +#: The dtype the provenance record identifier is owned as. The raw identifier +#: is an integer record id; a float copy of it would not be a faithful key. +PUF_GROWTH_PROVENANCE_RECID_DTYPE = "int64" + +#: Modules whose bytes are attested by every kernel's implementation hash. The +#: pure contract is included because a change there changes what these kernels +#: compute even when this file is untouched. +_IMPLEMENTATION_MODULES = ( + "microcosm.build.monetary_targets", + "microcosm.build.us_runtime.puf_growth", + "microcosm.build.us_runtime.puf_growth_graph", + "microcosm.build.us_runtime.puf_source_agi", +) + +_PARAMS = ("entity", "factor_table", "recipe") + + +class _PufGrowthKernel(KernelBase): + """Shared parameter contract and code identity for the three kernels.""" + + def implementation_hash(self) -> str: + # Keep the declared dependencies in the hash the way KernelBase does: + # without them a declared distribution would bind only its name into + # the node key, never its installed version. + return source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + + @staticmethod + def contract(params: Mapping[str, object]) -> tuple[str, CompiledPufGrowth]: + """Recompile the declared contract from the node's own parameters.""" + if set(params) != set(_PARAMS): + raise PufGrowthRefusalError("NODE_PARAMETER_CONTRACT") + entity = params["entity"] + recipe_document = params["recipe"] + factor_document = params["factor_table"] + if not ( + type(entity) is str + and type(recipe_document) is str + and type(factor_document) is str + ): + raise PufGrowthRefusalError("NODE_PARAMETER_CONTRACT") + recipe = parse_growth_recipe(recipe_document.encode("ascii")) + factors = GrowthFactorTable.from_bytes(factor_document.encode("ascii")) + return entity, compile_puf_growth(recipe, factors) + + +def _identifier_index(context: KernelContext, entity: str) -> pd.Index: + column = f"{entity}_id" + table = context.tables[entity] + if column not in table: + raise PufGrowthRefusalError("ENTITY_ID_COLUMN", entity) + return pd.Index(table[column], name=column) + + +def _declared_source(context: KernelContext, entity: str, fields: Sequence[str]): + table = context.tables[entity] + missing = [name for name in fields if name not in table.columns] + if missing: + raise PufGrowthRefusalError("MISSING_DECLARED_FIELD", missing[0]) + return table.loc[:, list(fields)] + + +def _contract_receipt(compiled: CompiledPufGrowth) -> dict[str, object]: + return { + "contract_sha256": compiled.sha256, + "factor_table_sha256": compiled.factors.sha256, + "factor_table_id": compiled.factors.table_id, + "factor_authority": str(compiled.factors.authority), + "release_eligible": compiled.release_eligible, + "target_year": compiled.target_year, + } + + +class PufGrowthBoundaryKernel(_PufGrowthKernel): + """Open a version for the rewrite, and publish the contract it will use.""" + + ref = "us.puf.growth.boundary@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + numeric=Numeric.BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + _, compiled = self.contract(context.params) + person = context.tables["person"] + return KernelResult( + keep=pd.Series( + True, index=pd.Index(person.person_id, name="person_id"), dtype=bool + ), + receipt={"puf_growth_contract": _contract_receipt(compiled)}, + ) + + +class PufGrowthMoneyKernel(_PufGrowthKernel): + """Grow every declared money field once, and copy the provenance columns.""" + + ref = "us.puf.growth.money@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + entity, compiled = self.contract(context.params) + source = _declared_source(context, entity, compiled.money_view_fields) + result = apply_puf_growth(source, compiled) + ids = _identifier_index(context, entity) + grown = result.table + if grown[PROVENANCE_RECID_COLUMN].dtype != PUF_GROWTH_PROVENANCE_RECID_DTYPE: + raise PufGrowthRefusalError( + "PROVENANCE_RECID_DTYPE", PROVENANCE_RECID_COLUMN + ) + columns = { + (entity, name): pd.Series( + grown[name].to_numpy(copy=True), index=ids, dtype="float64" + ) + for name in compiled.money_fields + } + columns[(entity, PROVENANCE_RECID_COLUMN)] = pd.Series( + grown[PROVENANCE_RECID_COLUMN].to_numpy(copy=True), + index=ids, + dtype=PUF_GROWTH_PROVENANCE_RECID_DTYPE, + ) + columns[(entity, PROVENANCE_SOURCE_AGI_COLUMN)] = pd.Series( + grown[PROVENANCE_SOURCE_AGI_COLUMN].to_numpy(copy=True), + index=ids, + dtype="float64", + ) + return KernelResult( + columns=columns, + receipt={"puf_growth": json.loads(result.receipt_bytes)}, + ) + + +class PufGrowthDesignWeightKernel(_PufGrowthKernel): + """Grow the raw design-weight column, and nothing else.""" + + ref = "us.puf.growth.design_weight@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + entity, compiled = self.contract(context.params) + rule = compiled.design_weight_rule + if rule is None: + raise PufGrowthRefusalError("NO_DESIGN_WEIGHT_RULE") + source = _declared_source(context, entity, (rule.field,)) + grown = apply_puf_design_weight_growth( + source[rule.field].to_numpy(dtype="float64"), compiled + ) + factor = compiled.factor_for(rule.field, rule.sign_branch) + return KernelResult( + columns={ + (entity, rule.field): pd.Series( + grown, index=_identifier_index(context, entity), dtype="float64" + ) + }, + receipt={ + "puf_design_weight_growth": { + **_contract_receipt(compiled), + "field": rule.field, + "series": factor.series, + "source_year": factor.source_year, + "ratio_bits": factor.ratio_bits, + "rows": int(len(grown)), + } + }, + ) + + +def us_puf_growth_nodes( + compiled: CompiledPufGrowth, + *, + base: str, + entity: str, + person_boundary_columns: Sequence[str], + stage: str = PUF_GROWTH_STAGE, +) -> tuple[Node, ...]: + """Declare the boundary, money and design-weight nodes for one contract. + + Args: + compiled: The contract these nodes carry. Its recipe identity and its + factor document become node parameters verbatim, so the node keys + move whenever the contract moves. + base: Id of the population version to filter. + entity: Entity whose table carries the raw source fields. + person_boundary_columns: Person columns the boundary reads. A ``FILTER`` + node returns a person mask, so it must declare a person slice. + stage: Node-id prefix, for a graph that runs more than one contract. + + Returns: + The boundary node, the money node, and the design-weight node. + + Raises: + PufGrowthRefusalError: If the contract declares no design weight, names + no person boundary column, or could not run at all + (:func:`~microcosm.build.us_runtime.puf_growth.require_transformable`). + """ + if type(compiled) is not CompiledPufGrowth: + raise PufGrowthRefusalError("COMPILED_TYPE") + weight_rule = compiled.design_weight_rule + if weight_rule is None: + raise PufGrowthRefusalError("NO_DESIGN_WEIGHT_RULE") + if not person_boundary_columns: + raise PufGrowthRefusalError("PERSON_BOUNDARY_COLUMNS") + # Everything the money kernel needs is checked here, at declaration time, + # rather than after a graph run has already loaded a source and written the + # design-weight column. + require_transformable(compiled) + params = { + "entity": entity, + "factor_table": compiled.factors.document.decode("ascii"), + "recipe": compiled.recipe.identity.decode("ascii"), + } + money_view = Slice(entity, compiled.money_view_fields) + boundary = f"{stage}.boundary" + return ( + Node( + boundary, + PufGrowthBoundaryKernel.ref, + base=base, + structural=StructuralDelta.FILTER, + inputs=(Slice("person", tuple(person_boundary_columns)),), + params=params, + description="Open a version so the declared money columns can be rewritten.", + ), + Node( + f"{stage}.money", + PufGrowthMoneyKernel.ref, + population=boundary, + inputs=(money_view,), + outputs=( + *( + Owned(entity, name, "float64", rewrite=True) + for name in compiled.money_fields + ), + Owned( + entity, + PROVENANCE_RECID_COLUMN, + PUF_GROWTH_PROVENANCE_RECID_DTYPE, + ), + Owned(entity, PROVENANCE_SOURCE_AGI_COLUMN, "float64"), + ), + params=params, + description="Grow declared money once and carry source provenance.", + ), + Node( + f"{stage}.design_weight", + PufGrowthDesignWeightKernel.ref, + population=boundary, + inputs=(Slice(entity, (weight_rule.field,)),), + outputs=(Owned(entity, weight_rule.field, "float64", rewrite=True),), + params=params, + description="Grow the raw design weight by its own declared series.", + ), + ) + + +def register_us_puf_growth_kernels(registry: KernelRegistry) -> None: + """Register the three PUF growth kernels.""" + registry.register(PufGrowthBoundaryKernel()) + registry.register(PufGrowthMoneyKernel()) + registry.register(PufGrowthDesignWeightKernel()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py new file mode 100644 index 000000000..73e16294b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py @@ -0,0 +1,689 @@ +"""The thirteen-field 2015 PUF monetary source projection: the twelve accepted +columns plus the publisher's reported ``E00100``. + +The reviewed twelve-field increment +(:mod:`microcosm.build.us_runtime.puf_monetary_source`) decodes wages, the two +interest columns, the two dividend columns, the Schedule C and Schedule F net +results, the two pension columns and the three Schedule D quantities. The PUF +growth contract additionally requires ``E00100`` as a declared money field +(:class:`~microcosm.build.us_runtime.puf_growth.GrowthRecipe`), and the +provenance AGI column that contract writes is a *copy of its own input*, not an +independent source. So a growth run over this lineage needs a projection that +carries reported AGI, and this module is that projection. + +Why a separate module, and what it preserves +-------------------------------------------- +The twelve-field kernel hashes the bytes of ``puf_monetary_source.py`` and +``puf_raw_source.py`` into its implementation hash, and the executor folds that +hash into the artifact store key. Editing either module would move the accepted +twelve-field artifact's key. A repository test also pins the twelve-field +generator's own sha256 to the packaged document. **Nothing here edits any of +them.** This module imports the reviewed decoder, the reviewed envelope +helpers and the reviewed fact derivation, so the thirteenth column is decoded +by exactly the code that decodes the other twelve rather than by a second +implementation of the same contract. The private names it imports are imported +deliberately, and each one is a piece of the reviewed contract this edition +must not restate. + +What is the same +---------------- +Every property of the twelve-field contract holds here unchanged: whole-dollar +signed integer tokens on individual returns, the separately bounded +disclosure-aggregate lexical grammar, the minimum-int64 out-of-universe +sentinel, the ``amount_known == (disclosure_aggregate == 0)`` rule derived from +the authenticated flag on every access, all 207,696 delivered rows retained, +the three disjoint row classes, the binding to the reviewed status artifact, +and ``target_year_use: refused_pending_independent_growth_decision`` on every +column including ``E00100``. + +What is different +----------------- +One more column, and a different artifact identity. The type is +``microcosm.us.puf_2015_monetary_agi_source_projection`` version 1, with its +own magic line. A twelve-field payload fed to this reader is refused **by +name** rather than reinterpreted, because the two artifacts carry different +column sets and silently reading one as the other is exactly the failure the +twelve-field module's superseded-magic list exists to prevent. Nothing here +derives, reconstructs or reconciles AGI: ``E00100`` is the delivered reported +field, and the document records that the booklets state no component identity +between it and the twelve. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from importlib import resources +from types import MappingProxyType + +import numpy as np + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SourceCodecRegistry, load_source_bytes + +# The reviewed twelve-field contract, imported rather than restated. The +# private names are deliberate: the thirteenth column is authenticated, +# encoded, read back and described by exactly the reviewed code, so a change +# there cannot leave a stale second copy of the same contract here. What this +# module owns is its own artifact identity, not its own parser. +from .puf_monetary_source import ( + _AGGREGATE_COLUMN_FACT_KEYS, + _ENVELOPE_FACT_KEYS, + _ENVELOPE_HEADER_KEYS, + _LEXICAL_KIND, + _RETURN_COLUMN_FACT_KEYS, + _STRUCTURAL_DTYPES, + _TYPED_KIND, + AGGREGATE_ROW_CLASS, + BODY_MAX_BYTES, + HEADER_MAX_BYTES, + MONETARY_PROJECTION_MAGIC, + ROW_CLASSES, + SUPERSEDED_MAGIC, + PufMonetaryProjection, + PufMonetarySourceProjection, + _check_envelope_sources, + _envelope_column_order, + _envelope_columns, + _envelope_int, + _exact_keys, + _facts_from_arrays, + _fixed_ascii, + _hex64, + _projection_from_document, + _read_fixed_ascii, + _refuse, + _thawed, + _verify_readback, + decode_puf_monetary_source, + packaged_projection, + payload_bound, +) +from .puf_raw_source import ( + PUF_RAW_SOURCE_NODE, + RETURN_STATUS_TYPE, + PufRawSourceDefinition, + csv_acceptance_profile, + decode_return_status, + puf_raw_source_codecs, +) +from .puf_raw_source import packaged_definition as packaged_raw_definition + +__all__ = [ + "AGI_FIELD", + "AGI_PROJECTION_EDITION", + "AGI_PROJECTION_MAGIC", + "AGI_PROJECTION_NODE", + "AGI_PROJECTION_STAGE", + "AGI_PROJECTION_TYPE", + "TWELVE_FIELD_MAGIC", + "USPufMonetaryAgiSourceKernel", + "decode_agi_projection", + "encode_agi_projection", + "fixture_agi_projection_document", + "packaged_agi_projection", + "register_us_puf_monetary_agi_source_kernels", + "us_puf_monetary_agi_source_node", +] + +#: The one column this edition adds to the accepted twelve. +AGI_FIELD = "E00100" + +#: The edition marker the packaged document must declare about itself. +AGI_PROJECTION_EDITION = "thirteen_field_reported_agi" + +AGI_PROJECTION_TYPE = ArtifactType( + "microcosm.us.puf_2015_monetary_agi_source_projection", 1 +) +AGI_PROJECTION_MAGIC = b"microcosm.us.puf_2015_monetary_agi_source_projection/1\n" + +#: Magic lines this reader refuses by name rather than reinterpreting. These +#: are the twelve-field artifact's own magics: those payloads carry a +#: different column set, and reading one here would report a twelve-column +#: artifact as a thirteen-column one. +TWELVE_FIELD_MAGIC = (MONETARY_PROJECTION_MAGIC, *SUPERSEDED_MAGIC) + +AGI_PROJECTION_STAGE = "us_puf_monetary_agi_source" +AGI_PROJECTION_NODE = f"{AGI_PROJECTION_STAGE}.projection" + +_AGI_RESOURCE = "puf_2015_monetary_agi_source_projection.json" +_AGI_PACKAGE = "microcosm.build.us_runtime" +_FIXTURE_AUTHORITY = "invented_fixture_nonauthority" + +_STATUS_ALIAS = "return_status" +_PROJECTION_ALIAS = "monetary_agi_projection" + + +# -------------------------------------------------------------------------- +# The packaged thirteen-field document +# -------------------------------------------------------------------------- + + +def _check_agi_edition( + document: Mapping[str, object], projection: PufMonetaryProjection +) -> None: + """Bind this edition to the accepted twelve-field document, at load time. + + The edition block is not decoration: it names the parent document by + digest and lists the parent's projected fields, so a thirteen-field + document that quietly dropped or renamed one of the accepted twelve — or + that was generated against a different parent — is refused here rather + than discovered downstream. + """ + + edition = document.get("projection_edition") + if not isinstance(edition, Mapping): + raise _refuse("AGI_PROJECTION_EDITION_MISSING") + if edition.get("edition") != AGI_PROJECTION_EDITION: + raise _refuse("AGI_PROJECTION_EDITION") + if list(edition.get("adds", ())) != [AGI_FIELD]: + raise _refuse("AGI_PROJECTION_EDITION_ADDS") + parent = packaged_projection() + if edition.get("parent_document_canonical_sha256") != parent.sha256: + raise _refuse("AGI_PROJECTION_PARENT_DOCUMENT") + if tuple(edition.get("parent_projected_fields", ())) != parent.fields: + raise _refuse("AGI_PROJECTION_PARENT_FIELDS") + fields = projection.fields + if AGI_FIELD not in fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + if set(fields) != set(parent.fields) | {AGI_FIELD}: + raise _refuse("AGI_PROJECTION_FIELD_SET") + if len(fields) != len(parent.fields) + 1: + raise _refuse("AGI_PROJECTION_FIELD_COUNT") + # Every accepted column must still declare exactly what it declared + # upstream. This edition adds a column; it does not restate the twelve. + upstream = {column.field: column for column in parent.columns} + for column in projection.columns: + if column.field == AGI_FIELD: + continue + if column != upstream[column.field]: + raise _refuse("AGI_PROJECTION_PARENT_COLUMN", column.field) + + +def _packaged_agi_bytes() -> bytes: + return resources.files(_AGI_PACKAGE).joinpath(_AGI_RESOURCE).read_bytes() + + +_PACKAGED_AGI: PufMonetaryProjection | None = None + + +def packaged_agi_projection() -> PufMonetaryProjection: + """The one closed thirteen-field document a production run may use.""" + + global _PACKAGED_AGI + if _PACKAGED_AGI is None: + document = json.loads(_packaged_agi_bytes().decode("utf-8")) + projection = _projection_from_document( + document, route="packaged", definition=packaged_raw_definition() + ) + _check_agi_edition(document, projection) + _PACKAGED_AGI = projection + return _PACKAGED_AGI + + +def fixture_agi_projection_document( + document: Mapping[str, object], definition: PufRawSourceDefinition +) -> PufMonetaryProjection: + """Build a thirteen-field projection on the explicit non-production route. + + Same rules as the twelve-field fixture route: ``route="test_fixture"``, + the invented-fixture authority, and no restatement of a packaged source + pin. It must still name ``E00100``, because a fixture without it would be + testing the twelve-field contract under this module's name. + """ + + projection = _projection_from_document( + document, route="test_fixture", definition=definition + ) + if document.get("authority") != _FIXTURE_AUTHORITY: + raise _refuse("FIXTURE_AUTHORITY") + packaged_raw = packaged_raw_definition() + genuine = {packaged_raw.main.sha256, packaged_raw.demographic.sha256} + stated = { + document["sources"]["main"]["sha256"], + document["sources"]["demographic"]["sha256"], + } + if stated & genuine: + raise _refuse("FIXTURE_RESTATES_PACKAGED_PIN") + if AGI_FIELD not in projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + return projection + + +# -------------------------------------------------------------------------- +# The typed envelope +# -------------------------------------------------------------------------- + + +def encode_agi_projection( + decoded: PufMonetarySourceProjection, + projection: PufMonetaryProjection, + definition: PufRawSourceDefinition, + *, + status_artifact_sha256: str, +) -> bytes: + """Encode a thirteen-field projection into its own bounded envelope. + + Byte-for-byte the twelve-field layout — magic line, big-endian header + length, canonical JSON header, little-endian typed bodies then fixed-width + lexical bodies — under a different magic and type, with the column bodies + the shared order helper produces for this projection. + """ + + if AGI_FIELD not in projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + rows = decoded.rows + total, sizes = payload_bound(rows, projection) + if total > BODY_MAX_BYTES: + raise _refuse("BODY_OVER_BOUND", total) + dtypes = { + **_STRUCTURAL_DTYPES, + **{column.field: column.numpy_dtype for column in projection.columns}, + } + widths = {column.field: column.lexical_width for column in projection.columns} + bodies: list[bytes] = [] + columns: list[dict[str, object]] = [] + for name in _envelope_column_order(projection): + if name.endswith("_lexical"): + field = name.removesuffix("_lexical") + body = _fixed_ascii(decoded.lexical[field], widths[field], field) + entry: dict[str, object] = { + "name": name, + "kind": _LEXICAL_KIND, + "width": widths[field], + } + else: + array = np.ascontiguousarray(decoded.typed[name], dtype=dtypes[name]) + if array.shape != (rows,): + raise _refuse("COLUMN_SHAPE", name) + body = array.tobytes() + entry = {"name": name, "kind": _TYPED_KIND, "dtype": dtypes[name]} + if len(body) != sizes[name]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + entry["bytes"] = len(body) + entry["sha256"] = hashlib.sha256(body).hexdigest() + columns.append(entry) + header = { + "schema_version": AGI_PROJECTION_TYPE.schema_version, + "type": [AGI_PROJECTION_TYPE.name, AGI_PROJECTION_TYPE.schema_version], + "rows": rows, + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "raw_source_definition_sha256": definition.sha256, + "status_artifact_sha256": _hex64( + status_artifact_sha256, "status_artifact_sha256" + ), + "sources": { + "main": { + "sha256": definition.main.sha256, + "git_blob_sha1": definition.main.git_blob_sha1, + "bytes": definition.main.bytes, + "header_record_canonical_sha256": ( + definition.main.header_record_canonical_sha256 + ), + }, + "demographic": { + "sha256": definition.demographic.sha256, + "git_blob_sha1": definition.demographic.git_blob_sha1, + "bytes": definition.demographic.bytes, + "header_record_canonical_sha256": ( + definition.demographic.header_record_canonical_sha256 + ), + }, + }, + "header_length_endianness": "big", + "body_endianness": "little", + "columns": columns, + "facts": dict(decoded.facts), + } + encoded = canonical_json(header) + if len(encoded) > HEADER_MAX_BYTES: + raise _refuse("HEADER_OVER_BOUND", len(encoded)) + return b"".join( + (AGI_PROJECTION_MAGIC, len(encoded).to_bytes(4, "big"), encoded, *bodies) + ) + + +def _resolve_agi_projection( + header: Mapping[str, object], projection: PufMonetaryProjection | None +) -> PufMonetaryProjection: + """The thirteen-field projection this payload may be checked against.""" + + route = header["projection_route"] + if route not in ("packaged", "test_fixture"): + raise _refuse("ENVELOPE_PROJECTION_ROUTE") + digest = _hex64(header["projection_sha256"], "envelope.projection_sha256") + if projection is not None: + if not isinstance(projection, PufMonetaryProjection): + raise _refuse("DECODE_PROJECTION_TYPE") + if projection.route != route or projection.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_MISMATCH") + if AGI_FIELD not in projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + return projection + if route == "packaged": + packaged = packaged_agi_projection() + if packaged.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_NOT_PACKAGED") + return packaged + # A fixture-route payload has no reconstructible projection document, so + # there is nothing to close its column set against. + raise _refuse("ENVELOPE_PROJECTION_REQUIRED") + + +def decode_agi_projection( + payload: bytes, + projection: PufMonetaryProjection | None = None, + *, + expected_status_artifact_sha256: str | None = None, +) -> PufMonetarySourceProjection: + """Decode a thirteen-field payload and re-prove it, never trusting it. + + Every reported fact is recomputed from the arrays actually read back by + the reviewed derivation and must agree exactly, and every row re-parses + under the universe its authenticated flag selects. + """ + + if not isinstance(payload, bytes): + raise _refuse("ENVELOPE_MAGIC") + if not payload.startswith(AGI_PROJECTION_MAGIC): + if any(payload.startswith(magic) for magic in TWELVE_FIELD_MAGIC): + raise _refuse("ENVELOPE_TWELVE_FIELD_MAGIC") + raise _refuse("ENVELOPE_MAGIC") + start = len(AGI_PROJECTION_MAGIC) + if len(payload) < start + 4: + raise _refuse("ENVELOPE_TRUNCATED") + length = int.from_bytes(payload[start : start + 4], "big") + if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: + raise _refuse("ENVELOPE_HEADER_LENGTH") + encoded = payload[start + 4 : start + 4 + length] + try: + header = json.loads(encoded.decode("utf-8")) + except (UnicodeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + try: + recanonical = canonical_json(header) + except (TypeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + if recanonical != encoded: + raise _refuse("ENVELOPE_HEADER_NOT_CANONICAL") + _exact_keys(header, _ENVELOPE_HEADER_KEYS, "header") + if header["schema_version"] != AGI_PROJECTION_TYPE.schema_version or header[ + "type" + ] != [AGI_PROJECTION_TYPE.name, AGI_PROJECTION_TYPE.schema_version]: + raise _refuse("ENVELOPE_TYPE") + if ( + header["header_length_endianness"] != "big" + or header["body_endianness"] != "little" + ): + raise _refuse("ENVELOPE_ENDIANNESS") + _hex64(header["raw_source_definition_sha256"], "raw_source_definition_sha256") + _hex64(header["status_artifact_sha256"], "status_artifact_sha256") + if expected_status_artifact_sha256 is not None: + _hex64(expected_status_artifact_sha256, "expected_status_artifact_sha256") + if header["status_artifact_sha256"] != expected_status_artifact_sha256: + raise _refuse("ENVELOPE_STATUS_ARTIFACT_MISMATCH") + rows = _envelope_int(header["rows"], "rows", minimum=1) + resolved = _resolve_agi_projection(header, projection) + if resolved.raw_definition_sha256 != header["raw_source_definition_sha256"]: + raise _refuse("ENVELOPE_RAW_DEFINITION_MISMATCH") + _check_envelope_sources(header, resolved) + columns = _envelope_columns(header, rows, resolved) + + cursor = start + 4 + length + typed: dict[str, np.ndarray] = {} + lexical: dict[str, tuple[str, ...]] = {} + for column in columns: + name = column["name"] + size = column["bytes"] + body = payload[cursor : cursor + size] + if len(body) != size: + raise _refuse("ENVELOPE_BODY_TRUNCATED", name) + if hashlib.sha256(body).hexdigest() != column["sha256"]: + raise _refuse("ENVELOPE_BODY_DIGEST", name) + cursor += size + if column["kind"] == _TYPED_KIND: + typed[name] = np.frombuffer(body, dtype=np.dtype(column["dtype"])) + else: + field = name.removesuffix("_lexical") + lexical[field] = _read_fixed_ascii(body, rows, column["width"], field) + if cursor != len(payload): + raise _refuse("ENVELOPE_TRAILING_BYTES") + + _verify_readback(typed, lexical, resolved) + facts = header["facts"] + _exact_keys(facts, _ENVELOPE_FACT_KEYS, "facts") + for field, classes in facts["column_facts"].items(): + if field not in resolved.fields: + raise _refuse("ENVELOPE_FACTS", "column_facts") + _exact_keys(classes, frozenset(ROW_CLASSES), f"column_facts.{field}") + for name, entry in classes.items(): + keys = ( + _AGGREGATE_COLUMN_FACT_KEYS + if name == AGGREGATE_ROW_CLASS + else _RETURN_COLUMN_FACT_KEYS + ) + _exact_keys(entry, keys, f"column_facts.{field}.{name}") + recomputed = _facts_from_arrays(typed, lexical, resolved) + if canonical_json(recomputed) != canonical_json(dict(facts)): + raise _refuse("ENVELOPE_FACTS_INCONSISTENT") + if recomputed["rows"] != rows: + raise _refuse("ENVELOPE_ROWS") + return PufMonetarySourceProjection( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(recomputed), + column_metadata=MappingProxyType(dict(recomputed["column_metadata"])), + ) + + +# -------------------------------------------------------------------------- +# The graph producer +# -------------------------------------------------------------------------- + +_IMPLEMENTATION_MODULES = ( + "microcosm.build.us_runtime.puf_raw_source", + "microcosm.build.us_runtime.puf_monetary_source", + "microcosm.build.us_runtime.puf_monetary_agi_projection", +) +_IMPLEMENTATION_IDENTITY_MAGIC = b"us.puf.monetary_agi_source/implementation/1\n" + + +class USPufMonetaryAgiSourceKernel(KernelBase): + """Read the pinned main delivery and produce the thirteen-field projection.""" + + ref = "us.puf.monetary_agi_source.projection@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__( + self, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, + ) -> None: + self.definition = ( + packaged_raw_definition() if definition is None else definition + ) + self.projection = ( + packaged_agi_projection() if projection is None else projection + ) + if self.projection.route not in ("packaged", "test_fixture"): + raise _refuse("KERNEL_PROJECTION_ROUTE") + if self.projection.route != self.definition.route: + raise _refuse("KERNEL_ROUTE_DISAGREEMENT") + if self.projection.raw_definition_sha256 != self.definition.sha256: + raise _refuse("KERNEL_DEFINITION_MISMATCH") + if AGI_FIELD not in self.projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + if self.projection.route == "packaged": + packaged = packaged_agi_projection() + if ( + self.projection.canonical != packaged.canonical + or self.projection.sha256 != packaged.sha256 + or canonical_json(_thawed(self.projection.document)) + != self.projection.canonical + ): + raise _refuse("KERNEL_PROJECTION_NOT_PACKAGED") + self.source_codecs = ( + puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self) -> str: + from importlib import import_module + + base = source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + digest = hashlib.sha256(_IMPLEMENTATION_IDENTITY_MAGIC) + digest.update(base.encode("ascii")) + digest.update(b"\0") + digest.update(canonical_json(csv_acceptance_profile())) + return digest.hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + definition = self.definition + projection = self.projection + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources != (definition.main.source_name,) + or context.node.inputs + or context.node.outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != RETURN_STATUS_TYPE + or context.node.artifact_outputs + != (ArtifactOutput(_PROJECTION_ALIAS, AGI_PROJECTION_TYPE),) + or set(context.params) != {"definition", "projection"} + ): + raise _refuse("NODE_DECLARATION") + if context.params["definition"] != definition.params_text: + raise _refuse("NODE_DEFINITION_PARAM") + if context.params["projection"] != projection.params_text: + raise _refuse("NODE_PROJECTION_PARAM") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != RETURN_STATUS_TYPE: + raise _refuse("STATUS_ARTIFACT_TYPE") + status = decode_return_status(value.payload, definition) + main_bytes = load_source_bytes( + definition.main.codec, + context.sources[definition.main.source_name], + registry=self.source_codecs, + ) + decoded = decode_puf_monetary_source(main_bytes, status, projection, definition) + payload = encode_agi_projection( + decoded, + projection, + definition, + status_artifact_sha256=hashlib.sha256(value.payload).hexdigest(), + ) + return KernelResult( + artifacts={_PROJECTION_ALIAS: payload}, + receipt={ + "puf_monetary_agi_source_projection": { + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "projection_edition": AGI_PROJECTION_EDITION, + "definition_sha256": definition.sha256, + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "status_artifact_sha256": hashlib.sha256(value.payload).hexdigest(), + "main_sha256": definition.main.sha256, + "csv_acceptance_profile": csv_acceptance_profile(), + "projected_fields": list(projection.fields), + "agi_field": AGI_FIELD, + "agi_is_derived": False, + **{ + key: value + for key, value in decoded.facts.items() + if key != "column_facts" + }, + } + }, + ) + + +def us_puf_monetary_agi_source_node( + *, + population: str, + producer: str = PUF_RAW_SOURCE_NODE, + producer_output: str = "return_status", + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + stage: str = AGI_PROJECTION_STAGE, +) -> Node: + """Declare the thirteen-field producer inside an explicitly named host.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + resolved_definition = ( + packaged_raw_definition() if definition is None else definition + ) + resolved = packaged_agi_projection() if projection is None else projection + if resolved.raw_definition_sha256 != resolved_definition.sha256: + raise _refuse("NODE_DEFINITION_MISMATCH") + if AGI_FIELD not in resolved.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + return Node( + f"{stage}.projection", + USPufMonetaryAgiSourceKernel.ref, + population=population, + sources=(resolved_definition.main.source_name,), + params={ + "definition": resolved_definition.params_text, + "projection": resolved.params_text, + }, + artifact_inputs=( + ArtifactInput(_STATUS_ALIAS, producer, producer_output, RETURN_STATUS_TYPE), + ), + artifact_outputs=(ArtifactOutput(_PROJECTION_ALIAS, AGI_PROJECTION_TYPE),), + description=( + "Decode the twelve accepted 2015 PUF monetary source columns plus " + "reported E00100 at return grain, bound to the reviewed status " + "artifact. Owns no population columns and admits no target year." + ), + ) + + +def register_us_puf_monetary_agi_source_kernels( + registry: KernelRegistry, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, +) -> KernelRegistry: + """Register the thirteen-field producer kernel into ``registry``.""" + + registry.register( + USPufMonetaryAgiSourceKernel( + projection=projection, + definition=definition, + source_codecs=source_codecs, + ) + ) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py new file mode 100644 index 000000000..5c91c5a55 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py @@ -0,0 +1,2036 @@ +"""The first monetary amounts typed out of the 2015 PUF delivery bytes. + +The reviewed raw source slice +(:mod:`microcosm.build.us_runtime.puf_raw_source`) types six of the 222 +delivered main columns and the six demographic codes, reports +``amounts_decoded: False``, and drops the other 216 main columns after +checking their header name, order and record width. This module is the next +bounded increment: it authenticates the **whole** delivered layout against the +retained IRS booklets and decodes a small, explicitly enumerated set of +monetary columns at source-return grain. + +Two things it produces, and one it refuses. + +*The full layout is authenticated, not merely counted.* The packaged document +``puf_2015_monetary_source_projection.json`` locates every one of the 222 main +columns and 7 demographic columns in the September 2022 booklet by the +publisher's own position number and name, retains that record-layout row +verbatim with its exact character span and span digest, and corroborates each +against the February 2023 edition. Nothing about a column's role is inferred +from an ``E``, ``P``, ``S`` or ``T`` prefix or from membership of the +amount-field list: the publisher's own "Misc Codes" block sits inside that list +and holds ``RECID``, ``S006``, ``S008``, ``S009``, ``WSAMP`` and ``TXRT``, none +of which is money and one of which (``TXRT``) carries an implied decimal point. + +*Twelve monetary columns are decoded, each with its own declaration.* Wages, +taxable and tax-exempt interest, ordinary and qualified dividends, the +Schedule C and Schedule F net results, pensions received and pensions in AGI, +and the three Schedule D quantities the booklets define. Each declares exact +units, sign evidence, missingness policy, return grain, source-period status +and source pin identity, and each is refused for target-year use. + +*No amount is grown, restated or combined.* The row period is unresolved — the +cover says Tax Year 2015, the code definitions print FLPDYR 2011-2014, the +disclosure section removed pre-2012 returns, and the reviewed header audit +observed 2012-2015 — so every column carries +``target_year_use: refused_pending_independent_growth_decision``. Component +relationships the booklets do not state (qualified dividends inside ordinary +dividends, pensions in AGI inside pensions received, ``E01000`` out of +``P22250`` and ``P23250``) are recorded as unresolved and never computed. +``E22250`` is absent from the delivered layout and from both booklets and is +never aliased onto ``P22250``. An uncapped self-employment earnings derivation +from ``E30400``/``E30500`` is refused with the publisher text that refuses it. + +Three properties are load-bearing. + +*Every delivered main record is projected.* All 207,696 rows, including the +88,021 with no demographic row and the four disclosure aggregate records. The +aggregate rows are tagged and every derived fact is reported for three +disjoint row classes — aggregate, demographic-matched and +demographic-unmatched — so an aggregate can never be averaged into a +per-return statistic and an unmatched row is never a zero. + +*The individual-return amount universe is explicit.* A row is inside it if and +only if its authenticated disclosure-aggregate flag is 0, and the decoded +object exposes that rule as a read-only ``amount_known`` mask derived from the +flag on every access. Individual returns keep the unchanged integer, +whole-dollar, twelve-digit grammar, and a fractional token on one of them +still refuses. The four disclosure aggregates are outside that universe: their +typed slots hold the minimum-int64 out-of-universe sentinel — never a zero, +which is a delivered value in these columns — and their delivered tokens are +retained exactly under a separate, explicitly bounded lexical grammar that is +an operational input bound for this pinned file rather than a publisher +specification. Nothing converts an aggregate token to a number, rounds it, +grows it or drops its row. The aggregate row class carries only token-class +counts and a token width, so no header, receipt or summary can state an +aggregate amount, and the exact row/token pairs are available only through a +local accessor on the decoded object. + +*The projection is bound to the reviewed status artifact.* The node takes +``microcosm.us.puf_2015_raw_return_status`` as an artifact input and refuses +unless its own ``RECID`` sequence and aggregate flags are element-wise +identical to that artifact's. ``demographic_status`` is carried from it rather +than re-joined, so the two artifacts cannot disagree about which returns +matched. + +*The raw producer has an explicit packaging revision.* This module imports +:mod:`puf_raw_source` for its definition, codec and artifact decoder. Moving +the unchanged definition bytes into ``us_runtime`` changed the raw resource +loader and its implementation identity. New runs use that revised identity; +previously accepted raw artifacts retain their original keys and evidence. +The CSV reader helpers here remain local to this monetary projection. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from importlib import resources +from types import MappingProxyType + +import numpy as np + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SourceCodecRegistry, load_source_bytes + +from .puf_raw_source import ( + PUF_AGGREGATE_RECIDS, + PUF_RAW_SOURCE_NODE, + RETURN_STATUS_TYPE, + PufRawSourceDefinition, + csv_acceptance_profile, + decode_return_status, + puf_raw_source_codecs, +) +from .puf_raw_source import packaged_definition as packaged_raw_definition + +__all__ = [ + "AGGREGATE_ROW_CLASS", + "AGGREGATE_SENTINEL", + "AMOUNT_KNOWN_RULE", + "BODY_MAX_BYTES", + "HEADER_MAX_BYTES", + "LEXICAL_WIDTH_MAX", + "MONETARY_PROJECTION_NODE", + "MONETARY_PROJECTION_STAGE", + "MONETARY_PROJECTION_SUMMARY_NODE", + "MONETARY_PROJECTION_TYPE", + "PufMonetaryProjection", + "PufMonetaryRefusalError", + "PufMonetarySourceDocument", + "ProjectedColumn", + "RETURN_ROW_CLASSES", + "SUPERSEDED_MAGIC", + "USPufMonetaryProjectionSummaryKernel", + "USPufMonetarySourceKernel", + "decode_monetary_projection", + "decode_puf_monetary_source", + "encode_monetary_projection", + "fixture_projection_document", + "packaged_projection", + "projection_document_json", + "register_us_puf_monetary_source_kernels", + "us_puf_monetary_projection_summary_node", + "us_puf_monetary_source_node", +] + +MONETARY_PROJECTION_TYPE = ArtifactType( + "microcosm.us.puf_2015_monetary_source_projection", 3 +) +MONETARY_PROJECTION_MAGIC = b"microcosm.us.puf_2015_monetary_source_projection/3\n" + +#: Magic lines this reader refuses by name instead of reinterpreting. No +#: genuine schema-1 or schema-2 monetary artifact was ever completed — the +#: first genuine run refused before writing one — but invented artifacts of +#: both do exist in test evidence, and the amount universe differs, so an old +#: payload gets its own refusal rather than falling through the generic magic +#: check as an unrecognised blob. +SUPERSEDED_MAGIC = ( + b"microcosm.us.puf_2015_monetary_source_projection/1\n", + b"microcosm.us.puf_2015_monetary_source_projection/2\n", +) + +MONETARY_PROJECTION_STAGE = "us_puf_monetary_source" +MONETARY_PROJECTION_NODE = f"{MONETARY_PROJECTION_STAGE}.projection" +MONETARY_PROJECTION_SUMMARY_NODE = f"{MONETARY_PROJECTION_STAGE}.projection_summary" + +#: Bound on the encoded artifact's column bodies, matching the raw slice's own +#: private bound. Twelve projected columns over the delivered 207,696 rows is +#: 262 bytes a row, so about 51.9 MiB; a thirteenth column of the same shape +#: would still fit and a much wider selection would not. +BODY_MAX_BYTES = 64 * 1024 * 1024 + +#: Bound on the canonical JSON header. The header carries per-column facts for +#: three row classes, so it is larger than the status artifact's. +HEADER_MAX_BYTES = 64 * 1024 + +#: The widest lexical allocation a projected column may declare. +LEXICAL_WIDTH_MAX = 64 + +_PROJECTION_RESOURCE = "puf_2015_monetary_source_projection.json" +_PROJECTION_PACKAGE = "microcosm.build.us_runtime" +_PROJECTION_SCHEMA = "microcosm.us.puf_2015_monetary_source_projection" +_FIXTURE_AUTHORITY = "invented_fixture_nonauthority" + +_STRUCTURAL_COLUMNS = ("RECID", "disclosure_aggregate", "demographic_status") +_STRUCTURAL_DTYPES = { + "RECID": " None: + super().__init__(" ".join((reason, *(str(part) for part in detail))).strip()) + self.reason = reason + + +def _refuse(reason: str, *detail: object) -> PufMonetaryRefusalError: + return PufMonetaryRefusalError(reason, *detail) + + +# -------------------------------------------------------------------------- +# The closed projection document +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ProjectedColumn: + """One enumerated monetary column, with everything the parse needs.""" + + field: str + concept: str + concept_group: str + delivered_index: int + publisher_position: int + publisher_label: str + dtype: str + lexical_width: int + grammar: str + max_digits: int + publisher_signed_marker: bool + negative_values: str + fit_admission: str + unresolved: tuple[str, ...] + + @property + def numpy_dtype(self) -> str: + return _DECLARED_DTYPE[self.dtype] + + +@dataclass(frozen=True) +class PufMonetaryProjection: + """A closed layout authentication plus the enumerated projection.""" + + route: str + document: Mapping[str, object] + canonical: bytes + sha256: str + columns: tuple[ProjectedColumn, ...] + raw_definition_sha256: str + main_delivered_header: tuple[str, ...] + demographic_delivered_header: tuple[str, ...] + + @property + def params_text(self) -> str: + """The canonical JSON the node carries verbatim as a parameter.""" + + return self.canonical.decode("utf-8") + + @property + def fields(self) -> tuple[str, ...]: + return tuple(column.field for column in self.columns) + + +#: Kept as a separate public name so a caller can talk about the document +#: without holding a parsed projection. +PufMonetarySourceDocument = Mapping[str, object] + + +def _frozen(value: object) -> object: + """A deeply immutable view, as the raw source definition already does.""" + + if isinstance(value, Mapping): + return MappingProxyType({key: _frozen(item) for key, item in value.items()}) + if isinstance(value, (list, tuple)): + return tuple(_frozen(item) for item in value) + return value + + +def _thawed(value: object) -> object: + if isinstance(value, Mapping): + return {key: _thawed(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thawed(item) for item in value] + return value + + +def _hex64(value: object, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise _refuse("PROJECTION_DIGEST", label) + return value + + +def _positive_int(value: object, label: str, *, minimum: int = 1) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise _refuse("PROJECTION_INTEGER", label) + return value + + +def _layout_names( + entry: Mapping[str, object], label: str, reasons: frozenset[str] +) -> tuple[str, ...]: + columns = entry["columns"] + if not isinstance(columns, (list, tuple)) or not columns: + raise _refuse("PROJECTION_LAYOUT", label) + names: list[str] = [] + for position, column in enumerate(columns): + if not isinstance(column, Mapping): + raise _refuse("PROJECTION_LAYOUT", label) + name = column["name"] + if not isinstance(name, str) or not name: + raise _refuse("PROJECTION_LAYOUT_NAME", label) + if column["delivered_index"] != position: + raise _refuse("PROJECTION_LAYOUT_INDEX", name) + admission = column["admission"] + if admission not in _ADMISSIONS: + raise _refuse("PROJECTION_LAYOUT_ADMISSION", name) + if admission == "not_selected": + # An unselected column carries a reason from the document's own + # closed list, so nothing is dropped by omission. + if column.get("not_selected_reason") not in reasons: + raise _refuse("PROJECTION_LAYOUT_NOT_SELECTED_REASON", name) + elif "not_selected_reason" in column: + raise _refuse("PROJECTION_LAYOUT_NOT_SELECTED_REASON", name) + row = column["layout_row"] + if not isinstance(row, Mapping): + raise _refuse("PROJECTION_LAYOUT_EVIDENCE", name) + text = row["text"] + if not isinstance(text, str) or not text: + raise _refuse("PROJECTION_LAYOUT_EVIDENCE", name) + if hashlib.sha256(text.encode("utf-8")).hexdigest() != row["text_sha256"]: + raise _refuse("PROJECTION_LAYOUT_EVIDENCE_DIGEST", name) + if name not in text: + raise _refuse("PROJECTION_LAYOUT_EVIDENCE_NAME", name) + names.append(name) + if entry["delivered_header_width"] != len(names): + raise _refuse("PROJECTION_LAYOUT_WIDTH", label) + if len(set(names)) != len(names): + raise _refuse("PROJECTION_LAYOUT_DUPLICATE", label) + return tuple(names) + + +def _projected_column( + entry: Mapping[str, object], layout: Mapping[str, Mapping[str, object]] +) -> ProjectedColumn: + field = entry["field"] + if not isinstance(field, str) or field not in layout: + raise _refuse("PROJECTION_UNDELIVERED_FIELD", field) + for key in _FORBIDDEN_COLUMN_KEYS: + if key in entry: + raise _refuse("PROJECTION_DERIVATION_NOT_SUPPORTED", field) + column = layout[field] + if column["admission"] != "monetary_source_projection": + raise _refuse("PROJECTION_ADMISSION_DISAGREEMENT", field) + if entry["delivered_index"] != column["delivered_index"]: + raise _refuse("PROJECTION_INDEX_DISAGREEMENT", field) + if entry["publisher_position"] != column["publisher_position"]: + raise _refuse("PROJECTION_POSITION_DISAGREEMENT", field) + if entry["publisher_label"] != column["publisher_label"]: + raise _refuse("PROJECTION_LABEL_DISAGREEMENT", field) + if entry["dtype"] not in _DECLARED_DTYPE: + raise _refuse("PROJECTION_DTYPE", field) + if entry["grammar"] != _GRAMMAR: + raise _refuse("PROJECTION_GRAMMAR", field) + width = _positive_int(entry["lexical_width"], f"{field}.lexical_width") + if width > LEXICAL_WIDTH_MAX: + raise _refuse("PROJECTION_LEXICAL_WIDTH", field) + units = entry["units"] + digits = _positive_int(units["field_width_digits"], f"{field}.field_width_digits") + if digits > _MAX_AMOUNT_DIGITS or width < digits + 1: + raise _refuse("PROJECTION_WIDTH_BELOW_GRAMMAR", field) + if units["currency"] != "USD" or units["scale"] != "whole_dollars": + raise _refuse("PROJECTION_UNITS", field) + if units["implied_decimals"] != 0: + raise _refuse("PROJECTION_IMPLIED_DECIMALS", field) + sign = entry["sign"] + marker = sign["publisher_signed_marker"] + if not isinstance(marker, bool) or marker != column["publisher_signed_marker"]: + raise _refuse("PROJECTION_SIGN_MARKER", field) + negative = sign["negative_values"] + if negative not in _NEGATIVE_STATES: + raise _refuse("PROJECTION_SIGN_STATE", field) + if marker and negative != "documented_by_record_layout_marker": + raise _refuse("PROJECTION_SIGN_STATE", field) + if sign["negative_zero"] != "refused": + raise _refuse("PROJECTION_NEGATIVE_ZERO_POLICY", field) + missing = entry["missingness"] + if missing["empty_token"] != "refused": + raise _refuse("PROJECTION_EMPTY_TOKEN_POLICY", field) + if missing["zero_meaning"] != "delivered_zero_in_edited_public_file": + raise _refuse("PROJECTION_ZERO_POLICY", field) + if entry["return_grain"]["grain"] != _GRAIN: + raise _refuse("PROJECTION_GRAIN", field) + if entry["source_period_status"]["status"] != _PERIOD_SEMANTICS: + raise _refuse("PROJECTION_PERIOD_STATUS", field) + if entry["target_year_use"] != _TARGET_YEAR_USE: + raise _refuse("PROJECTION_TARGET_YEAR_USE", field) + if entry["derived_combinations"] != "refused": + raise _refuse("PROJECTION_DERIVATION_NOT_SUPPORTED", field) + admission = entry["fit_admission"] + if admission not in _FIT_ADMISSIONS: + raise _refuse("PROJECTION_FIT_ADMISSION", field) + unresolved = tuple(item["code"] for item in entry["unresolved"]) + return ProjectedColumn( + field=field, + concept=entry["concept"], + concept_group=entry["concept_group"], + delivered_index=entry["delivered_index"], + publisher_position=entry["publisher_position"], + publisher_label=entry["publisher_label"], + dtype=entry["dtype"], + lexical_width=width, + grammar=entry["grammar"], + max_digits=digits, + publisher_signed_marker=marker, + negative_values=negative, + fit_admission=admission, + unresolved=unresolved, + ) + + +def _projection_from_document( + document: Mapping[str, object], + *, + route: str, + definition: PufRawSourceDefinition, +) -> PufMonetaryProjection: + """Validate a projection document against a raw source definition.""" + + if not isinstance(definition, PufRawSourceDefinition): + raise _refuse("PROJECTION_DEFINITION_TYPE") + if document.get("schema") != _PROJECTION_SCHEMA: + raise _refuse("PROJECTION_SCHEMA") + if document.get("schema_version") != 1: + raise _refuse("PROJECTION_SCHEMA_VERSION") + if document.get("route") != route: + raise _refuse("PROJECTION_ROUTE") + + raw = document["raw_source_definition"] + raw_sha = _hex64(raw["sha256"], "raw_source_definition.sha256") + if raw_sha != definition.sha256: + raise _refuse("PROJECTION_RAW_DEFINITION_MISMATCH") + + layout = document["layout"] + reasons = layout["not_selected_reasons"] + if not isinstance(reasons, Mapping) or not reasons: + raise _refuse("PROJECTION_NOT_SELECTED_REASONS") + allowed = frozenset(reasons) + main_names = _layout_names(layout["main"], "main", allowed) + demographic_names = _layout_names(layout["demographic"], "demographic", allowed) + if main_names != definition.main.delivered_header: + raise _refuse("PROJECTION_MAIN_LAYOUT_HEADER") + if demographic_names != definition.demographic.delivered_header: + raise _refuse("PROJECTION_DEMOGRAPHIC_LAYOUT_HEADER") + + for name, entry in ( + ("main", definition.main), + ("demographic", definition.demographic), + ): + pin = document["sources"][name] + if ( + pin["sha256"] != entry.sha256 + or pin["git_blob_sha1"] != entry.git_blob_sha1 + or pin["bytes"] != entry.bytes + or pin["header_record_canonical_sha256"] + != entry.header_record_canonical_sha256 + or pin["delivered_header_width"] != len(entry.delivered_header) + or pin["data_records"] != entry.data_records + ): + raise _refuse("PROJECTION_SOURCE_PIN", name) + + by_name = {column["name"]: column for column in layout["main"]["columns"]} + for refused in document["refused_fields"]: + if refused in main_names or refused in demographic_names: + raise _refuse("PROJECTION_REFUSED_FIELD_DELIVERED", refused) + + projection = document["projection"] + if projection["grain"] != _GRAIN: + raise _refuse("PROJECTION_GRAIN") + if projection["period_semantics"] != _PERIOD_SEMANTICS: + raise _refuse("PROJECTION_PERIOD_STATUS") + if projection["target_year_use"] != _TARGET_YEAR_USE: + raise _refuse("PROJECTION_TARGET_YEAR_USE") + if projection["amounts_decoded"] is not True: + raise _refuse("PROJECTION_AMOUNTS_DECODED") + + entries = projection["columns"] + if not isinstance(entries, (list, tuple)) or not entries: + raise _refuse("PROJECTION_EMPTY") + columns = tuple(_projected_column(entry, by_name) for entry in entries) + fields = [column.field for column in columns] + if len(set(fields)) != len(fields): + raise _refuse("PROJECTION_DUPLICATE_FIELD") + for field in fields: + if field in document["refused_fields"]: + raise _refuse("PROJECTION_REFUSED_FIELD", field) + admitted = { + name + for name, column in by_name.items() + if column["admission"] == "monetary_source_projection" + } + if admitted != set(fields): + raise _refuse("PROJECTION_ADMISSION_SET") + if projection["column_count"] != len(columns): + raise _refuse("PROJECTION_COLUMN_COUNT") + if sorted(fields, key=lambda name: by_name[name]["delivered_index"]) != fields: + raise _refuse("PROJECTION_COLUMN_ORDER") + + if any(value is not False for value in document["non_claims"].values()): + raise _refuse("PROJECTION_NON_CLAIMS") + + canonical = canonical_json(_thawed(document)) + return PufMonetaryProjection( + route=route, + document=_frozen(json.loads(canonical.decode("utf-8"))), + canonical=canonical, + sha256=hashlib.sha256(canonical).hexdigest(), + columns=columns, + raw_definition_sha256=raw_sha, + main_delivered_header=main_names, + demographic_delivered_header=demographic_names, + ) + + +def _packaged_bytes() -> bytes: + return ( + resources.files(_PROJECTION_PACKAGE).joinpath(_PROJECTION_RESOURCE).read_bytes() + ) + + +_PACKAGED: PufMonetaryProjection | None = None + + +def packaged_projection() -> PufMonetaryProjection: + """The one closed projection document a production run may use.""" + + global _PACKAGED + if _PACKAGED is None: + document = json.loads(_packaged_bytes().decode("utf-8")) + _PACKAGED = _projection_from_document( + document, route="packaged", definition=packaged_raw_definition() + ) + return _PACKAGED + + +def projection_document_json( + projection: PufMonetaryProjection | None = None, +) -> dict[str, object]: + """A fresh, plain, mutable JSON copy of a projection document.""" + + resolved = packaged_projection() if projection is None else projection + return json.loads(resolved.canonical.decode("utf-8")) + + +def fixture_projection_document( + document: Mapping[str, object], definition: PufRawSourceDefinition +) -> PufMonetaryProjection: + """Build a projection on the explicit non-production test route. + + A fixture projection must declare ``route="test_fixture"`` and the + invented-fixture authority, and it may not restate either packaged source + pin. It still has to authenticate the whole delivered layout, because a + fixture that dropped the layout would be testing a different contract. + """ + + projection = _projection_from_document( + document, route="test_fixture", definition=definition + ) + if document.get("authority") != _FIXTURE_AUTHORITY: + raise _refuse("FIXTURE_AUTHORITY") + packaged_raw = packaged_raw_definition() + genuine = {packaged_raw.main.sha256, packaged_raw.demographic.sha256} + stated = { + document["sources"]["main"]["sha256"], + document["sources"]["demographic"]["sha256"], + } + if stated & genuine: + raise _refuse("FIXTURE_RESTATES_PACKAGED_PIN") + return projection + + +# -------------------------------------------------------------------------- +# The structural decoder +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PufMonetarySourceProjection: + """Decoded source values with explicit, immutable fitting refusals.""" + + typed: Mapping[str, np.ndarray] + lexical: Mapping[str, tuple[str, ...]] + facts: Mapping[str, object] + column_metadata: Mapping[str, object] + + @property + def rows(self) -> int: + return int(self.facts["rows"]) + + @property + def amount_known(self) -> np.ndarray: + """Whether a row is inside the individual-return amount universe. + + Derived from the authenticated disclosure flag on every access and + handed back read-only, so there is no stored mask to tamper with and + nothing a caller holds can contradict the flags. A false entry means + *outside the universe*: the typed slot holds the sentinel and the + delivered token is the authority. + """ + + return _amount_known(self.typed["disclosure_aggregate"]) + + def aggregate_lexical_rows( + self, field: str | None = None + ) -> Mapping[str, tuple[tuple[int, str], ...]]: + """Exact ``(RECID, token)`` pairs for the disclosure-aggregate rows. + + A local accessor, for diagnosis at the console or in a test. Public + summaries, receipts and reports stay aggregate-only: nothing in the + kernels calls this. No token is parsed, totalled, ordered by value or + compared with another token. + """ + + if field is None: + fields = tuple(self.lexical) + elif field in self.lexical: + fields = (field,) + else: + raise _refuse("AGGREGATE_LEXICAL_FIELD", field) + aggregate = np.flatnonzero(~self.amount_known).tolist() + recids = self.typed["RECID"] + return MappingProxyType( + { + name: tuple( + (int(recids[position]), self.lexical[name][position]) + for position in aggregate + ) + for name in fields + } + ) + + +def _check_csv_profile(profile: Mapping[str, object]) -> None: + """Refuse when the effective acceptance state is not the pinned one.""" + + declared = profile["field_size_limit"] + if not isinstance(declared, int) or isinstance(declared, bool) or declared <= 0: + raise _refuse("DEFINITION_CSV_FIELD_SIZE_LIMIT") + if csv.field_size_limit() != declared: + raise _refuse("CSV_FIELD_SIZE_LIMIT") + if profile["quoting"] != "QUOTE_MINIMAL" or profile["strict"] is not True: + raise _refuse("DEFINITION_CSV_PROFILE") + if profile["encoding"] != "utf-8-strict" or profile["newline"] != "": + raise _refuse("DEFINITION_CSV_PROFILE") + + +def _reader(payload: bytes, profile: Mapping[str, object]): + if not isinstance(payload, bytes): + raise _refuse("DECODE_PAYLOAD_TYPE") + stream = io.TextIOWrapper( + io.BytesIO(payload), encoding="utf-8", errors="strict", newline="" + ) + return csv.reader( + stream, + delimiter=profile["delimiter"], + quotechar=profile["quotechar"], + doublequote=bool(profile["doublequote"]), + quoting=csv.QUOTE_MINIMAL, + strict=True, + ) + + +def _canonical_header_digest(header: Sequence[str]) -> str: + payload = json.dumps( + list(header), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + return hashlib.sha256(payload).hexdigest() + + +def _key_token(value: str, *, width: int) -> str: + """The unsigned decimal ``RECID`` token, as the raw slice reads it.""" + + if not isinstance(value, str): + raise _refuse("KEY_TOKEN_TYPE") + if not value.isascii(): + raise _refuse("KEY_TOKEN_NON_ASCII") + if "\x00" in value: + raise _refuse("KEY_TOKEN_EMBEDDED_NUL") + if not value: + raise _refuse("KEY_TOKEN_EMPTY") + if len(value) > width: + raise _refuse("KEY_TOKEN_OVER_WIDTH") + if any(character not in _DIGITS for character in value): + raise _refuse("KEY_TOKEN_GRAMMAR") + number = int(value) + if number > _INT64_MAX: + raise _refuse("KEY_TOKEN_INT64_OVERFLOW") + return value + + +def _amount(value: str, column: ProjectedColumn) -> int: + """Parse one delivered amount token, or refuse. + + The grammar is exactly what the retained booklets support: an optional + leading minus, then decimal digits, no wider than the publisher's declared + amount-field width. There is no missing-value sentinel anywhere in either + booklet, so an empty token is a refusal to be preserved and reviewed + rather than a zero to be invented, and a plus sign, a decimal point, + surrounding whitespace or an exponent is a refusal rather than a silently + widened grammar. ``-0`` is refused outright: a signed zero would compare + equal to zero while spelling differently. + """ + + label = column.field + if not isinstance(value, str): + raise _refuse("AMOUNT_TOKEN_TYPE", label) + if not value.isascii(): + raise _refuse("AMOUNT_TOKEN_NON_ASCII", label) + if "\x00" in value: + raise _refuse("AMOUNT_TOKEN_EMBEDDED_NUL", label) + if not value: + raise _refuse("AMOUNT_TOKEN_EMPTY", label) + if len(value) > column.lexical_width: + raise _refuse("AMOUNT_TOKEN_OVER_WIDTH", label) + digits = value[1:] if value.startswith("-") else value + if not digits: + raise _refuse("AMOUNT_TOKEN_GRAMMAR", label) + if any(character not in _DIGITS for character in digits): + raise _refuse("AMOUNT_TOKEN_GRAMMAR", label) + if len(digits) > column.max_digits: + raise _refuse("AMOUNT_TOKEN_OVER_DIGITS", label) + number = -int(digits) if value.startswith("-") else int(digits) + if number == 0 and value.startswith("-"): + raise _refuse("AMOUNT_TOKEN_NEGATIVE_ZERO", label) + if not _INT64_MIN <= number <= _INT64_MAX: + raise _refuse("AMOUNT_TOKEN_INT64_OVERFLOW", label) + return number + + +def _aggregate_token(value: str, column: ProjectedColumn) -> str: + """Validate one disclosure-aggregate token, and return it unchanged. + + This is a **separate, explicitly bounded operational grammar** for the + delivered aggregate rows of this pinned file. It is not a newly discovered + publisher specification, and it is never applied to an individual return. + Optional leading minus, one or more digits, at most one point followed by + one or more digits: no plus sign, exponent, whitespace, embedded NUL, + empty token or negative zero in either spelling, at most the column's + declared digit bound and at most its declared width. + + The token is returned exactly as delivered. Nothing here converts it to a + number, rounds it, or compares it against another token. + """ + + label = column.field + if not isinstance(value, str): + raise _refuse("AGGREGATE_TOKEN_TYPE", label) + if not value.isascii(): + raise _refuse("AGGREGATE_TOKEN_NON_ASCII", label) + if "\x00" in value: + raise _refuse("AGGREGATE_TOKEN_EMBEDDED_NUL", label) + if not value: + raise _refuse("AGGREGATE_TOKEN_EMPTY", label) + if len(value) > column.lexical_width: + raise _refuse("AGGREGATE_TOKEN_OVER_WIDTH", label) + negative = value.startswith("-") + body = value[1:] if negative else value + if not body: + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + if body.count(".") > 1: + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + whole, point, fraction = body.partition(".") + if point and (not whole or not fraction): + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + digits = whole + fraction + if not digits or any(character not in _DIGITS for character in digits): + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + if len(digits) > column.max_digits: + raise _refuse("AGGREGATE_TOKEN_OVER_DIGITS", label) + if negative and set(digits) == {"0"}: + raise _refuse("AGGREGATE_TOKEN_NEGATIVE_ZERO", label) + return value + + +def _amount_known(flags: np.ndarray) -> np.ndarray: + """The individual-return amount universe, derived from the flag alone. + + Returned read-only and recomputed at every call, so no caller can hold a + mask that contradicts the disclosure flags it came from. + """ + + known = np.asarray(flags, dtype=_STRUCTURAL_DTYPES["disclosure_aggregate"]) == 0 + # A read-only flag on an owning ndarray can be reversed with setflags. + # An immutable bytes backing store makes that reversal impossible. + return np.frombuffer(known.tobytes(), dtype=np.bool_) + + +def _typed_column( + tokens: Sequence[str], known: np.ndarray, column: ProjectedColumn +) -> np.ndarray: + """Parse by universe: an amount where known, the sentinel where not.""" + + if len(tokens) != int(known.shape[0]): + raise _refuse("COLUMN_LEXICAL_ROWS", column.field) + values: list[int] = [] + for position, token in enumerate(tokens): + if bool(known[position]): + values.append(_amount(token, column)) + else: + _aggregate_token(token, column) + values.append(AGGREGATE_SENTINEL) + return np.asarray(values, dtype=column.numpy_dtype) + + +def _check_header( + reader, projection: PufMonetaryProjection, definition: PufRawSourceDefinition +) -> list[str]: + pin = definition.main + profile = definition.document["csv_profile"] + try: + header = next(reader) + except StopIteration as error: + raise _refuse("HEADER_MISSING") from error + except (csv.Error, UnicodeError) as error: + raise _refuse("HEADER_UNREADABLE") from error + if len(header) > profile["header_field_cap"]: + raise _refuse("HEADER_WIDTH_LIMIT") + if tuple(header) != pin.delivered_header: + raise _refuse("HEADER_MISMATCH") + if tuple(header) != projection.main_delivered_header: + raise _refuse("HEADER_NOT_AUTHENTICATED_LAYOUT") + if _canonical_header_digest(header) != pin.header_record_canonical_sha256: + raise _refuse("HEADER_DIGEST") + refused = set(projection.document["refused_fields"]) & set(header) + if refused: + raise _refuse("HEADER_REFUSED_FIELD", sorted(refused)) + return header + + +def _records(reader, definition: PufRawSourceDefinition): + profile = definition.document["csv_profile"] + width = len(definition.main.delivered_header) + cap = profile["logical_record_character_cap"] + limit = profile["record_cap_per_file"] + count = 0 + while True: + try: + record = next(reader) + except StopIteration: + return + except (csv.Error, UnicodeError) as error: + raise _refuse("RECORD_UNREADABLE") from error + count += 1 + if count > limit: + raise _refuse("RECORD_CAP") + if len(record) != width: + raise _refuse("RECORD_WIDTH") + if sum(len(cell) for cell in record) > cap: + raise _refuse("RECORD_CHARACTER_CAP") + yield record + + +def decode_puf_monetary_source( + main_bytes: bytes, + status: object, + projection: PufMonetaryProjection, + definition: PufRawSourceDefinition, +) -> PufMonetarySourceProjection: + """Decode the enumerated monetary columns, bound to the status artifact. + + ``status`` is the decoded ``puf_2015_raw_return_status``. Its ``RECID`` + sequence and aggregate flags must be element-wise identical to this + parse's, and its ``demographic_status`` is carried through rather than + re-joined, so the two artifacts cannot disagree about which returns + matched. Every delivered main record is projected: nothing is filtered, + and the four disclosure aggregates stay tagged rather than dropped. + + Amount tokens are collected before that alignment runs and parsed after + it, by universe: the unchanged integer grammar on individual returns, the + bounded aggregate lexical grammar plus the out-of-universe sentinel on + disclosure aggregates. A parse therefore cannot decide which grammar a row + answers to before the status artifact has proved which row it is. + """ + + if not isinstance(projection, PufMonetaryProjection): + raise _refuse("DECODE_PROJECTION_TYPE") + if projection.raw_definition_sha256 != definition.sha256: + raise _refuse("DECODE_DEFINITION_MISMATCH") + profile = definition.document["csv_profile"] + _check_csv_profile(profile) + key_width = next( + field["lexical_width"] + for field in definition.document["fields"] + if field["name"] == "RECID" + ) + + reader = _reader(main_bytes, profile) + header = _check_header(reader, projection, definition) + positions = { + column.field: header.index(column.field) for column in projection.columns + } + for column in projection.columns: + if positions[column.field] != column.delivered_index: + raise _refuse("DELIVERED_INDEX_DISAGREEMENT", column.field) + recid_position = header.index("RECID") + + keys: list[int] = [] + seen: set[str] = set() + aggregates = frozenset(PUF_AGGREGATE_RECIDS) + flags: list[int] = [] + tokens: dict[str, list[str]] = {column.field: [] for column in projection.columns} + + # Tokens are collected first and parsed afterwards. Which grammar a row's + # amount answers to is decided only once the status artifact has proved + # the RECID sequence and the aggregate flags, so "authenticate the + # alignment before interpreting a row's amount universe" is the literal + # order of operations rather than a claim about one. + for record in _records(reader, definition): + key = _key_token(record[recid_position], width=key_width) + if key in seen: + raise _refuse("MAIN_DUPLICATE_LEXICAL_KEY") + seen.add(key) + recid = int(key) + keys.append(recid) + flags.append(1 if recid in aggregates else 0) + for column in projection.columns: + tokens[column.field].append(record[positions[column.field]]) + rows = len(keys) + if rows == 0: + raise _refuse("MAIN_NO_RECORDS") + if rows != definition.main.data_records: + raise _refuse("MAIN_RECORD_COUNT") + + typed: dict[str, np.ndarray] = { + "RECID": np.asarray(keys, dtype=_STRUCTURAL_DTYPES["RECID"]), + "disclosure_aggregate": np.asarray( + flags, dtype=_STRUCTURAL_DTYPES["disclosure_aggregate"] + ), + } + if len(np.unique(typed["RECID"])) != rows: + raise _refuse("MAIN_NUMERIC_KEY_COLLISION") + typed["demographic_status"] = _status_alignment(status, typed, rows) + known = _amount_known(typed["disclosure_aggregate"]) + for column in projection.columns: + typed[column.field] = _typed_column(tokens[column.field], known, column) + + lexical = {name: tuple(items) for name, items in tokens.items()} + facts = _facts_from_arrays(typed, lexical, projection) + # Match envelope readback's immutable byte-backed arrays. In particular, + # disclosure flags cannot change after an amount-known mask is returned. + typed = { + name: np.frombuffer(array.tobytes(), dtype=array.dtype) + for name, array in typed.items() + } + return PufMonetarySourceProjection( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(facts), + column_metadata=_frozen(facts["column_metadata"]), + ) + + +def _status_alignment( + status: object, typed: Mapping[str, np.ndarray], rows: int +) -> np.ndarray: + """Bind this parse to the reviewed status artifact, element by element.""" + + try: + status_typed = status.typed + except AttributeError as error: + raise _refuse("STATUS_ARTIFACT_TYPE") from error + for name in ("RECID", "disclosure_aggregate", "demographic_status"): + if name not in status_typed: + raise _refuse("STATUS_ARTIFACT_COLUMN", name) + if int(status_typed["RECID"].shape[0]) != rows: + raise _refuse("STATUS_ROW_COUNT") + if not np.array_equal( + np.asarray(status_typed["RECID"], dtype=" dict[str, np.ndarray]: + """The three disjoint row classes, which must partition the rows exactly.""" + + aggregate = typed["disclosure_aggregate"] == 1 + status = typed["demographic_status"] == 1 + masks = { + "disclosure_aggregate": aggregate, + "demographic_matched": (~aggregate) & status, + "demographic_unmatched": (~aggregate) & (~status), + } + total = sum(int(np.count_nonzero(mask)) for mask in masks.values()) + if total != int(typed["RECID"].shape[0]): + raise _refuse("ROW_CLASS_PARTITION") + return masks + + +def _column_facts(values: np.ndarray, mask: np.ndarray) -> dict[str, object]: + selected = values[mask] + count = int(selected.shape[0]) + if count == 0: + return { + "rows": 0, + "zero": 0, + "negative": 0, + "positive": 0, + "minimum": None, + "maximum": None, + "sum": 0, + } + total = int(sum(int(item) for item in selected.tolist())) + if not _INT64_MIN <= total <= _INT64_MAX: + raise _refuse("COLUMN_SUM_OVERFLOW") + return { + "rows": count, + "zero": int(np.count_nonzero(selected == 0)), + "negative": int(np.count_nonzero(selected < 0)), + "positive": int(np.count_nonzero(selected > 0)), + "minimum": int(selected.min()), + "maximum": int(selected.max()), + "sum": total, + } + + +def _aggregate_column_facts( + tokens: Sequence[str], mask: np.ndarray +) -> dict[str, object]: + """Token classes for the aggregate rows, and nothing that reads as money. + + Counts of token shapes and one token width. There is no zero, sign census, + minimum, maximum or sum key here, because the delivered token is the + authority for these rows and this module never interprets it. A diagnostic + may count token classes; it may not state an aggregate amount. + """ + + selected = [ + token for token, taken in zip(tokens, mask.tolist(), strict=True) if taken + ] + if not selected: + return { + "rows": 0, + "integer_tokens": 0, + "fractional_tokens": 0, + "negative_tokens": 0, + "max_token_width": None, + } + return { + "rows": len(selected), + "integer_tokens": sum(1 for token in selected if "." not in token), + "fractional_tokens": sum(1 for token in selected if "." in token), + "negative_tokens": sum(1 for token in selected if token.startswith("-")), + "max_token_width": max(len(token) for token in selected), + } + + +def _facts_from_arrays( + typed: Mapping[str, np.ndarray], + lexical: Mapping[str, Sequence[str]], + projection: PufMonetaryProjection, +) -> dict[str, object]: + """Derive every reported fact from the arrays themselves. + + The producer builds its facts here and the envelope readback rebuilds them + here, so a header can never report a count, a sum or a sign census the + arrays do not carry. Layout counts and column semantics come from the + authenticated projection document, rather than being inferred from rows. + The lexical arrays are an input because the aggregate class is described + from its delivered tokens and from nothing else. + """ + + rows = int(typed["RECID"].shape[0]) + masks = _row_masks(typed) + known = _amount_known(typed["disclosure_aggregate"]) + for column in projection.columns: + if len(lexical[column.field]) != rows: + raise _refuse("FACTS_LEXICAL_ROWS", column.field) + return { + "rows": rows, + "main_records": rows, + "projected_columns": len(projection.columns), + "layout_columns_authenticated": len(projection.main_delivered_header), + "demographic_layout_columns": len(projection.demographic_delivered_header), + "aggregate_records": int(np.count_nonzero(masks["disclosure_aggregate"])), + "demographic_matched_rows": int( + np.count_nonzero(typed["demographic_status"] == 1) + ), + "demographic_unmatched_rows": int( + np.count_nonzero(typed["demographic_status"] == 0) + ), + "row_class_rows": { + name: int(np.count_nonzero(masks[name])) for name in ROW_CLASSES + }, + "column_facts": { + column.field: { + AGGREGATE_ROW_CLASS: _aggregate_column_facts( + lexical[column.field], masks[AGGREGATE_ROW_CLASS] + ), + **{ + name: _column_facts(typed[column.field], masks[name]) + for name in RETURN_ROW_CLASSES + }, + } + for column in projection.columns + }, + "column_metadata": { + column.field: { + "concept": column.concept, + "grain": _GRAIN, + # The packaged projection document's integer, whole-dollar, + # twelve-digit semantics describe individual returns. That + # document's bytes are unchanged; this metadata says who they + # apply to, and what happens to the rows they do not. + "amount_units": _AMOUNT_UNITS, + "amount_units_scope": _AMOUNT_UNITS_SCOPE, + "amount_universe": _AMOUNT_UNITS_SCOPE, + "amount_known_rule": AMOUNT_KNOWN_RULE, + "aggregate_amount_universe": _AGGREGATE_UNIVERSE, + "aggregate_row_treatment": _AGGREGATE_TREATMENT, + "aggregate_typed_sentinel": AGGREGATE_SENTINEL, + "aggregate_token_grammar": _AGGREGATE_GRAMMAR, + "aggregate_token_max_digits": column.max_digits, + "aggregate_token_max_width": column.lexical_width, + "aggregate_token_grammar_authority": _AGGREGATE_GRAMMAR_AUTHORITY, + "aggregate_lexical_authority": _AGGREGATE_AUTHORITY, + "aggregate_numeric_interpretation": _AGGREGATE_INTERPRETATION, + "period_semantics": _PERIOD_SEMANTICS, + "fit_admission": column.fit_admission, + "unresolved": list(column.unresolved), + "negative_values": column.negative_values, + "zero_meaning": "delivered_zero_in_edited_public_file", + "target_year_use": _TARGET_YEAR_USE, + "projection_sha256": projection.sha256, + } + for column in projection.columns + }, + "grain": _GRAIN, + "period_semantics": _PERIOD_SEMANTICS, + "amounts_decoded": True, + "amount_units": _AMOUNT_UNITS, + "amount_units_scope": _AMOUNT_UNITS_SCOPE, + "amount_known_rule": AMOUNT_KNOWN_RULE, + "amount_known_rows": int(np.count_nonzero(known)), + "aggregate_amount_universe": _AGGREGATE_UNIVERSE, + "aggregate_row_treatment": _AGGREGATE_TREATMENT, + "aggregate_typed_sentinel": AGGREGATE_SENTINEL, + "aggregate_token_grammar": _AGGREGATE_GRAMMAR, + "aggregate_lexical_authority": _AGGREGATE_AUTHORITY, + "aggregate_numeric_interpretation": _AGGREGATE_INTERPRETATION, + "target_year_use": _TARGET_YEAR_USE, + } + + +# -------------------------------------------------------------------------- +# The typed envelope +# -------------------------------------------------------------------------- + + +def _fixed_ascii(values: Sequence[str], width: int, label: str) -> bytes: + body = bytearray(len(values) * width) + for position, value in enumerate(values): + raw = value.encode("ascii") + if len(raw) > width: + raise _refuse("LEXICAL_OVER_WIDTH", label) + if b"\x00" in raw: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + start = position * width + body[start : start + len(raw)] = raw + return bytes(body) + + +def _read_fixed_ascii( + body: bytes, rows: int, width: int, label: str +) -> tuple[str, ...]: + values = [] + for position in range(rows): + cell = body[position * width : (position + 1) * width] + stripped = cell.rstrip(b"\x00") + if b"\x00" in stripped: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + try: + values.append(stripped.decode("ascii")) + except UnicodeDecodeError as error: + raise _refuse("LEXICAL_NON_ASCII", label) from error + return tuple(values) + + +def payload_bound( + rows: int, projection: PufMonetaryProjection +) -> tuple[int, dict[str, int]]: + """The exact body size this artifact allocates, and its per-column parts.""" + + sizes = { + name: rows * np.dtype(_STRUCTURAL_DTYPES[name]).itemsize + for name in _STRUCTURAL_COLUMNS + } + for column in projection.columns: + sizes[column.field] = rows * np.dtype(column.numpy_dtype).itemsize + sizes[f"{column.field}_lexical"] = rows * column.lexical_width + return sum(sizes.values()), sizes + + +def _envelope_column_order(projection: PufMonetaryProjection) -> list[str]: + return [ + *_STRUCTURAL_COLUMNS, + *(column.field for column in projection.columns), + *(f"{column.field}_lexical" for column in projection.columns), + ] + + +def encode_monetary_projection( + decoded: PufMonetarySourceProjection, + projection: PufMonetaryProjection, + definition: PufRawSourceDefinition, + *, + status_artifact_sha256: str, +) -> bytes: + """Encode the projection into the bounded, self-describing envelope.""" + + rows = decoded.rows + total, sizes = payload_bound(rows, projection) + if total > BODY_MAX_BYTES: + raise _refuse("BODY_OVER_BOUND", total) + dtypes = { + **_STRUCTURAL_DTYPES, + **{column.field: column.numpy_dtype for column in projection.columns}, + } + widths = {column.field: column.lexical_width for column in projection.columns} + bodies: list[bytes] = [] + columns: list[dict[str, object]] = [] + for name in _envelope_column_order(projection): + if name.endswith("_lexical"): + field = name.removesuffix("_lexical") + body = _fixed_ascii(decoded.lexical[field], widths[field], field) + entry: dict[str, object] = { + "name": name, + "kind": _LEXICAL_KIND, + "width": widths[field], + } + else: + array = np.ascontiguousarray(decoded.typed[name], dtype=dtypes[name]) + if array.shape != (rows,): + raise _refuse("COLUMN_SHAPE", name) + body = array.tobytes() + entry = {"name": name, "kind": _TYPED_KIND, "dtype": dtypes[name]} + if len(body) != sizes[name]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + entry["bytes"] = len(body) + entry["sha256"] = hashlib.sha256(body).hexdigest() + columns.append(entry) + header = { + "schema_version": MONETARY_PROJECTION_TYPE.schema_version, + "type": [ + MONETARY_PROJECTION_TYPE.name, + MONETARY_PROJECTION_TYPE.schema_version, + ], + "rows": rows, + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "raw_source_definition_sha256": definition.sha256, + "status_artifact_sha256": _hex64( + status_artifact_sha256, "status_artifact_sha256" + ), + "sources": { + "main": { + "sha256": definition.main.sha256, + "git_blob_sha1": definition.main.git_blob_sha1, + "bytes": definition.main.bytes, + "header_record_canonical_sha256": ( + definition.main.header_record_canonical_sha256 + ), + }, + "demographic": { + "sha256": definition.demographic.sha256, + "git_blob_sha1": definition.demographic.git_blob_sha1, + "bytes": definition.demographic.bytes, + "header_record_canonical_sha256": ( + definition.demographic.header_record_canonical_sha256 + ), + }, + }, + "header_length_endianness": "big", + "body_endianness": "little", + "columns": columns, + "facts": dict(decoded.facts), + } + encoded = canonical_json(header) + if len(encoded) > HEADER_MAX_BYTES: + raise _refuse("HEADER_OVER_BOUND", len(encoded)) + return b"".join( + (MONETARY_PROJECTION_MAGIC, len(encoded).to_bytes(4, "big"), encoded, *bodies) + ) + + +def _exact_keys( + value: object, keys: frozenset[str], label: str +) -> Mapping[str, object]: + if not isinstance(value, dict) or set(value) != set(keys): + raise _refuse("ENVELOPE_SCHEMA", label) + return value + + +def _envelope_int(value: object, label: str, *, minimum: int = 0) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise _refuse("ENVELOPE_INTEGER", label) + return value + + +def _resolve_envelope_projection( + header: Mapping[str, object], + projection: PufMonetaryProjection | None, +) -> PufMonetaryProjection | None: + """The projection this payload may be checked against, or ``None``.""" + + route = header["projection_route"] + if route not in ("packaged", "test_fixture"): + raise _refuse("ENVELOPE_PROJECTION_ROUTE") + digest = _hex64(header["projection_sha256"], "envelope.projection_sha256") + if projection is not None: + if not isinstance(projection, PufMonetaryProjection): + raise _refuse("DECODE_PROJECTION_TYPE") + if projection.route != route or projection.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_MISMATCH") + return projection + if route == "packaged": + packaged = packaged_projection() + if packaged.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_NOT_PACKAGED") + return packaged + return None + + +def _check_envelope_sources( + header: Mapping[str, object], projection: PufMonetaryProjection | None +) -> None: + sources = _exact_keys( + header["sources"], frozenset({"main", "demographic"}), "sources" + ) + for name in ("main", "demographic"): + entry = _exact_keys(sources[name], _ENVELOPE_SOURCE_KEYS, f"sources.{name}") + _hex64(entry["sha256"], f"sources.{name}.sha256") + _envelope_int(entry["bytes"], f"sources.{name}.bytes", minimum=1) + if projection is None: + continue + pin = projection.document["sources"][name] + if ( + entry["sha256"] != pin["sha256"] + or entry["git_blob_sha1"] != pin["git_blob_sha1"] + or entry["bytes"] != pin["bytes"] + or entry["header_record_canonical_sha256"] + != pin["header_record_canonical_sha256"] + ): + raise _refuse("ENVELOPE_SOURCE_PIN", name) + + +def _envelope_columns( + header: Mapping[str, object], rows: int, projection: PufMonetaryProjection +) -> list[Mapping[str, object]]: + """Close the column schema: exact order, kind, dtype, width and size.""" + + columns = header["columns"] + if not isinstance(columns, list): + raise _refuse("ENVELOPE_COLUMNS") + declared = _envelope_column_order(projection) + if len(columns) != len(declared): + raise _refuse("ENVELOPE_COLUMNS") + dtypes = { + **_STRUCTURAL_DTYPES, + **{column.field: column.numpy_dtype for column in projection.columns}, + } + widths = {column.field: column.lexical_width for column in projection.columns} + resolved: list[Mapping[str, object]] = [] + total = 0 + for column, name in zip(columns, declared, strict=True): + if not isinstance(column, dict) or column.get("name") != name: + raise _refuse("ENVELOPE_COLUMNS") + if name.endswith("_lexical"): + field = name.removesuffix("_lexical") + entry = _exact_keys(column, _ENVELOPE_LEXICAL_COLUMN_KEYS, name) + if entry["kind"] != _LEXICAL_KIND: + raise _refuse("ENVELOPE_COLUMN_KIND", name) + width = _envelope_int(entry["width"], f"{name}.width", minimum=1) + if width > LEXICAL_WIDTH_MAX or width != widths[field]: + raise _refuse("ENVELOPE_COLUMN_WIDTH", name) + expected = rows * width + else: + entry = _exact_keys(column, _ENVELOPE_TYPED_COLUMN_KEYS, name) + if entry["kind"] != _TYPED_KIND: + raise _refuse("ENVELOPE_COLUMN_KIND", name) + # An exact dtype string closes width, signedness and endianness in + # one comparison, so `` BODY_MAX_BYTES: + raise _refuse("ENVELOPE_BODY_OVER_BOUND", total) + resolved.append(entry) + return resolved + + +def _verify_readback( + typed: Mapping[str, np.ndarray], + lexical: Mapping[str, tuple[str, ...]], + projection: PufMonetaryProjection, +) -> None: + """Re-prove the projection's own invariants from the arrays read back.""" + + rows = int(typed["RECID"].shape[0]) + status = typed["demographic_status"] + aggregate = typed["disclosure_aggregate"] + if not np.isin(status, (0, 1)).all(): + raise _refuse("READBACK_DEMOGRAPHIC_STATUS") + if not np.isin(aggregate, (0, 1)).all(): + raise _refuse("READBACK_AGGREGATE_FLAG") + recids = typed["RECID"] + if len(np.unique(recids)) != rows: + raise _refuse("READBACK_NUMERIC_KEY_COLLISION") + expected = np.isin(recids, np.asarray(PUF_AGGREGATE_RECIDS, dtype=" PufMonetarySourceProjection: + """Decode an artifact payload and re-prove it, never trusting its header. + + The header is a closed schema; every reported fact is recomputed from the + arrays actually read back and must agree exactly, so a payload cannot + report a row count, a negative census, a class partition or a column sum + it does not carry. + """ + + if not isinstance(payload, bytes): + raise _refuse("ENVELOPE_MAGIC") + if not payload.startswith(MONETARY_PROJECTION_MAGIC): + # A superseded magic is named, not reinterpreted: those payloads + # declare a different amount universe, and silently reading one under + # this contract would report aggregates as individual amounts. + if any(payload.startswith(magic) for magic in SUPERSEDED_MAGIC): + raise _refuse("ENVELOPE_SUPERSEDED_MAGIC") + raise _refuse("ENVELOPE_MAGIC") + start = len(MONETARY_PROJECTION_MAGIC) + if len(payload) < start + 4: + raise _refuse("ENVELOPE_TRUNCATED") + length = int.from_bytes(payload[start : start + 4], "big") + if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: + raise _refuse("ENVELOPE_HEADER_LENGTH") + encoded = payload[start + 4 : start + 4 + length] + try: + header = json.loads(encoded.decode("utf-8")) + except (UnicodeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + try: + recanonical = canonical_json(header) + except (TypeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + if recanonical != encoded: + raise _refuse("ENVELOPE_HEADER_NOT_CANONICAL") + _exact_keys(header, _ENVELOPE_HEADER_KEYS, "header") + if header["schema_version"] != MONETARY_PROJECTION_TYPE.schema_version or header[ + "type" + ] != [ + MONETARY_PROJECTION_TYPE.name, + MONETARY_PROJECTION_TYPE.schema_version, + ]: + raise _refuse("ENVELOPE_TYPE") + if ( + header["header_length_endianness"] != "big" + or header["body_endianness"] != "little" + ): + raise _refuse("ENVELOPE_ENDIANNESS") + _hex64(header["raw_source_definition_sha256"], "raw_source_definition_sha256") + _hex64(header["status_artifact_sha256"], "status_artifact_sha256") + if expected_status_artifact_sha256 is not None: + _hex64(expected_status_artifact_sha256, "expected_status_artifact_sha256") + if header["status_artifact_sha256"] != expected_status_artifact_sha256: + raise _refuse("ENVELOPE_STATUS_ARTIFACT_MISMATCH") + rows = _envelope_int(header["rows"], "rows", minimum=1) + resolved = _resolve_envelope_projection(header, projection) + if resolved is None: + # A fixture-route payload has no reconstructible projection document, + # so there is nothing to close its column set against. That is a + # caller error here rather than a weaker structural pass: the + # projection is exactly what makes the columns meaningful. + raise _refuse("ENVELOPE_PROJECTION_REQUIRED") + if resolved.raw_definition_sha256 != header["raw_source_definition_sha256"]: + raise _refuse("ENVELOPE_RAW_DEFINITION_MISMATCH") + _check_envelope_sources(header, resolved) + columns = _envelope_columns(header, rows, resolved) + + cursor = start + 4 + length + typed: dict[str, np.ndarray] = {} + lexical: dict[str, tuple[str, ...]] = {} + for column in columns: + name = column["name"] + size = column["bytes"] + body = payload[cursor : cursor + size] + if len(body) != size: + raise _refuse("ENVELOPE_BODY_TRUNCATED", name) + if hashlib.sha256(body).hexdigest() != column["sha256"]: + raise _refuse("ENVELOPE_BODY_DIGEST", name) + cursor += size + if column["kind"] == _TYPED_KIND: + typed[name] = np.frombuffer(body, dtype=np.dtype(column["dtype"])) + else: + field = name.removesuffix("_lexical") + lexical[field] = _read_fixed_ascii(body, rows, column["width"], field) + if cursor != len(payload): + raise _refuse("ENVELOPE_TRAILING_BYTES") + + _verify_readback(typed, lexical, resolved) + facts = header["facts"] + _exact_keys(facts, _ENVELOPE_FACT_KEYS, "facts") + for field, classes in facts["column_facts"].items(): + if field not in resolved.fields: + raise _refuse("ENVELOPE_FACTS", "column_facts") + _exact_keys(classes, frozenset(ROW_CLASSES), f"column_facts.{field}") + for name, entry in classes.items(): + keys = ( + _AGGREGATE_COLUMN_FACT_KEYS + if name == AGGREGATE_ROW_CLASS + else _RETURN_COLUMN_FACT_KEYS + ) + _exact_keys(entry, keys, f"column_facts.{field}.{name}") + recomputed = _facts_from_arrays(typed, lexical, resolved) + if canonical_json(recomputed) != canonical_json(dict(facts)): + raise _refuse("ENVELOPE_FACTS_INCONSISTENT") + if recomputed["rows"] != rows: + raise _refuse("ENVELOPE_ROWS") + return PufMonetarySourceProjection( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(recomputed), + column_metadata=_frozen(recomputed["column_metadata"]), + ) + + +# -------------------------------------------------------------------------- +# The graph producer, consumer and registry +# -------------------------------------------------------------------------- + +_IMPLEMENTATION_MODULES = ( + "microcosm.build.us_runtime.puf_raw_source", + "microcosm.build.us_runtime.puf_monetary_source", +) +#: The implementation identity moves with the contract: a node written +#: against the version-2 amount universe must not bind to this kernel. +_IMPLEMENTATION_IDENTITY_MAGIC = b"us.puf.monetary_source/implementation/3\n" + +_STATUS_ALIAS = "return_status" +_PROJECTION_ALIAS = "monetary_projection" + + +class USPufMonetarySourceKernel(KernelBase): + """Read the pinned main delivery and produce the monetary projection.""" + + ref = "us.puf.monetary_source.projection@3" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__( + self, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, + ) -> None: + self.definition = ( + packaged_raw_definition() if definition is None else definition + ) + self.projection = packaged_projection() if projection is None else projection + if self.projection.route not in ("packaged", "test_fixture"): + raise _refuse("KERNEL_PROJECTION_ROUTE") + if self.projection.route != self.definition.route: + raise _refuse("KERNEL_ROUTE_DISAGREEMENT") + if self.projection.raw_definition_sha256 != self.definition.sha256: + raise _refuse("KERNEL_DEFINITION_MISMATCH") + if self.projection.route == "packaged": + # The production wrapper, not the bare decoder, is the authority: + # the projection must be the packaged file's own canonical bytes, + # and the document it carries must still be those exact bytes. + packaged = packaged_projection() + if ( + self.projection.canonical != packaged.canonical + or self.projection.sha256 != packaged.sha256 + or canonical_json(_thawed(self.projection.document)) + != self.projection.canonical + ): + raise _refuse("KERNEL_PROJECTION_NOT_PACKAGED") + self.source_codecs = ( + puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self) -> str: + """Module source, dependency versions and the effective CSV profile.""" + + from importlib import import_module + + base = source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + digest = hashlib.sha256(_IMPLEMENTATION_IDENTITY_MAGIC) + digest.update(base.encode("ascii")) + digest.update(b"\0") + digest.update(canonical_json(csv_acceptance_profile())) + return digest.hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + definition = self.definition + projection = self.projection + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources != (definition.main.source_name,) + or context.node.inputs + or context.node.outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != RETURN_STATUS_TYPE + or context.node.artifact_outputs + != (ArtifactOutput(_PROJECTION_ALIAS, MONETARY_PROJECTION_TYPE),) + or set(context.params) != {"definition", "projection"} + ): + raise _refuse("NODE_DECLARATION") + if context.params["definition"] != definition.params_text: + raise _refuse("NODE_DEFINITION_PARAM") + if context.params["projection"] != projection.params_text: + raise _refuse("NODE_PROJECTION_PARAM") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != RETURN_STATUS_TYPE: + raise _refuse("STATUS_ARTIFACT_TYPE") + status = decode_return_status(value.payload, definition) + main_bytes = load_source_bytes( + definition.main.codec, + context.sources[definition.main.source_name], + registry=self.source_codecs, + ) + decoded = decode_puf_monetary_source(main_bytes, status, projection, definition) + payload = encode_monetary_projection( + decoded, + projection, + definition, + status_artifact_sha256=hashlib.sha256(value.payload).hexdigest(), + ) + return KernelResult( + artifacts={_PROJECTION_ALIAS: payload}, + receipt={ + "puf_monetary_source_projection": { + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "definition_sha256": definition.sha256, + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "status_artifact_sha256": hashlib.sha256(value.payload).hexdigest(), + "main_sha256": definition.main.sha256, + "csv_acceptance_profile": csv_acceptance_profile(), + "projected_fields": list(projection.fields), + **{ + key: value + for key, value in decoded.facts.items() + if key != "column_facts" + }, + } + }, + ) + + +class USPufMonetaryProjectionSummaryKernel(KernelBase): + """A downstream consumer: decode the projection and report its census.""" + + ref = "us.puf.monetary_source.projection_summary@3" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__(self, *, projection: PufMonetaryProjection | None = None) -> None: + self.projection = packaged_projection() if projection is None else projection + + def implementation_hash(self) -> str: + from importlib import import_module + + return source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + + def run(self, context: KernelContext) -> KernelResult: + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources + or context.node.inputs + or context.node.outputs + or context.node.artifact_outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != MONETARY_PROJECTION_TYPE + or context.params + ): + raise _refuse("SUMMARY_NODE_DECLARATION") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != MONETARY_PROJECTION_TYPE: + raise _refuse("SUMMARY_ARTIFACT_TYPE") + decoded = decode_monetary_projection(value.payload, self.projection) + facts = decoded.facts + # The per-return negative census covers the two return classes only. + # Summing it across all three would have counted a token class as a + # monetary sign, which is exactly the conflation this contract ends. + per_return_negatives = { + field: sum(int(classes[name]["negative"]) for name in RETURN_ROW_CLASSES) + for field, classes in facts["column_facts"].items() + } + # Aggregate-only: token-class counts and a width, never a token, a + # RECID or anything that could be read as an aggregate amount. The + # exact row/token pairs stay behind the local decoded accessor. + aggregate_token_classes = { + field: dict(classes[AGGREGATE_ROW_CLASS]) + for field, classes in facts["column_facts"].items() + } + return KernelResult( + receipt={ + "puf_monetary_projection_summary": { + "producer_key": value.producer_key, + "rows": decoded.rows, + "projected_columns": facts["projected_columns"], + "row_class_rows": dict(facts["row_class_rows"]), + "amount_known_rows": facts["amount_known_rows"], + "amount_known_rule": facts["amount_known_rule"], + "per_return_negative_values_by_field": per_return_negatives, + "aggregate_rows_are_lexical_only": True, + "aggregate_typed_sentinel": facts["aggregate_typed_sentinel"], + "aggregate_token_grammar": facts["aggregate_token_grammar"], + "aggregate_lexical_authority": facts["aggregate_lexical_authority"], + "aggregate_numeric_interpretation": facts[ + "aggregate_numeric_interpretation" + ], + "aggregate_lexical_token_classes": aggregate_token_classes, + "period_semantics": facts["period_semantics"], + "amount_units": facts["amount_units"], + "amount_units_scope": facts["amount_units_scope"], + "target_year_use": facts["target_year_use"], + "column_metadata": _thawed(decoded.column_metadata), + } + } + ) + + +def us_puf_monetary_source_node( + *, + population: str, + producer: str = PUF_RAW_SOURCE_NODE, + producer_output: str = "return_status", + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + stage: str = MONETARY_PROJECTION_STAGE, +) -> Node: + """Declare the producer node inside an explicitly named host population.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + resolved_definition = ( + packaged_raw_definition() if definition is None else definition + ) + resolved = packaged_projection() if projection is None else projection + if resolved.raw_definition_sha256 != resolved_definition.sha256: + raise _refuse("NODE_DEFINITION_MISMATCH") + return Node( + f"{stage}.projection", + USPufMonetarySourceKernel.ref, + population=population, + sources=(resolved_definition.main.source_name,), + params={ + "definition": resolved_definition.params_text, + "projection": resolved.params_text, + }, + artifact_inputs=( + ArtifactInput(_STATUS_ALIAS, producer, producer_output, RETURN_STATUS_TYPE), + ), + artifact_outputs=(ArtifactOutput(_PROJECTION_ALIAS, MONETARY_PROJECTION_TYPE),), + description=( + "Decode the enumerated 2015 PUF monetary source columns at return " + "grain, bound to the reviewed status artifact. Owns no population " + "columns and admits no target year." + ), + ) + + +def us_puf_monetary_projection_summary_node( + *, + population: str, + producer: str = MONETARY_PROJECTION_NODE, + stage: str = MONETARY_PROJECTION_STAGE, +) -> Node: + """Declare an ordinary consumer of the projection artifact.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + return Node( + f"{stage}.projection_summary", + USPufMonetaryProjectionSummaryKernel.ref, + population=population, + artifact_inputs=( + ArtifactInput( + _PROJECTION_ALIAS, + producer, + _PROJECTION_ALIAS, + MONETARY_PROJECTION_TYPE, + ), + ), + description="Read the monetary projection artifact and report its census.", + ) + + +def register_us_puf_monetary_source_kernels( + registry: KernelRegistry, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, +) -> KernelRegistry: + """Register the producer and consumer kernels into ``registry``.""" + + registry.register( + USPufMonetarySourceKernel( + projection=projection, + definition=definition, + source_codecs=source_codecs, + ) + ) + registry.register(USPufMonetaryProjectionSummaryKernel(projection=projection)) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py new file mode 100644 index 000000000..ba565c369 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py @@ -0,0 +1,1214 @@ +"""The developmental CPI-U price-restatement baseline for the 2015 PUF. + +One exact positive ratio of two published CPI-U all-items annual averages — +313689/237017, 2015 to 2024 — applied to every known money field of the +thirteen-field source projection. It restates 2015-file purchasing power at +2024 prices. It is **not** income aging: it carries no category-specific +nominal growth, no return-mass transition, and no claim that any field's real +level changed. ``S006`` is declared with an explicit identity series so the +no-mass-transition choice is recorded rather than implied. + +Three things this module is, and three it refuses +------------------------------------------------- +*It pins the resource.* The series is ``CUUR0000SA0``, U.S. city average, all +items, not seasonally adjusted, ``M13`` annual average, base 1982-84=100. The +levels are the publisher's own decimal text, copied verbatim, and the factor +table it builds carries the ``developmental_public_resource`` authority: the +retained bytes are checked against their named digest and their two records +against the declared level text. Retrieval location and time remain caller +metadata, not claims proved by hashing. The authority cannot become +release-eligible. + +*It declares the recipe.* One ``nominal_price_restatement`` series for all +thirteen money fields, one source reference year, one target year, ``RECID`` +as the identifier, ``S006`` as a design weight whose declared factor is +exactly one. + +*It adapts the artifact to the money view.* The projection is an artifact and +the growth transform consumes a table, so +:func:`adapt_projection_to_money_view` joins the projection to the reviewed +status artifact on ``RECID``, removes only the disclosure-aggregate rows, +records what it removed, and hands back ``FLPDYR``, the demographic status and +the unaltered integer ``S006`` separately. The growth itself is +:func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth`; nothing here +multiplies an amount. + +It refuses to interpret an aggregate token, to put the out-of-universe +sentinel anywhere near a numeric input, and to declare that this baseline +under- or over-states any 2024 amount. The public universes that would support +such a statement are not matched to this file. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import struct +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from .puf_growth import ( + DESIGN_WEIGHT_FIELD, + PROVENANCE_RECID_COLUMN, + PROVENANCE_SOURCE_AGI_COLUMN, + SOURCE_AGI_FIELD, + SOURCE_RECID_FIELD, + CompiledPufGrowth, + FactorAuthority, + FieldRole, + GrownPufTable, + GrowthFactorTable, + GrowthKind, + GrowthRecipe, + GrowthRule, + PufGrowthRefusalError, + SignBranch, + compile_puf_growth, +) +from .puf_monetary_agi_projection import AGI_FIELD +from .puf_monetary_source import AGGREGATE_SENTINEL, PufMonetarySourceProjection +from .puf_source_agi import PUF_SOURCE_YEAR + +__all__ = [ + "CPI_U_ANNUAL_AVERAGE_PERIOD", + "CPI_U_INDEX_BASE", + "CPI_U_SERIES_ID", + "CPI_U_SERIES_TITLE", + "DESIGN_WEIGHT_IDENTITY_SERIES", + "PRICE_BASELINE_LEVELS", + "PRICE_BASELINE_RATIO", + "PRICE_BASELINE_RECIPE_ID", + "PRICE_BASELINE_SERIES_NAME", + "PRICE_BASELINE_SOURCE_YEAR", + "PRICE_BASELINE_TABLE_ID", + "PRICE_BASELINE_TARGET_YEAR", + "PRICE_BASELINE_FACTOR_BITS", + "PAIRWISE_OBSERVATIONS", + "PufMoneyViewAdaptation", + "adapt_projection_to_money_view", + "compile_price_baseline", + "price_baseline_factor_document", + "price_baseline_invariants", + "price_baseline_recipe", + "require_price_baseline_contract", + "verify_cpi_resource_bytes", +] + +#: The primary BLS series this baseline is read from. +CPI_U_SERIES_ID = "CUUR0000SA0" +CPI_U_SERIES_TITLE = ( + "All items in U.S. city average, all urban consumers, not seasonally adjusted" +) +CPI_U_ANNUAL_AVERAGE_PERIOD = "M13" +CPI_U_INDEX_BASE = "1982-84=100" + +#: The source file's representative year, and the target the restatement +#: states amounts in. The source year is the growth slice's own PUF source +#: year, not a second opinion about it. +PRICE_BASELINE_SOURCE_YEAR = PUF_SOURCE_YEAR +PRICE_BASELINE_TARGET_YEAR = 2024 + +#: The publisher's own decimal text for the two annual averages. These are +#: level strings, never floats: the factor table stores text and the ratio is +#: computed once, in one place, by the growth contract. +PRICE_BASELINE_LEVELS = MappingProxyType( + {PRICE_BASELINE_SOURCE_YEAR: "237.017", PRICE_BASELINE_TARGET_YEAR: "313.689"} +) + +#: The exact rational factor, as ``(numerator, denominator)``. Both published +#: levels carry exactly three decimals, so the ratio of the thousandths-scaled +#: integers is the exact ratio of the published levels. +PRICE_BASELINE_RATIO = (313689, 237017) + +#: The one float64 the adopted factor is, as its little-endian bit pattern. +#: Derived from the exact rational here rather than written down, so it cannot +#: drift from :data:`PRICE_BASELINE_RATIO`; a test separately checks that the +#: division of the two published level *texts* lands on the same double. +#: :func:`require_price_baseline_contract` compares an incoming contract's +#: resolved bits against this, which is the check that makes every downstream +#: statement about "the factor" a statement about the contract in hand. +PRICE_BASELINE_FACTOR_BITS = struct.pack( + " None: + if not condition: + raise PufGrowthRefusalError(reason, field) + + +def _module_sha256() -> str: + """This module's own bytes, so the factor table names the code that built it.""" + return hashlib.sha256(Path(__file__).resolve().read_bytes()).hexdigest() + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +def _cpi_annual_average_records( + resource_bytes: bytes, years: Sequence[int] +) -> dict[int, bytes]: + """The CPI-U all-items ``M13`` level text for each of *years*, or fewer. + + BLS pads ``series_id`` and ``value`` with spaces for monospace column + alignment; the character that actually delimits a field is the tab, so + every field is stripped before it is compared or returned — the same + convention this lane's own retrieval script used to extract and hash + these two lines. A row whose first field is this series but which does + not split into the publisher's five tab-delimited fields + (``series_id``, ``year``, ``period``, ``value``, ``footnote_codes``) is + refused rather than silently skipped, because a truncated or reshuffled + row is exactly the failure mode this check exists to catch. Rows for a + year not in *years* are ignored: this baseline does not require the file + to be otherwise well-formed, only that its own two records are. + """ + series = CPI_U_SERIES_ID.encode("ascii") + period = CPI_U_ANNUAL_AVERAGE_PERIOD.encode("ascii") + wanted = {str(year).encode("ascii") for year in years} + found: dict[int, bytes] = {} + for line in resource_bytes.split(b"\n"): + if not line.strip(): + continue + fields = line.split(b"\t") + if fields[0].strip() != series: + continue + _require(len(fields) == 5, "PRICE_BASELINE_RESOURCE_LINE_SHAPE") + _, year_field, period_field, value_field, _ = ( + field.strip() for field in fields + ) + if period_field != period or year_field not in wanted: + continue + year = int(year_field) + _require( + year not in found, "PRICE_BASELINE_RESOURCE_DUPLICATE_RECORD", str(year) + ) + found[year] = value_field + return found + + +def verify_cpi_resource_bytes( + resource_bytes: bytes, *, resource_sha256: str +) -> Mapping[str, object]: + """Refuse unless *resource_bytes* is this baseline's named public resource. + + The whole-file digest is bound first — the caller's ``resource_sha256`` + is no longer a string nobody checks. Then the bytes themselves are parsed + with the publisher's own tab-delimited, space-padded convention (see + :func:`_cpi_annual_average_records`) and must contain **exactly** the two + ``CUUR0000SA0`` ``M13`` records this baseline reads, at the two years + :data:`PRICE_BASELINE_LEVELS` names, each with exactly + the declared decimal text after removing BLS column padding. Different + numeric spellings and near-equal floats are refused: source lexical + identity is stronger than arithmetic agreement. + + Args: + resource_bytes: The exact bytes retrieved from ``source_url``. + resource_sha256: The digest the caller claims for those bytes. + + Returns: + A receipt naming the series, period, verified digest and the two + matched records. + + Raises: + PufGrowthRefusalError: On a type, size, digest, shape, duplicate, + missing-record or value mismatch. + """ + _require( + isinstance(resource_bytes, (bytes, bytearray)), "PRICE_BASELINE_RESOURCE_TYPE" + ) + _require(len(resource_bytes) > 0, "PRICE_BASELINE_RESOURCE_EMPTY") + _require( + len(resource_bytes) <= _CPI_RESOURCE_MAX_BYTES, + "PRICE_BASELINE_RESOURCE_OVERSIZE", + ) + body = bytes(resource_bytes) + actual_sha256 = hashlib.sha256(body).hexdigest() + _require(actual_sha256 == resource_sha256, "PRICE_BASELINE_RESOURCE_DIGEST") + + years = tuple(PRICE_BASELINE_LEVELS) + found = _cpi_annual_average_records(body, years) + records: dict[str, object] = {} + for year in years: + _require(year in found, "PRICE_BASELINE_RESOURCE_RECORD_MISSING", str(year)) + _require( + found[year] == PRICE_BASELINE_LEVELS[year].encode("ascii"), + "PRICE_BASELINE_RESOURCE_RECORD_VALUE", + str(year), + ) + # Decode only after exact ASCII equality; malformed values must refuse + # by the same domain reason rather than leaking a parser exception. + level_text = found[year].decode("ascii") + found_value = float(level_text) + records[str(year)] = MappingProxyType( + {"level_text": level_text, "level_float64": found_value} + ) + return MappingProxyType( + { + "series_id": CPI_U_SERIES_ID, + "period": CPI_U_ANNUAL_AVERAGE_PERIOD, + "resource_sha256": actual_sha256, + "resource_bytes": len(body), + "records": MappingProxyType(records), + } + ) + + +def price_baseline_factor_document( + *, source_url: str, resource_sha256: str, retrieved_utc: str, resource_bytes: bytes +) -> bytes: + """The developmental factor table, as the exact bytes it hashes to. + + Args: + source_url: The publisher endpoint the levels were read from. + resource_sha256: Digest of the exact resource bytes that were read. + retrieved_utc: When they were read, as an ISO 8601 UTC instant. + resource_bytes: The exact bytes retrieved from ``source_url``. Checked + against ``resource_sha256`` and parsed for the two annual-average + records this baseline names — see + :func:`verify_cpi_resource_bytes`. Without this the two prior + arguments were caller assertions nothing in this module verified. + + Returns: + A canonical factor-table document carrying the price series and the + identity design-weight series, with the + ``developmental_public_resource`` authority. + """ + verify_cpi_resource_bytes(resource_bytes, resource_sha256=resource_sha256) + return _canonical( + { + "authority": "developmental_public_resource", + "provenance": { + "dependency_versions": {}, + "generated_by": ( + "microcosm.build.us_runtime.puf_price_baseline." + "price_baseline_factor_document" + ), + "generator_code_sha256": _module_sha256(), + "index_or_total": "index", + "parameter_paths": [], + "population_divisor": "none", + "resource_sha256": resource_sha256, + "retrieved_utc": retrieved_utc, + "rounding": ( + "none; the two published levels are copied verbatim as " + "decimal text and the factor is one float64 division of them" + ), + "source_url": source_url, + }, + "schema_version": 1, + "series": { + DESIGN_WEIGHT_IDENTITY_SERIES: { + "kind": str(GrowthKind.DESIGN_WEIGHT_GROWTH), + "levels": { + str(PRICE_BASELINE_SOURCE_YEAR): "1", + str(PRICE_BASELINE_TARGET_YEAR): "1", + }, + }, + PRICE_BASELINE_SERIES_NAME: { + "kind": str(GrowthKind.NOMINAL_PRICE_RESTATEMENT), + "levels": { + str(year): level + for year, level in PRICE_BASELINE_LEVELS.items() + }, + }, + }, + "table_id": PRICE_BASELINE_TABLE_ID, + "years": [PRICE_BASELINE_SOURCE_YEAR, PRICE_BASELINE_TARGET_YEAR], + } + ) + + +def price_baseline_recipe( + money_fields: Sequence[str], *, recipe_id: str = PRICE_BASELINE_RECIPE_ID +) -> GrowthRecipe: + """One price series for every money field, plus the two structural rules. + + Args: + money_fields: The projection's money fields. Must contain + ``E00100``, because the growth contract requires it, and must not + contain ``RECID`` or ``S006``, which are not money. + recipe_id: Recipe identity, for a caller running more than one. + + Raises: + PufGrowthRefusalError: If the field set is not a money field set. + """ + _require(bool(money_fields), "PRICE_BASELINE_NO_MONEY_FIELD") + fields = tuple(money_fields) + _require(len(set(fields)) == len(fields), "PRICE_BASELINE_DUPLICATE_FIELD") + _require(SOURCE_AGI_FIELD in fields, "PRICE_BASELINE_AGI_FIELD", SOURCE_AGI_FIELD) + for reserved in (SOURCE_RECID_FIELD, DESIGN_WEIGHT_FIELD): + _require(reserved not in fields, "PRICE_BASELINE_NON_MONEY_FIELD", reserved) + rules = [ + GrowthRule( + field, + FieldRole.MONEY, + PRICE_BASELINE_SOURCE_YEAR, + PRICE_BASELINE_TARGET_YEAR, + GrowthKind.NOMINAL_PRICE_RESTATEMENT, + SignBranch.ANY, + PRICE_BASELINE_SERIES_NAME, + _CITATION, + ) + for field in fields + ] + rules.append( + GrowthRule( + SOURCE_RECID_FIELD, + FieldRole.IDENTIFIER, + PRICE_BASELINE_SOURCE_YEAR, + PRICE_BASELINE_SOURCE_YEAR, + GrowthKind.NONE, + ) + ) + rules.append( + GrowthRule( + DESIGN_WEIGHT_FIELD, + FieldRole.DESIGN_WEIGHT, + PRICE_BASELINE_SOURCE_YEAR, + PRICE_BASELINE_TARGET_YEAR, + GrowthKind.DESIGN_WEIGHT_GROWTH, + SignBranch.ANY, + DESIGN_WEIGHT_IDENTITY_SERIES, + "declared identity: this baseline transitions no return mass", + ) + ) + return GrowthRecipe(recipe_id, tuple(rules)) + + +def require_price_baseline_contract( + compiled: CompiledPufGrowth, *, resource_bytes: bytes +) -> None: + """Refuse any contract that is not the adopted price baseline. + + :func:`adapt_projection_to_money_view` and + :func:`price_baseline_invariants` both call this before they record + anything, because both go on to state the source year, the exact rational + factor, the ``RECID`` identity and "this baseline transitions no return + mass" as facts about the run. A contract that named a different pair of + years, mixed a second factor series, aged rather than restated, split a + field across sign branches, or grew the design weight would leave every + other check in this module passing while those recorded statements were + false. The false receipt is the failure this refuses. + + What is checked is exactly what those receipts assert: + + * the ``developmental_public_resource`` authority, and no release + eligibility; + * every money rule at ``2015 -> 2024``, ``nominal_price_restatement``, + ``SignBranch.ANY``, on the one named CPI series, resolving to the + published level texts and to the exact float64 of ``313689/237017``; + * exactly one identifier rule, on ``RECID``, at the baseline source year; + * exactly one design-weight rule, on ``S006``, at the same two years, on + the declared identity series, whose resolved factor is exactly ``1.0`` + and whose two level texts are the same text — not two texts whose + float64 quotient happens to round to one; + * no rule in any other role; + * the exact resource bytes against the digest the contract names, and the + two source records against their declared series, years, period and text. + + The *field set* is deliberately not fixed: a contract over a subset of the + projection's money fields is still this baseline, and the adapter checks + the subset relation against the artifact separately. + + Args: + compiled: The contract a caller is about to run. + resource_bytes: The retained BLS body; checked against the contract's + own resource digest even if it came from the generic growth compiler. + + Raises: + PufGrowthRefusalError: With a ``PRICE_BASELINE_*`` reason naming the + first field that does not match. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + _require( + compiled.factors.authority is FactorAuthority.DEVELOPMENTAL_PUBLIC_RESOURCE, + "PRICE_BASELINE_AUTHORITY", + ) + _require(compiled.release_eligible is False, "PRICE_BASELINE_RELEASE_ELIGIBLE") + verify_cpi_resource_bytes( + resource_bytes, + resource_sha256=compiled.factors.provenance["resource_sha256"], + ) + + rules = compiled.recipe.rules + money = tuple(rule for rule in rules if rule.role is FieldRole.MONEY) + identifiers = tuple(rule for rule in rules if rule.role is FieldRole.IDENTIFIER) + weights = tuple(rule for rule in rules if rule.role is FieldRole.DESIGN_WEIGHT) + for rule in rules: + _require( + rule.role + in (FieldRole.MONEY, FieldRole.IDENTIFIER, FieldRole.DESIGN_WEIGHT), + "PRICE_BASELINE_ROLE", + rule.field, + ) + _require(bool(money), "PRICE_BASELINE_NO_MONEY_FIELD") + _require( + SOURCE_AGI_FIELD in {rule.field for rule in money}, + "PRICE_BASELINE_AGI_FIELD", + SOURCE_AGI_FIELD, + ) + source_level = PRICE_BASELINE_LEVELS[PRICE_BASELINE_SOURCE_YEAR] + target_level = PRICE_BASELINE_LEVELS[PRICE_BASELINE_TARGET_YEAR] + for rule in money: + field = rule.field + _require( + rule.source_reference_year == PRICE_BASELINE_SOURCE_YEAR, + "PRICE_BASELINE_SOURCE_YEAR", + field, + ) + _require( + rule.target_year == PRICE_BASELINE_TARGET_YEAR, + "PRICE_BASELINE_TARGET_YEAR", + field, + ) + _require( + rule.growth_kind is GrowthKind.NOMINAL_PRICE_RESTATEMENT, + "PRICE_BASELINE_GROWTH_KIND", + field, + ) + # ``SignBranch.ANY`` is the field's only rule, which is what lets one + # positive factor be checked per row rather than per branch. + _require( + rule.sign_branch is SignBranch.ANY, "PRICE_BASELINE_SIGN_BRANCH", field + ) + _require( + rule.factor_series == PRICE_BASELINE_SERIES_NAME, + "PRICE_BASELINE_SERIES", + field, + ) + resolved = compiled.factor_for(field, SignBranch.ANY) + # The substantive claim first — this is the one float64 — and then the + # two level texts it must have come from. The order matters because + # more than one pair of texts divides to the same double, and this + # baseline is the published pair, not merely a pair that agrees. + _require( + resolved.ratio_bits == PRICE_BASELINE_FACTOR_BITS, + "PRICE_BASELINE_FACTOR", + field, + ) + _require( + resolved.source_level == source_level, "PRICE_BASELINE_SOURCE_LEVEL", field + ) + _require( + resolved.target_level == target_level, "PRICE_BASELINE_TARGET_LEVEL", field + ) + + # ``GrowthRecipe`` already requires exactly one identifier field, and + # ``GrowthRule`` already fixes a non-money, non-weight role to + # ``GrowthKind.NONE`` with equal years. What is open here is which field + # carries the role and which year it names. + _require( + len(identifiers) == 1 and identifiers[0].field == SOURCE_RECID_FIELD, + "PRICE_BASELINE_IDENTIFIER", + ) + identifier = identifiers[0] + _require( + identifier.source_reference_year == PRICE_BASELINE_SOURCE_YEAR, + "PRICE_BASELINE_IDENTIFIER_YEAR", + identifier.field, + ) + + # Likewise the design-weight role already fixes the kind and the sign + # branch. What is open is the field, the series, and whether the declared + # factor really is one. + _require( + len(weights) == 1 and weights[0].field == DESIGN_WEIGHT_FIELD, + "PRICE_BASELINE_DESIGN_WEIGHT", + ) + weight = weights[0] + _require( + weight.factor_series == DESIGN_WEIGHT_IDENTITY_SERIES, + "PRICE_BASELINE_DESIGN_WEIGHT_SERIES", + weight.field, + ) + _require( + weight.source_reference_year == PRICE_BASELINE_SOURCE_YEAR + and weight.target_year == PRICE_BASELINE_TARGET_YEAR, + "PRICE_BASELINE_DESIGN_WEIGHT_YEAR", + weight.field, + ) + resolved_weight = compiled.factor_for(weight.field, SignBranch.ANY) + # The substantive claim first: the declared weight factor is exactly one. + # Then the two level texts, which must be the same text and not merely two + # texts whose float64 quotient rounds to one. + _require( + resolved_weight.value == 1.0, + "PRICE_BASELINE_DESIGN_WEIGHT_FACTOR", + weight.field, + ) + _require( + resolved_weight.source_level == resolved_weight.target_level, + "PRICE_BASELINE_DESIGN_WEIGHT_LEVEL", + weight.field, + ) + + +def compile_price_baseline( + money_fields: Sequence[str], + *, + source_url: str, + resource_sha256: str, + retrieved_utc: str, + resource_bytes: bytes, + recipe_id: str = PRICE_BASELINE_RECIPE_ID, +) -> CompiledPufGrowth: + """Compile the price-baseline contract; never release-eligible. + + ``resource_bytes`` is required and is checked against ``resource_sha256`` + by :func:`price_baseline_factor_document`; this function cannot hand back + a contract pinned to resource metadata nobody verified. + """ + recipe = price_baseline_recipe(money_fields, recipe_id=recipe_id) + factors = GrowthFactorTable.from_bytes( + price_baseline_factor_document( + source_url=source_url, + resource_sha256=resource_sha256, + retrieved_utc=retrieved_utc, + resource_bytes=resource_bytes, + ) + ) + compiled = compile_puf_growth(recipe, factors) + # The authority gate already guarantees this. Asserting it here means the + # one function a caller reaches for cannot hand back an eligible contract + # even if the gate is later widened by mistake. + _require(compiled.release_eligible is False, "PRICE_BASELINE_RELEASE_ELIGIBLE") + # The same gate the two entry points apply, so this function cannot hand + # back a contract they would decline. + require_price_baseline_contract(compiled, resource_bytes=resource_bytes) + return compiled + + +# -------------------------------------------------------------------------- +# The artifact-to-money-view adapter +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PufMoneyViewAdaptation: + """The money view the growth transform consumes, plus what stayed outside. + + Attributes: + money_view: One row per amount-known return, carrying ``RECID`` and + the declared money columns as int64. Never the design weight, and + never a disclosure-aggregate row. + recid: The kept rows' identifiers, in money-view order. + flpdyr: The kept rows' raw observation year, carried from the status + artifact and never restated. + demographic_status: The kept rows' demographic match flag. + design_weight: The kept rows' ``S006``, in the delivered integer + hundredths, unaltered and outside the money view. + excluded: The audit identity of the removed disclosure-aggregate rows. + receipt: What was joined, kept and removed. + """ + + money_view: pd.DataFrame + recid: np.ndarray + flpdyr: np.ndarray + demographic_status: np.ndarray + design_weight: np.ndarray + excluded: Mapping[str, object] + receipt: Mapping[str, object] + + +def _status_arrays(status: object, rows: int | None = None) -> Mapping[str, np.ndarray]: + """Every status column this adapter reads, present and the same length. + + The length check covers ``FLPDYR``, ``S006`` and the demographic status as + well as the two join keys, so a short or long side column is a refusal + rather than a silently misaligned slice later. + """ + try: + typed = status.typed + except AttributeError as error: + raise PufGrowthRefusalError("STATUS_ARTIFACT_TYPE") from error + # Presence first for every column, then length for every column, so a + # missing column is reported as missing rather than as a length mismatch. + for name in _STATUS_COLUMNS: + _require(name in typed, "STATUS_ARTIFACT_COLUMN", name) + if rows is not None: + for name in _STATUS_COLUMNS: + _require( + int(np.asarray(typed[name]).shape[0]) == rows, + "STATUS_COLUMN_ROWS", + name, + ) + return typed + + +def adapt_projection_to_money_view( + decoded: PufMonetarySourceProjection, + status: object, + compiled: CompiledPufGrowth, + *, + resource_bytes: bytes, +) -> PufMoneyViewAdaptation: + """Join the projection to the status artifact and drop only the aggregates. + + The join is by position **and** proved by value: the two artifacts' whole + ``RECID`` sequences and disclosure flags must be element-wise identical + before anything is selected, so a row's amounts and its status cannot come + from different returns. + + Exactly the rows outside the individual-return amount universe are + removed, and their identity is recorded. Every removed row must hold the + out-of-universe sentinel in every projected column, and no kept row may + hold it, so the sentinel cannot reach a numeric input in either direction. + + The contract is checked against the adopted baseline before anything is + recorded — see :func:`require_price_baseline_contract` — because the + receipt below states the source year, the design-weight identity and the + absence of a return-mass transition as facts about this run. + + Args: + decoded: A decoded thirteen-field projection. + status: The decoded ``puf_2015_raw_return_status`` artifact. + compiled: The compiled price-baseline contract. + resource_bytes: The retained BLS body authenticated by the entry gate. + + Returns: + The money view and everything deliberately kept out of it. + + Raises: + PufGrowthRefusalError: On any contract, join, universe or sentinel + violation. + """ + require_price_baseline_contract(compiled, resource_bytes=resource_bytes) + _require( + isinstance(decoded, PufMonetarySourceProjection), "PROJECTION_ARTIFACT_TYPE" + ) + projection_recid = np.asarray(decoded.typed[SOURCE_RECID_FIELD], dtype=" 0, "PROJECTION_NO_ROWS") + typed = _status_arrays(status, rows) + status_recid = np.asarray(typed[SOURCE_RECID_FIELD], dtype=" 0, "NO_AMOUNT_KNOWN_ROW") + _require( + removed == int(decoded.facts["row_class_rows"]["disclosure_aggregate"]), + "EXCLUDED_ROW_COUNT", + ) + + money_fields = compiled.money_fields + _require(set(money_fields) <= set(decoded.typed), "PROJECTION_MISSING_MONEY_FIELD") + _require(AGI_FIELD in money_fields, "PRICE_BASELINE_AGI_FIELD", AGI_FIELD) + columns: dict[str, np.ndarray] = {} + for field in money_fields: + values = np.asarray(decoded.typed[field], dtype=" int | None: + if greater not in frame.columns or lesser not in frame.columns: + return None + return int((frame[lesser] > frame[greater]).sum()) + + +def _bits(values: np.ndarray) -> np.ndarray: + """A float64 array's raw bit patterns, so ``-0.0`` is not ``0.0``.""" + return np.ascontiguousarray(values, dtype="float64").view(" Mapping[str, object]: + """Measure what one positive factor must leave true, and refuse otherwise. + + Three things happen here, in this order, and the order is deliberate. + + *The summaries.* Per column: dtype, finiteness, the sign set, the zero set, + no negative zero, and the exact integer sum of the input column times the + factor against the grown ``math.fsum``. Then the pairwise relations the + booklets leave open, counted before and after and required only to be + **equal**. These are the numbers a reader wants in the receipt, and each + names a specific symptom when it fires. + + *The exact row comparison.* Every kept row of every money column must be + bit-for-bit the contract's own resolved float64 factor applied to that + row's own delivered amount, and the two provenance columns must be + bit-for-bit their sources. This is the statement the summaries cannot + make: a coordinated rearrangement preserves every total, every sign, every + zero and every pairwise count, and is caught only here. It runs last so + that the narrower checks keep reporting the narrower diagnosis — and so + that each of them stays independently falsifiable. + + *The design weight.* Compared, if a caller supplies the weights it carried + forward, against the adaptation's own ``S006``. Not supplied, it is + reported as a declaration and **not** as a measurement. + + Args: + adaptation: The adaptation whose money view was grown. + grown: The result of :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth` + on that money view, under this same contract. + compiled: The contract, checked against the adopted baseline. + resource_bytes: The retained BLS body authenticated by the entry gate. + design_weight_after: The post-growth design weights, if the caller ran + :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_design_weight_growth`. + Omitted, ``design_weight_unchanged`` is ``None``, not ``True``. + relative_tolerance: Tolerance for the scaled-sum comparison only. + + Raises: + PufGrowthRefusalError: On any contract, identity, row, sum, pairwise or + weight violation. + """ + require_price_baseline_contract(compiled, resource_bytes=resource_bytes) + _require(type(grown) is GrownPufTable, "GROWN_TYPE") + _require(type(adaptation) is PufMoneyViewAdaptation, "ADAPTATION_TYPE") + # One contract, named the same way by all three objects. Without this the + # measurements below would describe a contract the table was not grown + # under, and every number in the receipt would be attributed to it. + contract_sha256 = compiled.sha256 + _require(grown.contract_sha256 == contract_sha256, "GROWN_CONTRACT_MISMATCH") + _require( + grown.receipt["contract_sha256"] == contract_sha256, + "GROWN_RECEIPT_CONTRACT_MISMATCH", + ) + _require( + adaptation.receipt["contract_sha256"] == contract_sha256, + "ADAPTATION_CONTRACT_MISMATCH", + ) + source = adaptation.money_view + table = grown.table + rows = len(source) + _require(len(table) == rows, "GROWN_ROWS") + _require( + bool( + np.array_equal( + table[SOURCE_RECID_FIELD].to_numpy(), + source[SOURCE_RECID_FIELD].to_numpy(), + ) + ), + "GROWN_RECID", + ) + factor = compiled.factor_for(compiled.money_fields[0], SignBranch.ANY).value + columns: dict[str, object] = {} + for field in compiled.money_fields: + resolved = compiled.factor_for(field, SignBranch.ANY) + _require( + resolved.ratio_bits + == compiled.factor_for(compiled.money_fields[0], SignBranch.ANY).ratio_bits, + "MIXED_FACTOR", + field, + ) + original = source[field].to_numpy() + result = table[field].to_numpy() + _require(result.dtype == np.dtype("float64"), "GROWN_DTYPE", field) + _require(bool(np.isfinite(result).all()), "GROWN_NONFINITE", field) + _require( + bool((np.sign(result) == np.sign(original.astype("float64"))).all()), + "SIGN_CHANGED", + field, + ) + zeros = result == 0.0 + _require(not bool(np.signbit(result[zeros]).any()), "NEGATIVE_ZERO", field) + _require(bool(np.array_equal(zeros, original == 0)), "ZERO_SET_CHANGED", field) + exact_sum = int(sum(int(value) for value in original.tolist())) + gross_sum = int(sum(abs(int(value)) for value in original.tolist())) + grown_sum = math.fsum(result.tolist()) + expected = exact_sum * factor + # Scaled by the column's **gross** magnitude, not its net total. The + # float error one positive factor can accumulate is bounded by the + # magnitude that passed through it; a signed column whose positives + # and negatives nearly cancel has a tiny net and the same gross, and + # scaling by the net alone would refuse a correct result for it. + net_scale = max(abs(expected), 1.0) + scale = max(abs(gross_sum * factor), net_scale) + _require( + abs(grown_sum - expected) <= relative_tolerance * scale, + "SCALED_SUM_DISAGREEMENT", + field, + ) + columns[field] = { + "rows": rows, + "exact_source_sum": exact_sum, + "gross_source_sum_absolute": gross_sum, + "grown_fsum": grown_sum, + "expected_grown_sum": expected, + "scale_basis": "gross_absolute_source_sum_times_factor", + "relative_difference": ( + abs(grown_sum - expected) / scale if scale else 0.0 + ), + "relative_difference_net_scaled": ( + abs(grown_sum - expected) / net_scale if net_scale else 0.0 + ), + "negative": int((original < 0).sum()), + "positive": int((original > 0).sum()), + "zero": int((original == 0).sum()), + "ratio_bits": resolved.ratio_bits, + "grown_minimum": float(result.min()), + "grown_maximum": float(result.max()), + } + pairwise = {} + for name, greater, lesser in PAIRWISE_OBSERVATIONS: + before = _violations(source, greater, lesser) + after = _violations(table, greater, lesser) + # A contract over a subset of the projection's fields can leave one + # side of a relation out of the money view entirely. ``None == None`` + # would then pass, and the entry would read as an observation that was + # never made, so the absence is named instead of being counted. + _require((before is None) == (after is None), "PAIRWISE_COLUMN_ASYMMETRY", name) + observable = before is not None + if observable: + _require(before == after, "PAIRWISE_VIOLATIONS_MOVED", name) + pairwise[name] = { + "greater": greater, + "lesser": lesser, + "violations_before": before, + "violations_after": after, + "status": ( + "observed_not_enforced" + if observable + else "not_observable_column_absent" + ), + "absent_columns": sorted( + column for column in (greater, lesser) if column not in source.columns + ), + } + + # The exact row comparison. ``apply_puf_growth`` selects ``values != 0.0`` + # for a SignBranch.ANY rule and writes ``values[selected] * factor``, so + # the expected array is reproduced here by the same two operations in the + # same order and compared on raw bits. Nothing is recomputed from the + # levels or the rational: the contract's own resolved float64 is the one + # multiplier, which is what makes this a check of the transform rather + # than a second opinion about the factor. + for field in compiled.money_fields: + resolved = compiled.factor_for(field, SignBranch.ANY) + original = source[field].to_numpy() + expected = original.astype("float64") + selected = expected != 0.0 + with np.errstate(over="ignore", under="ignore"): + expected[selected] = expected[selected] * resolved.value + result = table[field].to_numpy() + disagreeing = int(np.count_nonzero(_bits(result) != _bits(expected))) + _require(disagreeing == 0, "ROW_GROWTH_DISAGREEMENT", field) + + # The two columns ``apply_puf_growth`` copies rather than grows. A silent + # edit here would restate provenance while every money column still + # matched. + for column, origin in ( + (PROVENANCE_RECID_COLUMN, source[SOURCE_RECID_FIELD].to_numpy()), + ( + PROVENANCE_SOURCE_AGI_COLUMN, + source[SOURCE_AGI_FIELD].to_numpy().astype("float64"), + ), + ): + _require(column in table.columns, "PROVENANCE_COLUMN_MISSING", column) + carried = table[column].to_numpy() + if column == PROVENANCE_SOURCE_AGI_COLUMN: + _require( + bool((_bits(carried) == _bits(origin)).all()), + "PROVENANCE_DISAGREEMENT", + column, + ) + else: + _require( + bool(np.array_equal(carried, origin)), + "PROVENANCE_DISAGREEMENT", + column, + ) + + design_weight, design_weight_unchanged = _design_weight_observation( + adaptation, compiled, design_weight_after + ) + return MappingProxyType( + { + "rows": rows, + "factor_ratio_bits": compiled.factor_for( + compiled.money_fields[0], SignBranch.ANY + ).ratio_bits, + "factor_float64": factor, + "exact_rational": { + "numerator": PRICE_BASELINE_RATIO[0], + "denominator": PRICE_BASELINE_RATIO[1], + }, + "output_rounding": "none_unrounded_float64", + "columns": columns, + "pairwise_observations": pairwise, + "row_exact_growth": { + "rows": rows, + "fields": list(compiled.money_fields), + "comparison": ( + "bitwise float64 equality of every kept row against the " + "contract's own resolved factor applied to that row's " + "delivered amount, zeros left untouched" + ), + "disagreeing_rows": 0, + "provenance_columns_compared": list( + (PROVENANCE_RECID_COLUMN, PROVENANCE_SOURCE_AGI_COLUMN) + ), + }, + "contract_sha256": contract_sha256, + "contract_named_by": { + "compiled": contract_sha256, + "grown_result": grown.contract_sha256, + "grown_receipt": grown.receipt["contract_sha256"], + "adaptation_receipt": adaptation.receipt["contract_sha256"], + }, + "design_weight": design_weight, + # ``None`` when no post-growth weight was supplied. This is never + # ``True`` unless the arrays were actually compared. + "design_weight_unchanged": design_weight_unchanged, + "release_eligible": compiled.release_eligible, + } + ) + + +def _design_weight_observation( + adaptation: PufMoneyViewAdaptation, + compiled: CompiledPufGrowth, + design_weight_after: Sequence[float] | np.ndarray | None, +) -> tuple[dict[str, object], bool | None]: + """What the design weight did, separating the declaration from a measurement. + + The declaration is the contract's: an identity series whose two levels are + both one, so the declared factor is exactly ``1.0``. That is a statement + about the recipe, and it is reported as one. + + The measurement needs a second array. ``adaptation.design_weight`` is the + status artifact's own ``S006`` sliced to the kept rows — the before side, + with its digest recorded so the comparison is auditable. The after side is + whatever the caller carried forward. Without it there is no observation to + report, and this returns ``None`` rather than manufacturing one. + """ + rule = compiled.design_weight_rule + declared = compiled.factor_for(rule.field, SignBranch.ANY) + before = np.asarray(adaptation.design_weight) + _require(before.ndim == 1, "DESIGN_WEIGHT_SHAPE", rule.field) + record: dict[str, object] = { + "field": rule.field, + "rows": int(before.shape[0]), + "declared_series": rule.factor_series, + "declared_source_level": declared.source_level, + "declared_target_level": declared.target_level, + "declared_factor_float64": declared.value, + "declared_return_mass_transition": "none", + "before_source": ( + "adaptation.design_weight: the status artifact's own S006, in " + "delivered integer hundredths, sliced to the kept rows" + ), + "before_sha256": hashlib.sha256( + np.ascontiguousarray(before, dtype="= -(2**53)) & (values <= 2**53)).all()), + "QBI_INTEGER_PRECISION:" + name, + ) + values = values.astype(np.float64) + _require(bool(np.isfinite(values).all()), "QBI_NONFINITE:" + name) + return values + + +def _ids(recids): + values = np.asarray(recids) + _require( + values.ndim == 1 + and len(values) > 0 + and values.dtype.kind in "iu" + and bool(((values > 0) & (values <= np.iinfo(np.int64).max)).all()), + "QBI_RECID_PHYSICAL_TYPE", + ) + _require(len(np.unique(values)) == len(values), "QBI_DUPLICATE_RECID") + return values.astype(" 0 + if not positive.any(): + return 0.0 + term = slope * revenues[positive] + target_logit = np.log(target / (1 - target)) + low, high = target_logit - np.max(term) - 80, target_logit - np.min(term) + 80 + for _ in range(100): + mid = (low + high) / 2 + if _logistic(mid + term).mean() < target: + low = mid + else: + high = mid + return float((low + high) / 2) + + +@dataclass(frozen=True) +class QbiEmploymentCalibration: + """A reusable statistical fit; this numerical record is not source authority.""" + + version: str + parameters_sha256: str + seed: int + fitted_cohort_sha256: str + fitted_rows: int + positive_receipt_rows: int + intercept: float + + +@dataclass(frozen=True) +class QbiModelResult: + columns: Mapping[str, np.ndarray] + calibration: QbiEmploymentCalibration + receipt: Mapping[str, object] + + +def model_full_puf_qbi( + canonical, recids, *, known, input_money_year, seed=0, employment_calibration=None +): + """Model 15 QBI leaves and explicitly replace the ordinary Schedule C leaf. + + Knownness here means complete canonical input, possibly derived upstream; + it does not mean a modeled leaf was observed. Inputs and their RECIDs must + already belong to the ordinary-return cohort selected by the source owner. + Each RNG is keyed by RECID, seed, and the archived stream number. This is + intentionally not byte parity with the archive's row-position RNG. The + employee logit intercept is fit to the unweighted positive-receipt cohort + (the archive's 18% target); reuse the returned fit for stable subset/chunk + application. Fitting a different cohort is a different statistical model. + """ + ids = _ids(recids) + n = len(ids) + _require( + type(input_money_year) is int and input_money_year == 2015, + "QBI_INPUT_MONEY_YEAR", + ) + _require(type(seed) is int and 0 <= seed < 2**64, "QBI_SEED") + _require( + isinstance(canonical, Mapping) and isinstance(known, Mapping), + "QBI_INPUT_MAPPING", + ) + inputs = {} + for name in REQUIRED_INPUTS: + _require(name in canonical and name in known, "QBI_MISSING_INPUT:" + name) + mask = np.asarray(known[name]) + _require( + mask.shape == (n,) and mask.dtype.kind == "b", "QBI_KNOWNNESS_TYPE:" + name + ) + _require(bool(mask.all()), "QBI_UNKNOWN:" + name) + inputs[name] = _vector(canonical[name], name, n) + source = np.column_stack( + [inputs[name] for name in REQUIRED_INPUTS[:5]] + + [inputs["partnership_income"] + inputs["s_corp_income"]] + ) + _require(bool(np.isfinite(source).all()), "QBI_SOURCE_SUM_NONFINITE") + params = model_parameters() + qualified = np.zeros((n, 6), dtype=bool) + margins, labor = np.zeros((n, 6)), np.zeros((n, 6)) + employees_draw, capital_draw, normal_draw, sstb_draw = np.zeros((4, n)) + investment_draws = np.zeros((n, 3)) + # Generate each record independently; calculations below remain vectorized. + for row, recid in enumerate(ids): + + def rng(stream, recid=int(recid)): + return np.random.Generator( + np.random.PCG64(np.random.SeedSequence([seed, stream, recid])) + ) + + qualified[row] = rng(41).random(6) < params["qualification"] + business = rng(42) + for col, (a, b, scale, shift) in enumerate(params["profit_margin"]): + margins[row, col] = business.beta(a, b) * scale + shift + employees_draw[row] = business.random() + for col, (a, b, scale, shift) in enumerate(params["labor_ratio"]): + labor[row, col] = business.beta(a, b) * scale + shift + capital_draw[row] = business.random() + normal_draw[row] = business.standard_normal() + sstb_draw[row] = rng(64).random() + investment = rng(43) + for col, (_name, probability, a, b, scale) in enumerate( + params["reit_ptp"] + params["bdc"] + ): + receives = investment.random() < probability + share = np.clip(investment.beta(a, b) * scale, 0, 1) + investment_draws[row, col] = share if receives else 0.0 + components = source * qualified + positive = np.maximum(components, 0) + positive_total = positive.sum(axis=1) + _require(bool(np.isfinite(positive_total).all()), "QBI_POSITIVE_TOTAL_NONFINITE") + fractions = np.divide( + positive, + positive_total[:, None], + out=np.zeros_like(positive), + where=positive_total[:, None] > 0, + ) + qbi = components.sum(axis=1) + _require(bool(np.isfinite(qbi).all()), "QBI_NET_TOTAL_NONFINITE") + margin = (fractions * margins).sum(axis=1) + revenues = np.divide(np.maximum(qbi, 0), margin, out=np.zeros(n), where=margin > 0) + _require(bool(np.isfinite(revenues).all()), "QBI_REVENUE_NONFINITE") + cohort_sha = _digest_columns(ids, inputs) + logit = params["employee_logit"] + if employment_calibration is None: + # Sort by identity before reduction so input permutations replay exactly. + employment_calibration = QbiEmploymentCalibration( + VERSION, + PARAMETERS_SHA256, + seed, + cohort_sha, + n, + int((revenues > 0).sum()), + _employee_intercept( + revenues[np.argsort(ids)], + logit["slope_per_dollar"], + logit["target_share"], + ), + ) + _require( + isinstance(employment_calibration, QbiEmploymentCalibration) + and employment_calibration.version == VERSION + and employment_calibration.parameters_sha256 == PARAMETERS_SHA256 + and employment_calibration.seed == seed + and type(employment_calibration.intercept) is float + and np.isfinite(employment_calibration.intercept), + "QBI_EMPLOYMENT_CALIBRATION_IDENTITY", + ) + probability = np.where( + revenues > 0, + _logistic( + employment_calibration.intercept + logit["slope_per_dollar"] * revenues + ), + 0.0, + ) + wages = revenues * (fractions * labor).sum(axis=1) * (employees_draw < probability) + capital_probability = fractions @ params["ubia"]["capital_probability"] + mean_ubia = (fractions @ params["ubia"]["multiple"]) * np.maximum(qbi, 0) + ubia = np.where( + (capital_draw < capital_probability) & (mean_ubia > 0), + mean_ubia + * np.exp( + params["ubia"]["sigma"] * normal_draw - params["ubia"]["sigma"] ** 2 / 2 + ), + 0.0, + ) + # Canonical source names ensure the SSTB draw consumes qualification flags. + sstb_positions = [SOURCES.index(name) for name in params["sstb_sources"]] + sstb_sources = positive[:, sstb_positions] + largest = sstb_sources.argmax(axis=1) + sstb_probability = np.where( + sstb_sources.sum(axis=1) > 0, + np.asarray(params["sstb_probabilities"])[largest], + 0.0, + ) + is_sstb = sstb_draw < sstb_probability + output = { + name + "_would_be_qualified": qualified[:, i].copy() + for i, name in enumerate(SOURCES) + } + schedule_c = inputs["self_employment_income_before_lsr"] + output.update( + { + "business_is_sstb": is_sstb, + "self_employment_income_before_lsr": np.where(is_sstb, 0, schedule_c), + "sstb_self_employment_income_before_lsr": np.where(is_sstb, schedule_c, 0), + "self_employment_income_would_be_qualified": qualified[:, 0] & ~is_sstb, + "sstb_self_employment_income_would_be_qualified": qualified[:, 0] & is_sstb, + "w2_wages_from_qualified_business": wages, + "unadjusted_basis_qualified_property": ubia, + "sstb_w2_wages_from_qualified_business": np.where(is_sstb, wages, 0), + "sstb_unadjusted_basis_qualified_property": np.where(is_sstb, ubia, 0), + "qualified_reit_and_ptp_income": np.maximum( + inputs["non_qualified_dividend_income"], 0 + ) + * investment_draws[:, 0] + + np.maximum(source[:, 5], 0) * investment_draws[:, 1], + "qualified_bdc_income": np.maximum( + inputs["non_qualified_dividend_income"], 0 + ) + * investment_draws[:, 2], + } + ) + _require(set(output) == set(OUTPUTS), "QBI_OUTPUT_ROSTER") + for name, values in output.items(): + _require(bool(np.isfinite(values).all()), "QBI_OUTPUT_NONFINITE:" + name) + frozen = { + name: np.frombuffer( + np.asarray(output[name]).tobytes(), dtype=output[name].dtype + ) + for name in OUTPUTS + } + receipt = { + "version": VERSION, + "origin": "modeled_not_observed", + "release_eligible": False, + "archived_commit": ARCHIVED_COMMIT, + "archived_code_sha256": ARCHIVED_CODE_SHA256, + "archived_assumptions_sha256": ARCHIVED_ASSUMPTIONS_SHA256, + "parameters_sha256": PARAMETERS_SHA256, + "input_money_year": input_money_year, + "input_money_unit": "nominal USD", + "source_statistical_baseline": 2015, + "growth": "not applied; employee slope per dollar is bound to this basis", + "sources": list(REQUIRED_INPUTS), + "input_cohort_sha256": cohort_sha, + "rows": n, + "seed": seed, + "randomness": "PCG64 SeedSequence([seed, archived_stream, RECID]); streams 41/42/43/64", + "numpy_version": np.__version__, + "input_knownness": "complete_canonical_not_raw_observation", + "employment_fit": dict(employment_calibration.__dict__), + "applied_positive_receipt_rows": int((revenues > 0).sum()), + "applied_employee_expected_share": float(probability[revenues > 0].mean()) + if (revenues > 0).any() + else 0.0, + "employee_target_weighting": "unweighted positive simulated receipts, archived target 0.18", + "subset_replay": "reuse employment_calibration; refitting a subset changes the model", + "schedule_c": "all-or-nothing SSTB split preserving the signed input total", + "wages_and_ubia": "base leaves are total pools; SSTB leaves are conditional copies", + "output_columns": list(OUTPUTS), + "output_values_sha256": _digest_columns(ids, frozen), + } + receipt["sha256"] = _sha(_json(receipt).encode()) + return QbiModelResult( + MappingProxyType(frozen), employment_calibration, MappingProxyType(receipt) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_raw_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_raw_source.py new file mode 100644 index 000000000..c241ed011 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_raw_source.py @@ -0,0 +1,1726 @@ +"""The first typed artifact read straight from the 2015 PUF delivery bytes. + +One ordinary graph node reads the two accepted parent captures through a +dedicated pinned raw-byte codec and produces one opaque, typed artifact: +``microcosm.us.puf_2015_raw_return_status`` at source-return grain, 207,696 +rows in delivered main-file order. + +What it types is deliberately small. Six of the 222 main columns (``RECID``, +``FLPDYR``, ``FLPDMO``, ``MARS``, ``DSI``, ``S006``) and the six demographic +columns of the companion file. The other 216 main columns are checked for +header name, order and record width and then dropped: no amount is parsed, +no growth factor is admitted, no household or person is invented, and the +node owns no population columns at all. Its host population is whatever the +caller names. + +Three separations are load-bearing. + +*Structure is not provenance.* :func:`decode_puf_raw_source` validates CSV +structure against a definition it is handed. It authorizes nothing. The +production wrapper — :func:`puf_raw_source_codecs` and +:class:`USPufRawSourceKernel` — is what binds the closed packaged pins, and +it refuses any definition whose canonical bytes are not the packaged file's. + +*Fixtures take an explicit non-production route.* :func:`fixture_definition` +builds a definition marked ``route="test_fixture"``, which must carry the +invented-fixture authority and must not restate a packaged source pin. There +is no "expected hash omitted, so admit anything" path anywhere. + +*The private cap is private.* :data:`PUF_RAW_SOURCE_MAX_BYTES` is 128 MiB +because the pinned main delivery is 126,034,649 bytes. +:data:`microcosm.graph.codecs.RAW_BYTES_MAX_BYTES` stays at 64 MiB. + +Period semantics stay unresolved on purpose. The header audit observed +``FLPDYR`` in {2012, 2013, 2014, 2015} while the booklet prints 2011–2014; +the artifact records the raw code and the conflict, and resolves neither. +``S006`` is kept as exact integer hundredths and never divided. + +The four publisher aggregate records stay in the artifact, tagged +``disclosure_aggregate``, never disaggregated. The 88,021 returns with no +demographic record carry ``demographic_status = 0``, the ``-1`` sentinel and +empty lexical columns; absence is never spelled as a zero code. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import stat +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from functools import partial +from importlib import resources +from pathlib import Path +from types import MappingProxyType + +import numpy as np + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + SourceRef, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SourceCodecRegistry, load_source_bytes + +__all__ = [ + "BODY_MAX_BYTES", + "DEMOGRAPHIC_CODEC", + "DEMOGRAPHIC_SOURCE_NAME", + "HEADER_MAX_BYTES", + "MAIN_CODEC", + "MAIN_SOURCE_NAME", + "PUF_AGGREGATE_RECIDS", + "PUF_RAW_SOURCE_MAX_BYTES", + "PUF_RAW_SOURCE_NODE", + "PUF_RAW_SOURCE_STAGE", + "RETURN_STATUS_SUMMARY_NODE", + "RETURN_STATUS_TYPE", + "PufRawReturnStatus", + "PufRawSourceDefinition", + "PufRawSourceRefusalError", + "SourcePin", + "USPufRawReturnStatusSummaryKernel", + "USPufRawSourceKernel", + "csv_acceptance_profile", + "decode_puf_raw_source", + "definition_document_json", + "decode_return_status", + "encode_return_status", + "fixture_definition", + "load_pinned_puf_bytes", + "packaged_definition", + "puf_raw_source_codecs", + "register_us_puf_raw_source_codecs", + "register_us_puf_raw_source_kernels", + "us_puf_raw_return_status_summary_node", + "us_puf_raw_source_node", + "us_puf_raw_source_refs", +] + +#: The most this module reads from one PUF delivery file (128 MiB). Private: +#: the shared ``raw-bytes-v1`` cap is untouched at 64 MiB. +PUF_RAW_SOURCE_MAX_BYTES = 128 * 1024 * 1024 + +#: Bound on the encoded artifact's column bodies, below the 64 MiB a shared +#: opaque payload is expected to stay under. The full slice is ~35 MB. +BODY_MAX_BYTES = 64 * 1024 * 1024 + +#: Bound on the canonical JSON header, as in the ACS rent placement envelope. +HEADER_MAX_BYTES = 64 * 1024 + +RETURN_STATUS_TYPE = ArtifactType("microcosm.us.puf_2015_raw_return_status", 1) +RETURN_STATUS_MAGIC = b"microcosm.us.puf_2015_raw_return_status/1\n" + +MAIN_SOURCE_NAME = "puf_2015_main" +DEMOGRAPHIC_SOURCE_NAME = "puf_2015_demographic" +MAIN_CODEC = "us-puf-2015-main-raw-v1" +DEMOGRAPHIC_CODEC = "us-puf-2015-demographic-raw-v1" + +PUF_RAW_SOURCE_STAGE = "us_puf_raw_source" +PUF_RAW_SOURCE_NODE = f"{PUF_RAW_SOURCE_STAGE}.return_status" +RETURN_STATUS_SUMMARY_NODE = f"{PUF_RAW_SOURCE_STAGE}.return_status_summary" + +#: The publisher's four aggregate record identifiers. The literal values, not +#: an import of ``puf_aggregate_records``: that module pulls nine derivation +#: modules and the calibrate package behind it. +PUF_AGGREGATE_RECIDS = (999996, 999997, 999998, 999999) + +_DEFINITION_RESOURCE = "puf_2015_raw_source_definition.json" +_DEFINITION_PACKAGE = "microcosm.build.us_runtime" +_FIXTURE_AUTHORITY = "invented_fixture_nonauthority" + +_MAIN_FIELDS = ("RECID", "FLPDYR", "FLPDMO", "MARS", "DSI", "S006") +_DEMOGRAPHIC_FIELDS = ( + "AGEDP1", + "AGEDP2", + "AGEDP3", + "AGERANGE", + "EARNSPLIT", + "GENDER", +) +#: Typed column name to its explicit little-endian numpy dtype. Bodies are +#: written with these exact dtypes, so the payload is platform-independent. +_TYPED_DTYPES = { + "RECID": " None: + super().__init__(" ".join((reason, *(str(part) for part in detail))).strip()) + self.reason = reason + + +def _refuse(reason: str, *detail: object) -> PufRawSourceRefusalError: + return PufRawSourceRefusalError(reason, *detail) + + +# -------------------------------------------------------------------------- +# The closed definition +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class SourcePin: + """The closed content identity of one delivered PUF file.""" + + name: str + codec: str + source_name: str + bytes: int + sha256: str + git_blob_sha1: str + header_record_canonical_sha256: str + delivered_header: tuple[str, ...] + data_records: int + + +@dataclass(frozen=True) +class PufRawSourceDefinition: + """A closed source declaration plus the route that produced it.""" + + route: str + document: Mapping[str, object] + canonical: bytes + sha256: str + main: SourcePin + demographic: SourcePin + + @property + def params_text(self) -> str: + """The canonical JSON the node carries verbatim as a parameter.""" + + return self.canonical.decode("ascii") + + +def _hex64(value: object, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise _refuse("DEFINITION_DIGEST", label) + return value + + +def _hex40(value: object, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 40 + or any(character not in "0123456789abcdef" for character in value) + ): + raise _refuse("DEFINITION_DIGEST", label) + return value + + +def _pin(document: Mapping[str, object], name: str) -> SourcePin: + entry = document["sources"][name] + header = entry["delivered_header"] + if not isinstance(header, list) or not header: + raise _refuse("DEFINITION_HEADER", name) + if len(header) != entry["delivered_header_width"]: + raise _refuse("DEFINITION_HEADER_WIDTH", name) + if len(set(header)) != len(header): + raise _refuse("DEFINITION_HEADER_DUPLICATE", name) + if _canonical_header_digest(header) != entry["header_record_canonical_sha256"]: + raise _refuse("DEFINITION_HEADER_DIGEST", name) + size = entry["bytes"] + if not isinstance(size, int) or isinstance(size, bool) or size <= 0: + raise _refuse("DEFINITION_SOURCE_BYTES", name) + if size > PUF_RAW_SOURCE_MAX_BYTES: + raise _refuse("DEFINITION_SOURCE_OVER_CAP", name) + return SourcePin( + name=name, + codec=entry["codec"], + source_name=entry["source_name"], + bytes=size, + sha256=_hex64(entry["sha256"], f"{name}.sha256"), + git_blob_sha1=_hex40(entry["git_blob_sha1"], f"{name}.git_blob_sha1"), + header_record_canonical_sha256=_hex64( + entry["header_record_canonical_sha256"], f"{name}.header" + ), + delivered_header=tuple(header), + data_records=entry["data_records"], + ) + + +def _frozen(value: object) -> object: + """A deeply immutable view: mappings become proxies, sequences tuples. + + ``MappingProxyType`` only freezes the top level. ``definition.document`` + is reached through ordinary read-only attribute access, so handing out a + mutable nested container let a caller change a declared lexical width or + a CSV limit — changing what parses — while ``canonical``, ``sha256`` and + therefore ``params_text`` and the node key all stayed identical. Freezing + the whole tree closes that without any caller sandboxing: the document is + simply a value, as its canonical bytes always were. + """ + + if isinstance(value, Mapping): + return MappingProxyType({key: _frozen(item) for key, item in value.items()}) + if isinstance(value, (list, tuple)): + return tuple(_frozen(item) for item in value) + return value + + +def _canonical_header_digest(header: Sequence[str]) -> str: + """The audit's header identity: sha256 of the canonical JSON header list.""" + + payload = json.dumps( + list(header), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + return hashlib.sha256(payload).hexdigest() + + +def _definition_from_document( + document: Mapping[str, object], *, route: str +) -> PufRawSourceDefinition: + if document.get("schema") != "microcosm.us.puf_2015_raw_source_definition": + raise _refuse("DEFINITION_SCHEMA") + if document.get("schema_version") != 1: + raise _refuse("DEFINITION_SCHEMA_VERSION") + if document.get("route") != route: + raise _refuse("DEFINITION_ROUTE") + fields = document["fields"] + declared = tuple(field["name"] for field in fields) + if declared != _MAIN_FIELDS + _DEMOGRAPHIC_FIELDS: + raise _refuse("DEFINITION_FIELDS") + for field in fields: + width = field["lexical_width"] + if not isinstance(width, int) or isinstance(width, bool) or not 0 < width <= 64: + raise _refuse("DEFINITION_LEXICAL_WIDTH", field["name"]) + if field["grammar"] != "ascii_decimal_unsigned": + raise _refuse("DEFINITION_GRAMMAR", field["name"]) + if tuple(document["aggregates"]["recids"]) != PUF_AGGREGATE_RECIDS: + raise _refuse("DEFINITION_AGGREGATE_RECIDS") + canonical = canonical_json(dict(document)) + # The document is re-derived from the canonical bytes and then deeply + # frozen, so the two can never disagree and neither can be edited. + return PufRawSourceDefinition( + route=route, + document=_frozen(json.loads(canonical.decode("ascii"))), + canonical=canonical, + sha256=hashlib.sha256(canonical).hexdigest(), + main=_pin(document, "main"), + demographic=_pin(document, "demographic"), + ) + + +def _thawed(value: object) -> object: + """The plain-container form of a frozen document, for canonical rendering.""" + + if isinstance(value, Mapping): + return {key: _thawed(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thawed(item) for item in value] + return value + + +def _packaged_bytes() -> bytes: + return ( + resources.files(_DEFINITION_PACKAGE).joinpath(_DEFINITION_RESOURCE).read_bytes() + ) + + +_PACKAGED: PufRawSourceDefinition | None = None + + +def packaged_definition() -> PufRawSourceDefinition: + """The one closed definition a production run may use.""" + + global _PACKAGED + if _PACKAGED is None: + document = json.loads(_packaged_bytes().decode("ascii")) + _PACKAGED = _definition_from_document(document, route="packaged") + return _PACKAGED + + +def definition_document_json( + definition: PufRawSourceDefinition, +) -> dict[str, object]: + """A fresh, plain, mutable JSON copy of a definition's document. + + The definition's own document is deeply frozen. A caller that wants to + build a *different* document from this one — a fixture, say — starts here + and gets a copy it owns, rather than a handle on the original. + """ + + return json.loads(definition.canonical.decode("ascii")) + + +def fixture_definition(document: Mapping[str, object]) -> PufRawSourceDefinition: + """Build a definition on the explicit non-production test route. + + A fixture definition must declare ``route="test_fixture"`` and the + invented-fixture authority, and it may not restate either packaged source + pin. There is no route where an omitted or optional expected hash admits + a file: a fixture carries its own real pins over its own invented bytes. + """ + + definition = _definition_from_document(document, route="test_fixture") + if document.get("authority") != _FIXTURE_AUTHORITY: + raise _refuse("FIXTURE_AUTHORITY") + packaged = packaged_definition() + genuine = {packaged.main.sha256, packaged.demographic.sha256} + if {definition.main.sha256, definition.demographic.sha256} & genuine: + raise _refuse("FIXTURE_RESTATES_PACKAGED_PIN") + return definition + + +# -------------------------------------------------------------------------- +# The pinned raw-byte codec +# -------------------------------------------------------------------------- + + +def _identity(status: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_size, + status.st_mtime_ns, + ) + + +def _git_blob_sha1(payload: bytes) -> str: + header = b"blob " + str(len(payload)).encode("ascii") + b"\0" + return hashlib.sha1(header + payload).hexdigest() + + +def load_pinned_puf_bytes( + path: Path, *, store: object | None = None, pin: SourcePin +) -> bytes: + """Read one pinned PUF delivery file, or refuse. + + The codec authenticates the bytes itself: exact length, SHA-256 and git + blob SHA-1, all three from the descriptor it opened. It never consults a + capture receipt or run summary, so a forged receipt cannot admit changed + bytes. + + ``O_NOFOLLOW`` protects the path *this function receives*. The graph + executor resolves source paths strictly before codec lookup, so under the + executor the codec sees an already-resolved path and a user's symlink is + followed upstream; on a direct call a symlink is refused here. + ``O_NONBLOCK`` means a FIFO cannot make the codec wait. + """ + + del store # identity is content; no content store is consulted + if not isinstance(pin, SourcePin): + raise _refuse("CODEC_PIN_TYPE") + if pin.bytes > PUF_RAW_SOURCE_MAX_BYTES: + raise _refuse("CODEC_PIN_OVER_CAP", pin.name) + source = Path(path) + try: + descriptor = os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + except OSError as error: + raise _refuse("CODEC_OPEN", pin.name) from error + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise _refuse("CODEC_NOT_REGULAR_FILE", pin.name) + if before.st_size != pin.bytes: + raise _refuse("CODEC_SIZE", pin.name) + if _identity(source.lstat()) != _identity(before): + raise _refuse("CODEC_PATH_IDENTITY", pin.name) + digest = hashlib.sha256() + blob = hashlib.sha1(b"blob " + str(pin.bytes).encode("ascii") + b"\0") + chunks: list[bytes] = [] + total = 0 + limit = pin.bytes + 1 + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > limit: + raise _refuse("CODEC_GREW", pin.name) + digest.update(chunk) + blob.update(chunk) + chunks.append(chunk) + if total != pin.bytes: + raise _refuse("CODEC_LENGTH", pin.name) + if _identity(os.fstat(descriptor)) != _identity(before): + raise _refuse("CODEC_CHANGED_DURING_READ", pin.name) + if digest.hexdigest() != pin.sha256: + raise _refuse("CODEC_SHA256", pin.name) + if blob.hexdigest() != pin.git_blob_sha1: + raise _refuse("CODEC_GIT_BLOB_SHA1", pin.name) + finally: + os.close(descriptor) + return b"".join(chunks) + + +def register_us_puf_raw_source_codecs( + registry: SourceCodecRegistry, *, definition: PufRawSourceDefinition | None = None +) -> SourceCodecRegistry: + """Bind the two pinned byte codecs into ``registry`` through ``register_bytes``.""" + + resolved = packaged_definition() if definition is None else definition + registry.register_bytes( + resolved.main.codec, partial(load_pinned_puf_bytes, pin=resolved.main) + ) + registry.register_bytes( + resolved.demographic.codec, + partial(load_pinned_puf_bytes, pin=resolved.demographic), + ) + return registry + + +def puf_raw_source_codecs( + definition: PufRawSourceDefinition | None = None, +) -> SourceCodecRegistry: + """A private registry holding only this slice's two pinned byte codecs.""" + + return register_us_puf_raw_source_codecs( + SourceCodecRegistry(), definition=definition + ) + + +# -------------------------------------------------------------------------- +# The structural decoder +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PufRawReturnStatus: + """The decoded slice: typed arrays, lexical tokens and aggregate facts.""" + + typed: Mapping[str, np.ndarray] + lexical: Mapping[str, tuple[str, ...]] + facts: Mapping[str, object] + + @property + def rows(self) -> int: + return int(self.facts["rows"]) + + +def csv_acceptance_profile() -> dict[str, object]: + """The effective CSV acceptance state this decoder actually parses under. + + ``csv.field_size_limit`` is process-global state that silently changes + which deliveries parse: at a lowered limit a file that was admitted is + refused, and at a raised limit a file that was refused is admitted. It is + therefore part of the producer's identity, exactly as + :mod:`asec_income_observations` and :mod:`acs_housing_universe_source` + already bind it. This function only *reads* it — nothing in this module + ever sets it, so no other stage's acceptance is disturbed. + + The remaining entries are the reader semantics :func:`_reader` fixes + itself; the delimiter, quote character and doublequote flag come from the + closed definition and are checked against it at decode time. + """ + + return { + "engine": "stdlib_csv_reader", + "encoding": "utf-8-strict", + "newline": "", + "quoting": int(csv.QUOTE_MINIMAL), + "strict": True, + "csv.field_size_limit": csv.field_size_limit(), + } + + +def _check_csv_profile(profile: Mapping[str, object]) -> None: + """Refuse when the effective acceptance state is not the pinned one. + + The packaged definition states ``field_size_limit``. Before this check it + was a decorative fact: the decoder relied on the process default happening + to match. Now the pinned value is the value the parse actually ran under, + or the decode refuses — a lowered limit is a refusal, never a silent + reparse, and a raised limit is a refusal, never a silent widening. + """ + + declared = profile["field_size_limit"] + if not isinstance(declared, int) or isinstance(declared, bool) or declared <= 0: + raise _refuse("DEFINITION_CSV_FIELD_SIZE_LIMIT") + if csv.field_size_limit() != declared: + raise _refuse("CSV_FIELD_SIZE_LIMIT") + if profile["quoting"] != "QUOTE_MINIMAL" or profile["strict"] is not True: + raise _refuse("DEFINITION_CSV_PROFILE") + if profile["encoding"] != "utf-8-strict" or profile["newline"] != "": + raise _refuse("DEFINITION_CSV_PROFILE") + + +def _reader(payload: bytes, profile: Mapping[str, object]): + if not isinstance(payload, bytes): + raise _refuse("DECODE_PAYLOAD_TYPE") + stream = io.TextIOWrapper( + io.BytesIO(payload), encoding="utf-8", errors="strict", newline="" + ) + return csv.reader( + stream, + delimiter=profile["delimiter"], + quotechar=profile["quotechar"], + doublequote=bool(profile["doublequote"]), + quoting=csv.QUOTE_MINIMAL, + strict=True, + ) + + +def _token(value: str, *, width: int, label: str) -> str: + """An unsigned ASCII decimal token within its declared allocation.""" + + if not isinstance(value, str): + raise _refuse("TOKEN_TYPE", label) + raw = value.encode("ascii", errors="strict") if value.isascii() else None + if raw is None: + raise _refuse("TOKEN_NON_ASCII", label) + if b"\x00" in raw: + raise _refuse("TOKEN_EMBEDDED_NUL", label) + if not raw: + raise _refuse("TOKEN_EMPTY", label) + if len(raw) > width: + raise _refuse("TOKEN_OVER_WIDTH", label) + if any(byte not in _DIGITS for byte in raw): + raise _refuse("TOKEN_GRAMMAR", label) + return value + + +def _int64(token: str, label: str) -> int: + value = int(token) + if value > _INT64_MAX: + raise _refuse("TOKEN_INT64_OVERFLOW", label) + return value + + +def _widths(definition: PufRawSourceDefinition) -> dict[str, int]: + return { + field["name"]: field["lexical_width"] for field in definition.document["fields"] + } + + +def _domains(definition: PufRawSourceDefinition) -> dict[str, frozenset[int]]: + domains: dict[str, frozenset[int]] = {} + for field in definition.document["fields"]: + values = field.get("domain") + if values is not None: + domains[field["name"]] = frozenset(int(value) for value in values) + return domains + + +def _agerange_domains( + definition: PufRawSourceDefinition, +) -> dict[int, frozenset[int]]: + for field in definition.document["fields"]: + if field["name"] == "AGERANGE": + cases = field["conditional_domain"]["cases"] + return { + int(key): frozenset(int(value) for value in values) + for key, values in cases.items() + } + raise _refuse("DEFINITION_AGERANGE") + + +def _check_header( + reader, pin: SourcePin, *, profile: Mapping[str, object] +) -> list[str]: + try: + header = next(reader) + except StopIteration as error: + raise _refuse("HEADER_MISSING", pin.name) from error + except (csv.Error, UnicodeError) as error: + raise _refuse("HEADER_UNREADABLE", pin.name) from error + if len(header) > profile["header_field_cap"]: + raise _refuse("HEADER_WIDTH_LIMIT", pin.name) + if tuple(header) != pin.delivered_header: + raise _refuse("HEADER_MISMATCH", pin.name) + if _canonical_header_digest(header) != pin.header_record_canonical_sha256: + raise _refuse("HEADER_DIGEST", pin.name) + return header + + +def _records(reader, pin: SourcePin, *, profile: Mapping[str, object]): + width = len(pin.delivered_header) + cap = profile["logical_record_character_cap"] + limit = profile["record_cap_per_file"] + count = 0 + while True: + try: + record = next(reader) + except StopIteration: + return + except (csv.Error, UnicodeError) as error: + raise _refuse("RECORD_UNREADABLE", pin.name) from error + count += 1 + if count > limit: + raise _refuse("RECORD_CAP", pin.name) + if len(record) != width: + raise _refuse("RECORD_WIDTH", pin.name) + if sum(len(cell) for cell in record) > cap: + raise _refuse("RECORD_CHARACTER_CAP", pin.name) + yield record + + +def decode_puf_raw_source( + main_bytes: bytes, + demographic_bytes: bytes, + definition: PufRawSourceDefinition, +) -> PufRawReturnStatus: + """Decode the two delivery payloads into the typed status slice. + + This is structure only. It checks headers, widths, token grammar, closed + domains and the exact lexical join against the definition it is handed, + and it authorizes nothing about where the bytes came from. + """ + + if not isinstance(definition, PufRawSourceDefinition): + raise _refuse("DECODE_DEFINITION_TYPE") + profile = definition.document["csv_profile"] + _check_csv_profile(profile) + widths = _widths(definition) + domains = _domains(definition) + agerange = _agerange_domains(definition) + aggregates = frozenset(PUF_AGGREGATE_RECIDS) + + demographic = _decode_demographic( + demographic_bytes, definition, profile=profile, widths=widths + ) + main = _decode_main( + main_bytes, + definition, + profile=profile, + widths=widths, + domains=domains, + aggregates=aggregates, + ) + + rows = len(main["RECID_lexical"]) + orphans = set(demographic["by_key"]) - set(main["RECID_lexical"]) + if orphans: + raise _refuse("DEMOGRAPHIC_ORPHAN_KEYS", len(orphans)) + + typed: dict[str, np.ndarray] = { + name: np.empty(rows, dtype=_TYPED_DTYPES[name]) for name in _TYPED_COLUMNS + } + for name in _MAIN_FIELDS: + typed[name][:] = np.asarray(main[name], dtype=_TYPED_DTYPES[name]) + typed["disclosure_aggregate"][:] = np.asarray( + main["disclosure_aggregate"], dtype=_TYPED_DTYPES["disclosure_aggregate"] + ) + + lexical: dict[str, list[str]] = { + name: list( + main["RECID_lexical"] if name == "RECID" else main[f"{name}_lexical"] + ) + for name in _MAIN_FIELDS + } + for name in _DEMOGRAPHIC_FIELDS: + lexical[name] = [""] * rows + + status = np.zeros(rows, dtype=_TYPED_DTYPES["demographic_status"]) + for name in _DEMOGRAPHIC_FIELDS: + typed[name][:] = _ABSENT_SENTINEL + + aggregate_demographic_rows = 0 + for index, key in enumerate(main["RECID_lexical"]): + record = demographic["by_key"].get(key) + if record is None: + continue + status[index] = 1 + dsi = int(typed["DSI"][index]) + allowed = agerange.get(dsi) + if allowed is None: + raise _refuse("AGERANGE_CONDITION_UNKNOWN") + for name in _DEMOGRAPHIC_FIELDS: + token = record[name] + value = _int64(token, name) + permitted = allowed if name == "AGERANGE" else domains[name] + if value not in permitted: + raise _refuse("DEMOGRAPHIC_DOMAIN", name) + typed[name][index] = value + lexical[name][index] = token + if bool(typed["disclosure_aggregate"][index]): + aggregate_demographic_rows += 1 + typed["demographic_status"][:] = status + + del aggregate_demographic_rows # re-derived below from the arrays themselves + if demographic["records"] != int(status.sum()): + raise _refuse("DEMOGRAPHIC_RECORD_COUNT") + facts = _facts_from_arrays( + typed, + join_method=_join_method(definition), + untyped_main_columns=_untyped_main_columns(definition), + ) + _check_declared_expectations(facts, definition) + return PufRawReturnStatus( + typed=MappingProxyType({name: array for name, array in typed.items()}), + lexical=MappingProxyType( + {name: tuple(values) for name, values in lexical.items()} + ), + facts=MappingProxyType(facts), + ) + + +def _check_declared_expectations( + facts: Mapping[str, object], definition: PufRawSourceDefinition +) -> None: + """Cross-check the derived counts against the definition's declared ones. + + The packaged definition states how many records each delivery holds, how + many keys should match, how many main records should stay unmatched and + how many aggregate records to expect. Nothing compared them, so they were + documentation. The byte pins make them implied, but implied is not + checked: a delivery that hashed correctly and parsed to a different shape + would have gone unnoticed. + + This runs on the packaged route only. A fixture definition carries its own + invented counts over its own invented bytes, and inventing a fixture that + restates the genuine expectations would be the wrong kind of fidelity. + """ + + if definition.route != "packaged": + return + join = definition.document["join"] + for label, derived, declared in ( + ("rows", facts["rows"], definition.main.data_records), + ( + "demographic_records", + facts["demographic_records"], + definition.demographic.data_records, + ), + ("matched_keys", facts["matched_keys"], join["expected_matched_keys"]), + ( + "main_unmatched_records", + facts["main_unmatched_records"], + join["expected_main_unmatched_records"], + ), + ( + "aggregate_records", + facts["aggregate_records"], + definition.document["aggregates"]["expected_records"], + ), + ): + if derived != declared: + raise _refuse("DECLARED_EXPECTATION", label) + + +def _join_method(definition: PufRawSourceDefinition) -> str: + method = definition.document["join"]["method"] + if not isinstance(method, str) or not method: + raise _refuse("DEFINITION_JOIN_METHOD") + return method + + +def _untyped_main_columns(definition: PufRawSourceDefinition) -> int: + return len(definition.main.delivered_header) - len(_MAIN_FIELDS) + + +def _facts_from_arrays( + typed: Mapping[str, np.ndarray], + *, + join_method: str, + untyped_main_columns: int, +) -> dict[str, object]: + """Derive every reported fact from the typed arrays themselves. + + The producer builds its facts here and the envelope readback rebuilds them + here, so a header can never report a count the arrays do not carry. Only + ``join_method`` and ``untyped_main_columns`` come from outside: both are + properties of the definition, not of the decoded rows. + """ + + rows = int(typed["RECID"].shape[0]) + status = typed["demographic_status"] + aggregate = typed["disclosure_aggregate"] + matched = int(np.count_nonzero(status == 1)) + return { + "rows": rows, + "main_records": rows, + "demographic_records": matched, + "matched_keys": matched, + "main_unmatched_records": rows - matched, + "demographic_orphan_records": 0, + "aggregate_records": int(np.count_nonzero(aggregate == 1)), + "aggregate_demographic_records": int( + np.count_nonzero((aggregate == 1) & (status == 1)) + ), + "join_status": "one_to_one_subset_compatible", + "join_method": join_method, + "code_frequencies": _code_frequencies(typed), + "period_semantics": "unresolved_raw_FLPDYR", + "weight_units": "S006_hundredths", + "amounts_decoded": False, + "untyped_main_columns": untyped_main_columns, + } + + +def _code_frequencies(typed: Mapping[str, np.ndarray]) -> dict[str, dict[str, int]]: + counted = ("FLPDYR", "FLPDMO", "MARS", "DSI", "demographic_status") + frequencies: dict[str, dict[str, int]] = {} + for name in (*counted, *_DEMOGRAPHIC_FIELDS): + values, counts = np.unique(typed[name], return_counts=True) + frequencies[name] = { + str(int(value)): int(count) + for value, count in zip(values.tolist(), counts.tolist(), strict=True) + } + return frequencies + + +def _decode_main( + payload: bytes, + definition: PufRawSourceDefinition, + *, + profile, + widths, + domains, + aggregates, +) -> dict[str, object]: + pin = definition.main + reader = _reader(payload, profile) + header = _check_header(reader, pin, profile=profile) + index = {name: header.index(name) for name in _MAIN_FIELDS} + columns: dict[str, list] = {name: [] for name in _MAIN_FIELDS} + lexical: dict[str, list[str]] = {f"{name}_lexical": [] for name in _MAIN_FIELDS} + keys: list[str] = [] + numeric_keys: set[int] = set() + seen: set[str] = set() + aggregate_flags: list[int] = [] + for record in _records(reader, pin, profile=profile): + values: dict[str, int] = {} + for name in _MAIN_FIELDS: + token = _token(record[index[name]], width=widths[name], label=name) + lexical[f"{name}_lexical"].append(token) + values[name] = _int64(token, name) + key = lexical["RECID_lexical"][-1] + if key in seen: + raise _refuse("MAIN_DUPLICATE_LEXICAL_KEY") + seen.add(key) + recid = values["RECID"] + if recid in numeric_keys: + raise _refuse("MAIN_NUMERIC_KEY_COLLISION") + numeric_keys.add(recid) + keys.append(key) + for name in ("FLPDYR", "FLPDMO", "MARS", "DSI"): + if values[name] not in domains[name]: + raise _refuse("MAIN_DOMAIN", name) + aggregate = recid in aggregates + if aggregate != (values["MARS"] == 0): + raise _refuse("AGGREGATE_MARS_DISAGREEMENT") + aggregate_flags.append(1 if aggregate else 0) + for name in _MAIN_FIELDS: + columns[name].append(values[name]) + if not keys: + raise _refuse("MAIN_NO_RECORDS") + result: dict[str, object] = dict(columns) + result.update(lexical) + result["disclosure_aggregate"] = aggregate_flags + return result + + +def _decode_demographic( + payload: bytes, definition: PufRawSourceDefinition, *, profile, widths +) -> dict[str, object]: + pin = definition.demographic + reader = _reader(payload, profile) + header = _check_header(reader, pin, profile=profile) + index = {name: header.index(name) for name in ("RECID", *_DEMOGRAPHIC_FIELDS)} + by_key: dict[str, dict[str, str]] = {} + numeric_keys: set[int] = set() + records = 0 + for record in _records(reader, pin, profile=profile): + key = _token(record[index["RECID"]], width=widths["RECID"], label="RECID") + if key in by_key: + raise _refuse("DEMOGRAPHIC_DUPLICATE_LEXICAL_KEY") + recid = _int64(key, "RECID") + if recid in numeric_keys: + raise _refuse("DEMOGRAPHIC_NUMERIC_KEY_COLLISION") + numeric_keys.add(recid) + by_key[key] = { + name: _token(record[index[name]], width=widths[name], label=name) + for name in _DEMOGRAPHIC_FIELDS + } + records += 1 + return {"by_key": by_key, "records": records} + + +# -------------------------------------------------------------------------- +# The typed envelope +# -------------------------------------------------------------------------- + + +def _fixed_ascii(values: Sequence[str], width: int, label: str) -> bytes: + body = bytearray(len(values) * width) + for position, value in enumerate(values): + raw = value.encode("ascii") + if len(raw) > width: + raise _refuse("LEXICAL_OVER_WIDTH", label) + if b"\x00" in raw: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + start = position * width + body[start : start + len(raw)] = raw + return bytes(body) + + +def _read_fixed_ascii( + body: bytes, rows: int, width: int, label: str +) -> tuple[str, ...]: + values = [] + for position in range(rows): + cell = body[position * width : (position + 1) * width] + stripped = cell.rstrip(b"\x00") + if b"\x00" in stripped: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + try: + values.append(stripped.decode("ascii")) + except UnicodeDecodeError as error: + raise _refuse("LEXICAL_NON_ASCII", label) from error + return tuple(values) + + +def _payload_bound( + rows: int, definition: PufRawSourceDefinition +) -> tuple[int, dict[str, int]]: + """The exact body size this artifact will allocate, and its per-column parts.""" + + widths = _widths(definition) + sizes = { + name: rows * np.dtype(_TYPED_DTYPES[name]).itemsize for name in _TYPED_COLUMNS + } + sizes.update({f"{name}_lexical": rows * widths[name] for name in _LEXICAL_COLUMNS}) + return sum(sizes.values()), sizes + + +def encode_return_status( + status: PufRawReturnStatus, definition: PufRawSourceDefinition +) -> bytes: + """Encode the slice into the bounded, self-describing artifact envelope. + + Magic line, a 4-byte big-endian header length matching the ACS rent + placement precedent, a canonical JSON header, then explicit little-endian + integer bodies and fixed-width NUL-padded ASCII lexical bodies, in the + declared order. Every body carries its own SHA-256 in the header. + """ + + rows = status.rows + total, sizes = _payload_bound(rows, definition) + if total > BODY_MAX_BYTES: + raise _refuse("BODY_OVER_BOUND", total) + widths = _widths(definition) + bodies: list[bytes] = [] + columns: list[dict[str, object]] = [] + for name in _TYPED_COLUMNS: + array = np.ascontiguousarray(status.typed[name], dtype=_TYPED_DTYPES[name]) + if array.shape != (rows,): + raise _refuse("COLUMN_SHAPE", name) + body = array.tobytes() + if len(body) != sizes[name]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + columns.append( + { + "name": name, + "kind": "typed", + "dtype": _TYPED_DTYPES[name], + "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + } + ) + for name in _LEXICAL_COLUMNS: + body = _fixed_ascii(status.lexical[name], widths[name], name) + if len(body) != sizes[f"{name}_lexical"]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + columns.append( + { + "name": f"{name}_lexical", + "kind": "lexical_fixed_ascii_nul_padded", + "width": widths[name], + "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + } + ) + header = { + "schema_version": 1, + "type": [RETURN_STATUS_TYPE.name, RETURN_STATUS_TYPE.schema_version], + "rows": rows, + "definition_sha256": definition.sha256, + "definition_route": definition.route, + "sources": { + "main": { + "sha256": definition.main.sha256, + "git_blob_sha1": definition.main.git_blob_sha1, + "bytes": definition.main.bytes, + "header_record_canonical_sha256": ( + definition.main.header_record_canonical_sha256 + ), + }, + "demographic": { + "sha256": definition.demographic.sha256, + "git_blob_sha1": definition.demographic.git_blob_sha1, + "bytes": definition.demographic.bytes, + "header_record_canonical_sha256": ( + definition.demographic.header_record_canonical_sha256 + ), + }, + }, + "header_length_endianness": "big", + "body_endianness": "little", + "columns": columns, + "facts": dict(status.facts), + } + encoded = canonical_json(header) + if len(encoded) > HEADER_MAX_BYTES: + raise _refuse("HEADER_OVER_BOUND", len(encoded)) + return b"".join( + (RETURN_STATUS_MAGIC, len(encoded).to_bytes(4, "big"), encoded, *bodies) + ) + + +def _exact_keys( + value: object, keys: frozenset[str], label: str +) -> Mapping[str, object]: + """A mapping whose key set is exactly ``keys``. Nothing extra, nothing missing.""" + + if not isinstance(value, dict) or set(value) != set(keys): + raise _refuse("ENVELOPE_SCHEMA", label) + return value + + +def _envelope_int(value: object, label: str, *, minimum: int = 0) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise _refuse("ENVELOPE_INTEGER", label) + return value + + +def _resolve_envelope_definition( + header: Mapping[str, object], definition: PufRawSourceDefinition | None +) -> PufRawSourceDefinition | None: + """The definition this payload may be checked against, or ``None``. + + A caller may hand one in. Otherwise a payload that declares the *packaged* + route is checked against this module's own packaged definition, because + that one is closed and importable. A fixture-route payload has no + reconstructible definition, so it is checked structurally only: the + production and fixture boundaries stay exactly where they are. + """ + + route = header["definition_route"] + if route not in ("packaged", "test_fixture"): + raise _refuse("ENVELOPE_DEFINITION_ROUTE") + digest = _hex64(header["definition_sha256"], "envelope.definition_sha256") + if definition is not None: + if not isinstance(definition, PufRawSourceDefinition): + raise _refuse("DECODE_DEFINITION_TYPE") + if definition.route != route or definition.sha256 != digest: + raise _refuse("ENVELOPE_DEFINITION_MISMATCH") + return definition + if route == "packaged": + packaged = packaged_definition() + if packaged.sha256 != digest: + raise _refuse("ENVELOPE_DEFINITION_NOT_PACKAGED") + return packaged + return None + + +def _check_envelope_sources( + header: Mapping[str, object], definition: PufRawSourceDefinition | None +) -> None: + sources = _exact_keys( + header["sources"], frozenset({"main", "demographic"}), "sources" + ) + for name in ("main", "demographic"): + entry = _exact_keys(sources[name], _ENVELOPE_SOURCE_KEYS, f"sources.{name}") + _hex64(entry["sha256"], f"sources.{name}.sha256") + _hex40(entry["git_blob_sha1"], f"sources.{name}.git_blob_sha1") + _hex64(entry["header_record_canonical_sha256"], f"sources.{name}.header") + size = _envelope_int(entry["bytes"], f"sources.{name}.bytes", minimum=1) + if size > PUF_RAW_SOURCE_MAX_BYTES: + raise _refuse("ENVELOPE_SOURCE_OVER_CAP", name) + if definition is None: + continue + pin = definition.main if name == "main" else definition.demographic + if ( + entry["sha256"] != pin.sha256 + or entry["git_blob_sha1"] != pin.git_blob_sha1 + or size != pin.bytes + or entry["header_record_canonical_sha256"] + != pin.header_record_canonical_sha256 + ): + raise _refuse("ENVELOPE_SOURCE_PIN", name) + + +def _envelope_columns( + header: Mapping[str, object], + rows: int, + definition: PufRawSourceDefinition | None, +) -> tuple[list[Mapping[str, object]], dict[str, int]]: + """Close the column schema: exact order, kind, dtype, width and size.""" + + columns = header["columns"] + if not isinstance(columns, list): + raise _refuse("ENVELOPE_COLUMNS") + declared = [ + *_TYPED_COLUMNS, + *(f"{name}_lexical" for name in _LEXICAL_COLUMNS), + ] + if len(columns) != len(declared): + raise _refuse("ENVELOPE_COLUMNS") + widths = _widths(definition) if definition is not None else None + resolved: list[Mapping[str, object]] = [] + sizes: dict[str, int] = {} + total = 0 + for column, name in zip(columns, declared, strict=True): + if not isinstance(column, dict) or column.get("name") != name: + raise _refuse("ENVELOPE_COLUMNS") + if name in _TYPED_DTYPES: + entry = _exact_keys(column, _ENVELOPE_TYPED_COLUMN_KEYS, name) + if entry["kind"] != _TYPED_KIND: + raise _refuse("ENVELOPE_COLUMN_KIND", name) + # Exact dtype string: this closes width, signedness and endianness + # at once, so `` _LEXICAL_WIDTH_MAX: + raise _refuse("ENVELOPE_COLUMN_WIDTH", name) + if widths is not None and width != widths[field]: + raise _refuse("ENVELOPE_COLUMN_WIDTH", name) + expected = rows * width + sizes[field] = width + _hex64(entry["sha256"], f"{name}.sha256") + size = _envelope_int(entry["bytes"], f"{name}.bytes") + if size != expected: + raise _refuse("ENVELOPE_COLUMN_SIZE", name) + total += size + if total > BODY_MAX_BYTES: + raise _refuse("ENVELOPE_BODY_OVER_BOUND", total) + resolved.append(entry) + return resolved, sizes + + +def _verify_readback( + typed: Mapping[str, np.ndarray], + lexical: Mapping[str, tuple[str, ...]], + widths: Mapping[str, int], + definition: PufRawSourceDefinition | None, +) -> None: + """Re-prove the slice's own invariants from the arrays that were read back. + + Lexical and numeric spellings must agree, absence must stay absence rather + than a zero code, the aggregate flag must follow from ``RECID`` and agree + with ``MARS``, and the join keys must still be unique both ways. + """ + + rows = int(typed["RECID"].shape[0]) + status = typed["demographic_status"] + aggregate = typed["disclosure_aggregate"] + if not np.isin(status, (0, 1)).all(): + raise _refuse("READBACK_DEMOGRAPHIC_STATUS") + if not np.isin(aggregate, (0, 1)).all(): + raise _refuse("READBACK_AGGREGATE_FLAG") + if int(typed["S006"].min(initial=0)) < 0: + raise _refuse("READBACK_S006_NEGATIVE") + + present = status == 1 + for name in _MAIN_FIELDS: + tokens = lexical[name] + if len(tokens) != rows: + raise _refuse("READBACK_LEXICAL_ROWS", name) + width = widths[name] + values = typed[name] + for position, token in enumerate(tokens): + _token(token, width=width, label=name) + if int(token) != int(values[position]): + raise _refuse("READBACK_LEXICAL_DISAGREEMENT", name) + for name in _DEMOGRAPHIC_FIELDS: + tokens = lexical[name] + if len(tokens) != rows: + raise _refuse("READBACK_LEXICAL_ROWS", name) + width = widths[name] + values = typed[name] + for position, token in enumerate(tokens): + if not present[position]: + # Absence is the sentinel and an empty spelling, never a zero. + if token or int(values[position]) != _ABSENT_SENTINEL: + raise _refuse("READBACK_ABSENCE_NOT_EMPTY", name) + continue + _token(token, width=width, label=name) + if int(token) != int(values[position]): + raise _refuse("READBACK_LEXICAL_DISAGREEMENT", name) + + keys = lexical["RECID"] + if len(set(keys)) != rows: + raise _refuse("READBACK_DUPLICATE_LEXICAL_KEY") + recids = typed["RECID"] + if len(np.unique(recids)) != rows: + raise _refuse("READBACK_NUMERIC_KEY_COLLISION") + expected_aggregate = np.isin(recids, np.asarray(PUF_AGGREGATE_RECIDS, dtype=" PufRawReturnStatus: + """Decode an artifact payload and re-prove it, never trusting its header. + + The header is a closed schema: exact key sets, an exact dtype string for + every typed column, an exact width and body size for every lexical column, + and a total body bound. Every reported fact is then **recomputed** from the + arrays actually read back and must agree exactly, so a payload cannot + report a row count, a match count or a code frequency it does not carry. + + This stays structural. It is not source authority: it proves that a payload + is internally consistent and, when a definition is available, that it + matches that definition's widths, domains and source pins. It authorizes + nothing about where the delivery bytes came from — the codec does that. + """ + + if not isinstance(payload, bytes) or not payload.startswith(RETURN_STATUS_MAGIC): + raise _refuse("ENVELOPE_MAGIC") + start = len(RETURN_STATUS_MAGIC) + if len(payload) < start + 4: + raise _refuse("ENVELOPE_TRUNCATED") + length = int.from_bytes(payload[start : start + 4], "big") + if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: + raise _refuse("ENVELOPE_HEADER_LENGTH") + encoded = payload[start + 4 : start + 4 + length] + try: + header = json.loads(encoded.decode("ascii")) + except (UnicodeError, ValueError, RecursionError) as error: + # A deeply nested or malformed header is a refusal, not a stray + # ``RecursionError`` or ``UnicodeDecodeError`` out of the JSON reader. + raise _refuse("ENVELOPE_HEADER_JSON") from error + try: + recanonical = canonical_json(header) + except (TypeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + if recanonical != encoded: + raise _refuse("ENVELOPE_HEADER_NOT_CANONICAL") + _exact_keys(header, _ENVELOPE_HEADER_KEYS, "header") + if header["schema_version"] != 1 or header["type"] != [ + RETURN_STATUS_TYPE.name, + RETURN_STATUS_TYPE.schema_version, + ]: + raise _refuse("ENVELOPE_TYPE") + if ( + header["header_length_endianness"] != "big" + or header["body_endianness"] != "little" + ): + raise _refuse("ENVELOPE_ENDIANNESS") + rows = _envelope_int(header["rows"], "rows", minimum=1) + resolved = _resolve_envelope_definition(header, definition) + _check_envelope_sources(header, resolved) + columns, widths = _envelope_columns(header, rows, resolved) + + cursor = start + 4 + length + typed: dict[str, np.ndarray] = {} + lexical: dict[str, tuple[str, ...]] = {} + for column in columns: + name = column["name"] + size = column["bytes"] + body = payload[cursor : cursor + size] + if len(body) != size: + raise _refuse("ENVELOPE_BODY_TRUNCATED", name) + if hashlib.sha256(body).hexdigest() != column["sha256"]: + raise _refuse("ENVELOPE_BODY_DIGEST", name) + cursor += size + if column["kind"] == _TYPED_KIND: + typed[name] = np.frombuffer(body, dtype=np.dtype(column["dtype"])) + else: + field = name.removesuffix("_lexical") + lexical[field] = _read_fixed_ascii(body, rows, column["width"], field) + if cursor != len(payload): + raise _refuse("ENVELOPE_TRAILING_BYTES") + + _verify_readback(typed, lexical, widths, resolved) + facts = header["facts"] + _exact_keys(facts, _ENVELOPE_FACT_KEYS, "facts") + if resolved is not None: + join_method = _join_method(resolved) + untyped_main_columns = _untyped_main_columns(resolved) + else: + join_method = facts["join_method"] + if not isinstance(join_method, str) or not join_method: + raise _refuse("ENVELOPE_FACTS", "join_method") + untyped_main_columns = _envelope_int( + facts["untyped_main_columns"], "facts.untyped_main_columns" + ) + recomputed = _facts_from_arrays( + typed, + join_method=join_method, + untyped_main_columns=untyped_main_columns, + ) + if canonical_json(recomputed) != canonical_json(dict(facts)): + raise _refuse("ENVELOPE_FACTS_INCONSISTENT") + if recomputed["rows"] != rows: + raise _refuse("ENVELOPE_ROWS") + if resolved is not None: + _check_declared_expectations(recomputed, resolved) + return PufRawReturnStatus( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(recomputed), + ) + + +# -------------------------------------------------------------------------- +# The graph producer, consumer and registry +# -------------------------------------------------------------------------- + +_IMPLEMENTATION_MODULES = ("microcosm.build.us_runtime.puf_raw_source",) +_IMPLEMENTATION_IDENTITY_MAGIC = b"us.puf.raw_source/implementation/2\n" + + +class USPufRawSourceKernel(KernelBase): + """Read the two pinned deliveries and produce one typed status artifact.""" + + ref = "us.puf.raw_source.return_status@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__( + self, + *, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, + ) -> None: + self.definition = packaged_definition() if definition is None else definition + if self.definition.route not in ("packaged", "test_fixture"): + raise _refuse("KERNEL_DEFINITION_ROUTE") + if self.definition.route == "packaged": + # The production wrapper, not the bare decoder, is the authority: + # the definition must be the packaged file's own canonical bytes. + # Compare the bytes, not only their digest, and re-prove that the + # document this definition carries is still those exact bytes. + packaged = packaged_definition() + if ( + self.definition.canonical != packaged.canonical + or self.definition.sha256 != packaged.sha256 + or canonical_json(_thawed(self.definition.document)) + != self.definition.canonical + ): + raise _refuse("KERNEL_DEFINITION_NOT_PACKAGED") + self.source_codecs = ( + puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self) -> str: + """Module source, dependency versions **and** the effective CSV profile. + + The acceptance profile is process state the parse runs under, so it + belongs in the producer's identity: change it and this node's key + moves, which is a recompute or a ``require`` miss. It can no longer + reuse an admission made under a different acceptance. + """ + + from importlib import import_module + + base = source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + digest = hashlib.sha256(_IMPLEMENTATION_IDENTITY_MAGIC) + digest.update(base.encode("ascii")) + digest.update(b"\0") + digest.update(canonical_json(csv_acceptance_profile())) + return digest.hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + definition = self.definition + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources + != (definition.main.source_name, definition.demographic.source_name) + or context.node.inputs + or context.node.outputs + or context.node.artifact_inputs + or context.node.artifact_outputs + != (ArtifactOutput("return_status", RETURN_STATUS_TYPE),) + or set(context.params) != {"definition"} + ): + raise _refuse("NODE_DECLARATION") + if context.params["definition"] != definition.params_text: + raise _refuse("NODE_DEFINITION_PARAM") + main_bytes = load_source_bytes( + definition.main.codec, + context.sources[definition.main.source_name], + registry=self.source_codecs, + ) + demographic_bytes = load_source_bytes( + definition.demographic.codec, + context.sources[definition.demographic.source_name], + registry=self.source_codecs, + ) + status = decode_puf_raw_source(main_bytes, demographic_bytes, definition) + payload = encode_return_status(status, definition) + return KernelResult( + artifacts={"return_status": payload}, + receipt={ + "puf_raw_return_status": { + "definition_sha256": definition.sha256, + "definition_route": definition.route, + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "main_sha256": definition.main.sha256, + "demographic_sha256": definition.demographic.sha256, + "csv_acceptance_profile": csv_acceptance_profile(), + **{ + key: value + for key, value in status.facts.items() + if key != "code_frequencies" + }, + } + }, + ) + + +class USPufRawReturnStatusSummaryKernel(KernelBase): + """A downstream consumer: decode the typed artifact and report counts.""" + + ref = "us.puf.raw_source.return_status_summary@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def implementation_hash(self) -> str: + from importlib import import_module + + return source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + + def run(self, context: KernelContext) -> KernelResult: + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources + or context.node.inputs + or context.node.outputs + or context.node.artifact_outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != RETURN_STATUS_TYPE + or context.params + ): + raise _refuse("SUMMARY_NODE_DECLARATION") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != RETURN_STATUS_TYPE: + raise _refuse("SUMMARY_ARTIFACT_TYPE") + status = decode_return_status(value.payload) + return KernelResult( + receipt={ + "puf_raw_return_status_summary": { + "producer_key": value.producer_key, + "rows": status.rows, + "matched_keys": status.facts["matched_keys"], + "main_unmatched_records": status.facts["main_unmatched_records"], + "aggregate_records": status.facts["aggregate_records"], + "period_semantics": status.facts["period_semantics"], + "weight_units": status.facts["weight_units"], + "amounts_decoded": status.facts["amounts_decoded"], + } + } + ) + + +def us_puf_raw_source_refs( + definition: PufRawSourceDefinition | None = None, +) -> tuple[SourceRef, ...]: + """The two ``SourceRef`` declarations this slice's node reads.""" + + resolved = packaged_definition() if definition is None else definition + return ( + SourceRef( + resolved.main.source_name, + resolved.main.codec, + description="2015 PUF main delivery, pinned by content.", + ), + SourceRef( + resolved.demographic.source_name, + resolved.demographic.codec, + description="2015 PUF demographic delivery, pinned by content.", + ), + ) + + +def us_puf_raw_source_node( + *, + population: str, + definition: PufRawSourceDefinition | None = None, + stage: str = PUF_RAW_SOURCE_STAGE, +) -> Node: + """Declare the producer node inside an explicitly named host population.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + resolved = packaged_definition() if definition is None else definition + return Node( + f"{stage}.return_status", + USPufRawSourceKernel.ref, + population=population, + sources=(resolved.main.source_name, resolved.demographic.source_name), + params={"definition": resolved.params_text}, + artifact_outputs=(ArtifactOutput("return_status", RETURN_STATUS_TYPE),), + description=( + "Type the 2015 PUF return status slice from the pinned delivery " + "bytes. Owns no population columns." + ), + ) + + +def us_puf_raw_return_status_summary_node( + *, + population: str, + producer: str = PUF_RAW_SOURCE_NODE, + stage: str = PUF_RAW_SOURCE_STAGE, +) -> Node: + """Declare an ordinary consumer of the typed artifact.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + return Node( + f"{stage}.return_status_summary", + USPufRawReturnStatusSummaryKernel.ref, + population=population, + artifact_inputs=( + ArtifactInput( + "return_status", producer, "return_status", RETURN_STATUS_TYPE + ), + ), + description="Read the typed PUF status artifact and report its counts.", + ) + + +def register_us_puf_raw_source_kernels( + registry: KernelRegistry, + *, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, +) -> KernelRegistry: + """Register the producer and consumer kernels into ``registry``.""" + + registry.register( + USPufRawSourceKernel(definition=definition, source_codecs=source_codecs) + ) + registry.register(USPufRawReturnStatusSummaryKernel()) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py index d78c70a23..77178b006 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py @@ -29,13 +29,9 @@ puf_capital_gains_joint_metrics, puf_processed_capital_gains_stage, ) -from microcosm.build.us_runtime.puf_interest_components import ( - split_us_puf_e19200_by_agi_band, -) from microcosm.build.us_runtime.qbi_inputs import ( US_QBI_BOOLEAN_OUTPUT_COLUMNS, US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, - US_QBI_OUTPUT_COLUMNS, ) from microcosm.build.us_runtime.support_provenance import ( BASE_ASEC_SUPPORT_CHANNEL, @@ -54,6 +50,19 @@ from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights, wquantile from microcosm.frame.schema import EntitySchema +from .operator_column_contracts import ( + PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID as PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID, +) +from .operator_column_contracts import ( + PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS as PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, +) +from .operator_column_contracts import ( + PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS as PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, +) +from .operator_column_contracts import ( + US_PUF_SUPPORT_STAGE_NAME as US_PUF_SUPPORT_STAGE_NAME, +) + QRF: Any | None = None __all__ = [ @@ -91,7 +100,6 @@ "validate_puf_clone_attachment", ] -US_PUF_SUPPORT_STAGE_NAME = "puf_support_channel" #: Frozen receipt binding a seeded clone attachment (microcosm#578 revision #: item 3) to the live rows: fraction, seed, the floor-rule counts, and the @@ -207,54 +215,6 @@ class _PredictorSourcePlan: "puf_predictor_long_term_capital_gains", ) -PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS = ( - "employment_income_before_lsr", - "self_employment_income_before_lsr", - "taxable_interest_income", - "qualified_dividend_income", - "non_qualified_dividend_income", - "tax_exempt_interest_income", - "short_term_capital_gains", - "long_term_capital_gains_before_response", - "long_term_capital_gains_on_collectibles", - "non_sch_d_capital_gains", - "taxable_private_pension_income", - "taxable_ira_distributions", - "social_security_retirement", - "social_security_disability", - "social_security_dependents", - "social_security_survivors", - "alimony_income", - "alimony_expense", - "salt_refund_income", - "charitable_cash_donations", - "charitable_non_cash_donations", - "real_estate_taxes", - "home_mortgage_interest", - "investment_interest_expense", - "investment_income_elected_form_4952", - "student_loan_interest", - "educator_expense", - "qualified_tuition_expenses", - "casualty_loss", - "unreimbursed_business_employee_expenses", - # The engine owns the realized contribution amounts through the - # IRA-limit scale and self-employment caps; the persistable leaves are - # the desired contributions, equal to the PUF's observed deductions at - # baseline (issue #278). - "traditional_ira_contributions_desired", - "self_employed_pension_contributions_desired", - "rental_income", - "estate_income", - "farm_income", - "farm_operations_income", - "farm_rent_income", - "miscellaneous_income", - "partnership_income", - "s_corp_income", - "partnership_self_employment_net_earnings", - *US_QBI_OUTPUT_COLUMNS, -) PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS = ( "social_security_retirement", @@ -263,17 +223,6 @@ class _PredictorSourcePlan: "social_security_survivors", ) -PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS: tuple[str, ...] = ( - "domestic_production_ald", - "unrecaptured_section_1250_gain", - "first_home_mortgage_balance", - "second_home_mortgage_balance", - "first_home_mortgage_interest", - "second_home_mortgage_interest", - "first_home_mortgage_origination_year", - "second_home_mortgage_origination_year", - "health_savings_account_ald", -) _PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS = frozenset( { @@ -1505,6 +1454,16 @@ def _quarantine_us_puf_mortgage_fields( donor_build_summary["mortgage_field_quarantine"] = quarantine +def split_us_puf_e19200_by_agi_band( + total_interest_paid: Any, + adjusted_gross_income: Any, +) -> tuple[np.ndarray, np.ndarray]: + """Load the published component resource only for actual decomposition.""" + from .puf_interest_components import split_us_puf_e19200_by_agi_band as split + + return split(total_interest_paid, adjusted_gross_income) + + def _split_us_puf_e19200_components(donor: pd.DataFrame) -> None: """Split raw E19200 into mortgage and modeled non-mortgage components.""" @@ -2368,7 +2327,6 @@ def _id_multiplier_for_frame(frame: Frame) -> int: # indices up to 921 before int64 overflow — orders beyond any configured # clone count. Assembly enforces this bound; _remap_ids re-checks it so a # violation is a governed ValueError, never an OverflowError. -PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID = 10**15 - 1 _INT64_MAX = 2**63 - 1 @@ -2563,7 +2521,7 @@ def _formula_owned_engine() -> Any | None: except ImportError: return None try: - return PolicyEngineUSVariableMetadataIndex() + return PolicyEngineUSVariableMetadataIndex(include_consumers=False) except ImportError: return None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_target2024_growth.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_target2024_growth.py new file mode 100644 index 000000000..f68c0e1e0 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_target2024_growth.py @@ -0,0 +1,164 @@ +"""Source-specific, explicit PUF2015 to target2024 monetary transport. + +This is a data model. It computes no tax, deduction limit, benefit eligibility +or recipient membership. Receipt knownness is separate from raw observation. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +# Generated below from the reviewable, source-pinned GROWTH-RECIPE.json. +_RECIPE_JSON = '{\n "schema": "microcosm.us.puf2015_target2024_growth/1",\n "input_money_year": 2015,\n "output_money_year": 2024,\n "output_profile": "puf59_scf_mortgage_v1",\n "ordered_targets": [\n "employment_income_before_lsr",\n "self_employment_income_before_lsr",\n "taxable_interest_income",\n "qualified_dividend_income",\n "non_qualified_dividend_income",\n "tax_exempt_interest_income",\n "short_term_capital_gains",\n "long_term_capital_gains_before_response",\n "long_term_capital_gains_on_collectibles",\n "non_sch_d_capital_gains",\n "taxable_private_pension_income",\n "taxable_ira_distributions",\n "social_security_retirement",\n "social_security_disability",\n "social_security_dependents",\n "social_security_survivors",\n "alimony_income",\n "alimony_expense",\n "salt_refund_income",\n "charitable_cash_donations",\n "charitable_non_cash_donations",\n "real_estate_taxes",\n "home_mortgage_interest",\n "investment_interest_expense",\n "investment_income_elected_form_4952",\n "student_loan_interest",\n "educator_expense",\n "qualified_tuition_expenses",\n "casualty_loss",\n "unreimbursed_business_employee_expenses",\n "traditional_ira_contributions_desired",\n "self_employed_pension_contributions_desired",\n "rental_income",\n "estate_income",\n "farm_income",\n "farm_operations_income",\n "farm_rent_income",\n "miscellaneous_income",\n "partnership_income",\n "s_corp_income",\n "partnership_self_employment_net_earnings",\n "estate_income_would_be_qualified",\n "farm_operations_income_would_be_qualified",\n "farm_rent_income_would_be_qualified",\n "partnership_s_corp_income_would_be_qualified",\n "rental_income_would_be_qualified",\n "self_employment_income_would_be_qualified",\n "sstb_self_employment_income_would_be_qualified",\n "business_is_sstb",\n "qualified_bdc_income",\n "qualified_reit_and_ptp_income",\n "sstb_self_employment_income_before_lsr",\n "sstb_unadjusted_basis_qualified_property",\n "sstb_w2_wages_from_qualified_business",\n "unadjusted_basis_qualified_property",\n "w2_wages_from_qualified_business",\n "domestic_production_ald",\n "unrecaptured_section_1250_gain",\n "health_savings_account_ald"\n ],\n "money_fields": {\n "employment_income_before_lsr": {\n "positive_family": "awi",\n "negative_family": "awi",\n "positive_factor": "1.452153003110483604210764423019948801037",\n "negative_factor": "1.452153003110483604210764423019948801037",\n "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index."\n },\n "w2_wages_from_qualified_business": {\n "positive_family": "awi",\n "negative_family": "awi",\n "positive_factor": "1.452153003110483604210764423019948801037",\n "negative_factor": "1.452153003110483604210764423019948801037",\n "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index."\n },\n "sstb_w2_wages_from_qualified_business": {\n "positive_family": "awi",\n "negative_family": "awi",\n "positive_factor": "1.452153003110483604210764423019948801037",\n "negative_factor": "1.452153003110483604210764423019948801037",\n "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index."\n },\n "self_employment_income_before_lsr": {\n "positive_family": "business_profit",\n "negative_family": "business_loss",\n "positive_factor": "1.222517420922031096142334753675412822692",\n "negative_factor": "1.718898116890934040328182671660914217539",\n "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition."\n },\n "sstb_self_employment_income_before_lsr": {\n "positive_family": "business_profit",\n "negative_family": "business_loss",\n "positive_factor": "1.222517420922031096142334753675412822692",\n "negative_factor": "1.718898116890934040328182671660914217539",\n "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition."\n },\n "taxable_interest_income": {\n "positive_family": "taxable_interest",\n "negative_family": "taxable_interest",\n "positive_factor": "2.599750625636417399355437523686408804619",\n "negative_factor": "2.599750625636417399355437523686408804619",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "tax_exempt_interest_income": {\n "positive_family": "tax_exempt_interest",\n "negative_family": "tax_exempt_interest",\n "positive_factor": "0.9295395824924337172788943063597248917385",\n "negative_factor": "0.9295395824924337172788943063597248917385",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "qualified_dividend_income": {\n "positive_family": "qualified_dividends",\n "negative_family": "qualified_dividends",\n "positive_factor": "1.409891045424454325551259680558160347719",\n "negative_factor": "1.409891045424454325551259680558160347719",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "non_qualified_dividend_income": {\n "positive_family": "nonqualified_dividends",\n "negative_family": "nonqualified_dividends",\n "positive_factor": "2.483795459689869824933006427197633652403",\n "negative_factor": "2.483795459689869824933006427197633652403",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "non_sch_d_capital_gains": {\n "positive_family": "non_schedule_d_distributions",\n "negative_family": "non_schedule_d_distributions",\n "positive_factor": "1.120350061633613230084419890940102481805",\n "negative_factor": "1.120350061633613230084419890940102481805",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "taxable_private_pension_income": {\n "positive_family": "taxable_pensions",\n "negative_family": "taxable_pensions",\n "positive_factor": "1.327588034973940867230285994253866499704",\n "negative_factor": "1.327588034973940867230285994253866499704",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "taxable_ira_distributions": {\n "positive_family": "ira_distributions",\n "negative_family": "ira_distributions",\n "positive_factor": "1.510871073557979130794209158828995676924",\n "negative_factor": "1.510871073557979130794209158828995676924",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "alimony_income": {\n "positive_family": "alimony_income",\n "negative_family": "alimony_income",\n "positive_factor": "1.542033706590010885068081742929926580179",\n "negative_factor": "1.542033706590010885068081742929926580179",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "salt_refund_income": {\n "positive_family": "state_tax_refund",\n "negative_family": "state_tax_refund",\n "positive_factor": "0.9296526775720357466732616002244135476521",\n "negative_factor": "0.9296526775720357466732616002244135476521",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "short_term_capital_gains": {\n "positive_family": "short_current_gain",\n "negative_family": "short_current_loss",\n "positive_factor": "2.042865587775988756504710527637911842775",\n "negative_factor": "1.738139193325432816803529458382722438642",\n "assumption": "Current-year gross gains/loss components per all return proxy for signed net PUF amount; carried losses are excluded."\n },\n "long_term_capital_gains_before_response": {\n "positive_family": "long_current_gain",\n "negative_family": "long_current_loss",\n "positive_factor": "1.283710381837894882785950440755768785471",\n "negative_factor": "2.262384673681395979556694950087192296750",\n "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded."\n },\n "long_term_capital_gains_on_collectibles": {\n "positive_family": "long_current_gain",\n "negative_family": "long_current_loss",\n "positive_factor": "1.283710381837894882785950440755768785471",\n "negative_factor": "2.262384673681395979556694950087192296750",\n "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded."\n },\n "unrecaptured_section_1250_gain": {\n "positive_family": "long_current_gain",\n "negative_family": "long_current_loss",\n "positive_factor": "1.283710381837894882785950440755768785471",\n "negative_factor": "2.262384673681395979556694950087192296750",\n "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded."\n },\n "social_security_retirement": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "social_security_disability": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "social_security_dependents": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "social_security_survivors": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "rental_income": {\n "positive_family": "rental_royalty_profit",\n "negative_family": "rental_royalty_loss",\n "positive_factor": "1.476663829685885394438774259073237903799",\n "negative_factor": "1.690793641559507564235995133214906035185",\n "assumption": "Related rental/royalty reporting-return mean proxy; published total family includes farm rent and does not exactly equal E25850/E25860 component definitions."\n },\n "estate_income": {\n "positive_family": "estate_profit",\n "negative_family": "estate_loss",\n "positive_factor": "1.476331638358104936021886589313386353228",\n "negative_factor": "1.582810027948011789708036602150526302352",\n "assumption": ""\n },\n "farm_operations_income": {\n "positive_family": "farm_profit",\n "negative_family": "farm_loss",\n "positive_factor": "1.444720208215047581256304812994467826979",\n "negative_factor": "1.728383897337776476443532509645504817056",\n "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income."\n },\n "farm_income": {\n "positive_family": "farm_profit",\n "negative_family": "farm_loss",\n "positive_factor": "1.444720208215047581256304812994467826979",\n "negative_factor": "1.728383897337776476443532509645504817056",\n "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income."\n },\n "farm_rent_income": {\n "positive_family": "farm_rent_profit",\n "negative_family": "farm_rent_loss",\n "positive_factor": "1.629452810459537374541656575493532799084",\n "negative_factor": "1.150378978298943195689490419607276484593",\n "assumption": ""\n },\n "miscellaneous_income": {\n "positive_family": "other_property_gain",\n "negative_family": "other_property_loss",\n "positive_factor": "1.738242164922618767195190248113271033066",\n "negative_factor": "1.411645744283618865226902643798237455207",\n "assumption": "Canonical source E01200 is other property gains/losses, not Table1.4 other income."\n },\n "partnership_income": {\n "positive_family": "partnership_s_corp_profit",\n "negative_family": "partnership_s_corp_loss",\n "positive_factor": "1.595257264233857504616939193376066035754",\n "negative_factor": "1.968942227400735507640732147170530268700",\n "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy."\n },\n "s_corp_income": {\n "positive_family": "partnership_s_corp_profit",\n "negative_family": "partnership_s_corp_loss",\n "positive_factor": "1.595257264233857504616939193376066035754",\n "negative_factor": "1.968942227400735507640732147170530268700",\n "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy."\n },\n "partnership_self_employment_net_earnings": {\n "positive_family": "partnership_s_corp_profit",\n "negative_family": "partnership_s_corp_loss",\n "positive_factor": "1.595257264233857504616939193376066035754",\n "negative_factor": "1.968942227400735507640732147170530268700",\n "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy."\n },\n "qualified_bdc_income": {\n "positive_family": "nonqualified_dividends",\n "negative_family": "nonqualified_dividends",\n "positive_factor": "2.483795459689869824933006427197633652403",\n "negative_factor": "2.483795459689869824933006427197633652403",\n "assumption": "Modeled BDC component follows its parent nonqualified dividend pool; no independent source observation."\n },\n "qualified_reit_and_ptp_income": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real proxy for the combined modeled REIT/PTP output, which mixes nonqualified-dividend and pass-through components. No claim that this is a pure dividend pool."\n },\n "alimony_expense": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "charitable_cash_donations": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "charitable_non_cash_donations": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "real_estate_taxes": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "home_mortgage_interest": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "investment_interest_expense": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "investment_income_elected_form_4952": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "student_loan_interest": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "educator_expense": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "qualified_tuition_expenses": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "casualty_loss": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "unreimbursed_business_employee_expenses": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "traditional_ira_contributions_desired": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "self_employed_pension_contributions_desired": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "domestic_production_ald": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "health_savings_account_ald": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "unadjusted_basis_qualified_property": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "sstb_unadjusted_basis_qualified_property": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n }\n },\n "return_incidence_fields": [\n "business_is_sstb",\n "estate_income_would_be_qualified",\n "farm_operations_income_would_be_qualified",\n "farm_rent_income_would_be_qualified",\n "partnership_s_corp_income_would_be_qualified",\n "rental_income_would_be_qualified",\n "self_employment_income_would_be_qualified",\n "sstb_self_employment_income_would_be_qualified"\n ],\n "source_pins": {\n "NATIONAL-GROWTH-EXTRACT.json": "775d89d3fe7a3b1eb9a09c50f6077d41bf838cdb80d9e4aaa9f0ab80ada4e0c8",\n "INDEX-VALUES.json": "221f3786a462fe105d3da15df3624de5ad3e3f921b4d0205e096067685fd59b3",\n "extract_national.py": "b6c3a8e4fea52653c01d801d6efe9620cdba4a489d6d7ae8c95cf00ee6dfbe1b",\n "ACQUISITION.json": "e9a2c34c2ca06a229b2a790ed2987f25d31996653490f342b4b54af37d5f098c"\n },\n "sensitivity_cpi_factor": "1.323487344789614247079323424058189918867",\n "provenance": "All growth applications are modeled distribution transport from observed national series; source observed/derived/modeled flags remain separate.",\n "operation_order": "Decode source statistical2015 \\u2192 source canonical transformations \\u2192 QBI2015 model (all16 outputs) \\u2192 this growth transform once \\u2192 2024 canonical donor. Monetary recipient predictors remain 2024.",\n "noninterference": "RECID, S006, raw FLPDYR/FLPDMO, raw money, count source fields and actual survey membership are separate inputs and are not accepted or mutated by this transform.",\n "sensitivities": "cpi_only uses exactly the same source cohort, zeros, signs and return incidence; no new policy evaluation or calibration authority."\n}\n' +RECIPE_SHA256 = "f08bcc3c37434730bccbf81643c96089383d8e943c60f76ac8337f57a163e1f4" +if hashlib.sha256(_RECIPE_JSON.encode("utf-8")).hexdigest() != RECIPE_SHA256: + raise ValueError("PUF_GROWTH_RECIPE_IDENTITY") +VERSION = "microcosm.us.puf2015_target2024_growth/1" + + +def growth_recipe(): + return json.loads(_RECIPE_JSON) + + +_RECIPE = growth_recipe() +OUTPUTS = tuple(_RECIPE["ordered_targets"]) +INCIDENCE_FIELDS = tuple(_RECIPE["return_incidence_fields"]) + + +def _require(value, code): + if not value: + raise ValueError(code) + + +def _digest(columns): + h = hashlib.sha256() + for name in OUTPUTS: + value = columns[name] + h.update(name.encode("ascii") + b"\0") + h.update(np.asarray(value, dtype=" 0, "PUF_GROWTH_NONEMPTY_VECTOR") + n = len(first) + clean = {} + for name in OUTPUTS: + mask = np.asarray(known[name]) + _require( + mask.shape == (n,) and mask.dtype.kind == "b", + "PUF_GROWTH_KNOWNNESS_TYPE:" + name, + ) + _require(bool(mask.all()), "PUF_GROWTH_UNKNOWN:" + name) + value = np.asarray(canonical[name]) + allowed = "biu" if name in INCIDENCE_FIELDS else "ifu" + _require( + value.shape == (n,) and value.dtype.kind in allowed, + "PUF_GROWTH_PHYSICAL_TYPE:" + name, + ) + if value.dtype.kind in "iu": + _require( + bool(((value >= -(2**53)) & (value <= 2**53)).all()), + "PUF_GROWTH_INTEGER_PRECISION:" + name, + ) + _require(bool(np.isfinite(value).all()), "PUF_GROWTH_NONFINITE:" + name) + if name in INCIDENCE_FIELDS: + _require( + bool(((value == 0) | (value == 1)).all()), + "PUF_GROWTH_INCIDENCE_DOMAIN:" + name, + ) + clean[name] = value.astype(np.int64) + else: + clean[name] = value.astype(np.float64) + outputs = {} + for name, value in clean.items(): + if name in INCIDENCE_FIELDS: + grown = value.copy() + else: + rule = _RECIPE["money_fields"][name] + positive = float(rule["positive_factor"]) + negative = float(rule["negative_factor"]) + if scheme == "cpi_only": + positive = negative = float(_RECIPE["sensitivity_cpi_factor"]) + _require( + np.isfinite(positive) + and np.isfinite(negative) + and min(positive, negative) > 0, + "PUF_GROWTH_FACTOR", + ) + with np.errstate(over="ignore", invalid="ignore"): + grown = value * np.where(value < 0, negative, positive) + _require( + bool(np.isfinite(grown).all()), "PUF_GROWTH_OUTPUT_NONFINITE:" + name + ) + _require( + bool(np.array_equal(value == 0, grown == 0)) + and bool(np.array_equal(np.sign(value), np.sign(grown))), + "PUF_GROWTH_SIGN_OR_ZERO:" + name, + ) + outputs[name] = np.frombuffer(grown.tobytes(), dtype=grown.dtype) + receipt = { + "version": VERSION, + "recipe_sha256": RECIPE_SHA256, + "scheme": scheme, + "input_money_year": 2015, + "output_money_year": 2024, + "money_unit": "nominal USD", + "source_statistical_year": 2015, + "rows": n, + "origin": "modeled_distribution_transport", + "input_knownness": "complete_canonical_after_observed_derived_or_modeled_owner", + "source_pins": _RECIPE["source_pins"], + "monetary_columns": 51, + "unchanged_return_incidence_columns": 8, + "output_profile": _RECIPE["output_profile"], + "input_values_sha256": _digest(clean), + "output_values_sha256": _digest(outputs), + "no_weight_growth": True, + "no_extra_raw_row_year_cpi": True, + "release_eligible": False, + } + receipt["sha256"] = hashlib.sha256( + json.dumps( + receipt, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + return PufGrowthResult(MappingProxyType(outputs), 2024, MappingProxyType(receipt)) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py index bcbe95cdf..815081c95 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py @@ -4,25 +4,17 @@ Microdata Area — the finest geography the ACS PUMS publishes. A household that already knows its PUMA (an ACS-spine record) keeps it; a household that knows only its state (an ASEC-spine record) is assigned a PUMA within that state, -sampled proportional to 2020 PUMA population. Every coarser layer of the -ladder then derives from the PUMA by a population-weighted draw over the -PUMA's overlap with that layer: - -- **congressional district (119th)** — sampled within the PUMA proportional to - the ``(PUMA, CD)`` block-population overlap (a 2020 PUMA can span several - districts and a district several PUMAs; they do not nest). -- **county** — sampled within the PUMA proportional to the ``(PUMA, county)`` - block-population overlap. -- **tract** (behind ``assign_tract``) — sampled within the PUMA proportional to - the ``(PUMA, tract)`` block-population overlap; when tract is assigned the - county derives structurally from it (``county = tract // 10**6``) so the two - never disagree. Congressional district, county and state are the launch - requirement; tract is the stretch rung. - -One national dataset, filter by geography, at any grain (microcosm #275; no -per-area files, the standing rule): the dense ACS-spine artifact is filtered to -state / congressional district / county for local analysis because every record -carries them. +sampled proportional to 2020 PUMA population. The derived geography is one +population-weighted draw from the actual ``(PUMA, tract, 119th CD)`` block +join. County is the selected tract's prefix, so county and CD always have +joint source support. The retained marginal tables validate this joint +population; assignment never multiplies their probabilities. + +The internal tract identifies a supported crosswalk cell, not an observed +residence. ``assign_tract`` controls whether that assigned tract is exported; +it does not change the PUMA, county or CD draw. National/CD analysis still +requires its separate calibration, vintage and coverage gates. An assigned +county or tract is not certification for analysis at that grain. Vintage discipline follows the country-spec geography-spine schema (``vintage_policy: "error"``): the artifact records one vintage per derived @@ -45,6 +37,7 @@ import json from collections.abc import Mapping from dataclasses import dataclass +from io import BytesIO from pathlib import Path from typing import Any @@ -59,6 +52,7 @@ from microcosm.build.us_runtime.puma_ladder_sources import ( COUNTY_FROM_TRACT_DIVISOR, PUMA_GEOID_STATE_DIVISOR, + US_PUMA_LADDER_SCHEMA_VERSION, ) from microcosm.frame import Frame @@ -81,7 +75,6 @@ #: The finer tract rung, written only when ``assign_tract=True``. US_PUMA_LADDER_TRACT_COLUMN = "tract_geoid" -US_PUMA_LADDER_SCHEMA_VERSION = 1 US_PUMA_LADDER_KIND = "us_puma_ladder" PUMA_LADDER_ARTIFACT_SHA256_ATTR = "populace_puma_ladder_artifact_sha256" @@ -108,14 +101,18 @@ _REQUIRED_ARRAY_KEYS = ( *_ANCHOR_KEYS, *(key for keys in _OVERLAP_KEYS.values() for key in keys), + "joint_overlap_puma", + "joint_overlap_tract", + "joint_overlap_cd", + "joint_overlap_population", ) @dataclass(frozen=True) class UsPumaLadder: """The national PUMA ladder: one anchor row per populated 2020 PUMA plus - three ``(PUMA, layer_value, population)`` overlap tables sorted by - ``(puma, layer_value)``. + three marginal overlap tables and the actual joint ``(PUMA, tract, CD, + population)`` table. Joint rows are unique and sorted by ``(puma, tract, CD)``. Attributes: puma: 2020 PUMA geoids (``state_fips * 10**5 + PUMA5CE``) as ``int64``, @@ -134,7 +131,10 @@ class UsPumaLadder: the ``(PUMA, tract)`` population overlap; tract geoids are the 11-digit ``state+county+tract`` integer. metadata: the artifact's embedded metadata, including one vintage per - derived layer. + derived layer. Schema v2 requires the joint source table. + joint_overlap_puma / joint_overlap_tract / joint_overlap_cd / + joint_overlap_population: the supported populated-block join, + aggregated to unique PUMA/tract/CD cells; county is a tract prefix. """ puma: np.ndarray @@ -149,6 +149,10 @@ class UsPumaLadder: tract_overlap_tract: np.ndarray tract_overlap_population: np.ndarray metadata: Mapping[str, Any] + joint_overlap_puma: np.ndarray + joint_overlap_tract: np.ndarray + joint_overlap_cd: np.ndarray + joint_overlap_population: np.ndarray def __len__(self) -> int: return len(self.puma) @@ -186,6 +190,17 @@ def load_us_puma_ladder(path: str | Path) -> UsPumaLadder: source = Path(path) if not source.exists(): raise FileNotFoundError(f"US PUMA ladder artifact not found: {source}") + return _decode_us_puma_ladder_archive(source) + + +def decode_us_puma_ladder(payload: bytes) -> UsPumaLadder: + """Validate the same NPZ contract from immutable graph artifact bytes.""" + if not isinstance(payload, bytes): + raise TypeError("US PUMA ladder payload must be immutable bytes.") + return _decode_us_puma_ladder_archive(BytesIO(payload)) + + +def _decode_us_puma_ladder_archive(source: Path | BytesIO) -> UsPumaLadder: with np.load(source, allow_pickle=False) as payload: missing = [key for key in _REQUIRED_ARRAY_KEYS if key not in payload.files] if missing: @@ -274,7 +289,7 @@ def load_us_puma_ladder(path: str | Path) -> UsPumaLadder: raise ValueError("tract_overlap_tract geoids must be unique (tracts nest).") _assert_conservation(puma, population, tract_puma, tract_pop, layer="tract") - return UsPumaLadder( + ladder = UsPumaLadder( puma=puma, puma_population=population, cd_overlap_puma=cd_puma, @@ -287,7 +302,13 @@ def load_us_puma_ladder(path: str | Path) -> UsPumaLadder: tract_overlap_tract=tract_value, tract_overlap_population=tract_pop, metadata=metadata, + joint_overlap_puma=arrays["joint_overlap_puma"], + joint_overlap_tract=arrays["joint_overlap_tract"], + joint_overlap_cd=arrays["joint_overlap_cd"], + joint_overlap_population=arrays["joint_overlap_population"], ) + _validated_joint_support(ladder) + return ladder def assign_us_puma_ladder( @@ -304,10 +325,11 @@ def assign_us_puma_ladder( Records that already carry a valid ``puma`` (the ACS spine) keep it; records that carry none (the ASEC spine) draw a PUMA within their ``state_fips`` - proportional to 2020 PUMA population. Every record then draws a congressional - district and a county within its PUMA proportional to the block-population - overlap, and — when ``assign_tract`` — a tract from which the county derives - structurally. Draws are consumed from one seeded generator in a fixed sorted + proportional to 2020 PUMA population. Every record then draws one supported + joint tract/CD cell within its PUMA proportional to block population. County + derives from that tract. The tract is written only when ``assign_tract`` is + true; changing that flag leaves the same PUMA/county/CD assignments. + Draws are consumed from one seeded generator in a fixed sorted order (states, then PUMAs), so the result is reproducible from ``seed``. A household ``puma`` absent from the ladder, or a household ``state_fips`` @@ -319,6 +341,9 @@ def assign_us_puma_ladder( f"household table must contain {state_fips_column!r} before PUMA-" "ladder assignment." ) + joint_puma, joint_tract, joint_cd, joint_population = _validated_joint_support( + ladder + ) if expected_congressional_district_vintage is not None: ladder_cd_vintage = ladder.layer_vintages["congressional_district"] if ladder_cd_vintage != expected_congressional_district_vintage: @@ -343,23 +368,17 @@ def assign_us_puma_ladder( if unknown_mask.any(): _draw_puma_within_state(row_puma, unknown_mask, state_ints, ladder, rng) - cd_puma, cd_value, cd_pop = ladder._overlap("congressional_district") - cd_values = _draw_layer_values( - row_puma, cd_puma, cd_value, cd_pop, rng, layer="congressional_district" + joint_rows = _draw_layer_values( + row_puma, + joint_puma, + np.arange(len(joint_puma), dtype=np.int64), + joint_population, + rng, + layer="joint tract/congressional_district", ) - - tract_values: np.ndarray | None = None - if assign_tract: - tract_puma, tract_value, tract_pop = ladder._overlap("tract") - tract_values = _draw_layer_values( - row_puma, tract_puma, tract_value, tract_pop, rng, layer="tract" - ) - county_values = tract_values // COUNTY_FROM_TRACT_DIVISOR - else: - county_puma, county_value, county_pop = ladder._overlap("county") - county_values = _draw_layer_values( - row_puma, county_puma, county_value, county_pop, rng, layer="county" - ) + cd_values = joint_cd[joint_rows] + county_values = joint_tract[joint_rows] // COUNTY_FROM_TRACT_DIVISOR + tract_values = joint_tract[joint_rows] if assign_tract else None _assert_row_state_consistency( state_ints, @@ -383,6 +402,213 @@ def assign_us_puma_ladder( return assigned +def _validated_joint_support( + ladder: UsPumaLadder, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Require real joint support and exact agreement with every marginal. + + A frozen dataclass does not make its arrays immutable. Check the current + arrays when assigning or verifying, including normally constructed ladders. + Nothing in this function manufactures joint rows from marginal weights. + """ + + _validate_ladder_metadata(ladder.metadata) + if ladder.metadata.get("puma_vintage") != "2020_puma": + raise ValueError( + "Joint PUMA assignment requires the observed 2020 PUMA vintage." + ) + if any( + ladder.layer_vintages[layer] != "2020_census" for layer in ("county", "tract") + ): + raise ValueError( + "Joint PUMA assignment requires 2020 Census county/tract vintages." + ) + anchor = _int64_array(ladder.puma, label="puma") + if ( + not len(anchor) + or (np.diff(anchor) <= 0).any() + or ( + (anchor // PUMA_GEOID_STATE_DIVISOR < 1) + | (anchor // PUMA_GEOID_STATE_DIVISOR > 56) + ).any() + ): + raise ValueError( + "Joint PUMA anchors must be unique, sorted and within states 01–56." + ) + columns = tuple( + _int64_array(getattr(ladder, name), label=name) + for name in ( + "joint_overlap_puma", + "joint_overlap_tract", + "joint_overlap_cd", + "joint_overlap_population", + ) + ) + puma, tract, cd, population = columns + if not len(puma) or any(len(value) != len(puma) for value in columns): + raise ValueError("Joint PUMA overlap arrays must be nonempty and aligned.") + if (population <= 0).any(): + raise ValueError( + "Joint PUMA overlap populations must be positive integer counts." + ) + rows = list(zip(puma.tolist(), tract.tolist(), cd.tolist(), strict=True)) + if rows != sorted(set(rows)): + raise ValueError( + "Joint PUMA overlap rows must be unique and sorted by PUMA/tract/CD." + ) + tract_owner: dict[int, int] = {} + for row_puma, row_tract, _ in rows: + if tract_owner.setdefault(row_tract, row_puma) != row_puma: + raise ValueError("Joint PUMA overlap assigns one tract to multiple PUMAs.") + if ((tract < 10**9) | (tract >= 10**11)).any() or ((cd < 100) | (cd > 9999)).any(): + raise ValueError("Joint PUMA overlap contains an invalid tract or CD geoid.") + _assert_layer_state_matches( + puma, tract // COUNTY_FROM_TRACT_DIVISOR // 1000, layer="joint tract" + ) + _assert_layer_state_matches(puma, cd // 100, layer="joint CD") + + def counts(keys: list[tuple[int, ...]], values: np.ndarray, label: str) -> dict: + values = np.asarray(values) + if values.ndim != 1 or len(values) != len(keys): + raise ValueError(f"{label} population must align with its keys.") + result: dict[tuple[int, ...], int] = {} + for key, value in zip(keys, values.tolist(), strict=True): + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not np.isfinite(value) + or value <= 0 + or int(value) != value + ): + raise ValueError( + f"{label} population must contain positive integer counts." + ) + result[key] = result.get(key, 0) + int(value) + return result + + marginals = { + "anchor": [(int(key),) for key in puma], + "congressional_district": [ + (int(key), int(value)) for key, value in zip(puma, cd, strict=True) + ], + "county": [ + (int(key), int(value)) + for key, value in zip(puma, tract // COUNTY_FROM_TRACT_DIVISOR, strict=True) + ], + "tract": [ + (int(key), int(value)) for key, value in zip(puma, tract, strict=True) + ], + } + for layer, keys in marginals.items(): + if layer == "anchor": + reference_keys = [(int(key),) for key in anchor] + reference_pop = ladder.puma_population + else: + source_puma, source_value, reference_pop = ladder._overlap(layer) + source_puma = _int64_array(source_puma, label=layer + "_overlap_puma") + source_value = _int64_array(source_value, label=layer + "_overlap_value") + if len(source_puma) != len(source_value): + raise ValueError(f"{layer} overlap keys must align.") + reference_keys = list( + zip(source_puma.tolist(), source_value.tolist(), strict=True) + ) + if counts(keys, population, "joint") != counts( + reference_keys, reference_pop, layer + ): + raise ValueError( + f"Joint PUMA overlap does not exactly reproduce the {layer} marginal." + ) + return columns + + +def us_puma_ladder_joint_support_gate( + household: pd.DataFrame, + ladder: UsPumaLadder, + *, + assign_tract: bool = False, + expected_congressional_district_vintage: str | None = None, +) -> GateResult: + """Check current assigned rows against actual joint Census support. + + This structural support gate is independent of the national NYC mass gate + and of national/CD calibration; zero-weight clone rows must also be valid. + """ + + failures: list[str] = [] + details: dict[str, object] = {"household_rows": len(household)} + try: + puma, tract, cd, _ = _validated_joint_support(ladder) + if ( + expected_congressional_district_vintage is not None + and ladder.layer_vintages["congressional_district"] + != expected_congressional_district_vintage + ): + raise ValueError( + "Joint-support congressional-district vintage does not match the expected vintage." + ) + columns = ["puma", "county_fips", CONGRESSIONAL_DISTRICT_GEOID_COLUMN] + if assign_tract: + columns.append(US_PUMA_LADDER_TRACT_COLUMN) + missing = [name for name in columns if name not in household] + if missing: + raise ValueError(f"Joint support requires geography columns: {missing}.") + values = [] + for name in columns: + numeric = pd.to_numeric(household[name], errors="coerce").to_numpy( + dtype=np.float64, na_value=np.nan + ) + if not ( + np.isfinite(numeric) + & (numeric > 0) + & (numeric < 2**53) + & (numeric == np.floor(numeric)) + ).all(): + raise ValueError( + f"Joint support requires positive integral {name} geoids." + ) + values.append(numeric.astype(np.int64)) + support = set( + zip( + puma.tolist(), + (tract // COUNTY_FROM_TRACT_DIVISOR).tolist(), + cd.tolist(), + strict=True, + ) + ) + if assign_tract: + support = set( + zip( + puma.tolist(), + (tract // COUNTY_FROM_TRACT_DIVISOR).tolist(), + cd.tolist(), + tract.tolist(), + strict=True, + ) + ) + unsupported = [ + index + for index, row in enumerate( + zip(*(value.tolist() for value in values), strict=True) + ) + if row not in support + ] + details["unsupported_rows"] = len(unsupported) + if unsupported: + failures.append( + f"{len(unsupported)} household row(s) have no joint PUMA/county/CD support; row positions {unsupported[:5]}." + ) + details["joint_support_rows"] = len(puma) + details["layer_vintages"] = ladder.layer_vintages + except (TypeError, ValueError) as error: + failures.append(str(error)) + return GateResult( + name="us_puma_ladder_joint_support", + passed=not failures, + failures=tuple(failures), + details=details, + ) + + def with_household_us_puma_ladder( frame: Frame, ladder: UsPumaLadder, @@ -850,8 +1076,12 @@ def _assert_conservation( def _int64_array(values: np.ndarray, *, label: str) -> np.ndarray: array = np.asarray(values) + if array.ndim != 1: + raise ValueError(f"{label} must be a one-dimensional integer array.") if array.dtype.kind not in ("i", "u"): raise ValueError(f"{label} must be an integer array, got dtype {array.dtype}.") + if array.dtype.kind == "u" and (array > np.iinfo(np.int64).max).any(): + raise ValueError(f"{label} contains an integer outside signed int64.") return array.astype(np.int64, copy=False) @@ -898,5 +1128,6 @@ def _household_cd_state(values: Any) -> np.ndarray: "load_us_puma_ladder", "us_puma_ladder_assignment_summary", "us_puma_ladder_gate", + "us_puma_ladder_joint_support_gate", "with_household_us_puma_ladder", ] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py index bbf252e6b..1982f0b04 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py @@ -17,8 +17,10 @@ - ``block -> 119th CD`` and ``block -> POP100`` reuse the block-ladder parsers :func:`parse_national_cd_bef` and :func:`parse_pl_geo_blocks` unchanged. -Summing block populations by ``(PUMA, CD)``, ``(PUMA, county)`` and -``(PUMA, tract)`` yields the three overlap tables; summing by PUMA yields the +Summing block populations by ``(PUMA, tract, CD)`` preserves the joint support +needed to draw a consistent county/CD pair (county is the tract prefix). The +three marginal overlap tables are retained for validation and diagnostics; +summing by PUMA yields the anchor population used for the ASEC state -> PUMA draw. Every populated block contributes to exactly one PUMA, one CD, one county and one tract, so each overlap table conserves its PUMA's population exactly — a defect (a populated @@ -45,6 +47,7 @@ TRACT_FROM_BLOCK_DIVISOR = 10**4 COUNTY_FROM_BLOCK_DIVISOR = 10**10 COUNTY_FROM_TRACT_DIVISOR = 10**6 +US_PUMA_LADDER_SCHEMA_VERSION = 2 _TRACT_TO_PUMA_HEADER = ("STATEFP", "COUNTYFP", "TRACTCE", "PUMA5CE") @@ -120,16 +123,21 @@ def assemble_us_puma_ladder( For every populated block the block's tract (a structural prefix) must map to a PUMA and the block must carry a congressional district; either gap is - a source defect, not a skippable row, so the three overlap tables each - conserve their PUMA's population exactly. Returns arrays for the anchor - (``puma`` / ``puma_population``) and the three overlap tables, each sorted - by ``(puma, layer_value)`` for a stable, searchsorted-friendly artifact. + a source defect, not a skippable row. Schema v2 retains the actual joint + PUMA/tract/CD population before aggregation is lost; county is the tract + prefix. It also emits the anchor and three marginal tables for exact + conservation checks. Joint rows are sorted by ``(puma, tract, CD)`` and + marginal rows by ``(puma, layer_value)``. """ + if metadata.get("schema_version") != US_PUMA_LADDER_SCHEMA_VERSION: + raise ValueError("Joint PUMA source assembly requires schema_version 2.") + puma_population: dict[int, int] = {} puma_cd_population: dict[tuple[int, int], int] = {} puma_county_population: dict[tuple[int, int], int] = {} puma_tract_population: dict[tuple[int, int], int] = {} + joint_population: dict[tuple[int, int, int], int] = {} missing_puma: list[int] = [] missing_cd: list[int] = [] @@ -157,6 +165,8 @@ def assemble_us_puma_ladder( puma_tract_population[tract_key] = ( puma_tract_population.get(tract_key, 0) + population ) + joint_key = (puma, tract, cd) + joint_population[joint_key] = joint_population.get(joint_key, 0) + population if missing_puma: examples = [f"{block:015d}" for block in sorted(missing_puma)[:5]] @@ -180,6 +190,7 @@ def assemble_us_puma_ladder( cd_puma, cd_value, cd_pop = _overlap_arrays(puma_cd_population) county_puma, county_value, county_pop = _overlap_arrays(puma_county_population) tract_puma, tract_value, tract_pop = _overlap_arrays(puma_tract_population) + joint_keys = sorted(joint_population) _assert_conserves( pumas, anchor_population, cd_puma, cd_pop, layer="congressional_district" @@ -199,6 +210,16 @@ def assemble_us_puma_ladder( "tract_overlap_puma": tract_puma, "tract_overlap_tract": tract_value, "tract_overlap_population": tract_pop, + "joint_overlap_puma": np.asarray( + [key[0] for key in joint_keys], dtype=np.int64 + ), + "joint_overlap_tract": np.asarray( + [key[1] for key in joint_keys], dtype=np.int64 + ), + "joint_overlap_cd": np.asarray([key[2] for key in joint_keys], dtype=np.int64), + "joint_overlap_population": np.asarray( + [joint_population[key] for key in joint_keys], dtype=np.int64 + ), "metadata_json": np.asarray(json.dumps(dict(metadata), sort_keys=True)), } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py index 13f5c35c5..ca5dc186b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py @@ -40,6 +40,22 @@ from microcosm.build.us_runtime.take_up_contract import load_take_up_contract from microcosm.frame import US_SCHEMA, Frame +from .operator_column_contracts import ( + _GENERAL_QUALIFICATION_FLAGS as _GENERAL_QUALIFICATION_FLAGS, +) +from .operator_column_contracts import ( + _SSTB_QUALIFICATION_FLAG as _SSTB_QUALIFICATION_FLAG, +) +from .operator_column_contracts import ( + US_QBI_BOOLEAN_OUTPUT_COLUMNS as US_QBI_BOOLEAN_OUTPUT_COLUMNS, +) +from .operator_column_contracts import ( + US_QBI_NONNEGATIVE_OUTPUT_COLUMNS as US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, +) +from .operator_column_contracts import ( + US_QBI_OUTPUT_COLUMNS as US_QBI_OUTPUT_COLUMNS, +) + __all__ = [ "QBI_ARCHIVED_ASSUMPTIONS_URL", "QBI_ARCHIVED_CLONE_URL", @@ -87,40 +103,6 @@ US_QBI_STAGE_NAME = "puf_tax_detail" -_GENERAL_QUALIFICATION_FLAGS: tuple[str, ...] = ( - "estate_income_would_be_qualified", - "farm_operations_income_would_be_qualified", - "farm_rent_income_would_be_qualified", - "partnership_s_corp_income_would_be_qualified", - "rental_income_would_be_qualified", - "self_employment_income_would_be_qualified", -) -_SSTB_QUALIFICATION_FLAG = "sstb_self_employment_income_would_be_qualified" -US_QBI_BOOLEAN_OUTPUT_COLUMNS: tuple[str, ...] = ( - *_GENERAL_QUALIFICATION_FLAGS, - _SSTB_QUALIFICATION_FLAG, - # Keep the classifier last so the chained QRF can condition the SSTB draw - # on the qualification flags it must agree with. - "business_is_sstb", -) -US_QBI_NONNEGATIVE_OUTPUT_COLUMNS: tuple[str, ...] = ( - "qualified_bdc_income", - "qualified_reit_and_ptp_income", - "sstb_unadjusted_basis_qualified_property", - "sstb_w2_wages_from_qualified_business", - "unadjusted_basis_qualified_property", - "w2_wages_from_qualified_business", -) -US_QBI_OUTPUT_COLUMNS: tuple[str, ...] = ( - *US_QBI_BOOLEAN_OUTPUT_COLUMNS, - "qualified_bdc_income", - "qualified_reit_and_ptp_income", - "sstb_self_employment_income_before_lsr", - "sstb_unadjusted_basis_qualified_property", - "sstb_w2_wages_from_qualified_business", - "unadjusted_basis_qualified_property", - "w2_wages_from_qualified_business", -) US_QBI_NONCONSTANT_PERSON_COLUMNS = US_QBI_OUTPUT_COLUMNS _SELF_EMPLOYMENT_COLUMN = "self_employment_income_before_lsr" diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py index 3695df002..f4f391509 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py @@ -15,6 +15,7 @@ from __future__ import annotations +from collections.abc import Mapping from importlib.resources import files import numpy as np @@ -32,6 +33,9 @@ SourceRuntimeError, run_source_stage, ) +from microcosm.build.us_runtime._person_signal_summary import ( + validate_person_signal_summary, +) from microcosm.frame import Frame from microcosm.frame.units import US_SCHEMA @@ -41,6 +45,11 @@ "US_RELATIONSHIP_INPUTS_REQUIRED_SOURCE_COLUMNS", "US_RELATIONSHIP_INPUTS_STAGE_NAME", "derive_us_relationship_inputs_from_manifest", + "prepare_us_relationship_person", + "us_relationship_inputs_person_carries_signal", + "us_relationship_inputs_gate_from_summary", + "us_relationship_inputs_person_gate", + "us_relationship_inputs_person_summary", "us_relationship_inputs_signal_gate", "us_relationship_inputs_stage_spec", "us_relationship_inputs_summary", @@ -182,8 +191,13 @@ def derive_us_relationship_inputs_from_manifest( return result -def _relationship_surface_carries_signal(frame: Frame) -> bool: - person = frame.table("person") +def us_relationship_inputs_person_carries_signal(person: pd.DataFrame) -> bool: + """Return the exact incumbent pass-through decision from the real person table. + + All three outputs must be present and each must have multiple observed + values. Preserve the legacy null/constant rule; this does not validate raw + source columns or resolve weights, and is not a scientific signal gate. + """ if any(column not in person for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS): return False return all( @@ -192,22 +206,70 @@ def _relationship_surface_carries_signal(frame: Frame) -> bool: ) -def with_us_relationship_inputs( - frame: Frame, +def _relationship_surface_carries_signal(frame: Frame) -> bool: + return us_relationship_inputs_person_carries_signal(frame.table("person")) + + +def _validated_relationship_weights( + person: pd.DataFrame, weights: np.ndarray +) -> np.ndarray: + """Return ``weights`` as a float64 array aligned 1:1 with ``person``.""" + + array = np.asarray(weights, dtype=np.float64) + if array.ndim != 1 or len(array) != len(person): + raise ValueError( + "US relationship-input weights must be a 1-D array aligned 1:1 " + f"with the person table ({len(person)} row(s)); got shape " + f"{array.shape}." + ) + if not np.isfinite(array).all(): + raise ValueError("US relationship-input weights must be finite.") + if (array < 0.0).any(): + raise ValueError("US relationship-input weights must be nonnegative.") + return array + + +def prepare_us_relationship_person( + person: pd.DataFrame, + weights: np.ndarray, *, seed: int, time_period: int, -) -> Frame: - """Materialize measured ASEC relationship inputs on a US frame.""" - - if frame.schema != US_SCHEMA: - raise ValueError("US relationship inputs require the US schema.") - if _relationship_surface_carries_signal(frame): - return frame - - person = frame.table("person") +) -> pd.DataFrame: + """Derive measured ASEC relationship inputs onto a copy of ``person``. + + The deterministic table-helper behind :func:`with_us_relationship_inputs`: + it always runs the ``relationship_inputs`` source-manifest stage over + ``person`` and ``weights`` and returns the aligned result. It does not + decide whether the surface already carries signal — that pass-through + decision belongs to the Frame wrapper. + + Args: + person: The actual person table, carrying the raw ASEC source + columns (``PH_SEQ``, ``P_SEQ``, ``A_MARITL``), in its own index + and row order. + weights: Person weights aligned 1:1 with ``person``'s rows (same + length and row order; need not be reindexed by ``person_id``). + seed: Build-wide imputation seed threaded to the source-stage + runtime (the derivation itself is deterministic). + time_period: The dataset's time period. + + Returns: + A copy of ``person`` with ``is_household_head``, ``is_separated``, + and ``is_surviving_spouse`` attached as ``bool`` columns, in + ``person``'s original index and row order. + + Raises: + ValueError: If ``weights`` does not align 1:1 with ``person``, is + not finite/nonnegative, or the stage output does not cover + every person. + SourceRuntimeError: If required raw ASEC column(s) are missing or + malformed. + """ + + weight_values = _validated_relationship_weights(person, weights) stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = weight_values output = run_source_stage( us_relationship_inputs_stage_spec(), tables={"person": stage_person}, @@ -224,9 +286,34 @@ def with_us_relationship_inputs( f"for {column!r}." ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} + result = person.copy(deep=True) for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS: - tables["person"][column] = aligned[column].to_numpy(dtype=bool) + result[column] = aligned[column].to_numpy(dtype=bool) + return result + + +def with_us_relationship_inputs( + frame: Frame, + *, + seed: int, + time_period: int, +) -> Frame: + """Materialize measured ASEC relationship inputs on a US frame.""" + + if frame.schema != US_SCHEMA: + raise ValueError("US relationship inputs require the US schema.") + if _relationship_surface_carries_signal(frame): + return frame + + person = frame.table("person") + new_person = prepare_us_relationship_person( + person, + frame.resolve_weights("person").values, + seed=seed, + time_period=time_period, + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"] = new_person return Frame( tables, frame.schema, @@ -237,16 +324,35 @@ def with_us_relationship_inputs( ) -def us_relationship_inputs_summary(frame: Frame) -> dict[str, object]: - """Return weighted relationship shares and one-head invariants.""" +def us_relationship_inputs_person_summary( + person: pd.DataFrame, weights: np.ndarray +) -> dict[str, object]: + """Return weighted relationship shares and one-head invariants. - person = frame.table("person") - weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) - total_weight = float(weights.sum()) + The real-person-table counterpart of :func:`us_relationship_inputs_summary`, + for callers (e.g. graph adapters) that hold a person table and an + explicit weight vector without a :class:`~microcosm.frame.Frame`. + + Args: + person: The actual person table, already carrying + ``is_household_head``, ``is_separated``, and + ``is_surviving_spouse``. + weights: Person weights aligned 1:1 with ``person``'s rows. + + Returns: + The same summary payload as :func:`us_relationship_inputs_summary`. + """ + + weight_values = _validated_relationship_weights(person, weights) + total_weight = float(weight_values.sum()) def _share(column: str) -> float: values = person[column].fillna(False).astype(bool).to_numpy() - return float(weights[values].sum()) / total_weight if total_weight > 0 else 0.0 + return ( + float(weight_values[values].sum()) / total_weight + if total_weight > 0 + else 0.0 + ) household_column = ( "person_household_id" if "person_household_id" in person else "PH_SEQ" @@ -276,24 +382,50 @@ def _share(column: str) -> float: } -def us_relationship_inputs_signal_gate(frame: Frame) -> GateResult: - """Require plausible signal and exactly one ASEC head per household.""" +def us_relationship_inputs_summary(frame: Frame) -> dict[str, object]: + """Return weighted relationship shares and one-head invariants.""" - person = frame.table("person") - missing = [ - column - for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS - if column not in person - ] - if missing: - return GateResult( - name="relationship_inputs_signal", - passed=False, - failures=(f"person columns missing: {missing}.",), - details={"missing": missing}, - ) + return us_relationship_inputs_person_summary( + frame.table("person"), frame.resolve_weights("person").values + ) - summary = us_relationship_inputs_summary(frame) + +def us_relationship_inputs_gate_from_summary( + summary: Mapping[str, object], +) -> GateResult: + """Check relationship-input plausibility bands and invariants from a summary. + + The pure decision core of :func:`us_relationship_inputs_signal_gate`, + factored out so graph adapters can reuse the incumbent checks — same + bands, order, and meaning — against a summary computed off the real + person table (see :func:`us_relationship_inputs_person_summary`) + without a :class:`~microcosm.frame.Frame`. Assumes the caller has + already confirmed the three output columns are present; missing + columns are a separate failure mode (see + :func:`us_relationship_inputs_person_gate`). + + Raises: + ValueError: If required fields/counts are missing, measurements are + malformed, or supplied bands differ from the registered policy. + """ + + validate_person_signal_summary( + summary, + family="relationship", + outputs=US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS, + share_bands={ + "household_head_share": ( + "household_head_share_band", + _HOUSEHOLD_HEAD_SHARE_BAND, + ), + "separated_share": ("separated_share_band", _SEPARATED_SHARE_BAND), + "surviving_spouse_share": ( + "surviving_spouse_share_band", + _SURVIVING_SPOUSE_SHARE_BAND, + ), + }, + invariants=("households_without_exactly_one_head", "separated_and_surviving"), + ) failures: list[str] = [] for share_key, band_key, label in ( ( @@ -330,3 +462,51 @@ def us_relationship_inputs_signal_gate(frame: Frame) -> GateResult: failures=tuple(failures), details=summary, ) + + +def us_relationship_inputs_person_gate( + person: pd.DataFrame, weights: np.ndarray +) -> GateResult: + """Require plausible signal and exactly one ASEC head per household. + + The real-person-table counterpart of + :func:`us_relationship_inputs_signal_gate`, composing + :func:`us_relationship_inputs_person_summary` and + :func:`us_relationship_inputs_gate_from_summary` exactly as the Frame + wrapper does. + """ + + missing = [ + column + for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS + if column not in person + ] + if missing: + return GateResult( + name="relationship_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + summary = us_relationship_inputs_person_summary(person, weights) + return us_relationship_inputs_gate_from_summary(summary) + + +def us_relationship_inputs_signal_gate(frame: Frame) -> GateResult: + """Require plausible signal and exactly one ASEC head per household.""" + + person = frame.table("person") + missing = [ + column + for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS + if column not in person + ] + if missing: + return GateResult( + name="relationship_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + summary = us_relationship_inputs_summary(frame) + return us_relationship_inputs_gate_from_summary(summary) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/reported_coverage_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/reported_coverage_source.py new file mode 100644 index 000000000..5992b7c1d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/reported_coverage_source.py @@ -0,0 +1,448 @@ +"""Measured ASEC at-interview coverage recodes omitted by frozen H5 inputs. + +The pooled ``census_cps`` inputs carry ``NOW_GRP`` and ``NOW_MRK`` for every +income year, but their 2022 and 2023 person tables omit seven other recodes +consumed by :func:`derive_us_cps_carried_inputs`. Those omissions otherwise +turn measured coverage into structural false values for two pooled vintages. + +This module restores the seven fields from the same SHA-pinned official ASEC +person members already used for ``ED_VAL`` and ``PAW_TYP``. Restoration is an +exact ``(source_year, PERIDNUM)`` join with redundant Census identity checks; +it does not predict, infer, or substitute any coverage value. +""" + +from __future__ import annotations + +import hashlib +import zipfile +from collections.abc import Mapping +from pathlib import Path +from typing import BinaryIO + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime.education_assistance_source import ( + ASEC_EDUCATION_ASSISTANCE_ARCHIVES, + ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS, + AsecEducationArchive, + fetch_asec_education_assistance_source, +) + +__all__ = [ + "ASEC_REPORTED_COVERAGE_INCOME_YEARS", + "ASEC_REPORTED_COVERAGE_RAW_COLUMNS", + "ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS", + "fill_asec_reported_coverage_source", + "load_asec_reported_coverage_sources", +] + +# ``NOW_GRP`` and ``NOW_MRK`` are deliberately absent: they already survive +# in every frozen pooled source and remain native rather than sidecar-restored. +ASEC_REPORTED_COVERAGE_RAW_COLUMNS: tuple[str, ...] = ( + "NOW_NONM", + "NOW_MCAID", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", +) +ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS: tuple[str, ...] = ( + "PH_SEQ", + "P_SEQ", + "A_LINENO", + "PERIDNUM", + *ASEC_REPORTED_COVERAGE_RAW_COLUMNS, +) +ASEC_REPORTED_COVERAGE_INCOME_YEARS = ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS +_AUDIT_WEIGHT_COLUMN = "A_FNLWGT" +_VALID_CODES = frozenset({1, 2}) + + +def _sha256_stream(stream: BinaryIO, *, chunk_size: int) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + for chunk in iter(lambda: stream.read(chunk_size), b""): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +def _verify_member_path( + path: Path, + pins: AsecEducationArchive, + *, + chunk_size: int, +) -> None: + if not path.is_file(): + raise FileNotFoundError(path) + if path.stat().st_size != pins.member_size_bytes: + raise ValueError( + f"ASEC {pins.survey_year} person member byte length mismatch: " + f"expected {pins.member_size_bytes}, got {path.stat().st_size}." + ) + with path.open("rb") as stream: + digest, _ = _sha256_stream(stream, chunk_size=chunk_size) + if digest != pins.member_sha256: + raise ValueError( + f"ASEC {pins.survey_year} person member SHA-256 mismatch: " + f"expected {pins.member_sha256}, got {digest}." + ) + + +def _verified_zip_member( + archive: zipfile.ZipFile, + pins: AsecEducationArchive, +) -> zipfile.ZipInfo: + members = [info for info in archive.infolist() if info.filename == pins.member] + if len(members) != 1: + raise ValueError( + f"ASEC {pins.survey_year} archive must contain exactly one " + f"{pins.member!r} member; found {len(members)}." + ) + info = members[0] + if info.file_size != pins.member_size_bytes: + raise ValueError( + f"ASEC {pins.survey_year} person member byte length mismatch: " + f"expected {pins.member_size_bytes}, got {info.file_size}." + ) + actual_crc = f"{info.CRC:08x}" + if actual_crc != pins.member_crc32: + raise ValueError( + f"ASEC {pins.survey_year} person member CRC32 mismatch: expected " + f"{pins.member_crc32}, got {actual_crc}." + ) + return info + + +def _read_one_source( + path: Path, + pins: AsecEducationArchive, + *, + chunk_size: int, +) -> pd.DataFrame: + usecols = [*ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS, _AUDIT_WEIGHT_COLUMN] + if zipfile.is_zipfile(path): + if path.stat().st_size != pins.zip_size_bytes: + raise ValueError( + f"ASEC {pins.survey_year} archive byte length mismatch: expected " + f"{pins.zip_size_bytes}, got {path.stat().st_size}." + ) + with path.open("rb") as stream: + archive_digest, _ = _sha256_stream(stream, chunk_size=chunk_size) + if archive_digest != pins.zip_sha256: + raise ValueError( + f"ASEC {pins.survey_year} archive SHA-256 mismatch: expected " + f"{pins.zip_sha256}, got {archive_digest}." + ) + with zipfile.ZipFile(path) as archive: + info = _verified_zip_member(archive, pins) + with archive.open(info) as member: + member_digest, member_size = _sha256_stream( + member, + chunk_size=chunk_size, + ) + if ( + member_size != pins.member_size_bytes + or member_digest != pins.member_sha256 + ): + raise ValueError( + f"ASEC {pins.survey_year} person member identity mismatch." + ) + with archive.open(info) as member: + try: + return pd.read_csv( + member, + usecols=usecols, + dtype={"PERIDNUM": "string"}, + low_memory=False, + ) + except ValueError as error: + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source is " + f"missing required column(s): {error}." + ) from error + + _verify_member_path(path, pins, chunk_size=chunk_size) + try: + return pd.read_csv( + path, + usecols=usecols, + dtype={"PERIDNUM": "string"}, + low_memory=False, + ) + except ValueError as error: + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source is missing " + f"required column(s): {error}." + ) from error + + +def _fixed_width_peridnum(values: pd.Series, *, label: str) -> pd.Series: + if values.isna().any(): + rows = values.index[values.isna()].tolist()[:5] + raise ValueError(f"{label} PERIDNUM is missing at row(s): {rows}.") + decoded = values.map( + lambda value: ( + value.decode() + if isinstance(value, (bytes, bytearray, np.bytes_)) + else value + ) + ) + valid = decoded.map(lambda value: isinstance(value, str)) + valid &= decoded.astype("string").str.fullmatch(r"[0-9]{22}", na=False) + if not valid.all(): + rows = decoded.index[~valid].tolist()[:5] + raise ValueError( + f"{label} PERIDNUM must be an exact 22-digit string at row(s): {rows}." + ) + return decoded.astype(str) + + +def _validated_source_year(values: pd.Series, *, label: str) -> pd.Series: + numeric = pd.to_numeric(values, errors="coerce") + valid = numeric.notna() & np.isfinite(numeric) & numeric.eq(np.floor(numeric)) + if not valid.all(): + rows = values.index[~valid].tolist()[:5] + raise ValueError(f"{label} source_year is invalid at row(s): {rows}.") + return numeric.astype(np.int64) + + +def _validated_codes(values: pd.Series, *, label: str) -> np.ndarray: + numeric = pd.to_numeric(values, errors="coerce").to_numpy(dtype=np.float64) + boolean = values.map(lambda value: isinstance(value, (bool, np.bool_))).to_numpy() + valid = np.isfinite(numeric) & np.isin(numeric, sorted(_VALID_CODES)) & ~boolean + if not valid.all(): + rows = np.flatnonzero(~valid)[:5].tolist() + raise ValueError( + f"{label} must be complete integers in {sorted(_VALID_CODES)} at " + f"row(s): {rows}." + ) + return numeric.astype(np.int64) + + +def load_asec_reported_coverage_sources( + paths: Mapping[int, str | Path] | None = None, + *, + income_years: tuple[int, ...] = ASEC_REPORTED_COVERAGE_INCOME_YEARS, + chunk_size: int = 8 * 1024 * 1024, +) -> pd.DataFrame: + """Load the seven measured coverage recodes from pinned ASEC members.""" + + unknown = sorted(set(paths or ()) - set(ASEC_EDUCATION_ASSISTANCE_ARCHIVES)) + if unknown: + raise ValueError( + "No pinned ASEC reported-coverage archive covers income year(s) " + f"{unknown}; pinned income years: " + f"{list(ASEC_REPORTED_COVERAGE_INCOME_YEARS)}." + ) + if not income_years: + empty = pd.DataFrame( + columns=["source_year", *ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS] + ) + empty.attrs["source_audit"] = {} + return empty + + parts: list[pd.DataFrame] = [] + audits: dict[int, dict[str, object]] = {} + for income_year in income_years: + pins = ASEC_EDUCATION_ASSISTANCE_ARCHIVES.get(income_year) + if pins is None: + raise ValueError( + "No pinned ASEC reported-coverage archive covers income year " + f"{income_year}." + ) + provided = None if paths is None else paths.get(income_year) + path = ( + Path(provided).expanduser() + if provided is not None + else fetch_asec_education_assistance_source(income_year) + ) + raw = _read_one_source(path, pins, chunk_size=chunk_size) + if len(raw) != pins.rows: + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source row count " + f"mismatch: expected {pins.rows}, got {len(raw)}." + ) + raw["PERIDNUM"] = _fixed_width_peridnum( + raw["PERIDNUM"], + label=f"ASEC {pins.survey_year} reported-coverage source", + ) + if raw["PERIDNUM"].duplicated(keep=False).any(): + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source PERIDNUM " + "must be unique." + ) + weights = pd.to_numeric(raw[_AUDIT_WEIGHT_COLUMN], errors="coerce").to_numpy( + dtype=np.float64 + ) + if ( + not np.isfinite(weights).all() + or (weights < 0.0).any() + or float(weights.sum()) <= 0.0 + ): + raise ValueError( + f"ASEC {pins.survey_year} A_FNLWGT must be finite and " + "nonnegative with positive total mass." + ) + + column_audits: dict[str, dict[str, float | int]] = {} + scaled_weights = weights / 100.0 + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + values = _validated_codes( + raw[column], + label=f"ASEC {pins.survey_year} {column}", + ) + raw[column] = values + yes = values == 1 + column_audits[column] = { + "no_rows": int(np.count_nonzero(values == 2)), + "yes_rows": int(np.count_nonzero(yes)), + "weighted_yes_share": float( + scaled_weights[yes].sum() / scaled_weights.sum() + ), + } + audits[income_year] = { + "rows": int(len(raw)), + "columns": column_audits, + } + part = raw.loc[:, list(ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS)].copy() + part.insert(0, "source_year", np.int64(income_year)) + parts.append(part) + + result = pd.concat(parts, ignore_index=True) + result.attrs["source_audit"] = audits + return result + + +def fill_asec_reported_coverage_source( + person: pd.DataFrame, + source: pd.DataFrame, +) -> pd.DataFrame: + """Restore all seven recodes by exact ``(source_year, PERIDNUM)`` join.""" + + required_person = ("source_year", "PERIDNUM") + missing_person = [column for column in required_person if column not in person] + if missing_person: + raise ValueError( + "ASEC reported-coverage repair requires person column(s): " + f"{missing_person}." + ) + required_source = ("source_year", *ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS) + missing_source = [column for column in required_source if column not in source] + if missing_source: + raise ValueError( + f"ASEC reported-coverage sidecar missing column(s): {missing_source}." + ) + + donor = source.copy(deep=True) + donor["source_year"] = _validated_source_year( + donor["source_year"], label="ASEC reported-coverage sidecar" + ) + donor["PERIDNUM"] = _fixed_width_peridnum( + donor["PERIDNUM"], label="ASEC reported-coverage sidecar" + ) + duplicate = donor.duplicated(["source_year", "PERIDNUM"], keep=False) + if duplicate.any(): + keys = donor.loc[duplicate, ["source_year", "PERIDNUM"]].head(5) + raise ValueError( + "ASEC reported-coverage sidecar (source_year, PERIDNUM) keys must " + f"be unique; duplicate key(s): {keys.to_dict(orient='records')}." + ) + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + donor[column] = _validated_codes( + donor[column], label=f"ASEC reported-coverage sidecar {column}" + ) + + result = person.copy(deep=True) + person_years = _validated_source_year( + result["source_year"], label="ASEC reported-coverage person" + ) + result["PERIDNUM"] = _fixed_width_peridnum( + result["PERIDNUM"], label="ASEC reported-coverage frame" + ) + needed_years = sorted(int(year) for year in person_years.unique()) + covered_years = set(int(year) for year in donor["source_year"].unique()) + uncovered = [year for year in needed_years if year not in covered_years] + if uncovered: + raise ValueError( + "ASEC reported-coverage sidecar does not cover pooled income " + f"year(s): {uncovered}." + ) + + for year in needed_years: + year_mask = person_years.eq(year).to_numpy() + year_donor = donor.loc[donor["source_year"].eq(year)].set_index("PERIDNUM") + keys = result.loc[year_mask, "PERIDNUM"] + missing_keys = keys[~keys.isin(year_donor.index)].drop_duplicates() + if not missing_keys.empty: + raise ValueError( + "ASEC reported-coverage sidecar does not cover frame PERIDNUM " + f"key(s) for income year {year}: {missing_keys.tolist()[:5]}." + ) + aligned = year_donor.reindex(keys.to_numpy()) + aligned.index = result.index[year_mask] + + identity_pairs = ( + ( + "PH_SEQ", + "source_household_id" if "source_household_id" in result else "PH_SEQ", + ), + ("P_SEQ", "P_SEQ"), + ("A_LINENO", "A_LINENO"), + ) + for donor_column, frame_column in identity_pairs: + if frame_column not in result: + continue + observed = pd.to_numeric( + result.loc[year_mask, frame_column], errors="coerce" + ).to_numpy(dtype=np.float64) + expected = pd.to_numeric(aligned[donor_column], errors="coerce").to_numpy( + dtype=np.float64 + ) + mismatch = ( + ~np.isfinite(observed) | ~np.isfinite(expected) | (observed != expected) + ) + if mismatch.any(): + rows = result.index[year_mask].to_numpy()[mismatch][:5].tolist() + raise ValueError( + "ASEC reported-coverage redundant identity mismatch for " + f"{frame_column} against sidecar {donor_column} in income " + f"year {year} at row(s): {rows}." + ) + + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + expected = aligned[column].to_numpy(dtype=np.int64) + if column in result: + current = result.loc[year_mask, column] + present = current.notna().to_numpy() + numeric = pd.to_numeric(current, errors="coerce").to_numpy( + dtype=np.float64 + ) + invalid = present & ( + ~np.isfinite(numeric) + | ~np.isin(numeric, sorted(_VALID_CODES)) + | current.map( + lambda value: isinstance(value, (bool, np.bool_)) + ).to_numpy() + ) + mismatch = present & ~invalid & (numeric != expected) + if invalid.any() or mismatch.any(): + bad = invalid | mismatch + rows = result.index[year_mask].to_numpy()[bad][:5].tolist() + raise ValueError( + f"ASEC reported-coverage existing {column} disagrees " + f"with the pinned sidecar in income year {year} at " + f"row(s): {rows}." + ) + else: + result[column] = np.nan + result.loc[year_mask, column] = expected + + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + result[column] = pd.to_numeric(result[column], errors="raise").astype("int64") + result.attrs["reported_coverage_source_audit"] = source.attrs.get( + "source_audit", {} + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py b/packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py new file mode 100644 index 000000000..1f38f18b1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py @@ -0,0 +1,31 @@ +"""Bind source CSV readers to the real stdlib builtin, including caller aliases.""" + +import _csv +import csv +from types import BuiltinFunctionType + +_NATIVE_CSV = _csv +_CSV = csv +_READER = _csv.reader + + +def csv_reader_bound(module) -> bool: + """Refuse live rebinding, including wrappers installed before this import.""" + return ( + type(_READER) is BuiltinFunctionType + and _READER.__module__ == "_csv" + and _READER.__name__ == "reader" + and _READER.__self__ is _NATIVE_CSV + and module is _CSV + and getattr(module, "reader", None) is _READER + and getattr(_NATIVE_CSV, "reader", None) is _READER + ) + + +def capture_csv_reader(module): + """Return the checked builtin itself, never a later mutable alias lookup.""" + return _READER if csv_reader_bound(module) else None + + +if not csv_reader_bound(csv): + raise ValueError("SOURCE_CSV_READER_CHANGED") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py b/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py index ce6b6e296..4f39da34b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py @@ -8,14 +8,17 @@ from __future__ import annotations +import hashlib import re from collections.abc import Mapping +from dataclasses import asdict, dataclass +from types import MappingProxyType from typing import Any import numpy as np import pandas as pd -from microcosm.build.us_runtime.puf_support import ( +from microcosm.build.us_runtime.operator_column_contracts import ( PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID, ) from microcosm.build.us_runtime.support_provenance import ( @@ -36,7 +39,14 @@ Weights, ) -__all__ = ["assemble_spines"] +__all__ = [ + "SpinePreparation", + "SpineHarmonization", + "assemble_spines", + "prepare_spines", + "stack_survey_spines", + "harmonize_spine_weights", +] _CHANNEL_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$") _SUPPORT_CLONE_INDEX = 0 @@ -80,11 +90,146 @@ def assemble_spines( provenance, columns, or ID spaces violate the assembly contract. """ + prepared = _stack_spine_tables( + spines, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + harmonized = _harmonize_spine_values( + prepared.tables["household"], prepared.values, prepared.context + ) + # These are newly composed per-source records, not a carried Frame log. + # Table provenance remains owned by the independent stacking result. + assembled_mass_log = harmonized.mass_log + result = Frame( + prepared.tables, + US_SCHEMA, + {"household": harmonized.weights}, + prepared.strata, + mass_log=assembled_mass_log, + metadata=prepared.metadata, + ) + validate_assembly_provenance(result, boundary="spine assembly output") + anchor_mass = prepared.context["incoming_masses"][mass_anchor_channel] + if not np.isclose( + result.weights_for("household").total, anchor_mass, rtol=_SHARE_RTOL, atol=0.0 + ): + raise RuntimeError("Spine assembly failed to conserve anchor household mass.") + return result + + +@dataclass(frozen=True) +class SpinePreparation: + """Actual DESIGN-weight tables plus an immutable assembly context. + + The context schema determines which subsequent weight operation can use it. + This numerical container does not authenticate source issuance or membership. + """ + + frame: Frame + context: Mapping[str, object] + + def __post_init__(self) -> None: + if not isinstance(self.frame, Frame): + raise TypeError("SpinePreparation.frame must be a Frame.") + if self.frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError("Spine preparation requires actual DESIGN weights.") + if not isinstance(self.context, Mapping): + raise ValueError("Spine preparation context must be a mapping.") + object.__setattr__(self, "context", _freeze_context(self.context)) + + +@dataclass(frozen=True) +class SpineHarmonization: + """Computed importance weights and exact ordered legacy operator history.""" + + weights: Weights + mass_log: tuple[MassChangeRecord, ...] + + +@dataclass(frozen=True) +class _StackedTables: + tables: Mapping[str, pd.DataFrame] + strata: pd.Series + values: np.ndarray + context: Mapping[str, object] + metadata: Mapping[str, object] + mass_log: tuple[MassChangeRecord, ...] + + +def _freeze_context(value: object) -> object: + if isinstance(value, Mapping): + return MappingProxyType( + {key: _freeze_context(item) for key, item in value.items()} + ) + if isinstance(value, (list, tuple)): + return tuple(_freeze_context(item) for item in value) + return value + + +def _numeric_digest(values: np.ndarray, dtype: str) -> str: + array = np.ascontiguousarray(values, dtype=dtype) + return hashlib.sha256(memoryview(array).cast("B")).hexdigest() + + +def _channel_digest(values: np.ndarray) -> str: + digest = hashlib.sha256() + for value in values: + if not isinstance(value, str): + raise ValueError("Prepared household channels must be strings.") + encoded = value.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "little")) + digest.update(encoded) + return digest.hexdigest() + + +def _stack_spine_tables( + spines: Mapping[str, Frame], + *, + household_mass_shares: Mapping[str, float], + mass_anchor_channel: str, + require_design: bool = False, +) -> _StackedTables: ordered_channels = _validated_channels(spines, mass_anchor_channel) shares = _validated_shares(household_mass_shares, ordered_channels) frames = {channel: spines[channel] for channel in ordered_channels} + prepared = _stack_source_tables(frames, ordered_channels, require_design) + context = dict(prepared.context) + context.update( + schema="microcosm.us.spine-preparation.v1", + household_mass_shares=shares, + mass_anchor_channel=mass_anchor_channel, + incoming_masses={ + channel: float(frames[channel].weights_for("household").total) + for channel in ordered_channels + }, + ) + return _StackedTables( + prepared.tables, + prepared.strata, + prepared.values, + _freeze_context(context), + prepared.metadata, + prepared.mass_log, + ) + + +def _stack_source_tables( + spines: Mapping[str, Frame], + ordered_channels: tuple[str, ...], + require_design: bool, +) -> _StackedTables: + """Preserve source values while remapping only structural collisions.""" + frames = {channel: spines[channel] for channel in ordered_channels} for channel, frame in frames.items(): _validate_source_frame(frame, channel=channel) + if ( + require_design + and frame.weights_for("household").kind is not WeightKind.DESIGN + ): + raise ValueError( + f"Spine {channel!r} preparation requires actual DESIGN weights." + ) _validate_shared_column_dtypes(frames) offsets = _id_offsets(frames, ordered_channels) @@ -106,51 +251,286 @@ def assemble_spines( column_orders=column_orders, ) - anchor_mass = frames[mass_anchor_channel].weights_for("household").total - weights, mass_log = _assembled_household_weights( - frames, - ordered_channels=ordered_channels, - shares=shares, - anchor_mass=anchor_mass, + values = np.concatenate( + [ + frames[channel].weights_for("household").values + for channel in ordered_channels + ] ) - household_order = group_orders.get("household") - if household_order is not None: - weights = Weights(weights.values[household_order], weights.kind) - weights = _with_exact_total(weights, anchor_mass) - + order = group_orders.get("household") + if order is not None: + values = values[order] + household = tables["household"] + source_ids_digest = hashlib.sha256() + for channel in ordered_channels: + source_ids = np.ascontiguousarray( + prepared[channel]["household"]["household_id"].to_numpy(), dtype=" SpinePreparation: + """Stack actual DESIGN sources without performing importance allocation. + + Legacy ``assemble_spines`` continues to accept its existing mixed-kind + inputs. This new graph-facing seam never relabels such inputs as DESIGN. + """ + prepared = _stack_spine_tables( + spines, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + require_design=True, + ) + return _design_preparation(prepared) + + +def stack_survey_spines(spines: Mapping[str, Frame]) -> SpinePreparation: + """Stack original DESIGN survey weights before domain allocation. + + Channels are ordered lexically, independent of mapping insertion order. + Every household weight, source age and measured value is carried unchanged; + only structural IDs are remapped and source provenance is added. No survey + total is an anchor, no shares are assigned, and no mass is normalized. + + This is the numerical stacking seam. A graph source owner must separately + authenticate the original issuances, complete membership and publisher + anchors. Sampling and population-domain allocation are subsequent declared + operations. The returned v1 survey-stack context is deliberately refused + by the legacy anchor-total harmonizer. + """ + channels = _validated_source_channels(spines) + prepared = _stack_source_tables(spines, channels, require_design=True) + return _design_preparation(prepared) + + +def _design_preparation(prepared: _StackedTables) -> SpinePreparation: + frame = Frame( + prepared.tables, + US_SCHEMA, + {"household": Weights(prepared.values, WeightKind.DESIGN)}, + prepared.strata, + mass_log=prepared.mass_log, + metadata=prepared.metadata, ) - if not np.isclose( - result.weights_for("household").total, - anchor_mass, - rtol=_SHARE_RTOL, - atol=0.0, + validate_assembly_provenance(frame, boundary="spine preparation output") + return SpinePreparation(frame, prepared.context) + + +def harmonize_spine_weights( + *, + household: pd.DataFrame, + weights: Weights, + context: Mapping[str, object], +) -> SpineHarmonization: + """Compute importance allocation from declared DESIGN views and context. + + Input digests bind exact normalized values and ordered IDs/channels. + The explicit permutation restores source-order sums and correction rows. + The result includes legacy per-source logs, not an executor graph ledger. + """ + if not isinstance(weights, Weights) or weights.kind is not WeightKind.DESIGN: + raise ValueError("Spine harmonization requires actual DESIGN weights.") + if ( + not isinstance(context, Mapping) + or context.get("schema") != "microcosm.us.spine-preparation.v1" ): - raise RuntimeError("Spine assembly failed to conserve anchor household mass.") - return result + raise ValueError("Unsupported spine preparation context.") + kinds = context.get("source_weight_kinds") + ordered = context.get("ordered_channels") + if ( + not isinstance(kinds, Mapping) + or not isinstance(ordered, (tuple, list)) + or set(kinds) != set(ordered) + or any(kind != WeightKind.DESIGN.value for kind in kinds.values()) + ): + raise ValueError("Spine preparation context does not declare DESIGN sources.") + return _harmonize_spine_values(household, weights.values, context) + + +def _validated_spine_inputs( + household: pd.DataFrame, values: np.ndarray, context: Mapping[str, object] +) -> tuple: + if ( + not isinstance(context, Mapping) + or context.get("schema") != "microcosm.us.spine-preparation.v1" + ): + raise ValueError("Unsupported spine preparation context.") + if not isinstance(household, pd.DataFrame) or not { + "household_id", + support_channel_column("household"), + }.issubset(household): + raise ValueError("Harmonization requires household IDs and source channels.") + ids = household["household_id"].to_numpy() + channels = household[support_channel_column("household")].to_numpy() + if not np.issubdtype(ids.dtype, np.integer) or len(values) != len(ids): + raise ValueError("Prepared household IDs/weights are malformed.") + bindings = { + "household_ids_sha256": _numeric_digest(ids, " SpineHarmonization: + ordered, shares, counts, source_values, source_channels, order, anchor_mass = ( + _validated_spine_inputs(household, values, context) + ) + result, logs, allocated, start = [], [], 0.0, 0 + for index, channel in enumerate(ordered): + count = counts[channel] + existing_values = source_values[start : start + count] + if not np.all(source_channels[start : start + count] == channel): + raise ValueError( + "Prepared permutation does not restore source channel order." + ) + start += count + existing_mass = float(existing_values.sum()) + if existing_mass != float(context["incoming_masses"][channel]): + raise ValueError("Prepared source-order mass changed.") + target = ( + anchor_mass - allocated + if index == len(ordered) - 1 + else anchor_mass * shares[channel] + ) + scaled = _values_to_total(existing_values, target) + result.append(scaled) + allocated += float(scaled.sum()) + logs.extend( + MassChangeRecord(**dict(record)) + for record in context["source_mass_logs"][channel] + ) + logs.append( + MassChangeRecord( + entity="household", + old_total=existing_mass, + new_total=float(scaled.sum()), + declared_factor=target / existing_mass, + reason=f"allocated {channel!r} source mass in pre-operator spine assembly", + ) + ) + combined = np.concatenate(result) + if order is not None: + combined = combined[order] + return SpineHarmonization( + _with_exact_total(Weights(combined, WeightKind.IMPORTANCE), anchor_mass), + tuple(logs), + ) def _validated_channels( spines: Mapping[str, Frame], mass_anchor_channel: str, ) -> tuple[str, ...]: + channels = _validated_source_channels(spines) + if ( + not isinstance(mass_anchor_channel, str) + or _CHANNEL_PATTERN.fullmatch(mass_anchor_channel) is None + ): + raise ValueError( + "mass_anchor_channel must be a stable lower-snake-case identifier." + ) + if mass_anchor_channel not in channels: + raise ValueError( + f"mass_anchor_channel {mass_anchor_channel!r} is absent from spines." + ) + return ( + mass_anchor_channel, + *(channel for channel in channels if channel != mass_anchor_channel), + ) + + +def _validated_source_channels(spines: Mapping[str, Frame]) -> tuple[str, ...]: if not isinstance(spines, Mapping): raise TypeError(f"spines must be a mapping, got {type(spines).__name__}.") if len(spines) < 2: @@ -171,21 +551,7 @@ def _validated_channels( f"{PUF_TAX_DETAIL_SUPPORT_CHANNEL!r} is a clone operator channel, " "not a peer household spine." ) - if ( - not isinstance(mass_anchor_channel, str) - or _CHANNEL_PATTERN.fullmatch(mass_anchor_channel) is None - ): - raise ValueError( - "mass_anchor_channel must be a stable lower-snake-case identifier." - ) - if mass_anchor_channel not in spines: - raise ValueError( - f"mass_anchor_channel {mass_anchor_channel!r} is absent from spines." - ) - return ( - mass_anchor_channel, - *sorted(channel for channel in channels if channel != mass_anchor_channel), - ) + return tuple(sorted(channels)) def _validated_shares( @@ -394,6 +760,11 @@ def _id_offsets( ) offsets[channel][entity] = offset remapped = ids if offset == 0 else ids + offset + if int(remapped.max()) > PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID: + raise ValueError( + f"Spine {channel!r} {entity!r} collision remapping exceeds " + f"the clone-safe bound {PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID}." + ) accumulated[entity] = np.concatenate([used, remapped]) return offsets @@ -493,45 +864,6 @@ def _combined_tables( return tables, group_orders -def _assembled_household_weights( - frames: Mapping[str, Frame], - *, - ordered_channels: tuple[str, ...], - shares: Mapping[str, float], - anchor_mass: float, -) -> tuple[Weights, tuple[MassChangeRecord, ...]]: - values: list[np.ndarray] = [] - mass_log: list[MassChangeRecord] = [] - allocated = 0.0 - for index, channel in enumerate(ordered_channels): - existing = frames[channel].weights_for("household") - target = ( - anchor_mass - allocated - if index == len(ordered_channels) - 1 - else anchor_mass * shares[channel] - ) - scaled = _values_to_total(existing.values, target) - values.append(scaled) - allocated += float(scaled.sum()) - factor = target / existing.total - mass_log.extend(frames[channel].mass_log) - mass_log.append( - MassChangeRecord( - entity="household", - old_total=existing.total, - new_total=float(scaled.sum()), - declared_factor=factor, - reason=( - f"allocated {channel!r} source mass in pre-operator spine assembly" - ), - ) - ) - return ( - Weights(np.concatenate(values), WeightKind.IMPORTANCE), - tuple(mass_log), - ) - - def _values_to_total(values: np.ndarray, target: float) -> np.ndarray: result = np.asarray(values, dtype=np.float64) * ( target / float(np.asarray(values, dtype=np.float64).sum()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index d71320311..1e62b8c2d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -26,10 +26,11 @@ ``w_i' = w_i * share_s * M_anchor / M_s`` where ``M_s`` is arm ``s``'s incoming household mass and ``M_anchor`` is the -mass-anchor arm's incoming mass. For the seeded ACS sample, -``M_acs_sample ~= fraction * M_acs_full``, so the allocation factor contains -the inverse-sampling upweighting ``1 / fraction`` automatically; the realized -per-arm scale factors are receipted rather than assumed. +mass-anchor arm's incoming mass. Production sampling first normalizes each +source by its full-source mass divided by its realized sampled mass. For +unequal source weights, that ratio is not generally the inverse realized +household inclusion count ``N / n``. Legacy pilot controls allocate directly +from the sampled ACS mass. Both paths receipt their actual scale factors. """ from __future__ import annotations @@ -40,7 +41,6 @@ import math import os import pickle -import struct import sys from collections import Counter from collections.abc import Callable, Mapping, Sequence @@ -80,6 +80,12 @@ ) from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes from microcosm.build.source_manifest import load_source_manifest +from microcosm.build.table_identity import ( + TABLE_VALUES_DIGEST_CODEC as _LATE_TABLE_DIGEST_CODEC, +) +from microcosm.build.table_identity import ( + table_values_sha256 as _late_table_values_sha256, +) from microcosm.build.us_runtime import ( post_transfer_calibration as post_transfer_calibration_runtime, ) @@ -183,7 +189,13 @@ puf_tax_detail_tail_bound_quantiles_identity, validate_puf_clone_attachment, ) -from microcosm.build.us_runtime.spine_assembly import assemble_spines +from microcosm.build.us_runtime.spine_assembly import ( + _freeze_context, + _validated_spine_inputs, + assemble_spines, + harmonize_spine_weights, + prepare_spines, +) from microcosm.build.us_runtime.support_provenance import ( BASE_ASEC_SUPPORT_CHANNEL, PUF_TAX_DETAIL_CLONE_INDEX, @@ -220,7 +232,7 @@ us_late_producer_schedule_receipt, ) from microcosm.fit import Regime -from microcosm.frame import US_SCHEMA, Frame, WeightKind +from microcosm.frame import US_SCHEMA, Frame, MassChangeRecord, WeightKind, Weights __all__ = [ "ACS_STACKED_SUPPORT_CHANNEL", @@ -249,6 +261,11 @@ "StackedLateProducerResult", "StackedPostPufTransferResult", "StackedSpineResult", + "StackedSpinePreparation", + "StackedSpineHarmonization", + "prepare_stacked_spine", + "harmonize_stacked_spine_weights", + "finish_stacked_spine", "assemble_stacked_spine", "assert_stacked_tail_cells_preserved", "by_origin_battery", @@ -553,7 +570,47 @@ def _acs_native_group_quarters_receipt( } -def assemble_stacked_spine( +@dataclass(frozen=True) +class _SampledStack: + spines: Mapping[str, Frame] + shares: Mapping[str, float] + incoming_masses: Mapping[str, float] + sampling: Mapping[str, object] + manifest_version: int + mass_anchor_channel: str + + +@dataclass(frozen=True) +class StackedSpinePreparation: + """DESIGN preparation and a distinct pre-harmonization receipt.""" + + frame: Frame + receipt: Mapping[str, object] + + def __post_init__(self) -> None: + if not isinstance(self.frame, Frame) or not isinstance(self.receipt, Mapping): + raise TypeError( + "StackedSpinePreparation requires a Frame and receipt mapping." + ) + object.__setattr__(self, "receipt", _freeze_context(self.receipt)) + _validate_stacked_preparation(self.frame, self.receipt) + + +@dataclass(frozen=True) +class StackedSpineHarmonization: + """Actual weights and completed context, with ordered legacy history.""" + + weights: Weights + metadata: Mapping[str, object] + receipt: Mapping[str, object] + legacy_mass_log: tuple[MassChangeRecord, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", _freeze_context(self.metadata)) + object.__setattr__(self, "receipt", _freeze_context(self.receipt)) + + +def _sample_stacked_inputs( asec: Frame, acs: Frame, *, @@ -563,27 +620,8 @@ def assemble_stacked_spine( sample_seed: int | None = None, household_mass_shares: Mapping[str, float] | None = None, mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, -) -> StackedSpineResult: - """Assemble uniformly sampled ASEC and ACS survey arms into one spine. - - Production callers provide the single ``sample_fraction`` and - ``sample_seed`` scale-ladder controls. The exact same fraction is applied - independently to both survey arms at whole-household grain; each sampled - arm is then normalized back to its full-source household mass before the - reviewed :func:`assemble_spines` harmonization. This preserves each arm's - composition and prevents the anchor population from shrinking with the - rung. PUF donors are not accepted here and therefore remain unsampled. - - ``acs_sample_fraction``/``acs_sample_seed`` retain the reviewed version-1 - pilot contract for reproducibility only: ASEC remains full and ACS alone - is sampled. Supplying pilot and production controls together fails - closed. - - Returns: - A validated :class:`StackedSpineResult` whose receipt mirrors the - frozen manifest as a JSON-ready mapping. - """ - +) -> _SampledStack: + """Run the unchanged pilot or production sampling/normalization controls.""" shares = ( dict(DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES) if household_mass_shares is None @@ -688,6 +726,287 @@ def assemble_stacked_spine( }, } + return _SampledStack( + { + BASE_ASEC_SUPPORT_CHANNEL: sampled_asec, + ACS_STACKED_SUPPORT_CHANNEL: sampled_acs, + }, + shares, + incoming_masses, + sampling_manifest, + manifest_version, + mass_anchor_channel, + ) + + +def assemble_stacked_spine( + asec: Frame, + acs: Frame, + *, + acs_sample_fraction: float | None = None, + acs_sample_seed: int | None = None, + sample_fraction: float | None = None, + sample_seed: int | None = None, + household_mass_shares: Mapping[str, float] | None = None, + mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, +) -> StackedSpineResult: + """Compose actual sampling, stacking and importance harmonization. + + Production controls normalize each sampled source by M_full/M_sample. + This ratio is not generally inverse realized household inclusion N/n. + Legacy pilot controls and accepted source weight kinds remain unchanged. + """ + sampled = _sample_stacked_inputs( + asec, + acs, + acs_sample_fraction=acs_sample_fraction, + acs_sample_seed=acs_sample_seed, + sample_fraction=sample_fraction, + sample_seed=sample_seed, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + if all( + frame.weights_for("household").kind is WeightKind.DESIGN + for frame in sampled.spines.values() + ): + return finish_stacked_spine(_prepare_sampled_stack(sampled)) + # Existing direct callers may supply calibrated or mixed source kinds. + # They keep their historical path, never relabeled as a DESIGN anchor. + return _finish_legacy_sampled_stack(sampled) + + +def prepare_stacked_spine( + asec: Frame, + acs: Frame, + *, + acs_sample_fraction: float | None = None, + acs_sample_seed: int | None = None, + sample_fraction: float | None = None, + sample_seed: int | None = None, + household_mass_shares: Mapping[str, float] | None = None, + mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, +) -> StackedSpinePreparation: + """Sample/normalize actual DESIGN sources and stack them before allocation. + + Returns a dedicated preparation receipt, never an incomplete completed + stacked-spine manifest. No harmonized weights are precomputed. + """ + sampled = _sample_stacked_inputs( + asec, + acs, + acs_sample_fraction=acs_sample_fraction, + acs_sample_seed=acs_sample_seed, + sample_fraction=sample_fraction, + sample_seed=sample_seed, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + return _prepare_sampled_stack(sampled) + + +def _prepare_sampled_stack(sampled: _SampledStack) -> StackedSpinePreparation: + prepared = prepare_spines( + sampled.spines, + household_mass_shares=sampled.shares, + mass_anchor_channel=sampled.mass_anchor_channel, + ) + samples = sampled.sampling.get( + "survey_samples", + {ACS_STACKED_SUPPORT_CHANNEL: sampled.sampling.get("acs_sample")}, + ) + full_masses = { + channel: float(samples[channel]["incoming_household_mass"]) + if channel in samples + else float(sampled.incoming_masses[channel]) + for channel in sampled.spines + } + receipt = { + "schema": "microcosm.us.stacked-spine-preparation.v1", + "stacked_manifest_version": sampled.manifest_version, + "sampling": sampled.sampling, + "full_source_masses": full_masses, + "household_mass_shares": sampled.shares, + "mass_anchor_channel": sampled.mass_anchor_channel, + "assembly_context": prepared.context, + "assembly_metadata": prepared.frame.metadata, + "acs_native_group_quarters": _acs_native_group_quarters_receipt( + sampled.spines[ACS_STACKED_SUPPORT_CHANNEL], prepared.frame + ), + } + return StackedSpinePreparation(prepared.frame, receipt) + + +def _validate_stacked_preparation(frame: Frame, receipt: Mapping[str, object]) -> None: + boundary = "stacked spine preparation" + if frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError("Stacked spine preparation requires actual DESIGN weights.") + if STACKED_SPINE_MANIFEST_KEY in frame.metadata: + raise ValueError( + "A preparation must not carry a completed stacked spine manifest." + ) + validate_assembly_provenance(frame, boundary=boundary) + _validate_preparation_receipt(receipt) + if _json_ready(receipt["assembly_metadata"]) != _json_ready(frame.metadata): + raise ValueError("Prepared assembly metadata differs from the actual frame.") + _validated_spine_inputs( + frame.table("household"), + frame.weights_for("household").values, + receipt["assembly_context"], + ) + version, sampling = receipt["stacked_manifest_version"], receipt["sampling"] + production = version == _STACKED_SPINE_MANIFEST_VERSION + fraction = sampling["sample_fraction" if production else "acs_sample_fraction"] + seed = sampling["sample_seed" if production else "acs_sample_seed"] + _validate_fraction(fraction) + _validate_seed(seed) + samples = ( + sampling["survey_samples"] + if production + else {ACS_STACKED_SUPPORT_CHANNEL: sampling["acs_sample"]} + ) + expected = ( + {BASE_ASEC_SUPPORT_CHANNEL, ACS_STACKED_SUPPORT_CHANNEL} + if production + else {ACS_STACKED_SUPPORT_CHANNEL} + ) + if set(samples) != expected: + raise ValueError( + "Prepared sampling receipts do not cover their declared sources." + ) + for channel, sample in samples.items(): + _validate_survey_sample_receipt( + frame, + channel=channel, + fraction=fraction, + seed=seed, + sample=sample, + boundary=boundary, + require_normalization=production, + ) + if float(receipt["full_source_masses"][channel]) != float( + sample["incoming_household_mass"] + ): + raise ValueError( + "Prepared full-source mass differs from its sampling receipt." + ) + _validate_stacked_clone_role_lifecycle(frame, boundary=boundary) + _validated_acs_native_group_quarters_masks(frame, receipt, boundary=boundary) + + +def _validate_preparation_receipt(preparation: Mapping[str, object]) -> None: + if ( + not isinstance(preparation, Mapping) + or preparation.get("schema") != "microcosm.us.stacked-spine-preparation.v1" + ): + raise ValueError("Unsupported stacked-spine preparation receipt.") + if ( + preparation.get("stacked_manifest_version") + not in _SUPPORTED_STACKED_SPINE_MANIFEST_VERSIONS + ): + raise ValueError("Unsupported prepared stacked-spine manifest version.") + for field_name in ( + "sampling", + "full_source_masses", + "household_mass_shares", + "assembly_context", + "assembly_metadata", + "acs_native_group_quarters", + ): + if not isinstance(preparation.get(field_name), Mapping): + raise ValueError( + f"Prepared stacked-spine field {field_name!r} must be a mapping." + ) + context = preparation["assembly_context"] + if preparation.get("mass_anchor_channel") != context.get( + "mass_anchor_channel" + ) or preparation["household_mass_shares"] != context.get("household_mass_shares"): + raise ValueError("Prepared stack controls disagree with the assembly context.") + if set(preparation["full_source_masses"]) != { + BASE_ASEC_SUPPORT_CHANNEL, + ACS_STACKED_SUPPORT_CHANNEL, + }: + raise ValueError("Prepared full-source masses must name ASEC and ACS.") + + +def harmonize_stacked_spine_weights( + *, + household: pd.DataFrame, + weights: Weights, + preparation: Mapping[str, object], +) -> StackedSpineHarmonization: + """Compute importance weights and completed metadata from declared views. + + The preparation producer validates full source/person lineage. This helper + binds its household views and performs allocation without restacking or + reading a precomputed final-weight vector. The returned legacy mass log is + distinct from a graph executor's structural transition history. + """ + _validate_preparation_receipt(preparation) + result = harmonize_spine_weights( + household=household, weights=weights, context=preparation["assembly_context"] + ) + shares = preparation["household_mass_shares"] + incoming = preparation["assembly_context"]["incoming_masses"] + anchor = preparation["mass_anchor_channel"] + harmonization = _harmonization_receipt_from_views( + household, + result.weights, + shares=shares, + anchor_mass=incoming[anchor], + incoming_masses=incoming, + ) + manifest = { + "version": preparation["stacked_manifest_version"], + **preparation["sampling"], + "household_mass_shares": { + channel: float(share) for channel, share in shares.items() + }, + "mass_anchor_channel": anchor, + "weight_harmonization": harmonization, + "acs_native_group_quarters": preparation["acs_native_group_quarters"], + } + metadata = { + **preparation["assembly_metadata"], + STACKED_SPINE_MANIFEST_KEY: manifest, + } + return StackedSpineHarmonization( + result.weights, metadata, _json_ready(manifest), result.mass_log + ) + + +def finish_stacked_spine(prepared: StackedSpinePreparation) -> StackedSpineResult: + """Complete a validated preparation, preserving exact legacy Frame history.""" + if not isinstance(prepared, StackedSpinePreparation): + raise TypeError("finish_stacked_spine requires a StackedSpinePreparation.") + _validate_stacked_preparation(prepared.frame, prepared.receipt) + result = harmonize_stacked_spine_weights( + household=prepared.frame.table("household"), + weights=prepared.frame.weights_for("household"), + preparation=prepared.receipt, + ) + frame = Frame( + {entity: prepared.frame.table(entity) for entity in prepared.frame.entities}, + prepared.frame.schema, + {"household": result.weights}, + prepared.frame.strata, + mass_log=result.legacy_mass_log, + metadata=result.metadata, + ) + validated = validate_stacked_spine_frame( + frame, boundary="stacked spine assembly output" + ) + return StackedSpineResult(frame, _json_ready(validated)) + + +def _finish_legacy_sampled_stack(sampled: _SampledStack) -> StackedSpineResult: + sampled_asec, sampled_acs = ( + sampled.spines[BASE_ASEC_SUPPORT_CHANNEL], + sampled.spines[ACS_STACKED_SUPPORT_CHANNEL], + ) + shares, incoming_masses = sampled.shares, sampled.incoming_masses + sampling_manifest, manifest_version = sampled.sampling, sampled.manifest_version + mass_anchor_channel = sampled.mass_anchor_channel assembled = assemble_spines( { BASE_ASEC_SUPPORT_CHANNEL: sampled_asec, @@ -1632,9 +1951,25 @@ def _harmonization_receipt( anchor_mass: float, incoming_masses: Mapping[str, float], ) -> dict[str, dict[str, float]]: - household = assembled.table("household") + return _harmonization_receipt_from_views( + assembled.table("household"), + assembled.weights_for("household"), + shares=shares, + anchor_mass=anchor_mass, + incoming_masses=incoming_masses, + ) + + +def _harmonization_receipt_from_views( + household: pd.DataFrame, + household_weights: Weights, + *, + shares: Mapping[str, float], + anchor_mass: float, + incoming_masses: Mapping[str, float], +) -> dict[str, dict[str, float]]: channel_values = household[support_channel_column("household")].astype(str) - weights = np.asarray(assembled.weights_for("household").values, dtype=np.float64) + weights = np.asarray(household_weights.values, dtype=np.float64) receipt: dict[str, dict[str, float]] = {} for channel, share in shares.items(): incoming = float(incoming_masses[channel]) @@ -5104,7 +5439,6 @@ def validate_stacked_post_puf_transfer_receipt( ) -_LATE_TABLE_DIGEST_CODEC = "canonical_scalar_v1" _LATE_PRIMARY_QRF_INPUT_BINDING_ARTIFACT_KIND = ( "populace_us_stacked_late_primary_qrf_input_binding" ) @@ -5112,259 +5446,6 @@ def validate_stacked_post_puf_transfer_receipt( _LATE_RESOURCE_SEMANTICS_ARTIFACT_KIND = ( "populace_us_stacked_late_producer_resource_semantics" ) -_LATE_TABLE_DIGEST_CHUNK_ROWS = 65_536 - - -def _late_digest_part( - digest, - *, - domain: str, - payload: bytes | bytearray | memoryview | np.ndarray, -) -> None: - """Append one length-framed, domain-separated byte field to a digest.""" - - domain_bytes = domain.encode("utf-8") - payload_view = memoryview(payload) - if payload_view.format != "B" or payload_view.ndim != 1: - payload_view = payload_view.cast("B") - digest.update(struct.pack(" np.ndarray: - """Return a contiguous, explicitly little-endian numeric byte source.""" - - array = np.asarray(values) - array = array.astype(array.dtype.newbyteorder("<"), copy=False) - return np.ascontiguousarray(array) - - -def _late_scalar_bytes(value: object) -> bytes: - """Encode one supported object scalar without lossy intermediary hashes.""" - - missing = pd.isna(value) - if isinstance(missing, (bool, np.bool_)) and bool(missing): - return b"null" - if isinstance(value, (bool, np.bool_)): - return b"bool\x01" if bool(value) else b"bool\x00" - if isinstance(value, (int, np.integer)): - return b"integer\x00" + str(int(value)).encode("ascii") - if isinstance(value, (float, np.floating)): - if isinstance(value, np.floating) and value.dtype.itemsize > 8: - raise TypeError( - "US late-producer content digest does not support object " - f"floating scalar {value.dtype!s}." - ) - return b"float64\x00" + struct.pack(" 16: - raise TypeError( - "US late-producer content digest does not support object " - f"complex scalar {value.dtype!s}." - ) - numeric = complex(value) - return b"complex128\x00" + struct.pack(" None: - """Stream framed string or object scalars in bounded-memory chunks.""" - - for chunk_index, start in enumerate( - range(0, len(values), _LATE_TABLE_DIGEST_CHUNK_ROWS) - ): - stop = min(start + _LATE_TABLE_DIGEST_CHUNK_ROWS, len(values)) - lengths = np.zeros(stop - start, dtype=" None: - """Hash one ordered logical Series with explicit dtype and null domains.""" - - dtype = series.dtype - missing = series.isna().to_numpy(dtype=bool) - _late_digest_part( - digest, - domain=f"{domain}/dtype", - payload=str(dtype).encode("utf-8"), - ) - _late_digest_part( - digest, - domain=f"{domain}/row_count", - payload=struct.pack(" str: - """Hash ordered table scalars directly with typed, null-aware framing.""" - - values = ( - canonicalize_table_string_dtypes( - table, - boundary="late-producer content digest", - table_name="declared_surface", - ) - if normalize_strings - else table - ) - if isinstance(values.index, pd.MultiIndex): - index_levels = [ - pd.Series(values.index.get_level_values(level), copy=False) - for level in range(values.index.nlevels) - ] - else: - index_levels = [pd.Series(values.index, copy=False)] - header = { - "codec": _LATE_TABLE_DIGEST_CODEC, - "columns": [str(column) for column in values.columns], - "dtypes": [ - str(values.iloc[:, index].dtype) for index in range(values.shape[1]) - ], - "index_type": type(values.index).__name__, - "index_dtype": str(values.index.dtype), - "index_level_dtypes": [str(level.dtype) for level in index_levels], - "index_names": [ - None if name is None else str(name) for name in values.index.names - ], - } - digest = hashlib.sha256() - _late_digest_part( - digest, - domain="late_table_digest_codec", - payload=_LATE_TABLE_DIGEST_CODEC.encode("ascii"), - ) - _late_digest_part( - digest, - domain="late_table_header_json", - payload=json.dumps( - header, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8"), - ) - for level_index, level in enumerate(index_levels): - _late_digest_series_values( - digest, - level, - domain=f"index_level/{level_index}", - ) - for column_index in range(values.shape[1]): - _late_digest_series_values( - digest, - values.iloc[:, column_index], - domain=f"column/{column_index}", - ) - return digest.hexdigest() def _late_virtual_resource_kind(column: str) -> str: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py new file mode 100644 index 000000000..deffa57ee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py @@ -0,0 +1,220 @@ +"""Explicit S0101-only activation for the selected ACS/ASEC development path. + +A reviewed declaration pins two source responses. The caller's review and +guarded acquisition supply trust; this decoder is not independent attestation. +There is deliberately no default genuine declaration or mixed-inventory reader. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, replace +from pathlib import Path + +from microcosm.calibrate.registry import TargetRegistry, TargetSpec +from microcosm.graph.canonical import canonical_json + +from . import demographic_calibration_graph as demographic +from . import national_age_activation as age +from .survey_age_sources import _digest, read_survey_age_response, survey_age_requests + +PROFILE = "selected_survey_s0101_development_v1" +_CONVENTION = ( + "ACS 2024 observed interview ages and ASEC 2025 interview ages with income " + "year 2024; completed years used verbatim without temporal aging; calibration " + "to the published 2024 all-resident distribution does not establish that " + "the selected survey population covers that complete universe" +) + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ValueError("SURVEY_AGE_ACTIVATION_" + reason) + + +@dataclass(frozen=True, slots=True) +class SurveyAgeActivation: + """Exact source pins whose approval is a separate caller responsibility.""" + + metadata_sha256: str + data_sha256: str + + def __post_init__(self) -> None: + _digest(self.metadata_sha256) + _digest(self.data_sha256) + + +def activation_binding(declaration: SurveyAgeActivation) -> dict: + """Return the complete, detached semantic declaration, without source I/O.""" + _require(type(declaration) is SurveyAgeActivation, "DECLARATION") + declaration.__post_init__() + return { + "profile": PROFILE, + "requests": list(survey_age_requests()), + "metadata_sha256": declaration.metadata_sha256, + "data_sha256": declaration.data_sha256, + "period": "2024", + "entity": "household", + "universe": age.NATIONAL_AGE_ACTIVATION.universe, + "age_convention_id": "observed_interview_age_completed_years", + "age_convention": _CONVENTION, + "bands": [ + { + "variable": b.variable, + "label": b.label, + "low": b.low, + "high": b.high, + "column": b.column, + } + for b in age.NATIONAL_AGE_ACTIVATION.bands + ], + "published_total": "S0101_C01_001_consistency_only", + "consumes_se": False, + "release_eligible": False, + } + + +def declaration_from_binding(binding: object) -> SurveyAgeActivation: + """Validate value-level profile semantics; grant no source authorization.""" + _require(type(binding) is dict, "DECLARATION") + _require("metadata_sha256" in binding and "data_sha256" in binding, "DECLARATION") + declaration = SurveyAgeActivation( + binding["metadata_sha256"], binding["data_sha256"] + ) + _require( + canonical_json(binding) == canonical_json(activation_binding(declaration)), + "DECLARATION", + ) + return declaration + + +def activation_digest(declaration: SurveyAgeActivation) -> str: + """Bind source pins and all selected-survey conventions in one identifier.""" + return hashlib.sha256(canonical_json(activation_binding(declaration))).hexdigest() + + +def validate_survey_age_registry( + registry: TargetRegistry, binding: dict +) -> TargetRegistry: + """Check numerical profile metadata, not the truth of caller-supplied values.""" + declaration = declaration_from_binding(binding) + frozen = demographic._registry_from_json(demographic._registry_json(registry)) + bands = age.NATIONAL_AGE_ACTIVATION.bands + _require(len(frozen) == len(bands), "REGISTRY") + digest = activation_digest(declaration) + for spec, band in zip(frozen, bands, strict=True): + _require( + spec.name == band.variable + and spec.measure == band.column + and spec.metadata["table"] == "S0101" + and spec.metadata["reference_sha256"] == digest + and spec.metadata["evidence_scope"] == "source_documented", + "REGISTRY", + ) + return frozen + + +def activate_survey_age_targets( + source_dir: str | Path, *, declaration: SurveyAgeActivation +) -> TargetRegistry: + """Derive exactly 18 targets from the two explicitly pinned local responses.""" + binding = activation_binding(declaration) + _, metadata = read_survey_age_response( + source_dir, "metadata", declaration.metadata_sha256 + ) + _, data = read_survey_age_response(source_dir, "data", declaration.data_sha256) + # Reuse the established publisher/count/MOE interpretation without invoking + # its old inventory loader. This object supplies only pure field semantics. + semantics = replace(age.NATIONAL_AGE_ACTIVATION, age_convention=_CONVENTION) + row = age._national_row(data, semantics) + _require( + all( + name in {"GEO_ID", "NAME", "us"} or name.startswith("S0101_") + for name in row + ), + "RESPONSE_SCOPE", + ) + _require( + type(metadata) is dict and type(metadata.get("variables")) is dict, "METADATA" + ) + _require( + all( + type(name) is str + and (name in {"GEO_ID", "NAME", "us"} or name.startswith("S0101_")) + for name in metadata["variables"] + ), + "METADATA_SCOPE", + ) + + def cell(variable): + _require( + all(variable + suffix in row for suffix in ("E", "EA", "M", "MA")), "CELL" + ) + return { + "estimate": age.classify_acs_value( + row[variable + "E"], row[variable + "EA"] + ), + "moe": age.classify_acs_value(row[variable + "M"], row[variable + "MA"]), + } + + digest = activation_digest(declaration) + specs = [] + data_url = survey_age_requests()[1]["url"] + for band in semantics.bands: + published = age._published_variable(metadata, band.variable + "E", semantics) + _require( + published.get("label") == band.label + and published.get("predicateType") == "int", + "PUBLISHED_DEFINITION", + ) + derived = cell(band.variable) + specs.append( + TargetSpec( + name=band.variable, + entity="household", + measure=band.column, + value=float(age._count_estimate(derived, band.variable)), + period="2024", + se=age._standard_error(derived, band.variable, confidence_level=0.9), + source=f"{data_url} (S0101 {band.variable}E)", + family="acs.S0101", + hierarchy=age.demographic_target_hierarchy( + "S0101", band, geography="0100000US" + ), + notes=f"activation={digest}; label={band.label}; age_convention={_CONVENTION}; published 90% MOE divided by {age._MOE_90_TO_SE}, not consumed by current loss", + metadata={ + "table": "S0101", + "reference_sha256": digest, + "geography": "0100000US", + "universe": "population", + "role": "calibration", + "evidence_scope": "source_documented", + }, + ) + ) + total = age._published_variable(metadata, "S0101_C01_001E", semantics) + _require( + total.get("label") == "Estimate!!Total!!Total population" + and total.get("predicateType") == "int", + "PUBLISHED_TOTAL_DEFINITION", + ) + _require( + sum(int(s.value) for s in specs) + == age._count_estimate(cell("S0101_C01_001"), "S0101_C01_001"), + "PARTITION", + ) + return validate_survey_age_registry(TargetRegistry(specs, country="us"), binding) + + +def verify_survey_age_targets( + source_dir: str | Path, + *, + declaration: SurveyAgeActivation, + registry: TargetRegistry, +) -> None: + """Rederive the pinned target values; no stored registry substitutes for this.""" + actual = activate_survey_age_targets(source_dir, declaration=declaration) + _require( + demographic._registry_json(actual) == demographic._registry_json(registry), + "REGISTRY", + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py new file mode 100644 index 000000000..c2051861b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py @@ -0,0 +1,952 @@ +"""Country admission for survey age profiles with an explicit source prefix. + +Every call reconstructs the accepted source preparation. The numerical kernels +are values-only; source, full-population, group/row-cap and cache checks live +here. The original entry point remains invented-only; the named development +entry point derives its targets from an explicitly pinned S0101-only capture. +Both permit only the first IMPORTANCE-to-CALIBRATED transition and no release. +An optional atomic-geography recipe enriches the accepted allocation before +cloning; the sampling budget continues to retain the original raw allocation. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from dataclasses import dataclass, replace +from types import SimpleNamespace + +import numpy as np + +from microcosm.frame import Frame +from microcosm.graph import ( + CompiledGraph, + KernelRegistry, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.artifact_edges import typed_contracts +from microcosm.graph.canonical import canonical_json +from microcosm.graph.executor import ( + _all_node_keys, + _input_writers, + _project_context, + _source_paths_and_keys, + _tolerance_writer_payload, +) +from microcosm.graph.keys import ( + _capabilities_projection, + artifact_key, + frame_key, + opaque_artifact_key, + seed, + weights_key, +) +from microcosm.graph.manifest import _freeze_json +from microcosm.graph.population import ( + Population, + mass_record_receipt, + weight_cap_receipt, +) +from microcosm.graph.store import _verified_meta + +from . import graph_combined_clone as clone +from . import graph_survey_age_artifact as ages +from . import graph_survey_budget as transport +from . import graph_survey_calibration as numerical +from . import graph_survey_population as source_graph +from . import survey_age_activation as age_activation +from . import survey_calibration_diagnostics as diagnostic_check +from . import survey_origin_budget as budgets +from . import survey_population_replay as replay + +COUNT_NODE = "survey.age_count_matrix" + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_AGE_RUN_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _node_identity(node): + """Immutable snapshot excluding execution time and cache-hit bookkeeping.""" + return canonical_json( + { + "key": node.key, + "ref": node.kernel_ref, + "implementation": node.kernel_impl_hash, + "capabilities": _capabilities_projection(node.capabilities), + "typed_artifacts": node.typed_artifacts, + "seed": node.seed, + "frame_key": node.frame_key, + "weight_key": node.weight_key, + "artifacts": [ + [entity, column, key] + for (entity, column), key in sorted(node.artifacts.items()) + ], + "opaque_artifacts": node.opaque_artifacts, + "receipt": node.receipt, + "legacy_capabilities": node.legacy_capabilities, + } + ) + + +def _measurement(population): + node = ages.survey_age_count_artifact_node( + population=population.version, node_id=COUNT_NODE + ) + context = _project_context( + node, + population, + key="0" * 64, + sources={}, + tolerances={}, + numerics={}, + ) + measured = ages.SurveyAgeCountArtifactKernel().run(context) + payload = measured.artifacts["counts"] + values = ages.decode_survey_age_counts(payload) + _require( + tuple(values.household_ids) + == tuple(population.frame.table("household").household_id), + "MEASUREMENT_HOUSEHOLD_ORDER", + ) + return payload, json.loads(canonical_json(measured.receipt)) + + +def _measure(population): + return _measurement(population)[0] + + +def _load_diagnostics(store, key): + """Bound the diagnostic payload buffer before reading it. + + The existing store verifier still authenticates metadata and streams its + payload hashes. This does not claim a bound on that verifier's metadata + decoding or total I/O. Only the selected diagnostic payload is buffered here. + """ + path = store.object_path(key) + payload_path = path / "payload.bin" + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + fd = os.open(payload_path, flags) + try: + before = os.fstat(fd) + limit = diagnostic_check.MAX_DIAGNOSTIC_BYTES + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= limit, + "DIAGNOSTIC_STORE_LIMIT", + ) + metadata = _verified_meta(path, expected_kind="bytes") + _require( + set(metadata["payloads"]) == {"payload.bin"}, "DIAGNOSTIC_STORE_ROSTER" + ) + record = metadata["payloads"]["payload.bin"] + _require(record["size"] == before.st_size, "DIAGNOSTIC_STORE_SIZE") + with os.fdopen(fd, "rb", closefd=False) as stream: + payload = stream.read(limit + 1) + after = os.fstat(fd) + current = os.stat(payload_path, follow_symlinks=False) + + def seal(s): + return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns) + + _require( + seal(before) == seal(after) == seal(current) + and stat.S_ISREG(current.st_mode) + and len(payload) == record["size"] <= limit + and _sha(payload) == record["sha256"], + "DIAGNOSTIC_STORE_CHANGED", + ) + return payload + finally: + os.close(fd) + + +def _expected_nodes( + compiled: CompiledGraph, + kernels: KernelRegistry, + keys: dict[str, str], + implementations: dict[str, str], + prefix_identities: dict[str, bytes], + frame: Frame, +): + """Detach declaration expectations before any graph kernel/store execution.""" + states = {} + cells = tuple((e, str(c)) for e in frame.entities for c in frame.table(e)) + for node in compiled.graph.nodes: + if node.id in prefix_identities: + state = json.loads(prefix_identities[node.id]) + state["artifacts"] = {(e, c): k for e, c, k in state["artifacts"]} + # The accepted prefix runner already independently sealed this + # receipt. No receipt from the receiving run is trusted here. + states[node.id] = state + continue + key = keys[node.id] + structural = node.structural is not StructuralDelta.NONE + outputs = ( + cells if structural else tuple((o.entity, o.column) for o in node.outputs) + ) + states[node.id] = { + "key": key, + "ref": node.kernel, + "implementation": implementations[node.id], + "capabilities": json.loads( + canonical_json( + _capabilities_projection(kernels.get(node.kernel).capabilities) + ) + ), + "typed_artifacts": json.loads( + canonical_json(typed_contracts(compiled, node, keys, kernels)) + ), + "seed": seed(key), + "frame_key": frame_key(key) if structural else None, + "weight_key": weights_key(key, node.weights.entity) + if node.weights + else None, + "artifacts": {(e, c): artifact_key(key, e, c) for e, c in outputs}, + "opaque_artifacts": { + o.name: opaque_artifact_key(key, o.name) for o in node.artifact_outputs + }, + } + return states + + +def _complete_receipts( + compiled: CompiledGraph, + states: dict[str, dict[str, object]], + authored: dict[str, dict[str, object]], + observed: dict[str, Population], +): + """Normalize independently authored receipts through the executor's rules.""" + previous = {} + for node_id in compiled.order: + state = states[node_id] + if node_id in authored: + node = compiled.graph.node(node_id) + receipt = json.loads(canonical_json(authored[node_id])) + receipt["capabilities"] = dict(state["capabilities"]) + writers = _tolerance_writer_payload( + _input_writers(compiled, node_id, receipts=previous) + ) + if writers: + receipt["capabilities"]["tolerance_writers"] = writers + if node.structural not in {StructuralDelta.NONE, StructuralDelta.CREATE}: + receipt["mass"] = mass_record_receipt(observed[node_id].mass_ledger[-1]) + receipt.update(weight_cap_receipt(observed[node_id], node)) + state["receipt"] = json.loads(canonical_json(receipt)) + previous[node_id] = SimpleNamespace(receipt=state["receipt"]) + + +def _check_manifest(manifest, compiled, states): + _require( + manifest.country == compiled.graph.country and manifest.decisions == (), + "MANIFEST_COUNTRY_DECISIONS", + ) + _require( + set(manifest.populations) == set(compiled.versions.values()) + and set(manifest.mass_ledgers) == set(compiled.versions.values()), + "MANIFEST_POPULATION_VERSIONS", + ) + # Detached expectations use JSON arrays. NodeReceipt freezes those arrays + # as tuples, including nested capabilities and typed artifact descriptors. + # Match its representation without weakening the canonical value check. + frozen_states = { + node_id: { + **state, + "receipt": _freeze_json(state["receipt"]), + "typed_artifacts": _freeze_json(state["typed_artifacts"]), + } + for node_id, state in states.items() + } + source_graph._check_node_states(manifest, frozen_states) + for node_id, state in states.items(): + expected = {**state, "legacy_capabilities": False} + expected["artifacts"] = [ + [e, c, key] for (e, c), key in sorted(state["artifacts"].items()) + ] + _require( + _node_identity(manifest.node(node_id)) == canonical_json(expected), + "MANIFEST_CANONICAL_VALUES", + ) + + +@dataclass(frozen=True) +class SurveyAgeCalibrationRun: + """Values validated at return; retained source handles still require rechecks.""" + + manifest: object + compiled: object + budget: object + successor: object + diagnostics: dict + counts_sha256: str + numeric_bounds_sha256: str + release_eligible: bool = False + + +def run_survey_age_calibration( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed_value, + target_registry, + epochs, + learning_rate, + resume="auto", + geography_config=None, +): + """Reconstruct sources and admit a complete invented calibration graph. + + The source and store paths are the caller's explicitly approved local + inputs. Cache reuse never bypasses fresh source preparation or the complete + receiving-population and sampling-reference checks. This v1 does not admit + genuine targets, donor-detail descendants, pruning or repeated calibration. + """ + # Refuse unsupported targets/options before any source I/O. + calibration = numerical.survey_age_calibration_node( + target_registry, + base=clone.COMBINED_CLONE_NODE, + budget_node=transport.BUDGET_NODE, + count_node=COUNT_NODE, + epochs=epochs, + learning_rate=learning_rate, + ) + return _run_survey_age_calibration( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed_value=seed_value, + epochs=epochs, + learning_rate=learning_rate, + resume=resume, + calibration=calibration, + geography_config=geography_config, + ) + + +def run_survey_age_development( + source_dir, + *, + age_source_dir, + activation, + snapshot_root, + store_root, + fraction, + seed_value, + epochs, + learning_rate, + resume="auto", + geography_config=None, +): + """Run development calibration after deriving targets from exact source pins. + + The caller separately reviews the declaration and explicitly admits these + local source paths. There is no default capture, caller-supplied registry, + mixed-inventory fallback, production attestation or release permission. + """ + _require(type(resume) is str and resume in {"auto", "require"}, "RESUME") + _require(type(epochs) is int and 1 <= epochs <= 1000, "EPOCHS") + _require( + type(learning_rate) in (int, float) + and np.isfinite(learning_rate) + and 0 < learning_rate <= 1, + "LEARNING_RATE", + ) + binding = age_activation.activation_binding(activation) + registry = age_activation.activate_survey_age_targets( + age_source_dir, declaration=activation + ) + calibration = numerical.survey_age_development_node( + registry, + activation_binding=binding, + base=clone.COMBINED_CLONE_NODE, + budget_node=transport.BUDGET_NODE, + count_node=COUNT_NODE, + epochs=epochs, + learning_rate=learning_rate, + ) + return _run_survey_age_calibration( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed_value=seed_value, + epochs=epochs, + learning_rate=learning_rate, + resume=resume, + calibration=calibration, + age_source_dir=age_source_dir, + activation=activation, + geography_config=geography_config, + ) + + +def _run_survey_age_calibration( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed_value, + epochs, + learning_rate, + resume, + calibration, + age_source_dir=None, + activation=None, + geography_config=None, +): + """Shared population, budget, replay, diagnostic and final-return checks.""" + _require(type(resume) is str and resume in {"auto", "require"}, "RESUME") + activation_identity = None + calibration_binding = numerical.calibration_activation_binding(calibration.params) + _require( + (calibration_binding is None) == (activation is None), "ACTIVATION_REQUIRED" + ) + if activation is not None: + activation_identity = canonical_json( + age_activation.activation_binding(activation) + ) + _require( + canonical_json(calibration_binding) == activation_identity, + "ACTIVATION_BINDING", + ) + geography = None + geography_identity = None + geography_config_payload = None + prefix_populations = {} + if geography_config is None: + prefix = source_graph.run_authenticated_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed_value, + resume=resume, + clones=True, + return_values=True, + ) + else: + from . import graph_atomic_survey_population as atomic_source + + geography_config_payload = ( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + ) + prefix = atomic_source.run_atomic_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed_value, + resume=resume, + geography_config=geography_config, + return_values=True, + ) + geography = atomic_source.reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + replay.same_replayed_population( + geography.population, prefix.geography_population + ) + geography_identity = budgets._population_identity(prefix.geography_population) + prefix_populations = { + stage.node.id: stage.population for stage in geography.stages + } + source_owner, source_view = source_graph._checked_preparation(prefix.preparation) + preparation_entry = source_owner._ISSUED.get(id(prefix.preparation)) + instructions = source_graph.allocation_instructions( + source_view.selection_plan, source_view.receipt["origins"]["households"] + ) + _, allocated_context, allocation_payload, _, _ = source_graph._allocation_output( + source_view.frame, source_view.context, instructions, _sha(source_view.payload) + ) + prefix_artifacts = ( + ( + source_graph.CREATE_NODE, + "preparation", + source_graph.PREPARATION_TYPE, + source_view.payload, + source_graph.SurveyPopulationCreateKernel.capabilities, + ), + ( + source_graph.CREATE_NODE, + "frame_context", + source_graph.US_FRAME_CONTEXT_TYPE, + source_view.context, + source_graph.SurveyPopulationCreateKernel.capabilities, + ), + ( + source_graph.ALLOCATION_NODE, + "allocation", + source_graph.ALLOCATION_TYPE, + allocation_payload, + source_graph.SurveyPopulationAllocationKernel.capabilities, + ), + ( + source_graph.ALLOCATION_NODE, + "frame_context", + source_graph.US_FRAME_CONTEXT_TYPE, + allocated_context, + source_graph.SurveyPopulationAllocationKernel.capabilities, + ), + ) + if geography is not None: + prefix_artifacts += tuple( + ( + node.id, + "support", + atomic_source.ATOMIC_SUPPORT_TYPE, + geography.support_payload, + atomic_source.AtomicSupportImportKernel.capabilities, + ) + for node in geography.nodes + if node.kernel == atomic_source.AtomicSupportImportKernel.ref + ) + gate = geography.stages[-1] + prefix_artifacts += ( + ( + gate.node.id, + "validation", + atomic_source.reconstruction.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + gate.receipt, + atomic_source.reconstruction.atomic_graph.AtomicGeographyGateKernel.capabilities, + ), + ) + initial = prefix.clone_population + _require(initial is not None, "COMPLETE_CLONE_REQUIRED") + budget = budgets.freeze_survey_origin_budget( + prefix.preparation, + allocated_population=prefix.allocated_population, + clone_population=initial, + geography_config=geography_config, + ) + budget_view = budget.checked_view() + budget_payload = budget_view.payload + numeric_payload = transport.numeric_survey_budget_payload(budget_payload) + numeric_bounds = numerical.decode_numeric_survey_bounds(numeric_payload) + count_payload, count_receipt = _measurement(initial) + _require( + numeric_bounds.grouped.household_ids + == tuple(initial.frame.table("household").household_id), + "BUDGET_HOUSEHOLD_ORDER", + ) + prefix_identities = { + name: _node_identity(row) for name, row in prefix.manifest.nodes.items() + } + initial_identity = budgets._population_identity(initial) + allocation_identity = budgets._population_identity(prefix.allocated_population) + prefix_population_identities = { + name: budgets._population_identity(population) + for name, population in prefix_populations.items() + } + prefix_budget = budget + prefix_budget_entry = budgets._entry(prefix_budget, budgets.SamplingOriginBudget) + kernels = prefix.kernels + kernels.register(transport.SurveySamplingBudgetKernel(budget_payload)) + kernels.register(ages.SurveyAgeCountArtifactKernel()) + kernels.register(numerical.SurveyAgeCalibrationKernel()) + age_nodes = ( + transport.survey_sampling_budget_node(budget_sha256=_sha(budget_payload)), + ages.survey_age_count_artifact_node( + population=initial.version, node_id=COUNT_NODE + ), + calibration, + ) + graph = replace( + prefix.compiled.graph, + nodes=(*prefix.compiled.graph.nodes, *age_nodes), + ) + compiled = compile_graph(graph) + prefix_roster = set(prefix.compiled.order) + age_roster = {transport.BUDGET_NODE, COUNT_NODE, numerical.CALIBRATION_NODE} + _require( + set(prefix_identities) == prefix_roster + and not prefix_roster & age_roster + and tuple(node.id for node in age_nodes) + == (transport.BUDGET_NODE, COUNT_NODE, numerical.CALIBRATION_NODE) + and set(compiled.order) == prefix_roster | age_roster + and len(compiled.order) == len(prefix.compiled.order) + 3 + and tuple(name for name in compiled.order if name in prefix_roster) + == prefix.compiled.order, + "EXACT_GRAPH", + ) + _require( + clone.COMBINED_CLONE_CLAIM_NODE in compiled.predecessors[transport.BUDGET_NODE], + "OWNERSHIP_CLAIM_EDGE", + ) + _paths, source_keys = _source_paths_and_keys(compiled, prefix.sources, prefix.store) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + expected_nodes = _expected_nodes( + compiled, kernels, keys, implementations, prefix_identities, initial.frame + ) + observed, snapshots = {}, {} + admitted = None + successor_entry = None + successor_payload = None + receiving_initial = None + receiving_budget_entry = None + receiving_node = ( + clone.COMBINED_CLONE_CLAIM_NODE + if geography is None + else geography.stages[-1].node.id + ) + + def observe(node_id, population): + nonlocal admitted, budget, receiving_initial, receiving_budget_entry + nonlocal successor_entry, successor_payload + _require(node_id not in observed, "DUPLICATE_OBSERVATION") + if node_id == numerical.CALIBRATION_NODE: + _require(receiving_initial is not None, "RECEIVING_BUDGET_REQUIRED") + # Independent of the numerical kernel, on both execution and load. + numerical.check_numeric_survey_weights( + numeric_bounds, population.frame.weights_for("household").values + ) + admitted = budgets.admit_survey_weight_only_population( + budget, + previous=receiving_initial, + current=population, + ) + successor_entry = budgets._entry(admitted, budgets.SamplingOriginSuccessor) + successor_payload = admitted.payload + elif node_id in {transport.BUDGET_NODE, COUNT_NODE}: + # Read-only count/transport may precede the geography gate. Compare + # their complete current version, including whichever prefix writes + # have actually occurred. Calibration waits for every base member. + if geography is None: + replay.same_replayed_frame(initial.frame, population.frame) + else: + preceding = next( + name + for name in reversed(observed) + if name in prefix_populations + and compiled.versions[name] == initial.version + ) + replay.same_replayed_population( + prefix_populations[preceding], population + ) + _require( + population.frame.weights_for("household").values.tobytes() + == numeric_bounds.incoming.tobytes(), + "RECEIVING_INCOMING_WEIGHTS", + ) + if node_id == transport.BUDGET_NODE and geography is None: + _require(receiving_initial is not None, "RECEIVING_BUDGET_REQUIRED") + _require( + budgets._population_identity(population) + == snapshots[clone.COMBINED_CLONE_CLAIM_NODE], + "COMPLETE_CLAIMED_CLONE", + ) + elif node_id == COUNT_NODE: + _require(_measure(population) == count_payload, "REMEASURED_COUNTS") + elif node_id == source_graph.ALLOCATION_NODE: + replay.same_replayed_population(prefix.allocated_population, population) + elif node_id == receiving_node: + replay.same_replayed_population(initial, population) + _require(source_graph.ALLOCATION_NODE in observed, "ALLOCATION_REQUIRED") + # Actual source reconstruction rebinds an issued budget to this run's + # receiving objects. Candidate bytes grant no authority by themselves. + budget = budgets.freeze_survey_origin_budget( + prefix.preparation, + allocated_population=observed[source_graph.ALLOCATION_NODE], + clone_population=population, + candidate=budget_payload, + geography_config=geography_config, + ) + receiving_budget_entry = budgets._entry( + budget, budgets.SamplingOriginBudget + ) + receiving_initial = population + elif node_id == source_graph.CREATE_NODE: + replay.same_replayed_frame( + prefix.manifest.population(node_id), population.frame + ) + elif node_id == clone.COMBINED_CLONE_NODE: + actual_allocation = observed[ + source_graph.ALLOCATION_NODE + if geography is None + else atomic_source.projection.NODE + ] + source_graph._verify_cloned_frame( + actual_allocation.frame, + population.frame, + actual_allocation.design_weights["household"], + ) + if geography is None: + replay.same_replayed_frame(initial.frame, population.frame) + else: + replay.same_replayed_population(prefix_populations[node_id], population) + elif node_id in prefix_populations: + replay.same_replayed_population(prefix_populations[node_id], population) + else: + _require(False, "UNEXPECTED_NODE") + observed[node_id] = population + snapshots[node_id] = budgets._population_identity(population) + + manifest = run_graph( + compiled, + sources=prefix.sources, + store=prefix.store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + _require( + tuple(observed) == compiled.order and admitted is not None, + "OBSERVATION_COVERAGE", + ) + _require(set(manifest.nodes) == set(compiled.order), "MANIFEST_NODES") + for node in graph.nodes: + actual = manifest.node(node.id) + _require( + actual.key == keys[node.id] + and actual.kernel_ref == node.kernel + and actual.kernel_impl_hash == implementations[node.id] + and actual.typed_artifacts == expected_nodes[node.id]["typed_artifacts"] + and actual.seed == seed(keys[node.id]) + and actual.legacy_capabilities is False, + "MANIFEST_NODE_IDENTITY", + ) + if node.id in prefix_identities: + _require( + _node_identity(actual) == prefix_identities[node.id], + "PREFIX_RECEIPT_REPLAY", + ) + for node_id, name, type_, payload, capabilities in ( + *prefix_artifacts, + ( + transport.BUDGET_NODE, + "budget", + budgets.BUDGET_TYPE, + budget_payload, + transport.SurveySamplingBudgetKernel.capabilities, + ), + ( + transport.BUDGET_NODE, + "numeric_bounds", + numerical.BOUNDS_TYPE, + numeric_payload, + transport.SurveySamplingBudgetKernel.capabilities, + ), + ( + COUNT_NODE, + "counts", + ages.COUNTS_TYPE, + count_payload, + ages.SurveyAgeCountArtifactKernel.capabilities, + ), + ): + source_graph._final_artifact( + manifest, + prefix.store, + node_id=node_id, + name=name, + type_=type_, + payload=payload, + capabilities=capabilities, + ) + receipt = manifest.node(numerical.CALIBRATION_NODE) + diagnostic_payload = _load_diagnostics( + prefix.store, + expected_nodes[numerical.CALIBRATION_NODE]["opaque_artifacts"]["diagnostics"], + ) + current = observed[numerical.CALIBRATION_NODE] + anchors = { + "budget_sha256": _sha(budget_payload), + "numeric_bounds_sha256": _sha(numeric_payload), + "counts_sha256": _sha(count_payload), + "accepted_weight_sha256": _sha( + current.frame.weights_for("household").values.tobytes() + ), + "constraint_digest": numeric_bounds.grouped.digest, + "weight_anchor": calibration.params["weight_anchor"], + "cap_enforcement": calibration.params["cap_enforcement"], + "fixed_zero_rows": int(np.count_nonzero(numeric_bounds.incoming == 0)), + } + _require( + _sha(diagnostic_payload) == receipt.receipt.get("diagnostics_sha256"), + "DIAGNOSTICS_DIGEST", + ) + _require( + all(receipt.receipt.get(k) == v for k, v in anchors.items()), "RECEIPT_ANCHORS" + ) + registry = numerical.demographic._registry_from_json(calibration.params["registry"]) + diagnostics = diagnostic_check.validate_survey_calibration_diagnostics( + diagnostic_payload, + counts_payload=count_payload, + bounds_payload=numeric_payload, + weights=current.frame.weights_for("household").values, + registry=registry, + epochs=epochs, + learning_rate=learning_rate, + anchors=anchors, + activation_binding=calibration_binding, + ) + diagnostic_identity = canonical_json(diagnostics) + _complete_receipts( + compiled, + expected_nodes, + { + transport.BUDGET_NODE: { + "budget_sha256": _sha(budget_payload), + "numeric_bounds_sha256": _sha(numeric_payload), + "constraint_digest": numeric_bounds.grouped.digest, + "group_count": numeric_bounds.grouped.group_count, + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + COUNT_NODE: count_receipt, + numerical.CALIBRATION_NODE: { + **anchors, + "diagnostics_sha256": _sha(diagnostic_payload), + "scope": numerical.calibration_receipt_scope(calibration.params), + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + }, + observed, + ) + _check_manifest(manifest, compiled, expected_nodes) + result = SurveyAgeCalibrationRun( + manifest, + compiled, + budget, + admitted, + diagnostics, + _sha(count_payload), + _sha(numeric_payload), + ) + final_numbers = numerical.decode_numeric_survey_bounds(numeric_payload) + # A population version can be shared by several nonstructural nodes. Check + # the actual last receiving state for every version, including the prefix. + terminal_nodes = tuple({compiled.versions[n]: n for n in compiled.order}.values()) + terminal_frames = { + node_id: manifest.population(compiled.versions[node_id]) + for node_id in terminal_nodes + } + terminal_ledgers = { + node_id: manifest.mass_ledger(compiled.versions[node_id]) + for node_id in terminal_nodes + } + # Finish storage and key/implementation I/O before final source/target + # revalidation and complete-population checks. No store read follows. + final_keys, final_implementations = _all_node_keys(compiled, kernels, source_keys) + _require( + final_keys == keys and final_implementations == implementations, + "FINAL_IMPLEMENTATIONS", + ) + _require(_measure(initial) == count_payload, "FINAL_REMEASUREMENT") + _require( + _measure(receiving_initial) == count_payload, "FINAL_RECEIVING_MEASUREMENT" + ) + # Retain both authorities; the original source/planned budget is not waived + # when a candidate-identical handle is issued for the actual receiving run. + budgets.verify_survey_origin_budget(prefix_budget) + budgets.verify_survey_weight_only_successor(admitted) + if activation is not None: + # Recheck target evidence before the optional final support read; + # complete population, configuration and owner seals follow both. + age_activation.verify_survey_age_targets( + age_source_dir, declaration=activation, registry=registry + ) + if geography is not None: + _require( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + == geography_config_payload, + "FINAL_GEOGRAPHY_CONFIG", + ) + final_support, _support_identity = atomic_source.reconstruction._read_support( + geography_config + ) + _require(final_support == geography.support_payload, "FINAL_GEOGRAPHY_SUPPORT") + _require( + source_owner._ISSUED.get(id(prefix.preparation)) is preparation_entry + and preparation_entry is not None + and prefix.preparation.payload == preparation_entry[1], + "FINAL_SOURCE_ISSUANCE", + ) + source_owner._pure_final(preparation_entry[2]) + if geography is not None: + _require( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + == geography_config_payload + and budgets._population_identity(prefix.geography_population) + == geography_identity + and all( + budgets._population_identity(population) + == prefix_population_identities[name] + for name, population in prefix_populations.items() + ), + "FINAL_GEOGRAPHY_PREFIX", + ) + _require(budget.payload == budget_payload, "FINAL_BUDGET_BYTES") + _require( + budgets._population_identity(initial) == initial_identity + and budgets._population_identity(prefix.allocated_population) + == allocation_identity, + "FINAL_PREFIX_POPULATIONS", + ) + for node_id, population in observed.items(): + _require( + budgets._population_identity(population) == snapshots[node_id], + "FINAL_POPULATION_STATE", + ) + # A nonstructural node can precede the claim within the same version; + # its own snapshot is checked above. Terminal version state is checked + # on the declared weight transition and complete ownership claim. + if node_id in terminal_nodes: + replay.same_replayed_frame(population.frame, terminal_frames[node_id]) + _require( + terminal_ledgers[node_id] == population.mass_ledger, + "FINAL_MASS_LEDGER", + ) + numerical.check_numeric_survey_weights( + final_numbers, + current.frame.weights_for("household").values, + ) + if activation is not None: + _require( + canonical_json(age_activation.activation_binding(activation)) + == activation_identity + and calibration.params.get("activation") == activation_identity.decode(), + "FINAL_ACTIVATION_BINDING", + ) + # No decoding, store access, source borrow or implementation helper follows + # this seal. It covers the complete selected prefix and age trio, including + # scope and release flags, using expectations detached before execution. + _check_manifest(manifest, compiled, expected_nodes) + _require( + canonical_json(result.diagnostics) == diagnostic_identity, + "FINAL_DIAGNOSTICS_VALUES", + ) + # The verifications above perform I/O. Finish with the retained issuance + # entries, so the second verification cannot invalidate the first handle. + _require( + budgets._entry(prefix_budget, budgets.SamplingOriginBudget) + is prefix_budget_entry + and budgets._entry(budget, budgets.SamplingOriginBudget) + is receiving_budget_entry + and prefix_budget.payload == budget.payload == budget_payload, + "FINAL_BUDGET_ISSUANCE", + ) + _require( + budgets._entry(admitted, budgets.SamplingOriginSuccessor) is successor_entry + and admitted.payload == successor_payload, + "FINAL_SUCCESSOR_ISSUANCE", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py new file mode 100644 index 000000000..d4314af93 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py @@ -0,0 +1,181 @@ +"""Two explicit S0101-only source captures; no network or target activation. + +Callers supply already acquired response bytes. Reading an explicit capture +never discovers files or falls back to a mixed national reference inventory. +""" + +from __future__ import annotations + +import hashlib +import os +import stat +from pathlib import Path +from urllib.parse import urlencode + +from microcosm.graph.canonical import canonical_json + +from .cd_reference_sources import strict_json + +MAX_RESPONSE_BYTES = 8 * 1024**2 +MAX_DESCRIPTOR_BYTES = 4096 +_DESCRIPTOR_FIELDS = frozenset( + { + "table", + "year", + "dataset", + "geography", + "kind", + "url", + "sha256", + "size_bytes", + "path", + } +) + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ValueError("SURVEY_AGE_SOURCE_" + reason) + + +def survey_age_requests() -> tuple[dict[str, object], dict[str, object]]: + """Return detached descriptors of exactly two supported national requests.""" + base = "https://api.census.gov/data/2024/acs/acs1/subject" + common = { + "table": "S0101", + "year": 2024, + "dataset": "acs/acs1/subject", + "geography": "us:*", + } + return ( + {**common, "kind": "metadata", "url": f"{base}/groups/S0101.json"}, + { + **common, + "kind": "data", + "url": base + "?" + urlencode({"get": "group(S0101)", "for": "us:*"}), + }, + ) + + +def _request(value: object) -> dict[str, object]: + _require(type(value) is dict, "REQUEST") + for request in survey_age_requests(): + if set(value) == set(request) and all( + type(value[k]) is type(v) and value[k] == v for k, v in request.items() + ): + return request + _require(False, "REQUEST") + + +def _digest(value: object) -> str: + _require( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value), + "PIN", + ) + return value + + +def _path(value: str | Path) -> Path: + path = Path(value) + _require(".." not in path.parts, "PATH") + path = path.absolute() + for member in (*reversed(path.parents), path): + _require(not member.is_symlink(), "SYMLINK") + return path + + +def _read(path: Path, limit: int) -> bytes: + path = _path(path) + try: + # A FIFO must reach the regular-file check instead of waiting for a + # writer before fstat can reject it. Nonblocking leaves regular reads unchanged. + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + _require(stat.S_ISREG(before.st_mode), "REGULAR_FILE") + _require(0 < before.st_size <= limit, "SIZE") + payload = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + _require( + len(payload) == before.st_size + and (before.st_size, before.st_mtime_ns, before.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), + "CHANGED_FILE", + ) + return payload + except OSError: + raise ValueError("SURVEY_AGE_SOURCE_FILE") from None + + +def _write(path: Path, payload: bytes, limit: int) -> None: + path = _path(path) + if path.exists(): + _require(_read(path, limit) == payload, "IMMUTABLE") + return + path.parent.mkdir(parents=True, exist_ok=True) + _path(path) + with path.open("xb") as stream: + stream.write(payload) + + +def capture_survey_age_response( + root: str | Path, request: dict, payload: bytes +) -> dict: + """Store one caller-acquired response immutably; grant no target permission.""" + request = _request(request) + _require(type(payload) is bytes and 0 < len(payload) <= MAX_RESPONSE_BYTES, "SIZE") + strict_json(payload) + root = _path(root) + digest = hashlib.sha256(payload).hexdigest() + descriptor = { + **request, + "sha256": digest, + "size_bytes": len(payload), + "path": f"raw/{digest}.json", + } + key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor_path = root / "requests" / f"{key}.json" + encoded = canonical_json(descriptor) + # Refuse replacement before writing even an unreferenced new raw file. + if _path(descriptor_path).exists(): + _require(_read(descriptor_path, MAX_DESCRIPTOR_BYTES) == encoded, "IMMUTABLE") + _write(root / descriptor["path"], payload, MAX_RESPONSE_BYTES) + _write(descriptor_path, encoded, MAX_DESCRIPTOR_BYTES) + return descriptor + + +def read_survey_age_response( + root: str | Path, kind: str, expected_sha256: str +) -> tuple[bytes, object]: + """Read the fixed named response against a separately reviewed content pin.""" + _require(type(kind) is str and kind in {"metadata", "data"}, "KIND") + expected_sha256 = _digest(expected_sha256) + request = next(row for row in survey_age_requests() if row["kind"] == kind) + root = _path(root) + key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor = strict_json( + _read(root / "requests" / f"{key}.json", MAX_DESCRIPTOR_BYTES) + ) + _require( + type(descriptor) is dict and set(descriptor) == _DESCRIPTOR_FIELDS, "DESCRIPTOR" + ) + _request({key: descriptor[key] for key in request}) + _require( + descriptor["sha256"] == expected_sha256 + and descriptor["path"] == f"raw/{expected_sha256}.json", + "PIN", + ) + _require( + type(descriptor["size_bytes"]) is int + and 0 < descriptor["size_bytes"] <= MAX_RESPONSE_BYTES, + "SIZE", + ) + payload = _read(root / f"raw/{expected_sha256}.json", MAX_RESPONSE_BYTES) + _require( + len(payload) == descriptor["size_bytes"] + and hashlib.sha256(payload).hexdigest() == expected_sha256, + "RESPONSE_PIN", + ) + return payload, strict_json(payload) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py new file mode 100644 index 000000000..2beb174dc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py @@ -0,0 +1,639 @@ +"""Reconstruct observed constraints, initial clones and geography from live sources. + +The immutable recipe names normalized support bytes; their digest proves byte +integrity, not publisher provenance. A native source adapter must establish the +latter separately. This helper executes the maintained pure geography operators, +not graph kernels, and issues no source, Population, or release authority. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +from dataclasses import asdict, dataclass, replace +from io import BytesIO +from pathlib import Path +from zipfile import ZipFile + +import numpy as np +import pandas as pd + +from microcosm.build import atomic_geography as atomic +from microcosm.build import graph_atomic_geography as atomic_graph +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ( + Graph, + KernelResult, + Node, + SourceRef, + StructuralDelta, + compile_graph, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES +from microcosm.graph.executor import ( + _expand_declared_payload, + _expand_rewrite_coordinates, +) +from microcosm.graph.population import ( + Population, + _mass_record, + _storage_parts, + expand_lineage_receipt, + expand_writes_receipt, + patch, +) + +from . import atomic_block_support as blocks +from . import current_survey_geography as observed +from . import graph_atomic_survey_clone as composition +from . import graph_combined_clone as clone +from . import graph_current_survey_geography as observed_graph +from . import graph_survey_population as graph +from . import puf_support +from . import survey_population_preparation as source +from . import survey_population_replay as replay +from .graph_sources import frame_column_declarations + +PROTOCOL = "microcosm.us.atomic-survey-reconstruction.v2" +MAX_SUPPORT_BYTES = RAW_BYTES_MAX_BYTES +MAX_SUPPORT_EXPANDED_BYTES = 2 * 1024**3 +MAX_SUPPORT_MEMBERS = 32 + + +def _require(condition, reason): + if not condition: + raise ValueError("ATOMIC_SURVEY_RECONSTRUCTION_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyReconstruction: + """Exact immutable inputs; construction itself authenticates no source.""" + + support_path: str + support_sha256: str + source_ids: tuple[tuple[str, str], ...] + seed: int + + def __post_init__(self): + _config_bytes(self) + + def to_bytes(self): + """Revalidate and detach the recipe, including the literal source path.""" + return _config_bytes(self) + + +def _config_bytes(config): + _require(type(config) is AtomicSurveyReconstruction, "CONFIG_TYPE") + _require( + type(config.support_path) is str + and 0 < len(config.support_path) <= 4096 + and "\0" not in config.support_path + and Path(config.support_path).is_absolute() + and ".." not in Path(config.support_path).parts, + "SUPPORT_PATH", + ) + _require( + type(config.support_sha256) is str + and re.fullmatch(r"[0-9a-f]{64}", config.support_sha256) is not None, + "SUPPORT_DIGEST", + ) + _require( + type(config.source_ids) is tuple + and len(config.source_ids) == 3 + and all( + type(pair) is tuple + and len(pair) == 2 + and all(type(value) is str for value in pair) + and 0 < len(pair[1]) <= 4096 + and pair[1].strip() == pair[1] + for pair in config.source_ids + ) + and tuple(pair[0] for pair in config.source_ids) + == ("district", "population", "puma"), + "SOURCE_IDENTITIES", + ) + _require(type(config.seed) is int and 0 <= config.seed < 2**63, "SEED") + return canonical_json( + { + "protocol": PROTOCOL, + "support_path": config.support_path, + "support_sha256": config.support_sha256, + "source_ids": dict(config.source_ids), + "seed": config.seed, + } + ) + + +def _file_identity(value): + return tuple( + getattr(value, name) + for name in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + ) + + +def _read_support(config): + """Read one bounded regular file and verify its exact pinned bytes.""" + descriptor = os.open( + config.support_path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0), + ) + with os.fdopen(descriptor, "rb") as stream: + before = os.fstat(stream.fileno()) + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= MAX_SUPPORT_BYTES, + "SUPPORT_SIZE", + ) + payload = stream.read(before.st_size + 1) + after = os.fstat(stream.fileno()) + current = os.stat(config.support_path, follow_symlinks=False) + _require( + stat.S_ISREG(current.st_mode) + and _file_identity(before) == _file_identity(after) == _file_identity(current) + and len(payload) == before.st_size + and _sha(payload) == config.support_sha256, + "SUPPORT_CHANGED", + ) + return payload, _file_identity(current) + + +def _decode_support(payload): + # Bound declared decompressed bytes before numpy opens any array. The shared + # decoder still owns every schema, value, mapping and sampling-weight check. + with ZipFile(BytesIO(payload)) as archive: + members = archive.infolist() + _require( + 0 < len(members) <= MAX_SUPPORT_MEMBERS + and sum(member.file_size for member in members) + <= MAX_SUPPORT_EXPANDED_BYTES, + "SUPPORT_EXPANDED_SIZE", + ) + for member in members: + with archive.open(member) as stream: + version = np.lib.format.read_magic(stream) + if version == (1, 0): + shape, _fortran, dtype = np.lib.format.read_array_header_1_0(stream) + elif version == (2, 0): + shape, _fortran, dtype = np.lib.format.read_array_header_2_0(stream) + else: + raise ValueError("ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_NPY_VERSION") + # A small ZIP member must not claim a huge allocation in its + # NPY header. Validate the physical payload length before load. + _require( + not dtype.hasobject + and all(type(size) is int and size >= 0 for size in shape) + and math.prod(shape) * dtype.itemsize + == member.file_size - stream.tell(), + "SUPPORT_ARRAY_SIZE", + ) + return atomic.decode_atomic_support(payload) + + +def _copy_population(population): + frame = population.frame + return replace( + population, + frame=Frame( + {entity: frame.table(entity).copy(deep=True) for entity in frame.entities}, + frame.schema, + { + entity: Weights( + frame.weights_for(entity).values.copy(), + frame.weights_for(entity).kind, + ) + for entity in frame.weighted_entities + }, + frame.strata.copy(deep=True), + # Frame recursively freezes fresh metadata containers. The admitted + # mass records contain scalars and immutable tuples; replacing each + # record avoids deepcopy's reflective dataclass class-cache writes. + metadata=frame.metadata, + mass_log=tuple(replace(record) for record in frame.mass_log), + ), + owners=dict(population.owners), + weight_kind=dict(population.weight_kind), + design_weights={ + name: values.copy() for name, values in population.design_weights.items() + }, + mass_ledger=tuple(replace(record) for record in population.mass_ledger), + ) + + +def _population_stamp(population): + """Pure in-process seal, including storage beneath nullable masks. + + Never persist it or compare it across reconstructions: it folds physical + storage parts (masked ``_data`` under nulls, and before microcosm#907 + object-dtype pointer bytes), so equal content rebuilt elsewhere need not + match. A cross-call pin carries ``source._frame_identity`` plus the + version, owners, weight kinds, mass ledger and design weights instead. + """ + _require(type(population) is Population, "POPULATION_TYPE") + digest = hashlib.sha256() + frame = population.frame + digest.update( + canonical_json( + { + "frame": source._frame_identity(frame), + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": list(population.weight_kind.items()), + "mass_ledger": [asdict(record) for record in population.mass_ledger], + } + ) + ) + for series in ( + *( + frame.table(entity)[column] + for entity in frame.entities + for column in frame.table(entity) + ), + frame.strata, + ): + # Whole series: the slice selects every row in order, byte for byte. + for part in _storage_parts(series, slice(None)): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + for entity, values in population.design_weights.items(): + digest.update(canonical_json((entity, str(values.dtype), values.shape))) + digest.update(values.tobytes()) + return digest.hexdigest() + + +def _raw_allocation(view, population): + """Reconstruct the original allocation, retaining all existing checks.""" + _require(type(population) is Population, "RAW_POPULATION_TYPE") + instructions = graph.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _weights, _context, _allocation, receipt, expected = graph._allocation_output( + view.frame, view.context, instructions, _sha(view.payload) + ) + columns = frame_column_declarations(view.frame) + nodes = graph.survey_population_nodes( + columns, + preparation_sha256=_sha(view.payload), + fraction=view.selection_plan.fraction, + seed=view.selection_plan.seed, + ) + graph._same_frame(expected, population.frame) + replay.same_replayed_frame(expected, population.frame) + design = view.frame.weights_for("household").values + graph._check_design_anchors(population, design) + cells = ( + (entity, str(column)) + for entity in view.frame.entities + for column in view.frame.table(entity) + ) + ledger = ( + _mass_record( + view.frame, expected, nodes[1], KernelResult(receipt=receipt), "declared" + ), + ) + graph._check_population_state( + population, + version=graph.ALLOCATION_NODE, + owners=dict.fromkeys(cells, graph.ALLOCATION_NODE), + kind=WeightKind.IMPORTANCE, + ledger=ledger, + ) + return columns, nodes + + +def _clone_expectations(before, nodes, compiled): + """Independently reconstruct the complete clone, including inherited cells.""" + expanded = puf_support.clone_us_frame_for_puf_support( + before.frame, + clone_attachment_fraction=1.0, + clone_attachment_seed=0, + ) + design = graph._verify_cloned_frame( + before.frame, expanded, before.design_weights["household"] + ) + expand = next(n for n in nodes if n.structural is StructuralDelta.EXPAND) + claim = next(n for n in nodes if n.id == clone.COMBINED_CLONE_CLAIM_NODE) + ledger = ( + *before.mass_ledger, + _mass_record(before.frame, expanded, expand, KernelResult(), "conserve"), + ) + owners = { + (e, str(c)): expand.id for e in expanded.entities for c in expanded.table(e) + } + expected, receipts = {}, {} + lineage, facts = {}, {} + for entity in US_SCHEMA.entities: + lineage[entity], facts[entity] = clone._entity_lineage( + before.frame, expanded, entity + ) + authority = puf_support.validate_puf_clone_attachment( + expanded, boundary=graph.PHASE, expected_fraction=1.0, expected_seed=0 + ) + receipt = clone.USCombinedSurveyCloneExpandKernel._receipt( + before.frame, expanded, ("acs", "asec"), authority, facts + ) + receipt["expand"] = expand_lineage_receipt(lineage) + receipt["expand_declared"] = _expand_declared_payload(expand) + receipt["expand_writes"] = expand_writes_receipt( + before.frame, + expanded, + expand, + receipt, + rewrite_coordinates=_expand_rewrite_coordinates(compiled, expand), + ) + receipts[expand.id] = receipt + receipts[claim.id] = { + "phase": clone.COMBINED_CLONE_PHASE, + "claimed_cells": sorted(f"{o.entity}.{o.column}" for o in claim.outputs), + } + # Preserve exact structural and claim ownership before assignment. + for node_id in compiled.order: + if node_id not in receipts: + continue + if node_id == claim.id: + owners.update({(o.entity, o.column): claim.id for o in claim.outputs}) + expected[node_id] = Population.from_frame( + expanded, + expand.id, + owners=owners, + mass_ledger=ledger, + design_weights={"household": np.array(design, copy=True)}, + ) + return expected, receipts + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyGeographyStage: + """Expected complete Population and pure receipt, never executor evidence.""" + + node: Node + population: Population + receipt: bytes + artifacts: tuple[tuple[str, bytes], ...] = () + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyGeographyReconstruction: + """Detached reconstruction values; consumers must independently requalify.""" + + population: Population + observed_population: Population + expanded_population: Population + stages: tuple[AtomicSurveyGeographyStage, ...] + nodes: tuple[Node, ...] + definition: bytes + projection_receipt: bytes + support_payload: bytes + sources: tuple[tuple[str, str], ...] + config_sha256: str + receipt: bytes + + +def reconstruct_atomic_survey_geography(preparation, allocated_population, config): + """Qualify sources, complete initial clones, then assign on the clone version. + + Stage values follow the actual compiled order, including the structural + expansion. Support import and clone ownership may have either relative order. + Callers compare these complete expected Populations to executor observations + on cold and required-warm runs; these values do not replace those checks. + """ + config_bytes = _config_bytes(config) + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + payload, state = entry[1], entry[2] + view = source.CheckedSurveyPopulationView( + payload, state.context, state.frame, state.plan, json.loads(payload) + ) + columns, source_nodes = _raw_allocation(view, allocated_population) + original_stamp = _population_stamp(allocated_population) + current = _copy_population(allocated_population) + support_payload, support_identity = _read_support(config) + support = _decode_support(support_payload) + supports = {blocks.SYSTEM: support} + definition = blocks.assignment_definition( + identity=composition.ASSIGNMENT_IDENTITY, + state_column=observed.COLUMNS[1], + puma_column=observed.COLUMNS[2], + source_ids=dict(config.source_ids), + seed=config.seed, + ) + definition_bytes = canonical_json(definition) + projection = observed.qualify_current_survey_geography(preparation) + projection_receipt = projection.receipt + observed_graph._check_projection( + observed, projection, payload=payload, receipt_sha256=_sha(projection_receipt) + ) + _require( + projection.household.index.tolist() + == current.frame.table("household").household_id.tolist(), + "PROJECTION_ORDER", + ) + observation = observed_graph.current_survey_geography_node( + preparation_sha256=_sha(payload), + projection_receipt_sha256=_sha(projection_receipt), + population=graph.ALLOCATION_NODE, + ) + additions = composition.atomic_survey_clone_nodes( + definition, (*columns, *observation.outputs), base=graph.ALLOCATION_NODE + ) + nodes = (observation, *additions) + compiled = compile_graph( + Graph( + "us", + ( + SourceRef(graph.SOURCE_NAME, graph.SOURCE_CODEC), + SourceRef(blocks.SOURCE, "raw-bytes-v1"), + ), + (*source_nodes, *nodes), + ) + ) + wanted = {node.id for node in nodes} + ordered = tuple( + compiled.graph.node(name) for name in compiled.order if name in wanted + ) + stages = [] + cloned, clone_receipts = {}, {} + for node in ordered: + values, artifacts = {}, () + structural_or_claim = False + if node.id == observation.id: + values = { + ("household", name): projection.household[name].copy(deep=True) + for name in observed.COLUMNS + } + receipt = { + "phase": observed_graph.PHASE, + "preparation_sha256": _sha(payload), + "projection_receipt_sha256": _sha(projection_receipt), + "source_projection": json.loads(projection_receipt), + "population_admission_issued": False, + "release_eligible": False, + } + elif node.structural is StructuralDelta.EXPAND: + cloned, clone_receipts = _clone_expectations(current, nodes, compiled) + current = _copy_population(cloned[node.id]) + receipt = clone_receipts[node.id] + structural_or_claim = True + elif node.id == clone.COMBINED_CLONE_CLAIM_NODE: + _require(node.id in cloned, "CLONE_BEFORE_CLAIM") + current = _copy_population(cloned[node.id]) + receipt = clone_receipts[node.id] + structural_or_claim = True + elif node.kernel == atomic_graph.AtomicSupportImportKernel.ref: + _require( + support.metadata["system"] == node.params["system"], "SUPPORT_SYSTEM" + ) + artifacts = (("support", support_payload),) + receipt = { + "support_sha256": support.sha256, + "areas": len(support.arrays["area"]), + "metadata": support.metadata, + } + else: + households = current.frame.table("household") + if node.kernel == atomic_graph.AtomicAssignKernel.ref: + output = atomic.assign_atomic(households, definition, supports) + receipt = { + "scope": "atomic_area_assignment", + "households": len(output), + "support_sha256": { + name: value.sha256 for name, value in supports.items() + }, + } + elif node.kernel == atomic_graph.AtomicDeriveKernel.ref: + output = atomic.derive_geography(households, definition, supports) + receipt = { + "scope": "atomic_area_functional_lookup", + "layers": { + system["id"]: system["layers"] + for system in definition["systems"] + }, + } + else: + _require( + node.kernel == atomic_graph.AtomicGeographyGateKernel.ref, + "STAGE_ROSTER", + ) + output = None + receipt = atomic.validate_geography(households, definition, supports) + artifacts = (("validation", canonical_json(receipt)),) + if output is not None: + ids = pd.Index(households.household_id.to_numpy(), name="household_id") + values = { + ("household", name): pd.Series(output[name].array.copy(), index=ids) + for name in output + } + if not structural_or_claim: + current = _copy_population( + patch(current, node, KernelResult(columns=values)) + ) + stages.append( + AtomicSurveyGeographyStage( + node, current, canonical_json(receipt), artifacts + ) + ) + stage_tuple = tuple(stages) + by_id = {stage.node.id: stage for stage in stage_tuple} + observed_population = by_id[observation.id].population + expanded_population = by_id[clone.COMBINED_CLONE_CLAIM_NODE].population + validation_payload = by_id["geography.gate"].artifacts[0][1] + receipt = canonical_json( + { + "protocol": PROTOCOL, + "config_sha256": _sha(config_bytes), + "preparation_sha256": _sha(payload), + "projection_receipt_sha256": _sha(projection_receipt), + "support_sha256": _sha(support_payload), + "definition_sha256": _sha(definition_bytes), + "validation_receipt_sha256": _sha(validation_payload), + "assignment_identity": list(composition.ASSIGNMENT_IDENTITY), + "stages": [stage.node.id for stage in stage_tuple], + "frame_sha256": source._frame_identity(current.frame), + "population_sha256": _population_stamp(current), + "observed_population_sha256": _population_stamp(observed_population), + "expanded_population_sha256": _population_stamp(expanded_population), + "publisher_provenance_established": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + result = AtomicSurveyGeographyReconstruction( + population=_copy_population(current), + observed_population=observed_population, + expanded_population=expanded_population, + stages=stage_tuple, + nodes=nodes, + definition=definition_bytes, + projection_receipt=projection_receipt, + support_payload=support_payload, + sources=((blocks.SOURCE, config.support_path),), + config_sha256=_sha(config_bytes), + receipt=receipt, + ) + result_stamps = tuple( + _population_stamp(stage.population) for stage in result.stages + ) + stage_values = tuple( + (stage.node, stage.receipt, stage.artifacts) for stage in result.stages + ) + final_stamp = _population_stamp(result.population) + # Finish support and source-owner I/O before pure rechecks of every retained + # input and every detached expected Population. No callback is an authority. + _require( + _read_support(config) == (support_payload, support_identity), "FINAL_SUPPORT" + ) + final_entry = preparation._checked() + support_after = os.stat(config.support_path, follow_symlinks=False) + _require( + final_entry is entry + and source._ISSUED.get(id(preparation)) is entry + and preparation.payload == payload + and _config_bytes(config) == config_bytes, + "FINAL_ISSUANCE", + ) + _require( + stat.S_ISREG(support_after.st_mode) + and _file_identity(support_after) == support_identity, + "FINAL_SUPPORT_STAT", + ) + source._pure_final(state) + _raw_allocation(view, allocated_population) + _require( + _population_stamp(allocated_population) == original_stamp + and tuple(_population_stamp(stage.population) for stage in result.stages) + == result_stamps + and _population_stamp(result.population) == final_stamp, + "FINAL_POPULATIONS", + ) + observed_graph._check_projection( + observed, projection, payload=payload, receipt_sha256=_sha(projection_receipt) + ) + _require( + result.nodes == nodes + and result.observed_population is observed_population + and result.expanded_population is expanded_population + and tuple(stage.node for stage in result.stages) == ordered + and tuple( + (stage.node, stage.receipt, stage.artifacts) for stage in result.stages + ) + == stage_values + and result.definition == definition_bytes + and result.projection_receipt == projection_receipt + and result.support_payload == support_payload + and result.sources == ((blocks.SOURCE, config.support_path),) + and result.config_sha256 == _sha(config_bytes) + and result.receipt == receipt, + "FINAL_RESULT", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py new file mode 100644 index 000000000..033322cc3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py @@ -0,0 +1,207 @@ +"""Recompute supported age diagnostics without rerunning an optimizer. + +This is a value checker, not source authority. The caller supplies independently +verified ordered counts, sampling bounds and receiving weights. Optimizer history +has a bounded schema but cannot be reconstructed from those final values. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np + +from microcosm.calibrate import diagnostics_payload +from microcosm.calibrate.score import score_targets +from microcosm.frame import WeightKind, Weights +from microcosm.graph.canonical import canonical_json + +from . import graph_survey_calibration as numeric + +MAX_DIAGNOSTIC_BYTES = 8 * 1024**2 +HISTORY_FIELDS = ( + "initial_loss", + "loss_trajectory", + "options.matrix_format", + "options.grouped_upper_bounds.last_corrected_group_count", +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_DIAGNOSTICS_" + reason) + + +def _history(document, epochs, group_count): + options = document.get("options") + _require(type(options) is dict, "OPTIONS") + grouped = options.get("grouped_upper_bounds") + _require(type(grouped) is dict, "GROUPED_OPTIONS") + corrected = grouped.get("last_corrected_group_count") + _require(type(corrected) is int and 0 <= corrected <= group_count, "HISTORY_COUNT") + layout = options.get("matrix_format") + _require( + type(layout) is str and layout in {"dense", "sparse_csr"}, "HISTORY_LAYOUT" + ) + trajectory = document.get("loss_trajectory") + _require(type(trajectory) is list and len(trajectory) == epochs, "HISTORY_LENGTH") + _require( + all(type(v) is float and np.isfinite(v) and 0 <= v <= 10 for v in trajectory), + "HISTORY_LOSS", + ) + _require( + type(document.get("initial_loss")) is float + and document["initial_loss"] == trajectory[0], + "HISTORY_INITIAL", + ) + return corrected, layout, trajectory + + +def validate_survey_calibration_diagnostics( + payload, + *, + counts_payload, + bounds_payload, + weights, + registry, + epochs, + learning_rate, + anchors, + activation_binding=None, +): + """Return defensive schema-8 values plus an explicit verification annotation. + + Every field except the four named history fields is reconstructed using + score_targets and the package's diagnostics encoder. Only warning-free + schema 8 (every registry-backed target carrying its calibration hierarchy) + with target-loss attribution is supported in this first profile. + No convergence, optimality, target eligibility or release claim is granted. + """ + _require( + type(payload) is bytes and 0 < len(payload) <= MAX_DIAGNOSTIC_BYTES, + "PAYLOAD_LIMIT", + ) + try: + document = json.loads(payload) + except (ValueError, UnicodeError, RecursionError): + raise ValueError("SURVEY_DIAGNOSTICS_JSON") from None + _require(type(document) is dict, "DOCUMENT") + _require(canonical_json(document) == payload, "CANONICAL") + # Reuse the actual declaration's option/registry restrictions. This is a + # declaration and numeric work Frame, never a population/source substitute. + factory = numeric.survey_age_calibration_node + profile_options = {} + if activation_binding is not None: + factory = numeric.survey_age_development_node + profile_options["activation_binding"] = activation_binding + factory( + registry, + **profile_options, + base="diagnostic-values", + budget_node="bounds", + count_node="counts", + epochs=epochs, + learning_rate=learning_rate, + ) + bounds = numeric.decode_numeric_survey_bounds(bounds_payload) + counts = numeric.ages.decode_survey_age_counts(counts_payload) + _require( + counts.population == bounds.population + and tuple(counts.household_ids) == bounds.grouped.household_ids, + "ORDERED_POPULATION", + ) + numeric.check_numeric_survey_weights(bounds, weights) + accepted_weights = weights.tobytes() + corrected, layout, trajectory = _history( + document, epochs, bounds.grouped.group_count + ) + table = counts.counts.copy(deep=True) + table.insert(0, "household_id", counts.household_ids.copy()) + work = numeric.demographic.kernels_module._frame_from_context( + SimpleNamespace( + tables={"household": table}, + weights={ + "household": Weights(bounds.incoming.copy(), WeightKind.IMPORTANCE) + }, + ), + "household", + ) + targets = registry.to_target_set() + initial = score_targets( + work, targets, weights=bounds.incoming, target_loss_cap=10.0 + ) + final = score_targets(work, targets, weights=weights, target_loss_cap=10.0) + _require(not initial.skipped and not final.skipped, "SKIPPED_TARGETS") + options = { + # The closed grouped profile supplies no gates or record-budget search. + # Reconstruct these solver fields; never inherit them from the payload. + "gate_initialization_supplied": False, + "budget_basis": "nonzero_count", + "feasible_draw_pi_hi": None, + "budget_search": None, + "grouped_preserve_zeros": { + "enabled": True, + "fixed_zero_count": int(np.count_nonzero(bounds.incoming == 0)), + "ordered_zero_mask_sha256": hashlib.sha256( + np.asarray(bounds.incoming == 0, dtype=np.uint8).tobytes() + ).hexdigest(), + }, + "grouped_upper_bounds": bounds.grouped.diagnostics(weights, corrected), + "method": "adam", + "epochs": epochs, + "learning_rate": learning_rate, + # This profile always uses grouped bounds, whose accepted weights are + # the closing state. Ungrouped Adam's best-iterate history cannot apply. + "iterate_selection": "closing_state", + "iterate_selection_receipt": {}, + "mass": "free", + "mass_reason": None, + "max_weight_ratio": None, + "target_records": None, + "l1_lambda": 0.0, + "l1_penalty": "mean_initial_weight_ratio_abs", + "l2_lambda": 0.0, + "l2_anchor": "initial", + "l2_anchor_weights_supplied": False, + "l2_penalty": "mean_initial_pre_gate_weight_ratio_squared", + "seed": 0, + "target_loss_weights": final.options["target_loss_weights"], + "target_loss_scales": final.options["target_loss_scales"], + "warm_start_weights": {"enabled": False, "kind": None}, + "matrix_format": layout, + } + combined = replace( + final, + diagnostics=tuple( + replace(after, initial_estimate=before.final_estimate) + for before, after in zip( + initial.diagnostics, final.diagnostics, strict=True + ) + ), + options=options, + loss_trajectory=np.asarray(trajectory, dtype=np.float64), + ) + expected = diagnostics_payload(combined, target_registry=registry, build=anchors) + _require( + expected.get("schema_version") == 8 + and expected.get("diagnostic_warnings") == [] + and "target_loss_basis" in expected, + "UNSUPPORTED_DIAGNOSTIC_SCHEMA", + ) + _require(canonical_json(expected) == payload, "RECOMPUTED_VALUES") + # Make the distinction visible to downstream callers without altering the + # stored schema-8 evidence or its digest. + document["verification"] = { + "protocol": "microcosm.us.survey-age-diagnostics-verification.v1", + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "recomputed": "all artifact fields except the listed optimizer history", + "optimizer_history_schema_checked_only": list(HISTORY_FIELDS), + "optimizer_rerun": False, + "release_eligible": False, + } + numeric.check_numeric_survey_weights(bounds, weights) + _require(weights.tobytes() == accepted_weights, "FINAL_WEIGHT_BYTES") + return document diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py new file mode 100644 index 000000000..933e86851 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py @@ -0,0 +1,259 @@ +"""Plan one draw over complete supplied survey records, before native Frames. + +This is declaration/arithmetic only. Closed source owners must authenticate the +complete inputs and bind the result to the selected population. This module +reads no source, creates no Frame and never totals publisher weights. Original +anchors remain separate from the one-time share/inclusion multiplier. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from fractions import Fraction + +from microcosm.build.survey_domain_sample import select_domain_households + +from . import survey_population_domains as domains + +PROTOCOL = "microcosm.us.survey-catalogue-selection.v1" +_BATCH_HOUSEHOLDS = 10_000 +_BATCH_PEOPLE = 100_000 + + +class CatalogueSelectionError(ValueError): + """Static refusal, without raw source identities or observations.""" + + +def _require(condition, code): + if not condition: + raise CatalogueSelectionError(code) + + +@dataclass(frozen=True, slots=True) +class SelectedHousehold: + key: domains.HouseholdKey + domain: domains.Domain + statistical_unit: str + original_design_weight: Fraction + share: Fraction + inclusion_probability: Fraction + + @property + def importance_multiplier(self) -> Fraction: + return self.share / self.inclusion_probability + + +@dataclass(frozen=True, slots=True) +class ExcludedHousehold: + key: domains.HouseholdKey + reason: str + + +@dataclass(frozen=True, slots=True) +class SelectionCell: + source: domains.Source + domain: domains.Domain + statistical_unit: str + share: Fraction + eligible_households: int + selected_households: int + inclusion_probability: Fraction | None + + +@dataclass(frozen=True, slots=True) +class CatalogueSelectionPlan: + selected: tuple[SelectedHousehold, ...] + excluded: tuple[ExcludedHousehold, ...] + cells: tuple[SelectionCell, ...] + supplied_households: int + fraction: Fraction + seed: int + + @property + def source_authenticated(self) -> bool: + return False + + @property + def population_binding_authenticated(self) -> bool: + return False + + @property + def release_eligible(self) -> bool: + return False + + +def _sampling_key(key): + # Keep the original H_SEQ token in the result, but order ASEC keys + # numerically. All validated values fit five digits; padded strings allow + # both survey channels to use the shared selector's one literal-ID axis. + return ( + key.native_id + if key.source is domains.Source.ACS + else f"{int(key.native_id):05d}" + ) + + +def _ordered_key(key): + return key.source.value, _sampling_key(key) + + +def _original_anchor(decision): + row = decision.original + if row.key.source is domains.Source.ASEC: + return Fraction(int(row.hsup_wgt), 100) + if decision.statistical_unit == "person": + return Fraction(int(row.persons[0].pwgtp)) + return Fraction(int(row.wgtp)) + + +def plan_catalogue_selection(*, acs_households, asec_households, fraction, seed): + """Classify all supplied records and draw once in each positive-share cell. + + ``N`` counts complete eligible households in the supplied catalogue, never + selected Frame rows. Unknown classifications refuse before any draw, even + if a tentative share is zero. Known structural exclusions have a separate + ledger. Original zero ASEC anchors remain eligible; an all-zero selected + cell refuses without redrawing. No common-total normalization is performed. + + Full source completeness, immutable source authority, and original files + are obligations of the calling source orchestration, not this value API. + """ + _require( + type(acs_households) is tuple and type(asec_households) is tuple, + "CATALOGUE_TUPLES", + ) + _require(type(fraction) is Fraction and 0 < fraction <= 1, "FRACTION") + _require(type(seed) is int and 0 <= seed < 2**64, "SEED") + _require( + type(_BATCH_HOUSEHOLDS) is int + and 0 < _BATCH_HOUSEHOLDS <= min(10_000, domains.MAX_HOUSEHOLDS) + and type(_BATCH_PEOPLE) is int + and 0 < _BATCH_PEOPLE <= min(100_000, domains.MAX_TOTAL_MEMBERS), + "BATCH_LIMITS", + ) + rows, exclusions = [], [] + household_keys, person_keys = set(), set() + batch, batch_people = [], 0 + + def consume(): + for decision in domains.classify_households(tuple(batch)): + _require( + decision.status is not domains.Status.REVIEW_REQUIRED, + "SOURCE_CLASSIFICATION_REVIEW_REQUIRED", + ) + if decision.status is domains.Status.EXCLUDED: + _require(decision.share == 0, "EXCLUSION_SHARE") + exclusions.append( + ExcludedHousehold(decision.original.key, decision.reason) + ) + else: + _require( + decision.status is domains.Status.ELIGIBLE + and decision.share is not None + and decision.share > 0 + and decision.domain is not None, + "ELIGIBLE_CLASSIFICATION", + ) + # Retain only the compact household decision, not another full + # population of per-person decisions and observation objects. + rows.append( + ( + decision.original.key, + decision.domain, + decision.statistical_unit, + _original_anchor(decision), + decision.share, + ) + ) + + for source, expected, catalogue in ( + (domains.Source.ACS, domains.AcsHousehold, acs_households), + (domains.Source.ASEC, domains.AsecHousehold, asec_households), + ): + for row in catalogue: + _require(type(row) is expected, "CATALOGUE_SOURCE_TYPE") + canonical = domains._key(row.key) + _require( + canonical[0] is source + and row.key.source_year == 2024 + and row.key.survey_year + == (2024 if source is domains.Source.ACS else 2025), + "CATALOGUE_PERIOD_SOURCE", + ) + _require(canonical not in household_keys, "GLOBAL_HOUSEHOLD_COLLISION") + household_keys.add(canonical) + _require(type(row.persons) is tuple, "MEMBER_TUPLE") + _require( + len(row.persons) <= min(domains.MAX_MEMBERS, _BATCH_PEOPLE), + "HOUSEHOLD_MEMBER_BOUND", + ) + if source is domains.Source.ASEC: + for person in row.persons: + _require(type(person) is domains.AsecPerson, "PERSON_TYPE") + identity = (2024, person.peridnum) + _require(identity not in person_keys, "GLOBAL_PERSON_COLLISION") + person_keys.add(identity) + if batch and ( + len(batch) == _BATCH_HOUSEHOLDS + or batch_people + len(row.persons) > _BATCH_PEOPLE + ): + consume() + batch, batch_people = [], 0 + batch.append(row) + batch_people += len(row.persons) + if batch: + consume() + + declarations = { + (item.domain.value, source.value): (item.statistical_unit, share) + for item in domains.declaration() + for source, share in ( + (domains.Source.ACS, item.acs_share), + (domains.Source.ASEC, item.asec_share), + ) + if share > 0 + } + selection = select_domain_households( + row_ids=tuple(_sampling_key(row[0]) for row in rows), + source_channels=tuple(row[0].source.value for row in rows), + domain_keys=tuple(row[1].value for row in rows), + cells=tuple(declarations), + fraction=fraction, + seed=seed, + ) + chosen, cells = [], [] + for cell in selection: + unit, share = declarations[cell.domain, cell.source] + selected = [rows[position] for position in cell.positions] + _require( + not selected or any(row[3] > 0 for row in selected), + "SELECTED_CELL_HAS_NO_POSITIVE_ANCHOR", + ) + for key, domain, observed_unit, anchor, observed_share in selected: + _require( + (observed_unit, observed_share) == (unit, share), "DECLARATION_DRIFT" + ) + chosen.append( + SelectedHousehold( + key, domain, unit, anchor, share, cell.inclusion_probability + ) + ) + cells.append( + SelectionCell( + domains.Source(cell.source), + domains.Domain(cell.domain), + unit, + share, + cell.eligible_households, + len(selected), + cell.inclusion_probability, + ) + ) + return CatalogueSelectionPlan( + tuple(sorted(chosen, key=lambda row: _ordered_key(row.key))), + tuple(sorted(exclusions, key=lambda row: _ordered_key(row.key))), + tuple(cells), + len(household_keys), + fraction, + seed, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py new file mode 100644 index 000000000..c689b746f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py @@ -0,0 +1,254 @@ +"""Admit a checked financial graph result under its original sampling budget. + +This narrow IMPORTANCE-to-IMPORTANCE binding changes only the seven current +financial leaves authenticated by the actual twenty-node runner. It neither +changes the original budget's initial population nor admits arbitrary Frames. + +Age composition still needs an explicit dependency on financial ATTACH and a +checked binding to the actual receiving graph's Population before calibration +admission. This module does not supply those graph edges or a new age runner. +""" + +from __future__ import annotations + +import json +import weakref +from dataclasses import dataclass + +import numpy as np + +from microcosm.graph import ArtifactType + +PROTOCOL = "microcosm.us.sampling-origin-financial-successor.v1" +FINANCIAL_SUCCESSOR_TYPE = ArtifactType( + "microcosm.us.sampling_origin_financial_successor", 1 +) +MAX_PAYLOAD_BYTES = 256 * 1024 +_ISSUED = {} + + +class SurveyFinancialSuccessorError(ValueError): + """A financial result lacks its exact original budget/run ancestry.""" + + +def _require(condition, reason): + if not condition: + raise SurveyFinancialSuccessorError("SURVEY_FINANCIAL_SUCCESSOR_" + reason) + + +def _entry(value): + entry = _ISSUED.get(id(value)) + _require( + type(value) is SamplingOriginFinancialSuccessor + and entry is not None + and entry[0]() is value + and type(value.payload) is bytes + and value.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + + +@dataclass(frozen=True) +class _State: + budget: object + budget_entry: tuple + financial_run: object + run_entry: tuple + previous: object + current: object + previous_identity: tuple + current_identity: tuple + + +def _document(state, budget_view, run_view): + # Local imports avoid a budget -> financial owner -> runner -> host cycle. + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budget + + run, prefix = state.financial_run, state.financial_run.prefix + original = state.budget_entry[2] + _require( + budget._entry(state.budget, budget.SamplingOriginBudget) is state.budget_entry + and runner._run_entry(run) is state.run_entry + and original.preparation is prefix.preparation + and original.allocated is prefix.allocated_population + and original.expanded is prefix.clone_population + and original.geography_config is prefix.geography_config + and budget_view.preparation is prefix.preparation + and budget_view.allocated_population is prefix.allocated_population + and budget_view.initial_population is state.previous is prefix.clone_population + and budget_view.payload == state.budget_entry[1] + and budget_view.digest == budget._sha(state.budget_entry[1]) + and run_view.population is state.current is run.financial_population + and run_view.payload == state.run_entry[1] + and run_view.digest == budget._sha(state.run_entry[1]), + "ORIGINAL_BUDGET_RUN_ANCESTRY", + ) + previous, current = state.previous, state.current + _require( + previous.version == current.version == runner.atomic.clone.COMBINED_CLONE_NODE + and current.mass_ledger == previous.mass_ledger + and current.weight_kind == previous.weight_kind + and np.array_equal( + current.frame.weights_for("household").values, + previous.frame.weights_for("household").values, + ), + "UNCHANGED_SAMPLING_MASS", + ) + budget.graph._check_design_anchors(current, previous.design_weights["household"]) + owners = { + **previous.owners, + **{ + ("person", column): runner.financial.ATTACH_NODE + for column in runner.values.OUTPUTS + }, + } + budget.graph._check_population_state( + current, + version=previous.version, + owners=owners, + kind=budget.WeightKind.IMPORTANCE, + ledger=previous.mass_ledger, + ) + # The checked run freshly compares the complete result to source-derived + # current values and its pinned raw draws; no caller column table is used. + return budget._json( + { + "protocol": PROTOCOL, + "budget_sha256": budget._sha(state.budget_entry[1]), + "financial_run_sha256": budget._sha(state.run_entry[1]), + "financial_run": json.loads(state.run_entry[1]), + "previous_version": previous.version, + "current_version": current.version, + "previous_frame_sha256": budget.source._frame_identity(previous.frame), + "current_frame_sha256": budget.source._frame_identity(current.frame), + "owned_columns": list(runner.values.OUTPUTS), + "sampling_bounds_changed": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + + +def _pure_state(state): + """Final retained-owner checks; never opens an artifact or source file.""" + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budget + + _require( + budget._entry(state.budget, budget.SamplingOriginBudget) is state.budget_entry, + "FINAL_BUDGET_ISSUANCE", + ) + runner._pure_run(state.financial_run, state.run_entry) + _require( + budget._population_identity(state.previous) == state.previous_identity + and budget._population_identity(state.current) == state.current_identity, + "FINAL_POPULATION_SEAL", + ) + _require(budget._live() == budget._LIVE, "FINAL_PRODUCER_SEAL") + + +def _final_state(state): + from . import survey_origin_budget as budget + + # Complete the last budget/support I/O before the financial owner's pure + # seal, so that borrow cannot invalidate the already-checked financial run. + budget._final_budget_state(state.budget_entry[2]) + _pure_state(state) + + +@dataclass(frozen=True) +class SamplingOriginFinancialSuccessor: + payload: bytes + + def __post_init__(self): + raise SurveyFinancialSuccessorError("NO_PUBLIC_FINANCIAL_SUCCESSOR_CONSTRUCTOR") + + def checked_view(self): + from . import graph_atomic_survey_financial as runner + + entry = _entry(self) + state = entry[2] + _pure_state(state) + budget_view = state.budget.checked_view() + run_view = runner.check_atomic_survey_financial_run(state.financial_run) + _require(_document(state, budget_view, run_view) == entry[1], "RECONSTRUCTION") + _final_state(state) + _require(_entry(self) is entry, "FINAL_ISSUANCE") + # Do not expose a detached view to the final support-borrow callback. + # All owner/file I/O and retained-state seals are complete above. + return CheckedSamplingOriginFinancialSuccessor( + entry[1], + runner.codec.sha(entry[1]), + state.budget, + state.financial_run, + state.previous, + state.current, + ) + + def to_bytes(self): + return self.checked_view().payload + + +@dataclass(frozen=True) +class CheckedSamplingOriginFinancialSuccessor: + """Values borrowed from a checked handle, never decoded authority.""" + + payload: bytes + digest: str + budget: object + financial_run: object + previous: object + current: object + + +def admit_survey_financial_population(budget, *, financial_run, candidate=None): + """Admit only a live completed graph run under its own original budget.""" + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budgets + + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= MAX_PAYLOAD_BYTES), + "CANDIDATE_TYPE_OR_BOUND", + ) + # An unissued descriptive dataclass must fail before any owner/file borrow. + run_entry = runner._run_entry(financial_run) + budget_entry = budgets._entry(budget, budgets.SamplingOriginBudget) + budget_view = budget.checked_view() + run_view = runner.check_atomic_survey_financial_run(financial_run) + state = _State( + budget, + budget_entry, + financial_run, + run_entry, + budget_entry[2].expanded, + financial_run.financial_population, + budgets._population_identity(budget_entry[2].expanded), + budgets._population_identity(financial_run.financial_population), + ) + payload = _document(state, budget_view, run_view) + _require(len(payload) <= MAX_PAYLOAD_BYTES, "PAYLOAD_BOUND") + _require(candidate is None or candidate == payload, "CANDIDATE_RECONSTRUCTION") + result = object.__new__(SamplingOriginFinancialSuccessor) + object.__setattr__(result, "payload", payload) + identifier = id(result) + + def forget(reference): + entry = _ISSUED.get(identifier) + if entry is not None and entry[0] is reference: + _ISSUED.pop(identifier, None) + + reference = weakref.ref(result, forget) + _ISSUED[identifier] = (reference, payload, state) + entry = _entry(result) + _final_state(state) + _require(_entry(result) is entry, "FINAL_ISSUANCE") + return result + + +def verify_survey_financial_successor(binding): + _entry(binding) + binding.checked_view() + return binding diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py new file mode 100644 index 000000000..fde1388e8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py @@ -0,0 +1,96 @@ +"""Pure observed-age alias; no source, population or release authority. + +The caller authenticates the integer A_AGE observations. This rule changes no +observation date, top code or raw value and reads no source/weight/domain label. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +RULE = "microcosm.us.observed-age-normalization.v1" +AGE_CONVENTION = "observed_interview_age_completed_years" +MAX_ROWS = 2_000_000 +MAX_EXACT_FLOAT64_INTEGER = 2**53 + + +def rule_document() -> dict: + return { + "protocol": RULE, + "input": "A_AGE", + "output": "age", + "dtype": "float64", + "relation": "numeric_identity", + "age_convention": AGE_CONVENTION, + "existing_nonmissing": "must_equal_observed_age", + "temporal_adjustment": False, + "top_code_replacement": False, + } + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_OBSERVED_AGE_" + reason) + + +def normalize_observed_age( + raw_age: pd.Series, incumbent_age: pd.Series | None = None +) -> pd.Series: + """Return a defensive common age, accepting only verified numeric identity. + + Missing common cells may acquire the already observed value; missing or + malformed raw ages refuse. The float bound is a representation limit, not + a scientifically permitted maximum age. Measurement keeps its own range. + """ + _require(isinstance(raw_age, pd.Series), "RAW_SERIES") + _require(0 < len(raw_age) <= MAX_ROWS, "ROW_BOUND") + _require( + pd.api.types.is_integer_dtype(raw_age.dtype) + and not pd.api.types.is_bool_dtype(raw_age.dtype) + and not raw_age.isna().any(), + "RAW_INTEGER_REQUIRED", + ) + _require( + ((raw_age >= 0) & (raw_age <= MAX_EXACT_FLOAT64_INTEGER)).all(), + "RAW_REPRESENTATION", + ) + values = raw_age.to_numpy(dtype=np.float64, copy=True) + if incumbent_age is not None: + _require(isinstance(incumbent_age, pd.Series), "INCUMBENT_SERIES") + _require( + incumbent_age.index.identical(raw_age.index) + and len(incumbent_age) == len(raw_age), + "ORDERED_INDEX", + ) + present = incumbent_age.notna().to_numpy(dtype=bool) + if present.any(): + _require( + pd.api.types.is_numeric_dtype(incumbent_age.dtype) + and not pd.api.types.is_bool_dtype(incumbent_age.dtype) + and not pd.api.types.is_complex_dtype(incumbent_age.dtype), + "INCUMBENT_NUMERIC_REQUIRED", + ) + known = incumbent_age[present] + _require( + ((known >= 0) & (known <= MAX_EXACT_FLOAT64_INTEGER)).all(), + "INCUMBENT_REPRESENTATION", + ) + _require((known == raw_age[present]).all(), "INCUMBENT_CONFLICT") + if pd.api.types.is_integer_dtype(incumbent_age.dtype): + _require( + np.array_equal( + known.to_numpy(dtype=np.int64), + raw_age[present].to_numpy(dtype=np.int64), + ), + "INCUMBENT_CONFLICT", + ) + incumbent = incumbent_age.to_numpy(dtype=np.float64, na_value=np.nan) + _require( + np.isfinite(incumbent[present]).all() + and np.array_equal(incumbent[present], values[present]), + "INCUMBENT_CONFLICT", + ) + # Preserve the original float bits of every known incumbent cell. + values[present] = incumbent[present] + return pd.Series(values, index=raw_age.index.copy(deep=True), name="age") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py new file mode 100644 index 000000000..bdf2029fe --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py @@ -0,0 +1,1345 @@ +"""Source-qualified, sampling-aware budgets for the complete first survey clone. + +Original publisher d is never replaced. The sampling reference b=d/p and +incoming a=s*b are different quantities. These development coefficients are +provisional; this module grants neither graph execution nor release authority. +Decoded bytes cannot issue a budget or a weight-only successor. +An optional immutable geography recipe independently reconstructs the complete +postclone geography Frame; the retained allocation remains the raw source allocation. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import sys +import weakref +from dataclasses import asdict, dataclass +from fractions import Fraction +from pathlib import Path +from types import FunctionType + +import numpy as np +import pandas as pd + +from microcosm.calibrate import group_bounds +from microcosm.frame import Frame, WeightKind +from microcosm.graph import ( + ArtifactType, + KernelResult, + Node, + StructuralDelta, + WeightTransition, +) +from microcosm.graph.population import ( + Population, + _mass_record, + _storage_parts, + storage_equal, +) + +from . import graph_combined_clone as clone +from . import graph_survey_population as graph +from . import survey_atomic_geography as geography +from . import survey_financial_successor as financial_successor +from . import survey_population_preparation as source +from .graph_sources import frame_column_declarations +from .support_provenance import support_clone_index_column, support_source_id_column + +BUDGET_PROTOCOL = "microcosm.us.sampling-origin-budget.v1" +SUCCESSOR_PROTOCOL = "microcosm.us.sampling-origin-weight-only-successor.v1" +BUDGET_TYPE = ArtifactType("microcosm.us.sampling_origin_budget", 1) +SUCCESSOR_TYPE = ArtifactType("microcosm.us.sampling_origin_weight_only_successor", 1) +MAX_PAYLOAD_BYTES = 64 * 1024**2 +MAX_GROUPS = 1_000_000 +MAX_SCALAR_CHARS = 4096 +PRESCRIPTION = ( + "development-provisional-8b-4a-v1", + "allocation=float(exact d*s/p)", + "reference=float(exact d/p)", + "bounds=min(8.0*reference,4.0*actual_incoming)", +) +_ISSUED = {} + + +class SurveyOriginBudgetError(ValueError): + """Static refusal without source observations or identifiers.""" + + +def _require(condition, reason): + if not condition: + raise SurveyOriginBudgetError(reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _json(value): + return graph._bounded_json(value, MAX_PAYLOAD_BYTES) + + +def _pair(value): + _require(type(value) is Fraction, "EXACT_FRACTION_REQUIRED") + _require( + value.numerator.bit_length() <= 128 and value.denominator.bit_length() <= 128, + "FRACTION_BOUND", + ) + return [value.numerator, value.denominator] + + +def _reference(d, p, s, actual_incoming): + """Pure exact arithmetic, not source authority; no float-product shortcut.""" + for value in (d, p, s): + _pair(value) + _require( + d >= 0 and 0 < p <= 1 and s in (Fraction(1), Fraction(1, 2)), "REFERENCE_DOMAIN" + ) + _require(type(actual_incoming) is float, "INCOMING_FLOAT64_REQUIRED") + b, a = d / p, d * s / p + expected_a, reference = float(a), float(b) + _require( + math.isfinite(actual_incoming) + and actual_incoming.hex() == expected_a.hex() + and math.isfinite(reference), + "ALLOCATION_OPERATION_ORDER", + ) + design_bound, incoming_bound = 8.0 * reference, 4.0 * actual_incoming + _require( + all(math.isfinite(v) and v >= 0 for v in (design_bound, incoming_bound)), + "REFERENCE_OVERFLOW", + ) + upper = min(design_bound, incoming_bound) + _require(upper == incoming_bound, "CURRENT_SHARE_BOUND_RELATION") + return { + "d": _pair(d), + "p": _pair(p), + "s": _pair(s), + "b": _pair(b), + "a": _pair(a), + "original_float64_hex": float(d).hex(), + "reference_float64_hex": reference.hex(), + "actual_incoming_float64_hex": actual_incoming.hex(), + "design_bound_float64_hex": design_bound.hex(), + "incoming_bound_float64_hex": incoming_bound.hex(), + "upper_float64_hex": upper.hex(), + } + + +def _modules(): + modules = ( + sys.modules[__name__], + graph, + clone, + group_bounds, + sys.modules[Population.__module__], + sys.modules[Frame.__module__], + sys.modules[frame_column_declarations.__module__], + sys.modules[support_source_id_column.__module__], + sys.modules[clone.clone_us_frame_for_puf_support.__module__], + geography, + geography.replay, + financial_successor, + geography.blocks, + geography.atomic, + geography.atomic_graph, + geography.composition, + geography.observed, + geography.observed_graph, + geography.observed.demographics, + geography.observed.demographics.demographic, + geography.observed.demographics.household, + geography.observed.demographics.source_csv_builtin, + sys.modules[geography.observed.demographics.demographic._snapshot.__module__], + sys.modules[geography.atomic.keyed_uniform.__module__], + sys.modules[geography.canonical_json.__module__], + sys.modules[geography.compile_graph.__module__], + sys.modules[geography._expand_declared_payload.__module__], + ) + return tuple({module.__name__: module for module in modules}.values()) + + +def _live(): + result = {} + for module in _modules(): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[module.__name__, name] = source._function_seal(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + result[module.__name__, name] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[module.__name__, name, method] = source._function_seal( + function + ) + result["contract"] = ( + BUDGET_PROTOCOL, + SUCCESSOR_PROTOCOL, + (BUDGET_TYPE.name, BUDGET_TYPE.schema_version), + (SUCCESSOR_TYPE.name, SUCCESSOR_TYPE.schema_version), + MAX_PAYLOAD_BYTES, + MAX_GROUPS, + MAX_SCALAR_CHARS, + PRESCRIPTION, + np.__version__, + np.ndarray, + np.float64, + ) + result["geography_contract"] = source._runtime_marker( + ( + geography.PROTOCOL, + geography.MAX_SUPPORT_BYTES, + geography.MAX_SUPPORT_EXPANDED_BYTES, + geography.MAX_SUPPORT_MEMBERS, + geography.RAW_BYTES_MAX_BYTES, + geography.blocks.SYSTEM, + geography.blocks.SOURCE, + tuple(sorted(geography.blocks._STATES)), + tuple(sorted(geography.atomic.RELATIONS)), + geography.atomic._U53, + geography.observed.PROTOCOL, + geography.observed.COLUMNS, + geography.composition.ASSIGNMENT_IDENTITY, + geography.observed.MAX_HOUSEHOLDS, + geography.observed.MAX_RECEIPT_BYTES, + geography.observed_graph.NODE, + geography.observed_graph.REF, + geography.observed_graph.PHASE, + geography.observed_graph.OUTPUT_COLUMNS, + geography.observed.demographics.PROTOCOL, + geography.observed.demographics.STATE_COLUMNS, + geography.observed.demographics.STATE_CONTRACT, + ( + geography.atomic_graph.ATOMIC_SUPPORT_TYPE.name, + geography.atomic_graph.ATOMIC_SUPPORT_TYPE.schema_version, + ), + ( + geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE.name, + geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE.schema_version, + ), + geography.atomic_graph._DEPENDENCIES, + tuple( + (kernel.ref, asdict(kernel.capabilities)) + for kernel in ( + geography.atomic_graph.AtomicSupportImportKernel, + geography.atomic_graph.AtomicAssignKernel, + geography.atomic_graph.AtomicDeriveKernel, + geography.atomic_graph.AtomicGeographyGateKernel, + ) + ), + ) + ) + result["financial_successor_contract"] = ( + financial_successor.PROTOCOL, + financial_successor.FINANCIAL_SUCCESSOR_TYPE.name, + financial_successor.FINANCIAL_SUCCESSOR_TYPE.schema_version, + financial_successor.MAX_PAYLOAD_BYTES, + ) + return result + + +def _code_bytes(): + return {m.__name__: _sha(Path(m.__file__).read_bytes()) for m in _modules()} + + +def _producer(): + _require(_live() == _LIVE and _code_bytes() == _BYTES, "PRODUCER_CHANGED") + return _BYTES + + +def _population_identity(population): + _require(type(population) is Population, "LIVE_POPULATION_REQUIRED") + return ( + source._frame_identity(population.frame), + _physical_nonweight_identity(population.frame), + population.version, + tuple(sorted(population.owners.items())), + tuple(population.weight_kind.items()), + _json([asdict(record) for record in population.mass_ledger]), + tuple( + (e, str(v.dtype), v.shape, v.tobytes()) + for e, v in population.design_weights.items() + ), + ) + + +def _physical_nonweight_identity(frame): + """Detach actual storage, including masked numeric slots, one column at a time. + + Axis/schema/dtype identity is bound separately by the full Frame identity. + This is an in-process population seal, not a cache canonicalization boundary. + """ + digest = hashlib.sha256() + for series in ( + *( + frame.table(entity)[column] + for entity in frame.entities + for column in frame.table(entity) + ), + frame.strata, + ): + # Whole series: the slice selects every row in order, byte for byte. + for part in _storage_parts(series, slice(None)): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + return digest.hexdigest() + + +def _config_payload(config): + return ( + None + if config is None + else geography.AtomicSurveyReconstruction.to_bytes(config) + ) + + +def _geography_binding(value, config_payload, preparation_payload): + """Seal fresh reconstruction values; decoded receipts never issue authority.""" + _require( + type(value) is geography.AtomicSurveyGeographyReconstruction, + "GEOGRAPHY_RECONSTRUCTION_TYPE", + ) + document = json.loads(value.receipt) + population_sha256 = geography._population_stamp(value.population) + stages = {stage.node.id: stage for stage in value.stages} + _require( + value.observed_population is stages[geography.observed_graph.NODE].population + and value.expanded_population + is stages[clone.COMBINED_CLONE_CLAIM_NODE].population, + "GEOGRAPHY_STAGE_BINDINGS", + ) + validation = stages["geography.gate"].receipt + _require( + stages["geography.gate"].artifacts == (("validation", validation),), + "GEOGRAPHY_VALIDATION_ARTIFACT", + ) + _require( + value.config_sha256 == _sha(config_payload) + and document["protocol"] == geography.PROTOCOL + and document["preparation_sha256"] == _sha(preparation_payload) + and document["config_sha256"] == value.config_sha256 + and document["frame_sha256"] == source._frame_identity(value.population.frame) + and document["population_sha256"] == population_sha256 + and document["observed_population_sha256"] + == geography._population_stamp(value.observed_population) + and document["expanded_population_sha256"] + == geography._population_stamp(value.expanded_population) + and document["validation_receipt_sha256"] == _sha(validation) + and document["assignment_identity"] + == list(geography.composition.ASSIGNMENT_IDENTITY) + and document["projection_receipt_sha256"] == _sha(value.projection_receipt) + and document["definition_sha256"] == _sha(value.definition) + and document["support_sha256"] == _sha(value.support_payload) + and document["publisher_provenance_established"] is False + and document["source_admission_issued"] is False + and document["population_admission_issued"] is False + and document["release_eligible"] is False, + "GEOGRAPHY_RECONSTRUCTION_SEAL", + ) + # The full receipt above seals this detached object exactly. Frame-store + # replay can canonicalize storage beneath masked nulls, so its physical + # stamp cannot identify an independently reconstructed object across runs. + physical_fields = ( + "population_sha256", + "observed_population_sha256", + "expanded_population_sha256", + ) + semantic_receipt = { + key: item for key, item in document.items() if key not in physical_fields + } + population = value.population + semantic_population = { + "frame_sha256": document["frame_sha256"], + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": [ + (entity, kind.value) for entity, kind in population.weight_kind.items() + ], + "mass_ledger": [asdict(record) for record in population.mass_ledger], + "design_weights": [ + (entity, str(values.dtype), values.shape, values.tobytes().hex()) + for entity, values in population.design_weights.items() + ], + } + return _json( + { + "config_sha256": value.config_sha256, + "reconstruction_semantic_sha256": _sha(_json(semantic_receipt)), + "projection_receipt_sha256": _sha(value.projection_receipt), + "definition_sha256": _sha(value.definition), + "support_sha256": _sha(value.support_payload), + "postclone_geography_population_semantic_sha256": _sha( + _json(semantic_population) + ), + "validation_receipt": json.loads(validation), + "identity_scope": { + "reconstruction_omitted_fields": list(physical_fields), + "population_fields": list(semantic_population), + "null_backing": "retained_in_in_process_physical_seals_only", + }, + "publisher_provenance_established": False, + } + ) + + +def _initial( + view, + allocated, + expanded, + *, + preparation=None, + geography_config=None, + _with_geography_binding=False, +): + _require(type(_with_geography_binding) is bool, "GEOGRAPHY_BINDING_FLAG") + _require( + type(allocated) is Population and type(expanded) is Population, + "LIVE_POPULATION_REQUIRED", + ) + instructions = graph.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _require(0 < len(instructions) <= MAX_GROUPS, "GROUP_COUNT_BOUND") + design = view.frame.weights_for("household").values.copy() + _weights, _context, allocation, receipt, expected = graph._allocation_output( + view.frame, view.context, instructions, _sha(view.payload) + ) + columns = frame_column_declarations(view.frame) + nodes = graph.survey_population_nodes( + columns, + preparation_sha256=_sha(view.payload), + fraction=view.selection_plan.fraction, + seed=view.selection_plan.seed, + ) + graph._same_frame(expected, allocated.frame) + graph._check_design_anchors(allocated, design) + allocation_ledger = ( + _mass_record( + view.frame, expected, nodes[1], KernelResult(receipt=receipt), "declared" + ), + ) + cells = tuple((e, str(c)) for e in view.frame.entities for c in view.frame.table(e)) + graph._check_population_state( + allocated, + version=graph.ALLOCATION_NODE, + owners=dict.fromkeys(cells, graph.ALLOCATION_NODE), + kind=WeightKind.IMPORTANCE, + ledger=allocation_ledger, + ) + geography_binding = None + if geography_config is not None: + config_payload = geography.AtomicSurveyReconstruction.to_bytes(geography_config) + reconstructed = geography.reconstruct_atomic_survey_geography( + preparation, allocated, geography_config + ) + geography_binding = _geography_binding( + reconstructed, config_payload, view.payload + ) + # Verify the exact clone before assignment, then the complete geography + # population. Geography additions never relax the clone equality rule. + graph._verify_cloned_frame( + reconstructed.observed_population.frame, + reconstructed.expanded_population.frame, + design, + ) + geography.replay.same_replayed_population(reconstructed.population, expanded) + else: + clone_nodes = clone.us_combined_survey_clone_nodes( + columns, base=graph.ALLOCATION_NODE, source_channels=("acs", "asec") + ) + copied_design = graph._verify_cloned_frame(expected, expanded.frame, design) + graph._check_design_anchors(expanded, copied_design) + owners = { + (entity, str(column)): clone.COMBINED_CLONE_NODE + for entity in expected.entities + for column in expected.table(entity) + } + owners.update( + { + (o.entity, o.column): clone.COMBINED_CLONE_CLAIM_NODE + for o in clone_nodes[1].outputs + } + ) + graph._check_population_state( + expanded, + version=clone.COMBINED_CLONE_NODE, + owners=owners, + kind=WeightKind.IMPORTANCE, + ledger=( + *allocation_ledger, + _mass_record( + expected, expanded.frame, clone_nodes[0], KernelResult(), "conserve" + ), + ), + ) + if geography_config is not None: + _require( + geography.AtomicSurveyReconstruction.to_bytes(geography_config) + == config_payload + and _geography_binding(reconstructed, config_payload, view.payload) + == geography_binding, + "FINAL_GEOGRAPHY_RECONSTRUCTION_SEAL", + ) + # Current survey financial and diagnostic hosts consume the historical pair. + # Only the budget owner explicitly requests its additional reconstruction seal. + if _with_geography_binding: + return instructions, allocation, geography_binding + return instructions, allocation + + +def _document( + view, + allocated, + expanded, + instructions, + allocation, + producer, + geography_binding=None, +): + before, after = ( + allocated.frame.table("household"), + expanded.frame.table("household"), + ) + source_column = support_source_id_column("household") + role_column = support_clone_index_column("household") + positions = {int(value): i for i, value in enumerate(before[source_column])} + _require(len(positions) == len(instructions), "ROOT_SOURCE_ID_COLLISION") + members = [[] for _ in instructions] + ids, groups = [], [] + for row in after[["household_id", source_column, role_column]].itertuples( + index=False, name=None + ): + hh_id, source_id, role = (int(value) for value in row) + _require(source_id in positions and role in (0, 1), "CLONE_ORIGIN_ROLE") + group = positions[source_id] + members[group].append((hh_id, role)) + ids.append(hh_id) + groups.append(group) + _require( + len(ids) == 2 * len(instructions) and len(set(ids)) == len(ids), + "CLONE_CARDINALITY", + ) + incoming = allocated.frame.weights_for("household").values + clone_weights = expanded.frame.weights_for("household").values + by_id = {value: i for i, value in enumerate(ids)} + bounds = [] + selected_by_key = {row.key: row for row in view.selection_plan.selected} + + def records(): + for i, (instruction, group_members) in enumerate( + zip(instructions, members, strict=True) + ): + _require( + len(group_members) == 2 and {m[1] for m in group_members} == {0, 1}, + "COMPLETE_CLONE_ROLES", + ) + actual = float(incoming[i]) + _require( + math.fsum(float(clone_weights[by_id[m[0]]]) for m in group_members) + == actual, + "CLONE_INCOMING_CONSERVATION", + ) + reference = _reference( + instruction.original_anchor, + instruction.inclusion_probability, + instruction.share, + actual, + ) + bounds.append(float.fromhex(reference["upper_float64_hex"])) + key = instruction.key + _require( + type(key.native_id) is str and len(key.native_id) <= MAX_SCALAR_CHARS, + "NATIVE_KEY_BOUND", + ) + yield { + "source": key.source.value, + "source_year": key.source_year, + "survey_year": key.survey_year, + "raw_native_id": key.native_id, + "selected_receiving_household_id": instruction.selected_receiving_household_id, + "combined_household_id": instruction.household_id, + "statistical_unit": selected_by_key[instruction.key].statistical_unit, + "original_design_float64_bytes": allocated.design_weights["household"][ + i : i + 1 + ] + .tobytes() + .hex(), + **reference, + "members": [[hh_id, role] for hh_id, role in group_members], + "incoming_clone_float64_bytes": [ + clone_weights[by_id[hh_id] : by_id[hh_id] + 1].tobytes().hex() + for hh_id, _role in group_members + ], + } + + header = { + "protocol": BUDGET_PROTOCOL, + "prescription": PRESCRIPTION, + "coefficient_status": "provisional_development_not_scientific_approval", + "current_share_simplification": "min(8b,4a)=4a; 8b is redundant for current shares", + "preparation_sha256": _sha(view.payload), + "source_producer": view.receipt["producer"], + "source_native": view.receipt["native"], + "source_catalogues": view.receipt["catalogues"], + "selection_sha256": _sha(_json(view.receipt["selection"])), + "allocation_sha256": _sha(allocation), + "producer": producer, + "allocated_frame_sha256": source._frame_identity(allocated.frame), + "clone_frame_sha256": source._frame_identity(expanded.frame), + "household_ids": ids, + "group_indices": groups, + "group_count": len(instructions), + "release_eligible": False, + } + if geography_binding is not None: + header["atomic_geography"] = json.loads(geography_binding) + # Stream one bounded origin record at a time; never materialize an unbounded + # list of origin dictionaries before the transport cap is checked. + payload = bytearray() + + def append(piece): + _require(len(piece) <= MAX_PAYLOAD_BYTES - len(payload), "TRANSPORT_LIMIT") + payload.extend(piece) + + append(b"{") + keys = sorted((*header, "origins")) + for index, key in enumerate(keys): + if index: + append(b",") + append(_json(key) + b":") + if key == "origins": + append(b"[") + for position, record in enumerate(records()): + if position: + append(b",") + append(_json(record)) + append(b"]") + else: + append(_json(header[key])) + append(b"}") + constraint = group_bounds.GroupedUpperBounds(ids, groups, bounds) + constraint.check(clone_weights, positive=False) + return bytes(payload), constraint + + +@dataclass(frozen=True) +class _BudgetState: + preparation: object + preparation_entry: tuple + initial_view: source.CheckedSurveyPopulationView + allocated: Population + expanded: Population + preparation_payload: bytes + allocated_identity: tuple + expanded_identity: tuple + geography_config: geography.AtomicSurveyReconstruction | None + geography_config_payload: bytes | None + geography_binding: bytes | None + + +def _preparation_entry(preparation, payload, expected=None): + """Check retained issuance identity without repeating source-file reads.""" + entry = source._ISSUED.get(id(preparation)) + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation + and entry is not None + and (expected is None or entry is expected) + and entry[0]() is preparation + and type(preparation.payload) is bytes + and preparation.payload == entry[1] == payload, + "FINAL_PREPARATION_ISSUANCE", + ) + return entry + + +def _final_budget_state(state): + """Seal retained owners after helper/file I/O, with no recursive borrow. + + Optional support bytes are checked before the source owner's pure final + check. That pure check hashes retained Frames without rereading survey + files, issuing replacement capsules or granting decoded authority. + """ + _producer() + _require( + _config_payload(state.geography_config) == state.geography_config_payload, + "FINAL_GEOGRAPHY_CONFIG_SEAL", + ) + if state.geography_config is not None: + # Finish support I/O before the same pure owner/Population final seals. + # Re-reading exact pinned bytes grants no publisher provenance. + geography._read_support(state.geography_config) + _pure_budget_state(state) + + +def _pure_budget_state(state): + """Check retained source/Population state without a new file borrow.""" + entry = _preparation_entry( + state.preparation, state.preparation_payload, state.preparation_entry + ) + source._pure_final(entry[2]) + _require( + _population_identity(state.allocated) == state.allocated_identity + and _population_identity(state.expanded) == state.expanded_identity, + "FINAL_POPULATION_SEAL", + ) + _require( + ( + state.geography_config is None + and state.geography_config_payload is None + and state.geography_binding is None + ) + or ( + state.geography_config is not None + and geography.AtomicSurveyReconstruction.to_bytes(state.geography_config) + == state.geography_config_payload + ), + "FINAL_GEOGRAPHY_CONFIG_SEAL", + ) + _require(_live() == _LIVE, "FINAL_PRODUCER_SEAL") + _preparation_entry( + state.preparation, state.preparation_payload, state.preparation_entry + ) + + +def _validate_budget(state, payload): + _producer() + _require( + _config_payload(state.geography_config) == state.geography_config_payload, + "INITIAL_GEOGRAPHY_CONFIG_CHANGED", + ) + _require( + _population_identity(state.allocated) == state.allocated_identity + and _population_identity(state.expanded) == state.expanded_identity, + "INITIAL_POPULATION_CHANGED", + ) + # This private retained view is only a reconstruction input. The fresh + # source borrow below authenticates it after the first reconstruction. + view = state.initial_view + _require(view.payload == state.preparation_payload, "SOURCE_PREPARATION_CHANGED") + instructions, allocation, geography_binding = _initial( + view, + state.allocated, + state.expanded, + preparation=state.preparation, + geography_config=state.geography_config, + _with_geography_binding=True, + ) + _require(geography_binding == state.geography_binding, "GEOGRAPHY_RECONSTRUCTION") + expected, constraint = _document( + view, + state.allocated, + state.expanded, + instructions, + allocation, + _producer(), + geography_binding, + ) + _require(expected == payload, "BUDGET_RECONSTRUCTION") + final = source.AuthenticatedSurveyPopulationPreparation.checked_view( + state.preparation + ) + _require( + final.payload == view.payload + and final.frame is view.frame + and final.selection_plan is view.selection_plan, + "FINAL_SOURCE_SEAL", + ) + _instructions, _allocation, final_geography = _initial( + final, + state.allocated, + state.expanded, + preparation=state.preparation, + geography_config=state.geography_config, + _with_geography_binding=True, + ) + _require(final_geography == state.geography_binding, "FINAL_GEOGRAPHY_SEAL") + return constraint + + +def _entry(value, cls): + entry = _ISSUED.get(id(value)) + _require( + type(value) is cls + and entry is not None + and entry[0]() is value + and type(value.payload) is bytes + and value.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + + +def _issue(cls, payload, state): + value = object.__new__(cls) + object.__setattr__(value, "payload", payload) + identifier = id(value) + + def forget(reference): + current = _ISSUED.get(identifier) + if current is not None and current[0] is reference: + _ISSUED.pop(identifier, None) + + reference = weakref.ref(value, forget) + _ISSUED[identifier] = (reference, payload, state) + return value + + +def _check_view_constraint(constraint, document): + """Bind every derived numerical field to an already requalified issued view.""" + reason = "FINAL_BUDGET_VIEW_CONSTRAINT" + _require(type(constraint) is group_bounds.GroupedUpperBounds, reason) + ids = tuple(document["household_ids"]) + groups = document["group_indices"] + bounds_hex = [row["upper_float64_hex"] for row in document["origins"]] + # This is the maintained constraint's stable-ID reduction order. These + # expected buckets come from the issued document, not constructor output. + buckets = [[] for _ in bounds_hex] + for index in sorted( + range(len(ids)), key=lambda i: (isinstance(ids[i], str), ids[i]) + ): + buckets[groups[index]].append(index) + + def immutable_array(value, expected, dtype): + payload = np.asarray(expected, dtype=dtype).tobytes() + return ( + type(value) is np.ndarray + and value.dtype.str == dtype + and value.shape == (len(expected),) + and value.strides == (8,) + and not value.flags.writeable + and not value.flags.owndata + and type(value.base) is bytes + and value.base == payload + and value.tobytes() == payload + ) + + digest_payload = { + "contract": "stable-household-fsum-group-upper-v1", + "household_ids": ids, + "group_indices": groups, + "absolute_bounds_hex": bounds_hex, + } + expected_digest = _sha( + json.dumps( + digest_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + ) + _require( + type(constraint.household_ids) is tuple + and len(constraint.household_ids) == len(ids) + and all( + type(actual) is type(expected) and actual == expected + for actual, expected in zip(constraint.household_ids, ids, strict=True) + ) + and immutable_array(constraint.group_indices, groups, " bool: + return False + + @property + def population_binding_authenticated(self) -> bool: + return False + + @property + def release_eligible(self) -> bool: + return False + + @property + def required_bindings(self) -> tuple[str, ...]: + return ( + "original_source_bytes", + "complete_original_membership", + "original_publisher_weight_authority", + "source_period_and_residence_scope", + "full_receiving_population_and_producer_edges", + "actual_selection_probability", + ) + + @property + def analysis_year(self) -> int: + return 2024 + + @property + def income_reference(self) -> str: + if self.original.key.source is Source.ASEC: + return f"calendar_year_{self.original.key.source_year}" + return "previous_12_months_at_each_2024_interview" + + +def _require(condition: bool, code: str) -> None: + if not condition: + raise DomainInputError(code) + + +def _token(value: str) -> str: + _require(type(value) is str and len(value) <= MAX_TOKEN_CHARS, "LITERAL_TYPE_SIZE") + return value + + +def _unsigned(raw: str, digits: int, maximum: int) -> int | None: + _token(raw) + if re.fullmatch(rf"[0-9]{{1,{digits}}}", raw) is None: + return None + value = int(raw) + return value if value <= maximum else None + + +def _key(key: HouseholdKey) -> tuple[Source, int, int, str | int]: + _require(type(key) is HouseholdKey and type(key.source) is Source, "HOUSEHOLD_KEY") + _require( + type(key.source_year) is int and type(key.survey_year) is int, "COHORT_TYPE" + ) + _token(key.native_id) + if key.source is Source.ACS: + _require((key.source_year, key.survey_year) == (2024, 2024), "COHORT_IDENTITY") + _require( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", key.native_id) is not None + and int(key.native_id[-7:]) > 0, + "HOUSEHOLD_KEY", + ) + native = key.native_id + else: + _require( + key.source_year in (2022, 2023, 2024) + and key.survey_year == key.source_year + 1, + "COHORT_IDENTITY", + ) + native = _unsigned(key.native_id, MAX_TOKEN_CHARS, 99999) + _require(native is not None and native > 0, "HOUSEHOLD_KEY") + return key.source, key.source_year, key.survey_year, native + + +def declaration() -> tuple[DomainShare, ...]: + """The prospectively fixed coefficients, without applying them to weights.""" + return ( + DomainShare( + Domain.SHARED_HOUSING, + "occupied_housing_unit", + Fraction(1, 2), + Fraction(1, 2), + ), + DomainShare( + Domain.RESIDUAL_HOUSING, "occupied_housing_unit", Fraction(1), Fraction(0) + ), + DomainShare(Domain.INSTITUTIONAL_GQ, "person", Fraction(1), Fraction(0)), + DomainShare(Domain.NONINSTITUTIONAL_GQ, "person", Fraction(1), Fraction(0)), + ) + + +def reduce_qualifiers(values: tuple[Qualifier, ...]) -> Qualifier: + """Existential reduction over known roster members, not membership proof.""" + _require( + type(values) is tuple and 0 < len(values) <= MAX_MEMBERS, "QUALIFIER_ROSTER" + ) + _require(all(type(v) is Qualifier for v in values), "QUALIFIER_TYPE") + if Qualifier.TRUE in values: + return Qualifier.TRUE + if Qualifier.UNKNOWN in values: + return Qualifier.UNKNOWN + return Qualifier.FALSE + + +def _acs_state(age: str, raw: str, minimum: int, codes: tuple[str, ...]) -> str: + # Exact vocabulary of acs_person_coverage_columns._field_state. No owner import. + value = _unsigned(age, 2, 99) + if value is None: + return "age_unresolved" + if value < minimum: + return "outside_age_universe" if raw == "" else "value_below_age_universe" + if raw == "": + return "missing_in_universe" + return "observed_code" if raw in codes else "unlabelled_code" + + +def _asec_state(raw: str) -> str: + # Exact vocabulary of asec_person_coverage_source._state. + if raw in ("1", "2", "3"): + return "observed_code" + if raw == "": + return "blank_unresolved" + if raw in ("-4", "-3", "-2", "-1", "0"): + return "unlabelled_in_range" + if re.fullmatch(r"-?(?:0|[1-9][0-9]*)", raw) and raw != "-0": + return "out_of_range" + return "malformed_token" + + +def _person(person: AcsPerson | AsecPerson) -> PersonDecision: + is_acs = type(person) is AcsPerson + age = _unsigned(person.age, 2 if is_acs else 19, 99 if is_acs else 2**63 - 1) + observations = [ + Observation( + "AGEP" if is_acs else "A_AGE", + person.age, + "observed_integer" if age is not None else "age_unresolved", + ) + ] + diagnostics = [] + if is_acs: + _token(person.pwgtp) + for raw, state in ( + (person.esr, person.esr_state), + (person.mil, person.mil_state), + ): + _token(raw) + _token(state) + _require( + person.esr_state + == _acs_state(person.age, person.esr, 16, ("1", "2", "3", "4", "5", "6")), + "FIELD_STATE_MISMATCH", + ) + _require( + person.mil_state + == _acs_state(person.age, person.mil, 17, ("1", "2", "3", "4")), + "FIELD_STATE_MISMATCH", + ) + observations.extend( + ( + Observation("ESR", person.esr, person.esr_state), + Observation("MIL", person.mil, person.mil_state), + ) + ) + true = person.esr in ("1", "2", "3", "6") + false = person.esr in ("4", "5") + if ( + person.mil_state == "observed_code" + and person.esr_state == "observed_code" + and ( + (person.mil == "1" and true) + or (person.mil in ("2", "3", "4") and false) + ) + ): + diagnostics.append("mil_esr_disagreement") + if person.mil_state != "observed_code": + diagnostics.append("mil_" + person.mil_state) + contradiction = False + else: + _token(person.prpertyp) + _token(person.prpertyp_state) + _require( + person.prpertyp_state == _asec_state(person.prpertyp), + "FIELD_STATE_MISMATCH", + ) + observations.append( + Observation("PRPERTYP", person.prpertyp, person.prpertyp_state) + ) + true, false = person.prpertyp == "2", person.prpertyp == "3" + contradiction = age is not None and age >= 16 and person.prpertyp == "1" + if age is None: + qualifier, reason = Qualifier.UNKNOWN, "source_age_unresolved" + elif age < 16: + qualifier, reason = Qualifier.FALSE, "source_age_below_16" + elif contradiction: + qualifier, reason = Qualifier.UNKNOWN, "adult_age_with_child_type" + elif true: + qualifier, reason = Qualifier.TRUE, "source_civilian_proxy" + elif false: + qualifier, reason = Qualifier.FALSE, "source_armed_forces_proxy" + else: + qualifier, reason = Qualifier.UNKNOWN, "required_person_status_unresolved" + return PersonDecision( + person, qualifier, reason, tuple(observations), tuple(diagnostics) + ) + + +def _members(row: AcsHousehold | AsecHousehold) -> tuple[PersonDecision, ...]: + _require( + type(row.persons) is tuple and len(row.persons) <= MAX_MEMBERS, + "MEMBER_TUPLE_BOUND", + ) + household = _key(row.key) + seen_keys, seen_lines = set(), set() + expected = AcsPerson if type(row) is AcsHousehold else AsecPerson + for person in row.persons: + _require(type(person) is expected, "MEMBER_TYPE") + _require(_key(person.household_key) == household, "MEMBER_HOUSEHOLD_KEY") + raw_line = person.sporder if expected is AcsPerson else person.a_lineno + line = _unsigned( + raw_line, + 2 if expected is AcsPerson else 19, + 20 if expected is AcsPerson else 2**63 - 1, + ) + _require( + line is not None and line > 0 and line not in seen_lines, "MEMBER_LINE_KEY" + ) + seen_lines.add(line) + person_key = line if expected is AcsPerson else _token(person.peridnum) + if expected is AsecPerson: + _require( + re.fullmatch(r"[0-9]{22}", person_key) is not None, "MEMBER_PERSON_KEY" + ) + _require(person_key not in seen_keys, "MEMBER_DUPLICATE_KEY") + seen_keys.add(person_key) + return tuple(_person(person) for person in row.persons) + + +def _field( + name: str, raw: str, digits: int, maximum: int, minimum: int = 0 +) -> tuple[int | None, Observation]: + value = _unsigned(raw, digits, maximum) + known = value is not None and value >= minimum + state = ( + "valid" + if known + else "missing" + if raw == "" + else "malformed" + if re.fullmatch(rf"[0-9]{{1,{digits}}}", raw) is None + else "unlabelled" + ) + return value if known else None, Observation(name, raw, state) + + +def classify_household(row: AcsHousehold | AsecHousehold) -> HouseholdDecision: + """Classify supplied complete rows; unresolved results authorize no allocation. + + Key/tuple/count inconsistencies raise. Unknown required observations retain + their literals and produce REVIEW_REQUIRED, never residual by default. + Source and population authority cannot be supplied as a boolean argument. + """ + _require(type(row) in (AcsHousehold, AsecHousehold), "HOUSEHOLD_TYPE") + key = _key(row.key) + is_acs = type(row) is AcsHousehold + _require(key[0] is (Source.ACS if is_acs else Source.ASEC), "HOUSEHOLD_SOURCE") + people = _members(row) + observations = [] + zero = None + + def field(name, raw, digits, maximum, minimum=0): + value, observation = _field(name, raw, digits, maximum, minimum) + observations.append(observation) + return value + + def result(status, reason, domain=None, share=None, unit=None, housing_units=None): + return HouseholdDecision( + row, + status, + domain, + share, + reason, + unit, + housing_units, + zero, + people, + tuple(observations), + ( + Annotation( + DECLARATION, + "source_age_16_ESR_only_or_PRPERTYP", + "joint_MIL_ESR_or_other_age_threshold", + "prespecified_development_choice", + ), + Annotation( + DECLARATION, + "fixed_domain_shares_and_structural_exclusions", + "estimated_overlap_or_optimized_shares", + "not_empirically_validated", + ), + Annotation( + DECLARATION, + reason, + "no_silent_residual_or_donor_admission", + status.value, + ), + ), + ) + + count = field( + "NP" if is_acs else "H_NUMPER", + row.np if is_acs else row.h_numper, + 2, + 20 if is_acs else 16, + ) + if count is not None: + _require(count == len(row.persons), "REPORTED_MEMBERSHIP_COUNT") + if is_acs: + housing = field("TYPEHUGQ", row.typehugq, 1, 3, 1) + weight = field("WGTP", row.wgtp, 4, 9999) + if count is None or housing is None: + return result(Status.REVIEW_REQUIRED, "acs_household_condition_unresolved") + if row.key.native_id[4:6] != ("HU" if housing == 1 else "GQ"): + return result(Status.REVIEW_REQUIRED, "acs_serialno_housing_conflict") + if housing == 1 and count == 0: + return result( + Status.EXCLUDED, "acs_vacancy", share=Fraction(0), housing_units=0 + ) + if housing in (2, 3): + if count != 1 or weight != 0: + return result(Status.REVIEW_REQUIRED, "acs_gq_placeholder_conflict") + person_weight = field("PWGTP", row.persons[0].pwgtp, 4, 9999, 1) + if person_weight is None: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = False + return result( + Status.ELIGIBLE, + "acs_gq_allocation", + Domain.INSTITUTIONAL_GQ if housing == 2 else Domain.NONINSTITUTIONAL_GQ, + Fraction(1), + "person", + 0, + ) + if weight is None or weight <= 0: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = False + else: + interview = field("H_HHTYPE", row.h_hhtype, 1, 3, 1) + household_type = field("HRHTYPE", row.hrhtype, 2, 10) + living = field("H_LIVQRT", row.h_livqrt, 2, 12, 1) + weight = field("HSUP_WGT", row.hsup_wgt, MAX_TOKEN_CHARS, 999999999) + if count is None: + return result(Status.REVIEW_REQUIRED, "asec_membership_count_unresolved") + if row.key.source_year != 2024: + return result(Status.EXCLUDED, "asec_older_cohort", share=Fraction(0)) + if interview in (2, 3): + return result(Status.EXCLUDED, "asec_noninterview", share=Fraction(0)) + if interview is None or living is None or household_type is None: + return result(Status.REVIEW_REQUIRED, "asec_household_condition_unresolved") + if living == 11: + return result( + Status.REVIEW_REQUIRED, + "asec_student_quarters_source_review", + share=Fraction(0), + housing_units=0, + ) + if living >= 8: + return result( + Status.EXCLUDED, "asec_nonhousing", share=Fraction(0), housing_units=0 + ) + if household_type not in range(1, 9) or count == 0: + return result(Status.REVIEW_REQUIRED, "asec_housing_membership_conflict") + if weight is None: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = weight == 0 + predicate = reduce_qualifiers(tuple(person.qualifier for person in people)) + if predicate is Qualifier.UNKNOWN: + return result(Status.REVIEW_REQUIRED, "household_qualifier_unresolved") + if predicate is Qualifier.TRUE: + return result( + Status.ELIGIBLE, + "chosen_shared_housing", + Domain.SHARED_HOUSING, + Fraction(1, 2), + "occupied_housing_unit", + 1, + ) + return result( + Status.ELIGIBLE if is_acs else Status.EXCLUDED, + "acs_known_residual_housing" if is_acs else "asec_residual_zero_allocation", + Domain.RESIDUAL_HOUSING, + Fraction(1) if is_acs else Fraction(0), + "occupied_housing_unit", + 1, + ) + + +def classify_households( + rows: tuple[AcsHousehold | AsecHousehold, ...], +) -> tuple[HouseholdDecision, ...]: + """Classify a bounded batch, rejecting household and cohort-person collisions.""" + _require(type(rows) is tuple and 0 < len(rows) <= MAX_HOUSEHOLDS, "HOUSEHOLD_BATCH") + keys, members = set(), 0 + for row in rows: + _require(type(row) in (AcsHousehold, AsecHousehold), "HOUSEHOLD_TYPE") + key = _key(row.key) + _require(key not in keys, "DUPLICATE_HOUSEHOLD_KEY") + keys.add(key) + _require( + type(row.persons) is tuple and len(row.persons) <= MAX_MEMBERS, + "MEMBER_TUPLE_BOUND", + ) + members += len(row.persons) + _require(members <= MAX_TOTAL_MEMBERS, "BATCH_MEMBER_BOUND") + person_keys = set() + for row in rows: + if type(row) is AsecHousehold: + for person in row.persons: + _require(type(person) is AsecPerson, "MEMBER_TYPE") + literal = _token(person.peridnum) + _require( + re.fullmatch(r"[0-9]{22}", literal) is not None, + "MEMBER_PERSON_KEY", + ) + person_key = (row.key.source_year, literal) + _require(person_key not in person_keys, "DUPLICATE_COHORT_PERSON_KEY") + person_keys.add(person_key) + return tuple(classify_household(row) for row in rows) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py new file mode 100644 index 000000000..72428c0d1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -0,0 +1,1521 @@ +"""Authenticate complete catalogues, select once, and stack original anchors. + +The selection is a development declaration. Source issuers retain their own +exact-key/unknown-inclusion receipts; this successor binds the separately +demonstrated catalogue sampling relation. No allocation, calibration or donor +operation runs here. Source catalogues/parents still have full-source costs. +""" + +from __future__ import annotations + +import _csv +import csv +import hashlib +import json +import math +import os +import stat +import sys +import weakref +from contextlib import contextmanager +from dataclasses import InitVar, dataclass, fields +from enum import Enum +from fractions import Fraction +from pathlib import Path +from types import FunctionType + +import numpy as np +import pandas as pd + +from microcosm.build import survey_domain_sample +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import population as graph_population +from microcosm.graph.population import dtype_for_token, storage_equal + +from . import acs_native_coverage_binding as acs_native +from . import acs_population_catalogue as acs_catalogue +from . import asec_2024_native_population as asec_native +from . import asec_current_money as current_money +from . import asec_population_catalogue as asec_catalogue +from . import graph_context, graph_sources, spine_assembly +from . import survey_catalogue_selection as selection +from . import survey_observed_age as observed_age +from . import survey_population_domains as domains +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.survey-population-preparation.v2" +REQUEST_PROTOCOL = "microcosm.us.survey-population-request.v1" +SOURCE_CODEC = "us-survey-population-source-v1" +MAX_REQUEST_BYTES = 4096 +MAX_PAYLOAD_BYTES = 64 * 1024**2 +_MAX_SCALAR_BYTES = 1024**2 +_FRAME_FAST_STRING_CHARS = 4096 +_SOURCE_ROSTER = ( + "selection-request.json", + "acs/csv_hus.zip", + "acs/csv_pus.zip", + "asec/parent.h5", + "asec/household-attachment.h5", + "asec/person-income-attachment.h5", + "asec/pppub23.csv", + "asec/pppub24.csv", + "asec/pppub25.csv", + "asec/hhpub25.csv", +) +_TOKEN = object() +_ISSUED = {} + + +class SurveyPopulationPreparationError(ValueError): + """Static refusal; messages contain no source observations or paths.""" + + +def _require(condition, code): + if not condition: + raise SurveyPopulationPreparationError(code) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _check_scalars(value, depth=0): + _require(depth <= 64, "VALUE_DEPTH") + if type(value) is str: + _require(len(value) <= _MAX_SCALAR_BYTES, "SCALAR_LIMIT") + elif type(value) in (tuple, list): + for item in value: + _check_scalars(item, depth + 1) + elif type(value) is dict: + for key, item in value.items(): + _check_scalars(key, depth + 1) + _check_scalars(item, depth + 1) + + +def _chunks(value): + # A single primitive is bounded before JSON quoting. The encoder never + # builds an unbounded complete JSON string before enforcing the transport. + _check_scalars(value) + yield from json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ).iterencode(value) + + +def _encode(value, maximum=MAX_PAYLOAD_BYTES): + result = bytearray() + for chunk in _chunks(value): + encoded = chunk.encode("utf-8") + _require(len(result) + len(encoded) <= maximum, "PAYLOAD_LIMIT") + result.extend(encoded) + return bytes(result) + + +def _digest(value): + digest = hashlib.sha256() + for chunk in _chunks(value): + digest.update(chunk.encode("utf-8")) + return digest.hexdigest() + + +def _catalogue_fast_path(value): + """Recognize immutable, bounded raw records without changing admission.""" + if type(value) is not tuple or len(value) != 2: + return False + for group in value: + if type(group) is not tuple: + return False + for record in group: + if type(record) is not tuple or len(record) != 7: + return False + people = record[6] + if type(people) is not tuple or len(people) > 20: + return False + if any(type(person) is not tuple or len(person) != 9 for person in people): + return False + characters = 0 + for row in (record[:6], *people): + for item in row: + if type(item) is str: + if not item.isascii(): + return False + characters += len(item) + if characters > 65_536: + return False + elif type(item) is int: + if not -(2**63) <= item < 2**63: + return False + else: + return False + return True + + +def _catalogue_chunks(value): + # Decide for the whole value before encoding. Unexpected shapes retain + # the generic encoder's full-depth scalar pass and error precedence. + if not _catalogue_fast_path(value): + yield from _chunks(value) + return + _check_scalars(value) + encoder = json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ) + yield "[" + for group_index, group in enumerate(value): + if group_index: + yield "," + yield "[" + for record_index, record in enumerate(group): + if record_index: + yield "," + # encode uses the ordinary one-shot C provider when available. + # At most 186 leaves and 65,536 ASCII characters bound this + # individual JSON record below 400 KiB, including escaping. + yield encoder.encode(record) + yield "]" + yield "]" + + +def _catalogue_digest(value): + digest = hashlib.sha256() + for chunk in _catalogue_chunks(value): + digest.update(chunk.encode("utf-8")) + return digest.hexdigest() + + +def _catalogue_memo(value, expected_digest): + """Retain one proved immutable tree, never an unchecked caller digest. + + The exact-tuple/int/ASCII-str eligibility test also excludes mutable leaves + and subclasses. Strong references prevent identity reuse. Rehash once while + creating the memo to bind those exact immutable roots to the earlier seal. + A replacement tree or an ineligible shape keeps the original digest path. + """ + if not _catalogue_fast_path(value): + return None + if _catalogue_digest(value) != expected_digest: + return None + return (value[0], value[1], expected_digest) + + +def _memoized_catalogue_digest(value, memo=None, *, expected_digest=None): + """Reuse only the issued preparation's unchanged immutable record roots. + + This private memo is not source authority. The owning preparation still + performs every source/producer borrow and full physical Frame seal. Its + original nested digest is checked independently of the memo's digest. + """ + if ( + type(value) is tuple + and len(value) == 2 + and type(memo) is tuple + and len(memo) == 3 + and value[0] is memo[0] + and value[1] is memo[1] + and type(memo[2]) is str + and memo[2] == expected_digest + ): + return memo[2] + return _catalogue_digest(value) + + +def _value(value): + if isinstance(value, Enum): + return value.value + if type(value) is Fraction: + return [value.numerator, value.denominator] + if hasattr(type(value), "__dataclass_fields__"): + return { + field.name: _value(getattr(value, field.name)) for field in fields(value) + } + if type(value) in (tuple, list): + return [_value(item) for item in value] + if type(value) is dict: + return {key: _value(item) for key, item in value.items()} + if isinstance(value, np.generic): + return value.item() + _require(value is None or type(value) in (str, int, bool, float), "VALUE_TYPE") + return value + + +def _plan_document(plan): + _require(type(plan) is selection.CatalogueSelectionPlan, "SELECTION_TYPE") + result = { + "fraction": _value(plan.fraction), + "seed": plan.seed, + "supplied_households": plan.supplied_households, + } + budget = [0] + for name in ("selected", "excluded", "cells"): + result[name] = [] + for row in getattr(plan, name): + _bounded_append(result[name], _value(row), budget) + return result + + +def _root(value, *, allow_missing=False): + _require(isinstance(value, (str, Path)), "PATH_TYPE") + path = Path(value).absolute() + for component in (path, *path.parents): + if allow_missing: + try: + component.lstat() + except FileNotFoundError: + continue + _require(not stat.S_ISLNK(component.lstat().st_mode), "SOURCE_SYMLINK") + return path + + +def _stat_identity(info): + return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +@contextmanager +def _regular_reader(path, maximum): + """Never follow a replaced final symlink or block on a substituted FIFO.""" + _root(path.parent) + before = path.lstat() + _require( + stat.S_ISREG(before.st_mode) and 0 <= before.st_size <= maximum, + "SOURCE_REGULAR_FILE", + ) + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + try: + opened = os.fstat(descriptor) + _require( + stat.S_ISREG(opened.st_mode) + and _stat_identity(opened) == _stat_identity(before), + "SOURCE_DESCRIPTOR_CHANGED", + ) + with os.fdopen(descriptor, "rb", closefd=False) as stream: + yield stream, opened + after = os.fstat(descriptor) + current = path.lstat() + _require( + stat.S_ISREG(current.st_mode) + and _stat_identity(opened) + == _stat_identity(after) + == _stat_identity(current), + "SOURCE_CHANGED", + ) + finally: + os.close(descriptor) + + +def _request(source_dir): + root = _root(source_dir) + path = root / "selection-request.json" + info = path.lstat() + _require( + stat.S_ISREG(info.st_mode) and info.st_size <= MAX_REQUEST_BYTES, "REQUEST_FILE" + ) + with _regular_reader(path, MAX_REQUEST_BYTES) as (stream, _opened): + payload = stream.read(MAX_REQUEST_BYTES + 1) + _require(len(payload) <= MAX_REQUEST_BYTES, "REQUEST_LIMIT") + document = json.loads(payload) + _require( + type(document) is dict + and set(document) == {"protocol", "declaration", "fraction", "seed"}, + "REQUEST_FIELDS", + ) + _require( + document["protocol"] == REQUEST_PROTOCOL + and document["declaration"] == domains.DECLARATION, + "REQUEST_DECLARATION", + ) + pair = document["fraction"] + _require( + type(pair) is list + and len(pair) == 2 + and all(type(v) is int for v in pair) + and pair[1] > 0, + "REQUEST_FRACTION", + ) + fraction = Fraction(*pair) + _require( + 0 < fraction <= 1 and [fraction.numerator, fraction.denominator] == pair, + "REQUEST_FRACTION", + ) + seed = document["seed"] + _require(type(seed) is int and 0 <= seed < 2**64, "REQUEST_SEED") + _require(_encode(document, MAX_REQUEST_BYTES) == payload, "REQUEST_CANONICAL") + return root, document, payload, fraction, seed + + +def read_survey_population_request(source_dir): + """Decode bounded request arguments, without granting source authority.""" + try: + _path, _document, _payload, fraction, seed = _request(source_dir) + return fraction, seed + except SurveyPopulationPreparationError: + raise + except Exception: + raise SurveyPopulationPreparationError("REQUEST_REFUSED") from None + + +def _file_limits(): + # Limits bound only transport. Actual closed owners independently verify + # their pinned source bytes and stricter member/row/receipt contracts. + return { + **{name: asec_native._MAX_CHECKPOINT_BYTES for name in _SOURCE_ROSTER}, + **{ + f"asec/pppub{year}.csv": asec_native.coverage_owner.literal.MAX_MEMBER_BYTES + for year in (23, 24, 25) + }, + "asec/hhpub25.csv": asec_native.anchor_owner._MAX_MEMBER_BYTES, + "selection-request.json": MAX_REQUEST_BYTES, + "acs/csv_hus.zip": acs_catalogue._ARCHIVE_BYTES, + "acs/csv_pus.zip": acs_catalogue._ARCHIVE_BYTES, + } + + +def _source_files(root): + _root(root) + _require(stat.S_ISDIR(root.lstat().st_mode), "SOURCE_ROOT") + found = [] + for directory, expected in ( + (root, {"acs", "asec", "selection-request.json"}), + (root / "acs", {"csv_hus.zip", "csv_pus.zip"}), + ( + root / "asec", + {Path(n).name for n in _SOURCE_ROSTER if n.startswith("asec/")}, + ), + ): + _require(stat.S_ISDIR(directory.lstat().st_mode), "SOURCE_DIRECTORY") + children = tuple(directory.iterdir()) + _require( + {p.name for p in children} == expected and len(children) == len(expected), + "SOURCE_ROSTER", + ) + for path in children: + mode = path.lstat().st_mode + _require(not stat.S_ISLNK(mode), "SOURCE_SYMLINK") + if path.name in {"acs", "asec"} and path.parent == root: + _require(stat.S_ISDIR(mode), "SOURCE_DIRECTORY") + else: + _require(stat.S_ISREG(mode), "SOURCE_REGULAR_FILE") + found.append(path.relative_to(root).as_posix()) + _require(set(found) == set(_SOURCE_ROSTER), "SOURCE_ROSTER") + limits = _file_limits() + _require( + sum( + (root / name).lstat().st_size + for name in ("acs/csv_hus.zip", "acs/csv_pus.zip") + ) + <= acs_catalogue._ARCHIVE_BYTES, + "ARCHIVE_LIMIT", + ) + result = [] + for name in _SOURCE_ROSTER: + path = root / name + digest, size = hashlib.sha256(), 0 + with _regular_reader(path, limits[name]) as (stream, opened): + while block := stream.read(1024**2): + size += len(block) + _require(size <= limits[name], "SOURCE_FILE_LIMIT") + digest.update(block) + _require(size == opened.st_size, "SOURCE_CHANGED") + result.append([name, size, digest.hexdigest()]) + return tuple(tuple(row) for row in result) + + +def _file_stats(root): + _root(root) + result = [] + # Directory ctime/mtime also bind the exact admitted roster across the last + # producer I/O, including a newly added unlisted entry in either source arm. + for directory in (root, root / "acs", root / "asec"): + info = directory.lstat() + _require(stat.S_ISDIR(info.st_mode), "SOURCE_DIRECTORY") + result.append(_stat_identity(info)) + for name in _SOURCE_ROSTER: + path = root / name + _root(path.parent) + info = path.lstat() + _require(stat.S_ISREG(info.st_mode), "SOURCE_REGULAR_FILE") + result.append(_stat_identity(info)) + return tuple(result) + + +def _modules(): + modules = ( + sys.modules[__name__], + domains, + selection, + survey_domain_sample, + spine_assembly, + graph_sources, + graph_context, + graph_population, + observed_age, + acs_catalogue, + asec_catalogue, + acs_native, + acs_native.housing, + acs_native.coverage, + acs_native.coverage.literal, + sys.modules[acs_native.housing.build_acs_pums_unit_frame.__module__], + sys.modules[acs_native.housing.map_acs_native_inputs.__module__], + *asec_native._modules(), + ) + return tuple({module.__name__: module for module in modules}.values()) + + +def _runtime_marker(value, depth=0): + """Detach simple callable configuration, retaining opaque dependency identity.""" + _require(depth <= 16, "PRODUCER_CONFIGURATION_DEPTH") + if type(value) in (type(None), bool, int, float, str, bytes): + return type(value), value + if type(value) in (tuple, list): + return type(value), tuple(_runtime_marker(v, depth + 1) for v in value) + if type(value) is dict: + return tuple( + (_runtime_marker(k, depth + 1), _runtime_marker(v, depth + 1)) + for k, v in value.items() + ) + if isinstance(value, FunctionType): + return value, value.__code__ + return type(value), id(value) + + +def _function_seal(function, depth=0): + # contextmanager keeps executable code in both __wrapped__ and a closure; + # the public wrapper's __code__ alone cannot bind that implementation. + _require(depth <= 16, "PRODUCER_WRAPPER_DEPTH") + wrapped = getattr(function, "__wrapped__", None) + return ( + function, + function.__code__, + _runtime_marker(function.__defaults__), + _runtime_marker(function.__kwdefaults__), + tuple(_runtime_marker(c.cell_contents) for c in function.__closure__ or ()), + _function_seal(wrapped, depth + 1) + if isinstance(wrapped, FunctionType) + else _runtime_marker(wrapped), + ) + + +def _live(): + result = {} + for module in _modules(): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[(module.__name__, name)] = _function_seal(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + result[(module.__name__, name)] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[(module.__name__, name, method)] = _function_seal( + function + ) + result["rng"] = (np.random.Generator, np.random.PCG64, np.random.SeedSequence) + result["csv"] = (csv.reader, _csv.reader) + # The bounded catalogue path uses JSONEncoder.encode and its C provider; + # frame cells also use the same encode_basestring provider as _chunks. + # Bind their actual live identities through this existing seal. + result["json_catalogue"] = ( + json.JSONEncoder, + json.encoder, + tuple( + _function_seal(function) + if isinstance(function, FunctionType) + else _runtime_marker(function) + for function in ( + getattr(json.JSONEncoder, name, None) + for name in ("__init__", "encode", "iterencode") + ) + ), + getattr(json.encoder, "c_make_encoder", None), + getattr(json.encoder, "encode_basestring", None), + ) + result["contract"] = ( + PROTOCOL, + REQUEST_PROTOCOL, + SOURCE_CODEC, + MAX_REQUEST_BYTES, + MAX_PAYLOAD_BYTES, + _MAX_SCALAR_BYTES, + _FRAME_FAST_STRING_CHARS, + _SOURCE_ROSTER, + domains.DECLARATION, + domains.MAX_MEMBERS, + domains.MAX_HOUSEHOLDS, + domains.MAX_TOTAL_MEMBERS, + selection._BATCH_HOUSEHOLDS, + selection._BATCH_PEOPLE, + observed_age.RULE, + observed_age.AGE_CONVENTION, + observed_age.MAX_ROWS, + observed_age.MAX_EXACT_FLOAT64_INTEGER, + ) + return result + + +def _code_bytes(): + return { + module.__name__: _sha(Path(module.__file__).read_bytes()) + for module in _modules() + } + + +def _producer(): + _require(_live() == _LIVE and _code_bytes() == _BYTES, "PRODUCER_CHANGED") + return { + "implementation": _BYTES, + "acs": acs_catalogue._producer(), + "asec": _sha(_encode(asec_catalogue._implementation())), + "numpy": np.__version__, + "rng": "numpy.random.PCG64/SeedSequence", + "declaration": domains.DECLARATION, + } + + +def _authority(): + return _encode( + { + "asec": asec_catalogue._source_authority().hex(), + "acs": acs_native.housing._ARCHIVE_PINS, + } + ) + + +def _cell(value): + if ( + value is None + or value is pd.NA + or (isinstance(value, (float, np.floating)) and math.isnan(value)) + ): + return None + if isinstance(value, np.generic): + value = value.item() + _require(type(value) in (str, int, bool, float), "FRAME_CELL_TYPE") + if type(value) is float: + _require(math.isfinite(value), "FRAME_NONFINITE") + return ["float", value.hex()] + return value + + +def _frame_cell_encode(value, maximum=MAX_PAYLOAD_BYTES): + """Encode one normalized cell with the exact _encode preimage and limits. + + _cell admits only None, exact bool/int/str, or its own freshly constructed + ["float", finite_float.hex()] list. Keep normalization and the scalar walk + before encoding; large strings/integers retain the original slow path. + No Frame, Series, index, or normalized value is retained between calls. + """ + value = _cell(value) + kind = type(value) + if not ( + value is None + or kind is bool + or (kind is int and -(2**63) <= value < 2**64) + or (kind is str and len(value) <= _FRAME_FAST_STRING_CHARS) + or kind is list + ): + return _encode(value, maximum) + + _check_scalars(value) + if value is None: + encoded = b"null" + elif kind is bool: + encoded = b"true" if value else b"false" + elif kind is int: + # Match JSON's exact-int formatter, including its decimal digit policy. + encoded = int.__repr__(value).encode("ascii") + elif kind is str: + # _chunks selects this very provider with ensure_ascii=False. UTF-8 + # conversion still precedes the payload check, including surrogates. + # At most 6 * 4096 + 2 encoded bytes, even for all control characters. + encoded = json.encoder.encode_basestring(value).encode("utf-8") + else: + # Only _cell can construct this list: both strings are ASCII without + # JSON escapes. Keep hex spelling (especially -0.0), never decimalize. + encoded = b'["float","' + value[1].encode("ascii") + b'"]' + _require(len(encoded) <= maximum, "PAYLOAD_LIMIT") + return encoded + + +def _frame_identity(frame): + _require( + isinstance(frame, Frame) and frame.schema == US_SCHEMA and not frame.links, + "FRAME_TYPE", + ) + digest = hashlib.sha256() + + def update(value): + digest.update(_encode(value)) + digest.update(b"\n") + + def update_cell(value): + digest.update(_frame_cell_encode(value)) + digest.update(b"\n") + + update([list(frame.entities), list(frame.weighted_entities)]) + for entity in frame.entities: + table = frame.table(entity) + _require( + type(table) is pd.DataFrame and not table.columns.has_duplicates, + "FRAME_TABLE", + ) + update( + [ + entity, + list(table.columns), + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + ] + ) + for value in table.index: + update_cell(value) + for column in table: + series = table[column] + dtype = series.dtype + update( + [ + column, + str(dtype), + getattr(dtype, "storage", None), + str(getattr(dtype, "na_value", "")), + ] + ) + for value in series: + update_cell(value) + update( + [ + str(frame.strata.dtype), + type(frame.strata.index).__name__, + str(frame.strata.index.dtype), + list(frame.strata.index.names), + getattr(frame.strata.dtype, "storage", None), + frame.strata.name, + ] + ) + for value in frame.strata.index: + update_cell(value) + for value in frame.strata: + update_cell(value) + for entity in frame.weighted_entities: + weights = frame.weights_for(entity) + update( + [ + entity, + weights.kind.value, + str(weights.values.dtype), + list(weights.values.shape), + ] + ) + digest.update(weights.values.tobytes()) + digest.update(graph_context.encode_us_frame_context(frame)) + return digest.hexdigest() + + +def _copy_source(frame): + result = Frame( + {e: frame.table(e).copy(deep=True) for e in frame.entities}, + frame.schema, + { + e: Weights(frame.weights_for(e).values.copy(), frame.weights_for(e).kind) + for e in frame.weighted_entities + }, + frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + # Only physical string storage changes on owned copies, before the stack's + # exact shared-dtype check. Actual values and missing masks stay identical. + for entity in result.entities: + for column in result.table(entity): + original = result.table(entity)[column] + if isinstance(original.dtype, pd.StringDtype): + changed = original.astype(dtype_for_token("string")) + _same_values(original, changed) + result.table(entity)[column] = changed + return result + + +def _same_values(left, right): + _require(len(left) == len(right), "ROW_COUNT") + _require(list(map(_cell, left)) == list(map(_cell, right)), "SOURCE_VALUE_CHANGED") + + +def _normalized_source_copy(frame: Frame) -> Frame: + result = _copy_source(frame) + _require("A_AGE" in frame.person, "OBSERVED_AGE_REQUIRED") + result.person["age"] = observed_age.normalize_observed_age( + frame.person["A_AGE"], frame.person.get("age") + ) + _verify_normalized_copy(frame, result) + return result + + +def _verify_normalized_copy(original: Frame, normalized: Frame): + """Only the recorded age alias and existing string storage may differ.""" + _require( + original.schema == normalized.schema + and original.entities == normalized.entities + and original.links == normalized.links == () + and original.metadata == normalized.metadata + and original.mass_log == normalized.mass_log + and original.weighted_entities == normalized.weighted_entities, + "NORMALIZED_FRAME_CONTEXT", + ) + for entity in original.entities: + before, after = original.table(entity), normalized.table(entity) + expected_columns = list(before) + original_axis = after.columns + if entity == "person" and "age" not in before: + expected_columns.append("age") + original_axis = after.columns[:-1] + _require( + list(after) == expected_columns + and before.columns.identical(original_axis) + and before.index.identical(after.index), + "NORMALIZED_ORDERED_COLUMNS", + ) + for column in before: + if entity == "person" and column == "age": + continue # Independently reconstructed from raw observations below. + left, right = before[column], after[column] + if isinstance(left.dtype, pd.StringDtype): + _require( + right.dtype == dtype_for_token("string"), + "NORMALIZED_STRING_DTYPE", + ) + _same_values(left, right) + else: + _require(storage_equal(left, right), "NORMALIZED_SOURCE_STORAGE") + _require( + original.strata.index.identical(normalized.strata.index) + and original.strata.name == normalized.strata.name + and storage_equal(original.strata, normalized.strata), + "NORMALIZED_STRATA", + ) + for entity in original.weighted_entities: + left, right = original.weights_for(entity), normalized.weights_for(entity) + _require( + left.kind is right.kind + and left.values.dtype == right.values.dtype + and left.values.tobytes() == right.values.tobytes(), + "NORMALIZED_WEIGHTS", + ) + expected = observed_age.normalize_observed_age( + original.person["A_AGE"], original.person.get("age") + ) + _require( + storage_equal(expected, normalized.person["age"]), "NORMALIZED_AGE_IDENTITY" + ) + + +def _bounded_append(rows, row, budget): + size = len(_encode(row)) + 1 + _require(budget[0] + size <= MAX_PAYLOAD_BYTES, "ORIGIN_LIMIT") + budget[0] += size + rows.append(row) + + +def _origins(frame, sources, selected, native_receipts): + selected_by_key = {(r.key.source.value, r.key.native_id): r for r in selected} + native_hh, native_people = {}, {} + for row in native_receipts["asec"]["households"]: + native_hh[("asec", row["household_id"])] = row["source"]["H_SEQ"] + asec_rows = {row["person_id"]: row for row in native_receipts["asec"]["persons"]} + for channel, source in sources.items(): + for row in source.table("household").itertuples(index=False): + if channel == "acs": + native_hh[(channel, int(row.household_id))] = row.SERIALNO + for row in source.person.itertuples(index=False): + if channel == "acs": + person_key, line = str(row.SPORDER), str(row.SPORDER) + else: + original = asec_rows[int(row.person_id)] + person_key, line = original["PERIDNUM"], str(original["A_LINENO"]) + native_people[(channel, int(row.person_id))] = ( + person_key, + line, + int(row.person_household_id), + ) + maps, entities, households, people, budget = {}, {}, [], [], [0] + household_positions = {} + for channel, source in sources.items(): + ids = source.table("household").household_id + positions = {int(value): index for index, value in enumerate(ids)} + _require(len(positions) == len(ids), "SOURCE_ID_COLLISION") + household_positions[channel] = positions + for entity in frame.entities: + table = frame.table(entity) + ids = US_SCHEMA.entity_id_column(entity) + channels, previous = ( + support_channel_column(entity), + spine_source_id_column(entity), + ) + provenance = set(spine_assembly._support_metadata_columns(entity)) + expected_columns = provenance | { + column for source in sources.values() for column in source.table(entity) + } + _require(set(table) == expected_columns, "UNDECLARED_STACK_COLUMN") + records, mapping = [], {} + for new_id, channel, source_id in table[[ids, channels, previous]].itertuples( + index=False, name=None + ): + key = (channel, int(source_id)) + _require(key not in mapping, "SOURCE_ID_COLLISION") + mapping[key] = int(new_id) + _bounded_append(records, [int(new_id), channel, int(source_id)], budget) + expected = { + (channel, int(i)) + for channel, source in sources.items() + for i in source.table(entity)[ids] + } + _require(set(mapping) == expected, "COMPLETE_ENTITY_ORIGIN") + maps[entity], entities[entity] = mapping, records + for entity in frame.entities: + table = frame.table(entity) + id_column = US_SCHEMA.entity_id_column(entity) + for channel, source in sources.items(): + arm = table.loc[table[support_channel_column(entity)].eq(channel)] + old_ids = arm[spine_source_id_column(entity)].to_numpy() + original = ( + source.table(entity).set_index(id_column, drop=False).loc[old_ids] + ) + for column in original: + if column == id_column: + expected = [ + maps[entity][(channel, int(v))] for v in original[column] + ] + elif entity == "person" and column in { + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + }: + group = next( + e + for e in US_SCHEMA.group_entities + if US_SCHEMA.membership_column(e) == column + ) + expected = [ + maps[group][(channel, int(v))] for v in original[column] + ] + else: + expected = original[column] + _same_values(expected, arm[column]) + for column in ( + set(arm) + - set(original) + - set(spine_assembly._support_metadata_columns(entity)) + ): + _require(arm[column].isna().all(), "ABSENT_SOURCE_VALUE_INVENTED") + if entity == "person": + source_positions = { + int(value): index + for index, value in enumerate(source.person.person_id) + } + expected_strata = source.strata.iloc[ + [source_positions[int(value)] for value in old_ids] + ] + receiving_positions = np.flatnonzero( + table[support_channel_column(entity)].eq(channel).to_numpy() + ) + _same_values(expected_strata, frame.strata.iloc[receiving_positions]) + household_table = frame.table("household") + for position, (new_id, channel, source_id) in enumerate(entities["household"]): + raw = native_hh[(channel, source_id)] + chosen = selected_by_key.pop((channel, raw), None) + _require(chosen is not None, "SELECTED_SOURCE_IDENTITY") + original = chosen.original_design_weight + source = sources[channel] + source_position = household_positions[channel][source_id] + _require( + source.weights_for("household").kind is WeightKind.DESIGN + and frame.weights_for("household").kind is WeightKind.DESIGN, + "ORIGINAL_WEIGHT_KIND", + ) + expected = np.array([float(original)], dtype=np.float64).tobytes() + _require( + source.weights_for("household") + .values[source_position : source_position + 1] + .tobytes() + == expected + == frame.weights_for("household").values[position : position + 1].tobytes(), + "ORIGINAL_ANCHOR_CHANGED", + ) + _bounded_append( + households, + { + "household_id": new_id, + "source": channel, + "source_year": 2024, + "survey_year": 2024 if channel == "acs" else 2025, + "raw_native_id": raw, + "selected_receiving_household_id": source_id, + "original_anchor": _value(original), + }, + budget, + ) + _require( + not selected_by_key and len(households) == len(household_table), + "SELECTED_HOUSEHOLD_ROSTER", + ) + for new_id, channel, source_id in entities["person"]: + person_key, line, household_id = native_people[(channel, source_id)] + _bounded_append( + people, + [ + new_id, + channel, + 2024, + 2024 if channel == "acs" else 2025, + native_hh[(channel, household_id)], + person_key, + line, + source_id, + household_id, + maps["household"][(channel, household_id)], + ], + budget, + ) + return { + "households": households, + "entities": entities, + "persons": { + "columns": [ + "person_id", + "source", + "source_year", + "survey_year", + "raw_native_household_id", + "raw_native_person_id", + "native_line_numeric_original", + "selected_receiving_person_id", + "selected_receiving_household_id", + "household_id", + ], + "rows": people, + }, + } + + +@dataclass(frozen=True) +class _State: + root: Path + files: tuple + file_stats: tuple + producer: bytes + authority: bytes + catalogues: tuple + native: tuple + attached: tuple + source_frames: tuple + source_frame_seals: tuple + frame: Frame + identity: str + context: bytes + plan: selection.CatalogueSelectionPlan + plan_sha256: str + nested: tuple + acs_catalogue_memo: tuple | None = None + + +def _attached(state): + return tuple(value.payload for value in (*state.catalogues, *state.native)) + + +def _nested_seals( + catalogues, native, *, catalogue_memo=None, expected_catalogue_digest=None +): + """Pure final borrowed-object seals after all source/producer file checks. + + These inspect the actual retained issued owners, never construct issuer + authority from decoded records. The ancestor Frames still impose their + existing full-parent verification cost. + """ + acs = acs_native._owned(native[0]) + acs_cat = acs_catalogue._lookup(catalogues[0]) + values = [ + id(acs), + id(acs.prepared), + id(acs.prepared.source), + id(acs.literal), + acs.prepared.receipt_json, + acs.prepared.source.receipt_json, + acs.prepared.source.projection_json, + acs.literal.payload, + _encode( + [ + [str(path) for _, path in sorted(paths.items())] + for paths in acs.snapshots + ] + ), + _memoized_catalogue_digest( + (acs_cat.records, acs_cat.vacancies), + catalogue_memo, + expected_digest=expected_catalogue_digest, + ), + ] + for module, value in ((asec_catalogue, catalogues[1]), (asec_native, native[1])): + entry = module._ISSUED.get(id(value)) + _require( + entry is not None and entry[0]() is value and entry[1] == value.payload, + "ANCESTOR_ISSUANCE_CHANGED", + ) + state = entry[2] + attached = asec_native._attached_evidence( + state.parent, state.coverage, state.anchors, state.fields + ) + values.extend( + ( + id(state), + id(state.parent), + id(attached[0]), + *attached[1:], + asec_native._frame_identity(state.parent.frame), + ) + ) + if module is asec_catalogue: + values.append(asec_catalogue._current_records_identity(state)) + return tuple(values) + + +def _pure_final(state): + _require( + _live() == _LIVE and _authority() == state.authority, "FINAL_AUTHORITY_CHANGED" + ) + _require(_attached(state) == state.attached, "ATTACHED_EVIDENCE_CHANGED") + _require( + _nested_seals( + state.catalogues, + state.native, + catalogue_memo=state.acs_catalogue_memo, + expected_catalogue_digest=state.nested[9], + ) + == state.nested, + "NESTED_EVIDENCE_CHANGED", + ) + _require(_digest(_plan_document(state.plan)) == state.plan_sha256, "PLAN_CHANGED") + _require( + _frame_identity(state.frame) == state.identity + and graph_context.encode_us_frame_context(state.frame) == state.context, + "PREPARED_FRAME_CHANGED", + ) + _require( + tuple( + ( + acs_native._frame_sha256(frame) + if i == 0 + else asec_native._frame_identity(frame) + ) + for i, frame in enumerate(state.source_frames) + ) + == state.source_frame_seals, + "SOURCE_FRAME_CHANGED", + ) + + +def _validate(state): + _pure_final(state) + acs_catalogue.verify_acs_source_catalogue(state.catalogues[0]) + asec_catalogue.verify_asec_source_catalogue(state.catalogues[1]) + acs_native.verify_acs_native_coverage(state.native[0], state.source_frames[0]) + state.native[1].validate() + _require(_source_files(state.root) == state.files, "SOURCE_CHANGED") + _require(_encode(_producer()) == state.producer, "PRODUCER_CHANGED") + _require(_file_stats(state.root) == state.file_stats, "SOURCE_STAT_CHANGED") + _pure_final(state) + + +@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False) +class AuthenticatedSurveyPopulationPreparation: + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "ISSUANCE_CONSTRUCTOR") + + def _checked(self): + try: + entry = _ISSUED.get(id(self)) + _require( + type(self) is AuthenticatedSurveyPopulationPreparation + and entry is not None + and entry[0]() is self + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + _validate(entry[2]) + _require( + _ISSUED.get(id(self)) is entry + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + except SurveyPopulationPreparationError: + raise + except Exception: + raise SurveyPopulationPreparationError( + "PREPARATION_VERIFICATION_REFUSED" + ) from None + + def validate(self): + self._checked() + + def checked_view(self): + """Borrow one checked bundle; the view itself grants no authority.""" + _reference, payload, state = self._checked() + return CheckedSurveyPopulationView( + payload, state.context, state.frame, state.plan, json.loads(payload) + ) + + @property + def frame(self): + return self._checked()[2].frame + + @property + def context(self): + return self._checked()[2].context + + @property + def selection_plan(self): + return self._checked()[2].plan + + @property + def receipt(self): + return json.loads(self._checked()[1]) + + def to_bytes(self): + return self._checked()[1] + + +@dataclass(frozen=True, slots=True) +class CheckedSurveyPopulationView: + """A checked borrow's values, not an issued source or reusable certificate.""" + + payload: bytes + context: bytes + frame: Frame + selection_plan: selection.CatalogueSelectionPlan + receipt: dict + + +def verify_survey_population_preparation(value): + _require( + type(value) is AuthenticatedSurveyPopulationPreparation, "PREPARATION_TYPE" + ) + value._checked() + return value + + +def verify_materialized_survey_population(preparation, frame): + _require( + type(preparation) is AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + _require(_frame_identity(frame) == entry[2].identity, "MATERIALIZED_FRAME_CHANGED") + _validate(entry[2]) + _require(_frame_identity(frame) == entry[2].identity, "MATERIALIZED_FRAME_CHANGED") + _require(preparation.payload == entry[1], "UNISSUED_OR_CHANGED") + + +def prepare_authenticated_survey_population( + source_dir, *, snapshot_root, fraction, seed, candidate=None +): + """Reconstruct actual source authority before accepting candidate bytes.""" + try: + _require(type(candidate) is bytes or candidate is None, "CANDIDATE_TYPE") + _require( + candidate is None or len(candidate) <= MAX_PAYLOAD_BYTES, "CANDIDATE_LIMIT" + ) + root, request, request_bytes, requested_fraction, requested_seed = _request( + source_dir + ) + _require( + type(fraction) is Fraction + and fraction == requested_fraction + and type(seed) is int + and seed == requested_seed, + "REQUEST_ARGUMENTS", + ) + snapshots = _root(snapshot_root, allow_missing=True) + _require( + not snapshots.is_relative_to(root) and not root.is_relative_to(snapshots), + "SNAPSHOT_LOCATION", + ) + snapshots.mkdir(parents=True, exist_ok=True) + authority, files_before, producer = ( + _authority(), + _source_files(root), + _encode(_producer()), + ) + _require( + files_before[0] + == ("selection-request.json", len(request_bytes), _sha(request_bytes)), + "REQUEST_CHANGED", + ) + file_stats = _file_stats(root) + kwargs = dict( + parent_path=root / "asec/parent.h5", + household_attachment_path=root / "asec/household-attachment.h5", + person_income_attachment_path=root / "asec/person-income-attachment.h5", + person_member_paths={ + 2022: root / "asec/pppub23.csv", + 2023: root / "asec/pppub24.csv", + 2024: root / "asec/pppub25.csv", + }, + household_member_path=root / "asec/hhpub25.csv", + ) + acs = acs_catalogue.issue_acs_source_catalogue( + root / "acs", snapshot_root=snapshots + ) + asec = asec_catalogue.issue_asec_source_catalogue(**kwargs) + acs_rows, asec_rows = acs.households, asec.households + plan = selection.plan_catalogue_selection( + acs_households=acs_rows, + asec_households=asec_rows, + fraction=fraction, + seed=seed, + ) + acs_keys = tuple( + r.key.native_id for r in plan.selected if r.key.source is domains.Source.ACS + ) + asec_keys = tuple( + (2024, int(r.key.native_id)) + for r in plan.selected + if r.key.source is domains.Source.ASEC + ) + _require( + acs_keys and asec_keys and len(set(asec_keys)) == len(asec_keys), + "SELECTED_SOURCE_SUPPORT", + ) + actual_acs = acs_native.issue_acs_native_coverage( + root / "acs", snapshot_root=snapshots, serialnos=acs_keys + ) + actual_asec = asec_native.load_authenticated_asec_2024_native_population( + **kwargs, selected_households=asec_keys + ) + native_frames = actual_acs.frame, actual_asec.frame + source_seals = ( + acs_native._frame_sha256(native_frames[0]), + asec_native._frame_identity(native_frames[1]), + ) + source_copies = { + channel: _normalized_source_copy(frame) + for channel, frame in zip(("acs", "asec"), native_frames, strict=True) + } + stacked = spine_assembly.stack_survey_spines(source_copies) + frame = stacked.frame + transitions = graph_sources._canonical_assembly( + frame, tuple(source_copies.values()) + ) + context = graph_context.encode_us_frame_context(frame) + native_receipts = {"acs": actual_acs.receipt, "asec": actual_asec.receipt} + for channel, original in zip(("acs", "asec"), native_frames, strict=True): + _verify_normalized_copy(original, source_copies[channel]) + origins = _origins(frame, source_copies, plan.selected, native_receipts) + origins["storage_transitions"] = list(transitions) + origins["observed_age_normalization"] = { + "rule": observed_age.rule_document(), + "sources": { + channel: { + "native_frame_sha256": source_seals[i], + "normalized_frame_sha256": _frame_identity(source_copies[channel]), + "common_age_preexisting": "age" in native_frames[i].person, + } + for i, channel in enumerate(("acs", "asec")) + }, + } + documents = {"acs": acs.receipt, "asec": asec.receipt} + catalogues = { + "acs": { + "receipt_sha256": _sha(acs.to_bytes()), + "records_sha256": documents["acs"]["counts"]["canonical_record_sha256"], + "counts": documents["acs"]["counts"], + }, + "asec": { + "receipt_sha256": _sha(asec.to_bytes()), + "records_sha256": documents["asec"]["records"]["sha256"], + "counts": documents["asec"]["counts"], + }, + } + identity = _frame_identity(frame) + payload = _encode( + { + "protocol": PROTOCOL, + "request": request, + "request_sha256": _sha(request_bytes), + "source_files": files_before, + "producer": json.loads(producer), + "catalogues": catalogues, + "native": { + channel: { + "receipt_sha256": _sha(value.payload), + "frame_sha256": source_seals[i], + "households": native_frames[i].n("household"), + "persons": native_frames[i].n("person"), + } + for i, (channel, value) in enumerate( + (("acs", actual_acs), ("asec", actual_asec)) + ) + }, + "selection": _plan_document(plan), + "origins": origins, + "frame_sha256": identity, + "context_sha256": _sha(context), + "release_eligible": False, + } + ) + nested = _nested_seals((acs, asec), (actual_acs, actual_asec)) + acs_owned = acs_catalogue._lookup(acs) + catalogue_memo = _catalogue_memo( + (acs_owned.records, acs_owned.vacancies), nested[9] + ) + state = _State( + root, + files_before, + file_stats, + producer, + authority, + (acs, asec), + (actual_acs, actual_asec), + tuple(value.payload for value in (acs, asec, actual_acs, actual_asec)), + native_frames, + source_seals, + frame, + identity, + context, + plan, + _digest(_plan_document(plan)), + nested, + catalogue_memo, + ) + _validate(state) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + result = AuthenticatedSurveyPopulationPreparation(payload, _token=_TOKEN) + key = id(result) + + def cleanup(reference): + entry = _ISSUED.get(key) + if entry is not None and entry[0] is reference: + del _ISSUED[key] + + _ISSUED[key] = (weakref.ref(result, cleanup), payload, state) + _pure_final(state) + _require( + type(result.payload) is bytes + and result.payload == payload + and _ISSUED[key][0]() is result, + "UNISSUED_OR_CHANGED", + ) + return result + except SurveyPopulationPreparationError: + raise + except Exception: + raise SurveyPopulationPreparationError("PREPARATION_ISSUANCE_REFUSED") from None + + +_BYTES = _code_bytes() + + +def _current_survey_wage_projection(preparation, entry): + """Project one just-checked retained owner; bytes grant no source authority. + + The caller must have obtained ``entry`` from ``preparation._checked()`` in + the current operation, and must seal its receiving Populations afterwards. + The one ready() call retains the existing all-field/full-parent admission. + No old reported-income contract or prior-wage column is manufactured. + """ + _require( + type(preparation) is AuthenticatedSurveyPopulationPreparation + and _ISSUED.get(id(preparation)) is entry + and entry[0]() is preparation + and type(preparation.payload) is bytes + and preparation.payload == entry[1], + "WAGE_PREPARATION_ISSUANCE", + ) + state = entry[2] + native_entry = asec_native._ISSUED.get(id(state.native[1])) + _require( + native_entry is not None + and native_entry[0]() is state.native[1] + and state.native[1].payload == native_entry[1], + "WAGE_NATIVE_ISSUANCE", + ) + parent = native_entry[2].parent + ready = parent.ready() # Exactly once, outside both row and feature loops. + field = ready.field("WSAL_VAL") + domain = next(d for d in parent.spec.fields if d.name == "WSAL_VAL") + scope = parent.scope + positions = {pid: i for i, pid in enumerate(scope.person_ids)} + _require(len(positions) == len(scope.person_ids), "WAGE_PARENT_ROSTER") + native_ids = set(state.source_frames[1].person.person_id) + rows, budget = [], [0] + people = state.frame.person + selected = people.loc[people[support_channel_column("person")].eq("asec")] + _require( + set(selected[spine_source_id_column("person")]) == native_ids, + "WAGE_SELECTED_ROSTER", + ) + for stacked, native in selected[ + ["person_id", spine_source_id_column("person")] + ].itertuples(index=False, name=None): + _require(int(native) in positions, "WAGE_PARENT_MEMBER") + i = positions[int(native)] + _require(scope.person_years[i] == 2024, "WAGE_CURRENT_COHORT") + amount = field.amount_bytes[8 * i : 8 * (i + 1)] + number = np.frombuffer(amount, dtype="= 0, + "WAGE_CURRENT_FEATURE", + ) + _bounded_append( + rows, + [ + int(stacked), + int(native), + 2024, + amount.hex(), + field.status_bytes[i], + field.validity_bytes[i], + field.zero_origin_bytes[i], + ], + budget, + ) + document = { + "preparation_sha256": _sha(entry[1]), + "asec_native_sha256": _sha(native_entry[1]), + "money_header": json.loads(ready.header), + "money_header_sha256": _sha(ready.header), + "domain": {f.name: getattr(domain, f.name) for f in fields(domain)}, + "zero_origin_code": int(current_money._origin_code(parent.spec, "WSAL_VAL")), + "columns": [ + "stacked_person_id", + "native_person_id", + "income_year", + "amount_f64le_hex", + "status", + "validity", + "zero_origin", + ], + "rows": rows, + } + payload = _encode(document) + # ready() may perform I/O. Recheck the actual retained entries and all pure + # owner seals after it; this does not reread the national catalogues. + _pure_final(state) + _require( + _ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1] + and asec_native._ISSUED.get(id(state.native[1])) is native_entry + and state.native[1].payload == native_entry[1] + and native_entry[2].parent is parent, + "WAGE_FINAL_ISSUANCE", + ) + return payload + + +_LIVE = _live() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py new file mode 100644 index 000000000..dd55a5e65 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py @@ -0,0 +1,208 @@ +"""Directional ContentStore replay checks, never source or population authority. + +The caller must separately retain the original source/population seals. These +comparisons permit the store's missing-buffer normalization and scalar object +encoding only at a named materialization boundary. They are not mutation seals. +""" + +from __future__ import annotations + +from dataclasses import asdict + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, MassChangeRecord, WeightKind +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import MassRecord, Population +from microcosm.graph.store import ( + _axis_name_payload, + _encode_frame_metadata, + _encode_object_scalar, +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_POPULATION_REPLAY_" + reason) + + +def _array_bytes_equal(left, right): + return ( + type(left) is np.ndarray + and type(right) is np.ndarray + and left.dtype == right.dtype + and not left.dtype.hasobject + and left.shape == right.shape + and left.tobytes() == right.tobytes() + ) + + +def _object_bytes(value): + try: + return _encode_object_scalar(value) + except (TypeError, ValueError, OverflowError, UnicodeError): + raise ValueError("SURVEY_POPULATION_REPLAY_UNSUPPORTED_OBJECT") from None + + +def _series(expected, actual): + _require( + type(expected) is pd.Series + and type(actual) is pd.Series + and type(expected.dtype) is type(actual.dtype) + and expected.dtype == actual.dtype + and len(expected) == len(actual), + "SERIES_DTYPE_OR_LENGTH", + ) + dtype = expected.dtype + left, right = expected.array, actual.array + if isinstance(dtype, pd.BooleanDtype) or ( + isinstance(dtype, pd.api.extensions.ExtensionDtype) + and pd.api.types.is_integer_dtype(dtype) + ): + # The v1 country profile supports pandas' actual masked integer/bool + # storage. Other extension-array implementations must be reviewed first. + ld, rd = getattr(left, "_data", None), getattr(right, "_data", None) + lm, rm = getattr(left, "_mask", None), getattr(right, "_mask", None) + _require( + _array_bytes_equal(lm, rm) + and lm.dtype == np.dtype(np.bool_) + and lm.shape == (len(expected),) + and type(ld) is np.ndarray + and type(rd) is np.ndarray + and ld.dtype == rd.dtype + and not ld.dtype.hasobject + and ld.shape == rd.shape == lm.shape, + "MASKED_STORAGE", + ) + _require(_array_bytes_equal(ld[~lm], rd[~rm]), "PRESENT_BITS") + _require( + _array_bytes_equal(ld, rd) or bool(np.all(rd[rm] == 0)), + "NONCANONICAL_NULL_BACKING", + ) + return + if isinstance(dtype, pd.StringDtype): + _require( + dtype.storage == actual.dtype.storage + and (dtype.na_value is pd.NA) == (actual.dtype.na_value is pd.NA), + "STRING_POLICY", + ) + lm = expected.isna().to_numpy(dtype=np.bool_, copy=False) + rm = actual.isna().to_numpy(dtype=np.bool_, copy=False) + _require(_array_bytes_equal(lm, rm), "STRING_MASK") + for missing, a, b in zip(lm, expected, actual, strict=True): + if not missing: + _require(type(a) is str and type(b) is str and a == b, "STRING_VALUE") + return + if pd.api.types.is_object_dtype(dtype): + # Reuse the store's typed scalar bytes. None/pd.NA/NaT, bool/int, + # float signed zero and NaN payloads remain distinguished by that codec. + # Object pointers and np scalar wrappers are not portable identities. + for a, b in zip(expected, actual, strict=True): + _require(_object_bytes(a) == _object_bytes(b), "OBJECT_VALUE") + return + _require( + not isinstance(dtype, pd.api.extensions.ExtensionDtype), + "UNSUPPORTED_EXTENSION_DTYPE", + ) + _require( + _array_bytes_equal(expected.to_numpy(copy=False), actual.to_numpy(copy=False)), + "NATIVE_BITS", + ) + + +def _name_bytes(value): + try: + return canonical_json(_axis_name_payload(value)) + except (TypeError, ValueError, OverflowError, UnicodeError): + raise ValueError("SURVEY_POPULATION_REPLAY_UNSUPPORTED_AXIS_NAME") from None + + +def _axis(expected, actual): + _require( + type(expected) is type(actual) + and expected.identical(actual) + and not isinstance(expected, pd.MultiIndex), + "AXIS", + ) + _require(_name_bytes(expected.name) == _name_bytes(actual.name), "AXIS_NAME") + _series(pd.Series(expected.array, copy=False), pd.Series(actual.array, copy=False)) + + +def same_replayed_frame(expected: Frame, actual: Frame) -> None: + """Compare actual Frames, allowing only the named store representation rule. + + No I/O or allocation of another Frame. Temporary buffers are proportional + to individual columns and context metadata, without a total input RAM cap. + """ + _require(isinstance(expected, Frame) and isinstance(actual, Frame), "FRAME_TYPE") + _require( + expected.schema == actual.schema + and expected.entities == actual.entities + and expected.links == actual.links == () + and canonical_json(_encode_frame_metadata(expected.metadata)) + == canonical_json(_encode_frame_metadata(actual.metadata)) + and all( + type(r) is MassChangeRecord for r in (*expected.mass_log, *actual.mass_log) + ) + and canonical_json([asdict(r) for r in expected.mass_log]) + == canonical_json([asdict(r) for r in actual.mass_log]) + and expected.weighted_entities == actual.weighted_entities, + "FRAME_CONTEXT", + ) + for entity in expected.entities: + left, right = expected.table(entity), actual.table(entity) + _require( + type(left) is pd.DataFrame + and type(right) is pd.DataFrame + and left.flags == right.flags, + "TABLE_TYPE_OR_FLAGS", + ) + _axis(left.index, right.index) + _axis(left.columns, right.columns) + for column in left: + _series(left[column], right[column]) + _axis(expected.strata.index, actual.strata.index) + _require( + _name_bytes(expected.strata.name) == _name_bytes(actual.strata.name), + "STRATA_NAME", + ) + _series(expected.strata, actual.strata) + for entity in expected.weighted_entities: + left, right = expected.weights_for(entity), actual.weights_for(entity) + _require( + left.kind is right.kind and _array_bytes_equal(left.values, right.values), + "WEIGHT_BYTES", + ) + + +def same_replayed_population(expected: Population, actual: Population) -> None: + """Require complete receiving state as well as the directional Frame check.""" + _require( + type(expected) is Population and type(actual) is Population, "POPULATION_TYPE" + ) + same_replayed_frame(expected.frame, actual.frame) + _require( + type(expected.version) is type(actual.version) is str + and expected.version == actual.version + and tuple(sorted(expected.owners.items())) + == tuple(sorted(actual.owners.items())) + and all(type(v) is str for v in actual.owners.values()) + and tuple(expected.weight_kind.items()) == tuple(actual.weight_kind.items()) + and all(type(v) is WeightKind for v in actual.weight_kind.values()) + and type(expected.mass_ledger) is type(actual.mass_ledger) is tuple + and all( + type(r) is MassRecord for r in (*expected.mass_ledger, *actual.mass_ledger) + ) + and canonical_json([asdict(r) for r in expected.mass_ledger]) + == canonical_json([asdict(r) for r in actual.mass_ledger]) + and tuple(expected.design_weights) == tuple(actual.design_weights), + "POPULATION_CONTEXT", + ) + for entity in expected.design_weights: + _require( + _array_bytes_equal( + expected.design_weights[entity], actual.design_weights[entity] + ), + "DESIGN_BYTES", + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py new file mode 100644 index 000000000..adf73f3ab --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py @@ -0,0 +1,176 @@ +"""Explicit source-report Social Security component basis. + +These are numerical owners, not source admission APIs. A graph host must bind +the live source projection and any actual fitted completion artifact before +using the returned values. Missing components never become zero. Report-record totals are not individual +beneficiary incidence. This module neither replaces totals with PUF draws nor +allocates combined family payments to individual beneficiaries. +""" + +from __future__ import annotations + +import numpy as np + +COMPONENTS = ( + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", +) +PROTOCOL = "microcosm.us.survey-social-security-basis.v1" +REASON_COMPONENTS = { + 1: (0,), + 2: (1,), + 3: (3,), + 4: (2,), + 5: (3,), + 6: (2,), + # The published code mixes surviving, dependent and disabled children. + 7: (1, 2, 3), + 8: (0, 1, 2, 3), +} + + +def require(condition, reason): + if not condition: + raise ValueError("SURVEY_SOCIAL_SECURITY_" + reason) + + +def _float64(values, *, shape, nullable=False): + require(type(values) is np.ndarray, "ARRAY_TYPE") + require(values.dtype == np.dtype("float64") and values.shape == shape, "ARRAY_AXIS") + require(not np.isinf(values).any(), "NONFINITE") + require(nullable or not np.isnan(values).any(), "UNKNOWN") + require((values[~np.isnan(values)] >= 0).all(), "NEGATIVE") + return values.copy() + + +def asec_reason_basis(total, reason_1, reason_2): + """Map a source total and reasons without inventing component amounts. + + Callers qualify the actual money and literal reason observations. Codes + 3/5 share survivors and 4/6 share dependents. Multiple distinct components, + code 7, code 8, or missing reasons leave the positive amount unallocated. + A single component receives the total as an explicit mapping judgment. + No age fallback or priority between reported reasons is applied. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + reasons = [] + for values in (reason_1, reason_2): + arr = _float64(values, shape=total.shape, nullable=True) + require( + ( + (arr[np.isfinite(arr)] == np.floor(arr[np.isfinite(arr)])) + & (arr[np.isfinite(arr)] <= 8) + ).all(), + "REASON_DOMAIN", + ) + reasons.append(arr) + out = np.full((len(amount), 4), np.nan, dtype=np.float64) + allowed = np.ones((len(amount), 4), dtype=np.bool_) + labels = [] + for i, value in enumerate(amount): + if value == 0: + out[i] = 0 + allowed[i] = False + labels.append("known_total_zero") + continue + codes = [arr[i] for arr in reasons] + if any(np.isnan(code) for code in codes): + labels.append("unresolved_missing_reason") + continue + nonzero = [int(code) for code in codes if code != 0] + if not nonzero: + labels.append("unresolved_niu_reason_with_positive_total") + continue + possible = set().union(*(REASON_COMPONENTS[code] for code in nonzero)) + allowed[i] = False + allowed[i, list(possible)] = True + if len(possible) == 1: + out[i] = 0 + out[i, next(iter(possible))] = value + labels.append("source_reason_total_allocation") + else: + labels.append("unresolved_component_split") + return out, allowed, tuple(labels) + + +def asec_reporting_basis(total, age, recipiency, reason_1, reason_2): + """Respect the age-15 reporting universe without inferring beneficiaries. + + SS_VAL=0 outside the question universe is a source sentinel, not observed + absence of benefits. Retain that literal upstream and return unknown total + and shares here. A reporting adult may receive combined family payments; + the returned component basis remains at that source report's grain. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + ages = _float64(age, shape=total.shape) + receipt = _float64(recipiency, shape=total.shape, nullable=True) + require(((ages == np.floor(ages)) & (ages <= 99)).all(), "AGE_DOMAIN") + require( + np.isin(receipt[np.isfinite(receipt)], [0, 1, 2]).all(), "RECIPIENCY_DOMAIN" + ) + eligible = ages >= 15 + require( + ((amount[~eligible] == 0) & (receipt[~eligible] == 0)).all(), + "OUTSIDE_REPORTING_UNIVERSE_OBSERVATION", + ) + require(np.isin(receipt[eligible], [1, 2]).all(), "IN_UNIVERSE_RECIPIENCY_UNKNOWN") + require(not ((amount > 0) & (receipt != 1)).any(), "RECIPIENCY_CONTRADICTION") + basis, allowed, labels = asec_reason_basis(amount, reason_1, reason_2) + amount[~eligible] = np.nan + basis[~eligible] = np.nan + allowed[~eligible] = True + labels = tuple( + label if inside else "asec_below15_outside_reporting_universe" + for label, inside in zip(labels, eligible, strict=True) + ) + return amount, basis, allowed, labels + + +def complete_positive_basis(total, source_basis, allowed, modeled_scores): + """Convert actual model scores to shares only for unresolved positive rows. + + The graph owner must authenticate the fitted artifact. This numerical API + cannot do so. Existing complete rows must have no modeled scores; eligible + rows require a complete nonnegative four-vector with positive allowed mass. + Source totals and excluded component possibilities are retained exactly. + A missing/zero model vector refuses instead of creating equal shares. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + shape = (len(amount), 4) + basis = _float64(source_basis, shape=shape, nullable=True) + scores = _float64(modeled_scores, shape=shape, nullable=True) + require( + type(allowed) is np.ndarray + and allowed.dtype == np.dtype("bool") + and allowed.shape == shape, + "ALLOWED_AXIS", + ) + complete = np.isfinite(basis).all(axis=1) + require((complete | np.isnan(basis).all(axis=1)).all(), "PARTIAL_COMPONENT_BASIS") + require(np.isnan(scores[complete]).all(), "MODELED_SOURCE_OVERWRITE") + require( + np.array_equal(basis[complete].sum(axis=1), amount[complete]), + "SOURCE_TOTAL_IDENTITY", + ) + require((basis[complete][~allowed[complete]] == 0).all(), "SOURCE_SUPPORT") + missing = ~complete + require((amount[missing] > 0).all(), "UNRESOLVED_ZERO_TOTAL") + require(np.isfinite(scores[missing]).all(), "COMPLETION_UNKNOWN") + selected = scores[missing].copy() + selected[~allowed[missing]] = 0 + mass = selected.sum(axis=1) + require(np.isfinite(mass).all() and (mass > 0).all(), "COMPLETION_NO_ALLOWED_MASS") + filled = selected / mass[:, None] * amount[missing, None] + # Put the floating residual on the largest supported component. This keeps + # excluded components exactly zero and avoids systematic lost pennies. + if len(filled): + j = filled.argmax(axis=1) + filled[np.arange(len(filled)), j] += amount[missing] - filled.sum(axis=1) + require(np.isfinite(filled).all() and (filled >= 0).all(), "COMPLETION_ARITHMETIC") + basis[missing] = filled + return basis diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index d9332b8f2..1ce17fcd8 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "751d5ef5d25406bbae1798667f0e29890d4aad933d323c12912c7d45d8809bb9" + "14d7f749f14e1dc1fa32a064c13f50a76b73bf03ddd9b6c0534d9ede2d9ff44e" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-build/tests/test_atomic_geography.py b/packages/microcosm-build/tests/test_atomic_geography.py new file mode 100644 index 000000000..ccc4b172c --- /dev/null +++ b/packages/microcosm-build/tests/test_atomic_geography.py @@ -0,0 +1,511 @@ +"""Invented support controls; no survey, Census or administrative files are read.""" + +from copy import deepcopy + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import atomic_geography as geo +from microcosm.build.graph_atomic_geography import ( + ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + atomic_geography_nodes, + register_atomic_geography_kernels, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + + +def support_parts(system="invented_block", *, district=True): + columns = { + name: { + "kind": "code", + "source": "invented-lookup-v1", + "vintage": "invented-2020", + "relation": "official_tabulation" if name == "district" else "exact", + } + for name in ("area", "region", "puma", *(("district",) if district else ())) + } + columns.update( + { + name: {"kind": "weight", "source": "invented-counts-v1", "basis": basis} + for name, basis in (("population", "persons"), ("households", "households")) + } + ) + arrays = { + "area": np.asarray(["0001", "0002", "0003", "0004"]), + "region": np.asarray(["R", "R", "R", "S"]), + "puma": np.asarray(["A", "A", "B", "C"]), + "population": np.asarray([0, 2, 6, 1]), + "households": np.asarray([1, 3, 4, 1]), + } + if district: + arrays["district"] = np.asarray(["01", "02", "02", "03"]) + return { + "version": 1, + "system": system, + "level": "block", + "code_system": "invented", + "vintage": "invented-2020", + "columns": columns, + }, arrays + + +def fixture(system="invented_block", *, district=True): + metadata, arrays = support_parts(system, district=district) + payload = geo.encode_atomic_support(metadata, arrays) + support = geo.decode_atomic_support(payload) + spec = { + "version": 1, + "identity": ["source_record"], + "stream": ["sha256-u53-v1", "invented-geography", 0, 21], + "outputs": {"area": "block", "system": "area_system", "basis": "area_basis"}, + "systems": [ + { + "id": system, + "level": "block", + "code_system": "invented", + "vintage": "invented-2020", + "source": system + "_source", + "selector": {}, + "constraints": [ + {"input": "observed_region", "support": "region", "required": True}, + {"input": "observed_puma", "support": "puma", "required": False}, + ], + "observed_area": None, + "stages": [{"level": "area", "weight": "population"}], + "layers": ( + [ + { + "input": "district", + "output": "district_code", + "vintage": "invented-2020", + "relation": "official_tabulation", + "source": "invented-lookup-v1", + } + ] + if district + else [] + ), + } + ], + } + households = pd.DataFrame( + { + "household_id": np.arange(1, 9, dtype=np.int64), + "source_record": pd.array( + ["source-" + str(i) for i in range(8)], dtype="string" + ), + "observed_region": pd.array(["R"] * 7 + ["S"], dtype="string"), + "observed_puma": pd.array( + [None, "A", "B", None, "A", None, "B", "C"], dtype="string" + ), + } + ) + return households, spec, {system: support}, payload + + +def complete(households, spec, supports): + assigned = pd.concat( + [households, geo.assign_atomic(households, spec, supports)], axis=1 + ) + return pd.concat([assigned, geo.derive_geography(assigned, spec, supports)], axis=1) + + +def test_codec_is_deterministic_and_decoded_values_are_immutable(): + metadata, arrays = support_parts() + first = geo.encode_atomic_support(metadata, arrays) + assert first == geo.encode_atomic_support( + metadata, dict(reversed(list(arrays.items()))) + ) + support = geo.decode_atomic_support(first) + assert support.arrays["area"].tolist() == ["0001", "0002", "0003", "0004"] + with pytest.raises(ValueError): + support.arrays["area"].setflags(write=True) + with pytest.raises(TypeError): + support.metadata["columns"]["area"]["source"] = "changed" + + +def test_integer_inverse_cdf_respects_zero_mass_and_boundaries(): + _, _, supports, _ = fixture() + index = geo._SupportIndex(supports["invented_block"]) + stage = {"level": "area", "weight": "population"} + cell = (("region", "R"),) + assert index.pick(cell, stage, 0.0) == "0002" + assert index.pick(cell, stage, 0.25 - 1 / 2**53) == "0002" + assert index.pick(cell, stage, 0.25) == "0003" + assert index.pick(cell, stage, 1 - 1 / 2**53) == "0003" + + +def test_draws_are_stable_under_order_subset_extra_columns_and_unrelated_rows(): + households, spec, supports, _ = fixture() + before = households.copy(deep=True) + expected = geo.assign_atomic(households, spec, supports) + reordered = households.iloc[[7, 2, 0, 5]] + actual = geo.assign_atomic(reordered.assign(unrelated=42), spec, supports) + pd.testing.assert_frame_equal(actual, expected.loc[reordered.index]) + extra = households.iloc[[0]].copy() + extra.index = [100] + extra["source_record"] = "unrelated-record" + grown = geo.assign_atomic(pd.concat([households, extra]), spec, supports) + pd.testing.assert_frame_equal(grown.loc[households.index], expected) + pd.testing.assert_frame_equal(households, before) + + +def test_stage_law_uses_declared_weights_at_each_level(): + households, spec, supports, _ = fixture() + spec["systems"][0]["stages"] = [ + {"level": "puma", "weight": "households"}, + {"level": "area", "weight": "population"}, + ] + result = complete(households, spec, supports) + assert result.loc[1, "block"] == "0002" + assert result.loc[2, "block"] == "0003" + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + # A/B household mass is 4/4, but population mass is 2/6. + index = geo._SupportIndex(supports["invented_block"]) + assert index.pick((("region", "R"),), spec["systems"][0]["stages"][0], 0.49) == "A" + assert index.pick((("region", "R"),), spec["systems"][0]["stages"][0], 0.5) == "B" + + +def test_qualified_observed_area_is_retained_even_with_zero_sampling_mass(): + households, spec, supports, _ = fixture() + households["observed_block"] = pd.array(["0001"] + [None] * 7, dtype="string") + spec["systems"][0]["observed_area"] = "observed_block" + result = complete(households, spec, supports) + assert result.loc[0, "block"] == "0001" + assert result.loc[0, "area_basis"] == "observed" + assert result.loc[0, "district_code"] == "01" + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + + +@pytest.mark.parametrize( + "defect", + [ + "missing_region", + "wrong_region", + "inconsistent_puma", + "duplicate_identity", + "null_identity", + "existing_output", + "unknown_observed_area", + "conflicting_observed_area", + ], +) +def test_assignment_refuses_invalid_or_conflicting_households(defect): + households, spec, supports, _ = fixture() + if defect == "missing_region": + households.loc[0, "observed_region"] = pd.NA + elif defect == "wrong_region": + households.loc[0, "observed_region"] = "missing" + elif defect == "inconsistent_puma": + households.loc[0, "observed_puma"] = "C" + elif defect == "duplicate_identity": + households.loc[0, "source_record"] = households.loc[1, "source_record"] + elif defect == "null_identity": + households.loc[0, "source_record"] = pd.NA + elif defect == "existing_output": + households["block"] = "untouched" + else: + households["observed_block"] = pd.array( + ["unknown" if defect == "unknown_observed_area" else "0004"] + [None] * 7, + dtype="string", + ) + spec["systems"][0]["observed_area"] = "observed_block" + with pytest.raises(ValueError): + geo.assign_atomic(households, spec, supports) + + +@pytest.mark.parametrize( + "defect", + [ + "duplicate_area", + "negative_weight", + "float_weight", + "bool_weight", + "overflow_weight", + "all_zero_weight", + "unknown_relation", + "wrong_area_vintage", + "empty_code", + "wrong_length", + ], +) +def test_support_refuses_ambiguous_or_invalid_artifacts(defect): + metadata, arrays = support_parts() + if defect == "duplicate_area": + arrays["area"][1] = arrays["area"][0] + elif defect == "negative_weight": + arrays["population"][0] = -1 + elif defect == "float_weight": + arrays["population"] = arrays["population"].astype(float) + elif defect == "bool_weight": + arrays["population"] = arrays["population"].astype(bool) + elif defect == "overflow_weight": + arrays["population"] = np.asarray([2**63] * 4, dtype=np.uint64) + elif defect == "all_zero_weight": + arrays["population"] *= 0 + elif defect == "unknown_relation": + metadata["columns"]["district"]["relation"] = "exact_enough" + elif defect == "wrong_area_vintage": + metadata["columns"]["area"]["vintage"] = "different" + elif defect == "empty_code": + arrays["area"][0] = "" + elif defect == "wrong_length": + arrays["region"] = arrays["region"][:-1] + with pytest.raises(ValueError): + geo.encode_atomic_support(metadata, arrays) + + +@pytest.mark.parametrize("field", ["source", "relation", "vintage"]) +def test_layer_metadata_must_match_the_pinned_support(field): + households, spec, supports, _ = fixture() + spec["systems"][0]["layers"][0][field] = ( + "exact" if field == "relation" else "different" + ) + with pytest.raises(ValueError): + geo.assign_atomic(households, spec, supports) + + +def test_three_nation_systems_use_the_same_operations_and_mark_absent_layers(): + households, spec, supports, _ = fixture() + households["nation"] = pd.array( + ["one"] * 3 + ["two"] * 3 + ["three"] * 2, dtype="string" + ) + spec["systems"][0]["selector"] = {"nation": ["one"]} + for name, selector, district in ( + ("invented_oa", "two", True), + ("invented_dz", "three", False), + ): + _, other, added, _ = fixture(name, district=district) + system = other["systems"][0] + system["selector"] = {"nation": [selector]} + spec["systems"].append(system) + supports.update(added) + result = complete(households, spec, supports) + assert ( + result["area_system"].tolist() + == ["invented_block"] * 3 + ["invented_oa"] * 3 + ["invented_dz"] * 2 + ) + assert result.loc[[6, 7], "district_code"].isna().all() + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + spec["systems"][1]["selector"] = {"nation": ["one", "two"]} + with pytest.raises(ValueError, match="exactly one"): + geo.assign_atomic(households, spec, supports) + + +def test_clone_inheritance_and_pruned_mapping_checks_do_not_redraw(): + households, spec, supports, _ = fixture() + original = complete(households, spec, supports) + clone = original.iloc[[2, 0, 5]].copy() + clone["household_id"] += 100 + clone["source_record"] += "-clone" + assert geo.validate_geography(clone, spec, supports)["households"] == 3 + clone.iloc[0, clone.columns.get_loc("district_code")] = "wrong" + with pytest.raises(ValueError, match="derived geography differs"): + geo.validate_geography(clone, spec, supports) + + +@pytest.mark.parametrize("defect", ["atomic", "basis", "observed", "system"]) +def test_final_integrity_gate_refuses_changed_geography(defect): + households, spec, supports, _ = fixture() + result = complete(households, spec, supports) + if defect == "atomic": + result.loc[0, "block"] = "missing" + elif defect == "basis": + result.loc[0, "area_basis"] = "observed" + elif defect == "observed": + result.loc[0, "observed_region"] = "wrong" + elif defect == "system": + result.loc[0, "area_system"] = pd.NA + with pytest.raises(ValueError): + geo.validate_geography(result, spec, supports) + + +class _InventedSpine(KernelBase): + ref = "invented.atomic_spine@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + households, _, _, _ = fixture() + person = pd.DataFrame( + { + "person_id": np.arange(1, 9, dtype=np.int64), + "person_household_id": households["household_id"], + "age": np.arange(21, 29, dtype=np.int64), + } + ) + return KernelResult( + frame=Frame( + {"person": person, "household": households}, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.ones(8), WeightKind.DESIGN)}, + pd.Series(["invented"] * 8, name="stratum"), + ) + ) + + +@pytest.mark.parametrize("emit_validation_artifact", (False, True)) +def test_real_executor_cold_warm_and_changed_source_identity( + tmp_path, emit_validation_artifact +): + households, spec, supports, payload = fixture() + columns = ( + Owned("person", "age", "int64"), + *( + Owned("household", name, "string") + for name in ("source_record", "observed_region", "observed_puma") + ), + ) + base = Node( + "spine", + _InventedSpine.ref, + sources=("invented_spine",), + structural=StructuralDelta.CREATE, + outputs=columns, + ) + nodes = atomic_geography_nodes( + spec, columns, base=base.id, emit_validation_artifact=emit_validation_artifact + ) + graph = compile_graph( + Graph( + "invented", + ( + SourceRef("invented_spine", "raw-bytes-v1"), + SourceRef("invented_block_source", "raw-bytes-v1"), + ), + (base, *nodes), + ) + ) + registry = KernelRegistry() + registry.register(_InventedSpine()) + register_atomic_geography_kernels(registry) + raw_spine = tmp_path / "invented-spine.bin" + raw_spine.write_bytes(b"invented test source declaration") + raw_support = tmp_path / "invented-support.npz" + raw_support.write_bytes(payload) + sources = {"invented_spine": raw_spine, "invented_block_source": raw_support} + store = ContentStore(tmp_path / "store") + cold = run_graph(graph, sources=sources, store=store, kernels=registry) + warm = run_graph( + graph, sources=sources, store=store, kernels=registry, resume="require" + ) + actual = cold.population(base.id).table("household") + expected = complete(households, spec, supports) + for column in expected: + left, right = actual[column], expected[column] + if isinstance(expected[column].dtype, pd.StringDtype): + # Source checkpoints and emitted columns may use different string + # backends. Check the nullable-string contract, then compare every + # value and missing cell using one explicit physical representation. + assert isinstance(left.dtype, pd.StringDtype) + assert left.dtype.na_value is pd.NA + left = left.astype(pd.StringDtype(storage="python")) + right = right.astype(pd.StringDtype(storage="python")) + pd.testing.assert_series_equal(left, right, check_index_type=False) + assert cold.nodes["geography.assign"].key == warm.nodes["geography.assign"].key + assert cold.nodes["geography.gate"].receipt["outcome"] == "pass" + gate = cold.nodes["geography.gate"] + if emit_validation_artifact: + output = graph.graph.node("geography.gate").artifact_outputs + assert len(output) == 1 and output[0].type == ATOMIC_GEOGRAPHY_VALIDATION_TYPE + key = gate.opaque_artifacts["validation"] + assert key == opaque_artifact_key(gate.key, "validation") + assert store.load_bytes(key) == canonical_json( + geo.validate_geography(expected, spec, supports) + ) + assert warm.nodes["geography.gate"].opaque_artifacts["validation"] == key + else: + assert not gate.opaque_artifacts + + np.testing.assert_array_equal( + cold.population(base.id).weights_for("household").values, np.ones(8) + ) + metadata, arrays = support_parts() + arrays["population"][1] += 1 + raw_support.write_bytes(geo.encode_atomic_support(metadata, arrays)) + changed = run_graph(graph, sources=sources, store=store, kernels=registry) + assert changed.nodes["geography.assign"].key != cold.nodes["geography.assign"].key + + +def test_graph_builder_refuses_input_overwrites_and_duplicate_inventory(): + _, spec, _, _ = fixture() + columns = tuple( + Owned("household", c, "string") + for c in ("source_record", "observed_region", "observed_puma") + ) + with pytest.raises(ValueError, match="overwrite"): + atomic_geography_nodes( + spec, (*columns, Owned("household", "block", "string")), base="spine" + ) + with pytest.raises(ValueError, match="repeats"): + atomic_geography_nodes(spec, (*columns, columns[0]), base="spine") + with pytest.raises(ValueError, match="missing"): + atomic_geography_nodes(spec, columns[1:], base="spine") + altered = deepcopy(spec) + altered["outputs"]["area"] = "observed_region" + with pytest.raises(ValueError, match="overwritten"): + atomic_geography_nodes(altered, columns, base="spine") + + +@pytest.mark.parametrize("defect", ("missing", "type")) +def test_typed_geography_gate_edge_refuses_invalid_producer_contract(defect): + _, spec, _, _ = fixture() + columns = tuple( + Owned("household", name, "string") + for name in ("source_record", "observed_region", "observed_puma") + ) + base = Node( + "spine", + _InventedSpine.ref, + sources=("invented_spine",), + structural=StructuralDelta.CREATE, + outputs=columns, + ) + nodes = atomic_geography_nodes( + spec, columns, base=base.id, emit_validation_artifact=(defect != "missing") + ) + edge_type = ( + ArtifactType("invented.wrong_gate", 1) + if defect == "type" + else ATOMIC_GEOGRAPHY_VALIDATION_TYPE + ) + consumer = Node( + "consumer", + "invented.consumer@1", + population=base.id, + artifact_inputs=( + ArtifactInput("validation", "geography.gate", "validation", edge_type), + ), + ) + with pytest.raises(ValueError, match="no declared artifact|type does not match"): + compile_graph( + Graph( + "invented", + ( + SourceRef("invented_spine", "raw-bytes-v1"), + SourceRef("invented_block_source", "raw-bytes-v1"), + ), + (base, *nodes, consumer), + ) + ) diff --git a/packages/microcosm-build/tests/test_frame_checkpoint.py b/packages/microcosm-build/tests/test_frame_checkpoint.py index 7c79ba3f8..d5e052a40 100644 --- a/packages/microcosm-build/tests/test_frame_checkpoint.py +++ b/packages/microcosm-build/tests/test_frame_checkpoint.py @@ -576,6 +576,205 @@ def test_schema_3_checkpoint_requires_nullable_boolean_encoding( load_frame_checkpoint(path) +@pytest.mark.parametrize( + "dtype", ["Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64"] +) +@pytest.mark.parametrize("has_missing", [False, True]) +def test_nullable_integer_round_trip_preserves_width_values_and_null_mask( + tmp_path: Path, dtype: str, has_missing: bool +) -> None: + frame = _nullable_boolean_checkpoint_frame() + numpy_dtype = pd.api.types.pandas_dtype(dtype).numpy_dtype + bounds = np.iinfo(numpy_dtype) + frame.person["nullable_integer"] = pd.Series( + [int(bounds.min), pd.NA if has_missing else 0, int(bounds.max)], + index=frame.person.index, + dtype=dtype, + ) + first_path = tmp_path / "integer-first.h5" + second_path = tmp_path / "integer-second.h5" + + write_frame_checkpoint(first_path, frame) + loaded = load_frame_checkpoint(first_path) + write_frame_checkpoint(second_path, loaded.frame) + + assert first_path.read_bytes() == second_path.read_bytes() + pd.testing.assert_frame_equal(loaded.frame.person, frame.person, check_exact=True) + pd.testing.assert_frame_equal(loaded.frame.link("jobs"), frame.link("jobs")) + h5py = pytest.importorskip("h5py") + with h5py.File(first_path, mode="r") as h5: + metadata, spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + assert metadata["schema_version"] == 4 + assert spec == { + "name": "nullable_integer", + "dtype": dtype, + "encoding": "nullable_integer_v1", + "has_null_mask": has_missing, + } + values = np.asarray(group["values"]) + assert values.dtype == numpy_dtype + assert values.tolist() == [int(bounds.min), 0, int(bounds.max)] + if has_missing: + mask = np.asarray(group["null_mask"]) + assert mask.dtype == np.dtype(np.uint8) + assert mask.tolist() == [0, 1, 0] + else: + assert "null_mask" not in group + + +def _nullable_integer_checkpoint_frame() -> Frame: + frame = _checkpoint_frame() + frame.person["nullable_integer"] = pd.Series( + [1, pd.NA, 3], index=frame.person.index, dtype="Int64" + ) + return frame + + +def test_nullable_integer_masked_storage_is_canonical(tmp_path: Path) -> None: + paths = [tmp_path / "hidden-zero.h5", tmp_path / "hidden-nonzero.h5"] + for hidden, path in zip((0, 97), paths, strict=True): + frame = _nullable_integer_checkpoint_frame() + frame.person["nullable_integer"] = pd.Series( + pd.arrays.IntegerArray( + np.asarray([1, hidden, 3], dtype=np.int64), + np.asarray([False, True, False], dtype=np.bool_), + ), + index=frame.person.index, + ) + write_frame_checkpoint(path, frame) + assert paths[0].read_bytes() == paths[1].read_bytes() + + +@pytest.mark.parametrize( + "damage", + ["missing", "all_zero", "nonbinary", "wrong_length", "wrong_dtype", "wrong_rank"], +) +def test_nullable_integer_checkpoint_rejects_malformed_null_mask( + tmp_path: Path, damage: str +) -> None: + path = tmp_path / "malformed-integer-mask.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + del group["null_mask"] + if damage != "missing": + masks = { + "all_zero": np.asarray([0, 0, 0], dtype=np.uint8), + "nonbinary": np.asarray([0, 2, 0], dtype=np.uint8), + "wrong_length": np.asarray([0, 1], dtype=np.uint8), + "wrong_dtype": np.asarray([0, 1, 0], dtype=np.int16), + "wrong_rank": np.asarray([[0, 1, 0]], dtype=np.uint8), + } + group.create_dataset("null_mask", data=masks[damage], track_times=False) + with pytest.raises(ValueError, match="null mask"): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize("damage", ["wrong_dtype", "wrong_rank", "hidden_nonzero"]) +def test_nullable_integer_checkpoint_rejects_malformed_values( + tmp_path: Path, damage: str +) -> None: + path = tmp_path / "malformed-integer-values.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + del group["values"] + values = { + "wrong_dtype": np.asarray([1, 0, 3], dtype=np.uint64), + "wrong_rank": np.asarray([[1, 0, 3]], dtype=np.int64), + "hidden_nonzero": np.asarray([1, 2, 3], dtype=np.int64), + } + group.create_dataset("values", data=values[damage], track_times=False) + message = ( + "noncanonical nonzero" + if damage == "hidden_nonzero" + else "nullable integer values" + ) + with pytest.raises(ValueError, match=message): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize("version", [2, 3]) +def test_legacy_checkpoint_cannot_smuggle_nullable_integer_encoding( + tmp_path: Path, version: int +) -> None: + path = tmp_path / "forged-legacy-integer.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata, _spec, _group = _checkpoint_series_group( + root, table="person", column="nullable_integer" + ) + metadata["schema_version"] = version + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match=f"schema version {version}.*nullable integer"): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("has_null_mask", "yes", "has_null_mask"), + ("dtype", "int64", "must pair a supported pandas integer dtype"), + ("encoding", "numpy", "must pair a supported pandas integer dtype"), + ], +) +def test_nullable_integer_checkpoint_rejects_malformed_spec( + tmp_path: Path, field: str, value: object, message: str +) -> None: + path = tmp_path / "malformed-integer-spec.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata, spec, _group = _checkpoint_series_group( + root, table="person", column="nullable_integer" + ) + spec[field] = value + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match=message): + load_frame_checkpoint(path) + + +def test_maskless_nullable_integer_rejects_unexpected_null_mask(tmp_path: Path) -> None: + frame = _nullable_integer_checkpoint_frame() + frame.person["nullable_integer"] = frame.person["nullable_integer"].fillna(0) + path = tmp_path / "unexpected-integer-mask.h5" + write_frame_checkpoint(path, frame) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + group.create_dataset( + "null_mask", data=np.zeros(3, dtype=np.uint8), track_times=False + ) + with pytest.raises(ValueError, match="unexpected null mask"): + load_frame_checkpoint(path) + + +def test_schema_4_checkpoint_requires_nullable_integer_encoding(tmp_path: Path) -> None: + path = tmp_path / "forged-schema-4.h5" + write_frame_checkpoint(path, _nullable_boolean_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata = json.loads(np.asarray(root["metadata_json"]).tobytes()) + metadata["schema_version"] = 4 + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match="schema version 4.*nullable integer"): + load_frame_checkpoint(path) + + def test_frame_checkpoint_fsyncs_parent_directory_after_rename( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -656,10 +855,9 @@ def test_frame_checkpoint_preserves_range_indexes_and_column_axis_name( @pytest.mark.parametrize( "unsupported", [ - pd.Series([1, pd.NA, 3], dtype="Int64"), pd.Series(["a", "b", "a"], dtype="category"), ], - ids=["nullable_integer", "categorical"], + ids=["categorical"], ) def test_frame_checkpoint_rejects_unsupported_dtype_without_replacing_destination( tmp_path: Path, diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index fecadaba5..5c0ecaa3c 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,10 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "c4163f9ab3577cd5b6d509c6fe091d0f95ae7892a4746562a57a62a3f8b3be8e" +# Resolved country identities include the shared legacy-v1 seed protocol. +# Reviewed snapshot iteration labels in solve.py change its source attestation; +# AM/BE/UK authored resources and draw-site bindings remain unchanged. +AM_SPEC_SHA256 = "ab8458f8520ffe8325bf9193a7c4f3cb76bade3944522fba5f9ab3f9c4b235b6" @pytest.mark.parametrize( @@ -45,7 +48,7 @@ ), ( "be", - "fe25bbc48c785801bc7f518380eff9f528a3ceab4fa8501e1b527362c9c579b6", + "44cabc2b42e47090605a9e9947eb71e0b894d008babdfe7415faf894aa330bf2", { "household.household_id", "person.person_id", @@ -55,7 +58,7 @@ ), ( "uk", - "3ea1b53abb0ceef57b6ddef6d49251579e633bfe7d5ae7ec5a2152c0858445af", + "fcaa57d66c3831e174917e64319cd7c651b5d8aad89dd13327ad6f7b4d5c85f3", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 240228a0b..1f4f3c754 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "b465989064184f03ecc296c13bec8ffc5a70d5dd4cf30d42487b399633d32a62" + "a866bfe36a9eeb3b9a9888466b4b906faf4d8da57daf380d9bbc8ccf22e1e048" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py b/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py new file mode 100644 index 000000000..16e62dd45 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py @@ -0,0 +1,130 @@ +"""Source-only context formatting: no audit install or denied file operation.""" + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + root = Path(__file__).resolve().parents[3] + spec = importlib.util.spec_from_file_location( + "spec_seed_diagnostic_context_test", + root / "tools/spec_seed_identity_diagnostics.py", + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("value", "scope", "shown"), + ( + ("/repo/tools/example.py", "repository", "tools/example.py"), + ( + "/repo/packages/example/schema.json", + "repository", + "packages/example/schema.json", + ), + ("/repo/data/invented.csv", "repository", None), + ("/repo/.env", "repository", None), + ("/repo/credentials.json", "repository", None), + ("/repo/example.err.log", "repository", None), + ("/unlisted/identity/example.py", "unlisted", "example.py"), + ("/opt/python/lib/example.py", "system_opt", "python/lib/example.py"), + ), +) +def test_path_classification_is_bounded_and_redacts_data_and_credentials( + diagnostic, value, scope, shown +): + result = diagnostic.refusal_path_context( + value, (("repository", Path("/repo")), ("system_opt", Path("/opt"))) + ) + assert result["scope"] == scope + assert result["path"] == shown + assert result["name_redacted"] is (shown is None) + + +def test_requested_and_resolved_code_scopes_remain_distinct_without_io(diagnostic): + # Literal examples only: no symlink or path is created, resolved or opened. + payload = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/opt/python/lib/loader.py"), + "/environment/lib/loader.py", + (("environment", Path("/environment")), ("system_opt", Path("/opt"))), + ) + value = json.loads(payload) + assert value["requested"]["scope"] == "environment" + assert value["resolved"]["scope"] == "system_opt" + assert value["paths_are_metadata_only"] is True + assert len(payload) <= 4096 and len(value["frames"]) <= 8 + + +def test_context_never_serializes_locals_or_exception_text(diagnostic): + private_local = "invented-secret-value-that-must-never-be-exported" + error = RuntimeError(private_local) + payload = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/repo/credentials.json"), + "/repo/credentials.json", + (("repository", Path("/repo")),), + ) + assert str(error).encode() not in payload + value = json.loads(payload) + assert value["requested"]["path"] is None + assert value["resolved"]["path"] is None + assert value["code"] == "READ_SCOPE" and value["event"] == "open" + assert all(set(frame) == {"file", "function", "line"} for frame in value["frames"]) + assert len(payload) <= 4096 + + +@pytest.mark.parametrize( + ("path", "label"), + ( + ("/etc/os-release", "etc_os_release"), + ("/usr/lib/os-release", "usr_lib_os_release"), + ("/lib/os-release", "lib_os_release"), + ("/etc/localtime", "etc_localtime"), + ( + f"/runtime/lib/python{sys.version_info.major}{sys.version_info.minor}.zip", + "python_stdlib_zip_candidate", + ), + ("/runtime/lib/invented-data.zip", None), + ), +) +def test_fixed_os_metadata_labels_do_not_expose_data_path_text(diagnostic, path, label): + value = diagnostic.refusal_path_context(path, (("runtime", Path("/runtime")),)) + assert value["known_path_label"] == label + assert value["path"] is None and value["name_redacted"] is True + + +def test_terminal_refusal_retains_own_context_after_an_earlier_probe(diagnostic): + first = diagnostic.encoded({"code": "DATA_FILE", "event": "open"}) + last = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/usr/lib/os-release"), + "/etc/os-release", + (("system_usr", Path("/usr")), ("system_etc", Path("/etc"))), + ) + swallowed = diagnostic.RefusalError("DATA_FILE", boundary_context=first) + terminal = diagnostic.RefusalError("READ_SCOPE", boundary_context=last) + assert str(swallowed) == "DATA_FILE" and str(terminal) == "READ_SCOPE" + assert type(terminal.boundary_context) is bytes + assert terminal.boundary_context == last and swallowed.boundary_context == first + context = json.loads(terminal.boundary_context) + assert context["requested"]["known_path_label"] == "etc_os_release" + assert context["resolved"]["known_path_label"] == "usr_lib_os_release" + assert context["code"] != json.loads(swallowed.boundary_context)["code"] + assert diagnostic.RefusalError("WALL").boundary_context is None + + +@pytest.mark.parametrize("value", (bytearray(b"{}"), b"x" * 4097)) +def test_exception_context_requires_immutable_bounded_bytes(diagnostic, value): + with pytest.raises(TypeError, match="REFUSAL_CONTEXT"): + diagnostic.RefusalError("READ_SCOPE", boundary_context=value) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py b/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py new file mode 100644 index 000000000..9c31d579c --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py @@ -0,0 +1,226 @@ +"""Invented bootstrap controls; no Torch, Microcosm, engines or data imported.""" + +from __future__ import annotations + +import importlib +import importlib.machinery +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + # Normal stdlib-only module loading; main/audit/derive are never called. + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("cpu_bootstrap_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def fresh_import_state(monkeypatch): + # Keep the process import roster isolated without importing any dependency. + for name in tuple(sys.modules): + if ( + name == "torch" + or name == "cuda.bindings" + or name.startswith("cuda.bindings.") + ): + monkeypatch.delitem(sys.modules, name) + monkeypatch.setattr(sys, "meta_path", list(sys.meta_path)) + + +@pytest.mark.parametrize( + "name", ["json", "torch", "cuda", "cuda.pathfinder", "cuda.bindings_other"] +) +def test_non_cuda_bindings_imports_pass_through(diagnostic, name): + finder = diagnostic._CpuOnlyCudaBindingsFinder() + assert finder.find_spec(name, ["invented"], object()) is None + assert finder.attempts == 0 + + +def test_only_exact_optional_bindings_are_excluded(diagnostic): + finder = diagnostic._CpuOnlyCudaBindingsFinder() + with pytest.raises(ModuleNotFoundError) as error: + finder.find_spec("cuda.bindings") + assert error.value.name == "cuda.bindings" + assert finder.attempts == 1 + + +@pytest.mark.parametrize("fails", [False, True]) +def test_finder_removed_on_success_and_failure(diagnostic, fresh_import_state, fails): + before = tuple(sys.meta_path) + other = object() + evidence = {} + + class InventedFailureError(Exception): + pass + + try: + with diagnostic.cpu_only_torch_import(evidence): + finder = sys.meta_path[0] + assert type(finder) is diagnostic._CpuOnlyCudaBindingsFinder + assert tuple(sys.meta_path[1:]) == before + sys.meta_path.append(other) + with pytest.raises(ModuleNotFoundError): + finder.find_spec("cuda.bindings") + if fails: + raise InventedFailureError + except InventedFailureError: + assert fails + assert tuple(sys.meta_path) == (*before, other) + assert evidence == { + "cuda_bindings_import_attempts": 1, + "temporary_finder_removed": True, + } + + +@pytest.mark.parametrize("name", ["torch", "cuda.bindings", "cuda.bindings.runtime"]) +def test_preloaded_optional_path_refuses_before_finder( + diagnostic, fresh_import_state, monkeypatch, name +): + monkeypatch.setitem(sys.modules, name, SimpleNamespace()) + before = tuple(sys.meta_path) + with pytest.raises(diagnostic.RefusalError, match="^TORCH_BOOTSTRAP_PRELOADED$"): + with diagnostic.cpu_only_torch_import({}): + pytest.fail("preloaded dependency was accepted") + assert tuple(sys.meta_path) == before + + +def test_actual_fallback_contract_is_required( + diagnostic, fresh_import_state, monkeypatch +): + evidence = { + "temporary_finder_removed": True, + "cuda_bindings_import_attempts": 1, + } + monkeypatch.setitem( + sys.modules, + "torch.cuda._utils", + SimpleNamespace(_HAS_CUDA_BINDINGS=False, _cuda_bindings_runtime=None), + ) + diagnostic.verify_cpu_only_torch_import(evidence) + assert evidence["torch_optional_bindings_fallback_verified"] is True + + +@pytest.mark.parametrize("bindings_present", [False, True]) +def test_unconfirmed_or_loaded_bindings_refuse( + diagnostic, fresh_import_state, monkeypatch, bindings_present +): + evidence = { + "temporary_finder_removed": True, + "cuda_bindings_import_attempts": 1 if bindings_present else 0, + } + monkeypatch.setitem( + sys.modules, + "torch.cuda._utils", + SimpleNamespace(_HAS_CUDA_BINDINGS=False, _cuda_bindings_runtime=None), + ) + if bindings_present: + monkeypatch.setitem(sys.modules, "cuda.bindings", SimpleNamespace()) + with pytest.raises(diagnostic.RefusalError, match="^TORCH_CPU_FALLBACK$"): + diagnostic.verify_cpu_only_torch_import(evidence) + + +def _candidate(monkeypatch): + monkeypatch.setattr(sys, "base_prefix", "/invented-bootstrap-prefix") + return str( + Path(sys.base_prefix) + / "lib" + / f"python{sys.version_info.major}{sys.version_info.minor}.zip" + ) + + +def test_only_exact_confirmed_absent_zip_entry_is_removed(diagnostic, monkeypatch): + candidate = _candidate(monkeypatch) + others = ["", "/invented/code", "/invented/unrelated.zip", candidate + "-other"] + monkeypatch.setattr(sys, "path", [candidate, *others, candidate]) + checked = [] + + def absent(path): + checked.append(path) + raise FileNotFoundError(path) + + monkeypatch.setattr(diagnostic.os, "lstat", absent) + assert diagnostic.omit_absent_stdlib_zip() is True + assert checked == [candidate] + assert sys.path == others + + +@pytest.mark.parametrize("kind", ["regular", "broken_symlink"]) +def test_existing_zip_or_symlink_remains_subject_to_original_guard( + diagnostic, monkeypatch, kind +): + candidate = _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", [candidate, "other"]) + # Any successful lstat, including a broken symlink, prevents omission. + monkeypatch.setattr(diagnostic.os, "lstat", lambda path: SimpleNamespace(kind=kind)) + assert diagnostic.omit_absent_stdlib_zip() is False + assert sys.path == [candidate, "other"] + + +def test_other_stat_errors_do_not_justify_omission(diagnostic, monkeypatch): + candidate = _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", [candidate]) + + def denied(path): + raise PermissionError(path) + + monkeypatch.setattr(diagnostic.os, "lstat", denied) + with pytest.raises(PermissionError): + diagnostic.omit_absent_stdlib_zip() + assert sys.path == [candidate] + + +def test_unlisted_stdlib_candidate_is_not_probed(diagnostic, monkeypatch): + _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", ["other"]) + monkeypatch.setattr( + diagnostic.os, "lstat", lambda path: pytest.fail("unexpected metadata probe") + ) + assert diagnostic.omit_absent_stdlib_zip() is False + assert sys.path == ["other"] + + +def test_real_non_cuda_import_keeps_original_loader( + diagnostic, fresh_import_state, monkeypatch +): + name = "invented_non_cuda_bootstrap_control" + monkeypatch.delitem(sys.modules, name, raising=False) + loaded = [] + + class Loader: + def create_module(self, spec): + return None + + def exec_module(self, module): + loaded.append(module.__name__) + module.marker = "ordinary_loader" + + loader = Loader() + + class Finder: + def find_spec(self, fullname, path=None, target=None): + if fullname == name: + return importlib.machinery.ModuleSpec(fullname, loader) + return None + + finder = Finder() + sys.meta_path.insert(0, finder) + before = tuple(sys.meta_path) + try: + with diagnostic.cpu_only_torch_import({}): + module = importlib.import_module(name) + assert module.marker == "ordinary_loader" + assert module.__loader__ is loader + assert loaded == [name] + assert tuple(sys.meta_path) == before + finally: + sys.modules.pop(name, None) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py b/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py new file mode 100644 index 000000000..2771d8988 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py @@ -0,0 +1,186 @@ +"""Pinned engine parameter files and the urllib3 probe: invented hooks only. + +The derive phase imports policyengine-us, which reads eight public parameter +CSVs at module import. The boundary may open exactly those files, only when +their bytes hash to the pins verified before the hook is armed; every other +data-suffixed open still refuses. urllib3's import-time IPv6 probe creates one +socket and binds it to loopback from ``_has_ipv6``; nothing else on a socket is +accepted. +""" + +import base64 +import hashlib +import importlib.util +import os +import types +from importlib import metadata +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("engine_parameter_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _boundary(diagnostic, monkeypatch, allowed): + hooks, first, distinct, contexts = [], [], [], {} + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(Path, "resolve", lambda path: path) + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + diagnostic.install_boundary( + Path("/invented/repo"), + Path("/invented/owned"), + Path("/invented/output"), + first_refusal=first, + distinct_refusals=distinct, + code_contexts=contexts, + allowed_data_files=allowed, + ) + assert len(hooks) == 1 + return hooks[0], distinct + + +@pytest.mark.requires_us +def test_pins_match_the_installed_distribution_record(diagnostic): + """Every pin is a regular file in the distribution whose RECORD digest agrees.""" + + dist = metadata.distribution(diagnostic.ENGINE_PUBLIC_PARAMETER_DISTRIBUTION) + root = Path(dist.locate_file("")).resolve() + record = {} + for entry in dist.files or (): + if entry.hash is not None and entry.hash.mode == "sha256": + padded = entry.hash.value + "=" * (-len(entry.hash.value) % 4) + record[str(entry)] = base64.urlsafe_b64decode(padded).hex() + assert len(diagnostic.ENGINE_PUBLIC_PARAMETER_FILES) == 8 + for relative, expected, size in diagnostic.ENGINE_PUBLIC_PARAMETER_FILES: + path = root / relative + assert path.is_file() and not path.is_symlink(), relative + raw = path.read_bytes() + assert len(raw) == size and hashlib.sha256(raw).hexdigest() == expected + assert record[relative] == expected + report, allowed = diagnostic.pinned_engine_parameter_files() + assert report["policy"] == "pinned_engine_public_parameter_files_v1" + assert report["count"] == 8 and report["version"] == dist.version + assert allowed == frozenset( + (root / relative).resolve() + for relative, _digest, _size in diagnostic.ENGINE_PUBLIC_PARAMETER_FILES + ) + + +def test_a_pin_whose_bytes_differ_refuses_before_arming(diagnostic, tmp_path): + """Verification fails closed on bytes, size, RECORD, or a missing file.""" + + relative = "invented_engine/parameters/table.csv" + root = tmp_path / "site-packages" + (root / "invented_engine/parameters").mkdir(parents=True) + raw = b"invented,public,parameter\n1,2,3\n" + (root / relative).write_bytes(raw) + good = hashlib.sha256(raw).hexdigest() + encoded = base64.urlsafe_b64encode(hashlib.sha256(raw).digest()).rstrip(b"=") + + class _Hash: + mode = "sha256" + + def __init__(self, value: str) -> None: + self.value = value + + class _Entry(str): + hash = _Hash(encoded.decode()) + + class _Distribution: + version = "0.0.0-invented" + files = [_Entry(relative)] + + def locate_file(self, _name): + return root + + def distribution(name): + assert name == "invented-engine" + return _Distribution() + + real = metadata.distribution + try: + metadata.distribution = distribution + report, allowed = diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=((relative, good, len(raw)),) + ) + assert report["count"] == 1 and allowed == {(root / relative).resolve()} + for pins in ( + ((relative, "0" * 64, len(raw)),), + ((relative, good, len(raw) + 1),), + (("invented_engine/parameters/absent.csv", good, len(raw)),), + (("../outside.csv", good, len(raw)),), + ): + with pytest.raises(diagnostic.RefusalError, match="^ENGINE_PARAMETER_PIN$"): + diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=pins + ) + _Entry.hash = _Hash("AAAA") # RECORD disagrees with the file bytes. + with pytest.raises(diagnostic.RefusalError, match="^ENGINE_PARAMETER_PIN$"): + diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=((relative, good, len(raw)),) + ) + finally: + metadata.distribution = real + + +def test_hook_opens_only_the_allowed_data_files(diagnostic, monkeypatch): + allowed = frozenset({Path("/invented/site-packages/engine/parameters/pinned.csv")}) + hook, distinct = _boundary(diagnostic, monkeypatch, allowed) + hook("open", (str(next(iter(allowed))), "r", os.O_RDONLY)) + assert distinct == [] + for path in ( + "/invented/site-packages/engine/parameters/other.csv", + "/invented/site-packages/engine/parameters/pinned.csv.bak.csv", + "/invented/repo/pinned.csv", + ): + with pytest.raises(diagnostic.RefusalError, match="^DATA_FILE$"): + hook("open", (path, "r", os.O_RDONLY)) + assert distinct == ["DATA_FILE"] + + +def test_proc_stat_is_readable_metadata(diagnostic, monkeypatch): + hook, distinct = _boundary(diagnostic, monkeypatch, frozenset()) + hook("open", ("/proc/stat", "r", os.O_RDONLY)) + assert distinct == [] + with pytest.raises(diagnostic.RefusalError, match="^READ_SCOPE$"): + hook("open", ("/proc/version", "r", os.O_RDONLY)) + + +def _call_as_has_ipv6(diagnostic, event, args): + """Invoke the hook from a frame named like urllib3's probe function.""" + + code = compile( + "def _has_ipv6(hook, event, args):\n return hook(event, args)\n", + "/invented/site-packages/urllib3/util/connection.py", + "exec", + ) + namespace: dict[str, object] = {} + exec(code, namespace) + return namespace["_has_ipv6"](diagnostic, event, args) + + +def test_urllib3_ipv6_probe_is_the_only_accepted_socket_use(diagnostic, monkeypatch): + hook, distinct = _boundary(diagnostic, monkeypatch, frozenset()) + sock = types.SimpleNamespace() + _call_as_has_ipv6(hook, "socket.__new__", (sock, 10, 1, 0)) + _call_as_has_ipv6(hook, "socket.bind", (sock, ("::1", 0))) + _call_as_has_ipv6(hook, "socket.bind", (sock, ("127.0.0.1", 0))) + assert distinct == [] + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + _call_as_has_ipv6(hook, "socket.bind", (sock, ("0.0.0.0", 0))) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + _call_as_has_ipv6(hook, "socket.connect", (sock, ("::1", 80))) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("socket.__new__", (sock, 10, 1, 0)) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("socket.bind", (sock, ("::1", 0))) + assert distinct == ["NETWORK_OR_CHILD"] diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_lock.py b/packages/microcosm-build/tests/test_spec_seed_identity_lock.py new file mode 100644 index 000000000..07764e2f8 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_lock.py @@ -0,0 +1,40 @@ +"""Checked-in source hashing only; no diagnostic bootstrap or runtime execution.""" + +import hashlib +import importlib.util +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("seed_lock_controls", source) + module = importlib.util.module_from_spec(spec) + # Read the checked-in source directly; do not consume a cached bytecode file. + exec(compile(source.read_bytes(), str(source), "exec"), module.__dict__) + return module + + +def test_checked_in_lock_passes_actual_source_stamps(diagnostic): + root = Path(__file__).resolve().parents[3] + stamps = diagnostic.source_stamps(root) + assert ( + stamps["uv.lock"] == hashlib.sha256((root / "uv.lock").read_bytes()).hexdigest() + ) + assert set(stamps) == { + *diagnostic.SOURCE_PATHS, + "tools/spec_seed_identity_diagnostics.py", + "uv.lock", + ".github/workflows/test.yml", + } + + +def test_actual_source_stamps_refuses_an_incorrect_lock_pin(diagnostic, monkeypatch): + root = Path(__file__).resolve().parents[3] + monkeypatch.setattr(diagnostic, "LOCK_SHA256", "0" * 64) + with pytest.raises(diagnostic.RefusalError, match="^LOCK$"): + diagnostic.source_stamps(root) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py b/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py new file mode 100644 index 000000000..a24788b54 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py @@ -0,0 +1,107 @@ +"""Formatting-only outer-failure controls; no main/audit/dependency execution.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("outer_failure_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("kind", [TypeError, PermissionError]) +def test_outer_error_keeps_fixed_type_and_code_coordinates_without_message( + diagnostic, kind +): + try: + raise kind("secret-password.csv user-input-payload") + except kind as error: + payload = diagnostic.outer_failure_context(error) + value = json.loads(payload) + assert value["exception_type"] == kind.__name__ + assert value["code"] == "OUTER_FAILURE" + assert value["completed"] is value["coverage_pass"] is False + assert value["artifact_status_not_modified"] is True + assert b"secret-password" not in payload + assert b"user-input-payload" not in payload + assert len(payload) <= 8192 + + +def test_actual_short_named_traceback_frame_is_retained(diagnostic): + def cleanup_probe(): + raise TypeError("not emitted") + + try: + cleanup_probe() + except TypeError as error: + result = json.loads(diagnostic.outer_failure_context(error)) + assert result["traceback_code_frames"][-1]["function"] == "cleanup_probe" + assert result["traceback_code_frames"][-1]["line"] > 0 + + +def test_fixed_refusal_retains_attached_sanitized_context(diagnostic): + context = {"code": "READ_SCOPE", "event": "open", "requested": {"path": None}} + error = diagnostic.RefusalError( + "READ_SCOPE", boundary_context=diagnostic.encoded(context) + ) + result = json.loads(diagnostic.outer_failure_context(error)) + assert result["exception_type"] == "RefusalError" + assert result["code"] == "READ_SCOPE" + assert result["boundary_refusal"] == context + + +def test_unknown_refusal_and_exception_names_are_fixed_labels(diagnostic): + class SecretNamedError(Exception): + pass + + errors = ( + diagnostic.RefusalError("arbitrary-secret-code"), + SecretNamedError("secret-message"), + ) + for error in errors: + payload = diagnostic.outer_failure_context(error) + assert b"secret" not in payload.lower() + assert b"SecretNamedError" not in payload + result = json.loads(payload) + assert result["code"] in {"OUTER_FAILURE", "OTHER_FIXED_REFUSAL"} + + +def test_traceback_scan_and_output_are_bounded(diagnostic): + def recurse(depth): + if depth: + recurse(depth - 1) + else: + raise ValueError("unprinted") + + try: + recurse(40) + except ValueError as error: + payload = diagnostic.outer_failure_context(error) + result = json.loads(payload) + assert result["traceback_truncated"] is True + assert len(result["traceback_code_frames"]) <= 8 + assert len(payload) <= 8192 + + +def test_exception_only_metadata_does_not_open_or_resolve_files( + diagnostic, monkeypatch +): + def forbidden(*args, **kwargs): + pytest.fail("outer report attempted file I/O or resolution") + + monkeypatch.setattr(Path, "read_bytes", forbidden) + monkeypatch.setattr(Path, "read_text", forbidden) + monkeypatch.setattr(Path, "resolve", forbidden) + monkeypatch.setattr(Path, "stat", forbidden) + monkeypatch.setattr(Path, "open", forbidden) + payload = diagnostic.outer_failure_context(TypeError("no traceback")) + assert json.loads(payload)["traceback_code_frames"] == [] diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py b/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py new file mode 100644 index 000000000..2fe9d1739 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py @@ -0,0 +1,60 @@ +"""Owned temporary-directory controls; no audit, main, Torch or Microcosm run.""" + +import importlib.util +import stat +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("owned_temp_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("refuses", [False, True]) +def test_owned_fixture_survives_success_and_exact_refusal( + diagnostic, tmp_path, refuses +): + context = b'{"code":"READ_SCOPE","event":"open"}\n' + error = diagnostic.RefusalError("READ_SCOPE", boundary_context=context) + caught = None + try: + with diagnostic.retained_owned_directory(tmp_path) as owned: + assert owned.parent == tmp_path + assert owned.name.startswith("spec-seed-owned-") + assert stat.S_IMODE(owned.stat().st_mode) == 0o700 + fixture = owned / "invented-spec.txt" + fixture.write_text("invented fixture only") + if refuses: + raise error + except diagnostic.RefusalError as actual: + caught = actual + assert caught is (error if refuses else None) + if caught is not None: + assert caught.boundary_context is context + assert fixture.read_text() == "invented fixture only" + assert owned.is_dir() + + +def test_each_workspace_is_distinct_and_has_no_implicit_cleanup( + diagnostic, tmp_path, monkeypatch +): + def forbidden(*args, **kwargs): + pytest.fail("implicit TemporaryDirectory cleanup was registered") + + monkeypatch.setattr(diagnostic.tempfile, "TemporaryDirectory", forbidden) + with diagnostic.retained_owned_directory(tmp_path) as first: + (first / "invented.txt").write_text("first") + with diagnostic.retained_owned_directory(tmp_path) as second: + (second / "invented.txt").write_text("second") + assert first != second + assert first.parent == second.parent == tmp_path + assert (first / "invented.txt").read_text() == "first" + assert (second / "invented.txt").read_text() == "second" diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py b/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py new file mode 100644 index 000000000..0ee55f9a9 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py @@ -0,0 +1,228 @@ +"""Invented audit calls only; no real hook, source read, child or derivation.""" + +import importlib.util +import json +import os +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("per_code_context_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def audited(diagnostic, monkeypatch): + hooks, first, distinct, contexts = [], [], [], {} + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(Path, "resolve", lambda path: path) + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + refusals = diagnostic.install_boundary( + Path("/invented/repo"), + Path("/invented/owned"), + Path("/invented/output"), + first_refusal=first, + distinct_refusals=distinct, + code_contexts=contexts, + ) + assert len(hooks) == 1 + return hooks[0], refusals, first, distinct, contexts + + +def test_caught_middle_child_denial_keeps_own_context(diagnostic, audited): + hook, refusals, first, distinct, contexts = audited + terminal = None + secret = "invented-secret-never-retain" + for event, args, code in ( + ("open", ("/proc/version", "r", os.O_RDONLY), "READ_SCOPE"), + ( + "subprocess.Popen", + (secret, [secret], secret, {secret: secret}), + "NETWORK_OR_CHILD", + ), + ("open", ("/invented/repo/never-open.csv", "r", os.O_RDONLY), "DATA_FILE"), + ): + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$") as error: + hook(event, args) + terminal = error.value + # The historical refusal counter remains first-code-only. Every operation + # still raises; retaining context must not turn a swallowed denial into success. + assert refusals == ["READ_SCOPE"] + assert distinct == ["READ_SCOPE", "NETWORK_OR_CHILD", "DATA_FILE"] + assert list(contexts) == distinct + assert first == [contexts["READ_SCOPE"]] + assert terminal.boundary_context == contexts["DATA_FILE"] + child = json.loads(contexts["NETWORK_OR_CHILD"]) + assert child["code"] == "NETWORK_OR_CHILD" and child["event"] == "subprocess.Popen" + assert child["frames"] and len(child["frames"]) <= 8 + assert child["requested"]["path"] is None and child["resolved"]["path"] is None + for payload in contexts.values(): + assert type(payload) is bytes and len(payload) <= 4096 + assert secret.encode() not in payload and b"never-open.csv" not in payload + + +def test_repeated_code_retains_first_record_but_exception_has_fresh_context( + diagnostic, audited +): + hook, refusals, first, distinct, contexts = audited + + def original_caller(): + hook("subprocess.Popen", ("invented", [], None, None)) + + def later_caller(): + hook("subprocess.Popen", ("invented", [], None, None)) + + with pytest.raises(diagnostic.RefusalError) as original: + original_caller() + retained = contexts["NETWORK_OR_CHILD"] + with pytest.raises(diagnostic.RefusalError) as later: + later_caller() + assert contexts == {"NETWORK_OR_CHILD": retained} + assert first == [retained] and refusals == distinct == ["NETWORK_OR_CHILD"] + assert original.value.boundary_context == retained + assert later.value.boundary_context != retained + assert "original_caller" in [f["function"] for f in json.loads(retained)["frames"]] + assert "later_caller" in [ + f["function"] for f in json.loads(later.value.boundary_context)["frames"] + ] + + +def test_eight_record_capacity_never_discards_or_grows(diagnostic, audited): + hook, refusals, first, distinct, contexts = audited + # A full invented collector exercises the bound even though the current + # boundary emits only seven different fixed codes. + codes = ( + "WRITE_SCOPE", + "DATA_FILE", + "FILESYSTEM_PATH", + "FILESYSTEM_DESCRIPTOR", + "FILESYSTEM_LINK", + "NETWORK_OR_CHILD", + "WALL", + "OUTPUT_CHANGED", + ) + contexts.update({code: diagnostic.encoded({"code": code}) for code in codes}) + retained = tuple(contexts.items()) + with pytest.raises(diagnostic.RefusalError, match="^READ_SCOPE$") as error: + hook("open", ("/proc/version", "r", os.O_RDONLY)) + assert tuple(contexts.items()) == retained and len(contexts) == 8 + assert refusals == distinct == ["READ_SCOPE"] + assert first == [error.value.boundary_context] + + +@pytest.mark.parametrize( + ("event", "label"), + ( + ("os.posix_spawn", "os.posix_spawn"), + ("socket.connect", "network_or_child_event"), + ("subprocess.invented-secret-event", "network_or_child_event"), + ), +) +def test_network_context_event_is_fixed_and_never_serializes_arguments( + diagnostic, audited, event, label +): + hook, _, _, _, contexts = audited + secret = "invented-secret-arguments" + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook(event, (secret, {secret: secret})) + payload = contexts["NETWORK_OR_CHILD"] + assert json.loads(payload)["event"] == label + assert secret.encode() not in payload and b"invented-secret-event" not in payload + + +def test_context_capture_reads_no_source_files_or_locals( + diagnostic, audited, monkeypatch +): + hook, _, _, _, contexts = audited + private_local = "invented-private-local-not-evidence" + + def forbidden(*args, **kwargs): + pytest.fail("context formatting attempted a source or filesystem read") + + with monkeypatch.context() as patch: + for name in ("open", "read_bytes", "read_text", "stat", "lstat"): + patch.setattr(Path, name, forbidden) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("subprocess.Popen", (private_local,)) + payload = contexts["NETWORK_OR_CHILD"] + assert private_local.encode() not in payload + assert all( + set(frame) == {"file", "function", "line"} + for frame in json.loads(payload)["frames"] + ) + + +def test_context_capture_failure_does_not_replace_denial_with_exception_text( + diagnostic, audited, monkeypatch +): + hook, refusals, first, _, contexts = audited + + def unavailable(*args): + raise RuntimeError("invented-sensitive-exception-text") + + monkeypatch.setattr(diagnostic, "read_refusal_context", unavailable) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$") as error: + hook("subprocess.Popen", ("invented",)) + expected = diagnostic.encoded( + {"code": "NETWORK_OR_CHILD", "context": "unavailable"} + ) + assert refusals == ["NETWORK_OR_CHILD"] + assert first == [expected] and contexts == {"NETWORK_OR_CHILD": expected} + assert error.value.boundary_context == expected + + +def test_all_maximum_records_fit_status_cap_and_preserve_aggregate_cap(diagnostic): + base = diagnostic.encoded({"code": "READ_SCOPE", "context": ""}) + maximum = diagnostic.encoded( + {"code": "READ_SCOPE", "context": "x" * (4096 - len(base))} + ) + assert len(maximum) == 4096 + status = diagnostic.encoded( + { + "completed": False, + "coverage_pass": False, + "first_boundary_refusal": json.loads(maximum), + "terminal_boundary_refusal": json.loads(maximum), + "boundary_refusal_contexts": [json.loads(maximum)] * 8, + "boundary_refusal_context_limit": {"codes": 8, "bytes_per_code": 4096}, + } + ) + assert 16 * 1024 < len(status) < 64 * 1024 + assert diagnostic.CAPS == { + "candidate-digests.json": 64 * 1024, + "seed-protocol.json": 256 * 1024, + "seed-map.json": 256 * 1024, + "seed-bindings.json": 128 * 1024, + "environment-and-source.json": 128 * 1024, + "diagnostic-status.json": 64 * 1024, + } + assert diagnostic.MAX_TOTAL == 1024 * 1024 + payloads = {name: b"{}" for name in diagnostic.CAPS} + payloads["diagnostic-status.json"] = status + diagnostic.validate_payloads(payloads) + + +@pytest.mark.parametrize( + "name", + ( + "candidate-digests.json", + "seed-protocol.json", + "seed-map.json", + "seed-bindings.json", + "environment-and-source.json", + "diagnostic-status.json", + ), +) +def test_each_existing_output_cap_still_refuses_before_publication(diagnostic, name): + payloads = {key: b"{}" for key in diagnostic.CAPS} + payloads[name] = b"x" * (diagnostic.CAPS[name] + 1) + with pytest.raises(diagnostic.RefusalError, match="^OUTPUT_CAP$"): + diagnostic.validate_payloads(payloads) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py new file mode 100644 index 000000000..8b4c89b98 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py @@ -0,0 +1,353 @@ +"""Invented metadata controls; no main, engine, data, child or real audit hook.""" + +import importlib.util +import os +import platform +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("system_metadata_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def invented_uname(diagnostic, monkeypatch): + observed = SimpleNamespace( + sysname="Linux", + nodename="invented-host", + release="invented-release", + version="invented-version", + machine="invented_arch", + ) + info = platform.uname_result( + observed.sysname, + observed.nodename, + observed.release, + observed.version, + observed.machine, + ) + monkeypatch.setattr(diagnostic.sys, "platform", "linux") + monkeypatch.setattr(diagnostic.os, "uname", lambda: observed) + monkeypatch.setattr(diagnostic.platform, "uname", lambda: info) + + def forbidden(*args, **kwargs): + pytest.fail("processor metadata attempted a subprocess") + + monkeypatch.setattr(subprocess, "check_output", forbidden) + return info, observed + + +def test_real_cached_property_uses_machine_without_replacing_stdlib( + diagnostic, invented_uname +): + info, observed = invented_uname + before = ( + platform.uname, + platform.processor, + platform.uname_result, + vars(type(info))["processor"], + ) + bootstrap = {} + state = diagnostic.prime_processor_metadata(bootstrap) + assert platform.processor() == observed.machine + assert tuple(info) == ( + observed.sysname, + observed.nodename, + observed.release, + observed.version, + observed.machine, + observed.machine, + ) + assert bootstrap["system_metadata"]["uname_p_parity_claimed"] is False + assert bootstrap["system_metadata"]["source"] == "os.uname.machine" + assert before == ( + platform.uname, + platform.processor, + platform.uname_result, + vars(type(info))["processor"], + ) + diagnostic.verify_processor_metadata(state, bootstrap) + # An already-cached identical machine value is not silently altered. + again = diagnostic.prime_processor_metadata(bootstrap) + assert again == state + + +def test_conflicting_observed_processor_cache_is_not_overwritten( + diagnostic, invented_uname +): + info, _ = invented_uname + info.processor = "different_observed_processor" + bootstrap = {} + with pytest.raises(diagnostic.RefusalError, match="^PROCESSOR_CACHE_CONFLICT$"): + diagnostic.prime_processor_metadata(bootstrap) + assert info.processor == "different_observed_processor" + assert bootstrap == {} + + +@pytest.mark.parametrize("change", ("cached_value", "os_machine", "evidence")) +def test_metadata_change_refuses(diagnostic, invented_uname, change): + info, observed = invented_uname + bootstrap = {} + state = diagnostic.prime_processor_metadata(bootstrap) + if change == "cached_value": + info.processor = "changed" + elif change == "os_machine": + observed.machine = "changed" + else: + bootstrap["system_metadata"]["uname_p_parity_claimed"] = True + with pytest.raises(diagnostic.RefusalError, match="^PROCESSOR_METADATA_CHANGED$"): + diagnostic.verify_processor_metadata(state, bootstrap) + + +@pytest.fixture +def audited(diagnostic, monkeypatch): + hooks = [] + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(diagnostic.os, "getpid", lambda: 321) + monkeypatch.setattr(diagnostic.os, "readlink", lambda _: "pipe:[invented]") + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + + def resolve(path): + return Path("/proc/321/maps") if path == Path("/proc/self/maps") else path + + monkeypatch.setattr(Path, "resolve", resolve) + refusals = diagnostic.install_boundary( + Path("/invented/repo"), Path("/invented/owned"), Path("/invented/output") + ) + assert len(hooks) == 1 + return hooks[0], refusals + + +@pytest.mark.parametrize("path", ("/proc/self/maps", "/proc/321/maps")) +def test_only_logical_and_resolved_own_maps_are_admitted(audited, path): + hook, refusals = audited + hook("open", (path, "r", os.O_RDONLY)) + assert refusals == [] + + +@pytest.mark.parametrize( + "kind", ("other_process", "other_file", "relative", "descriptor", "child") +) +def test_maps_alias_does_not_admit_other_paths_descriptors_or_children( + diagnostic, audited, kind +): + hook, refusals = audited + path = { + "other_process": "/proc/322/maps", + "other_file": "/proc/321/mem", + "relative": "maps", + "descriptor": 55, + "child": None, + }[kind] + code = "NETWORK_OR_CHILD" if kind == "child" else "READ_SCOPE" + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$"): + if kind == "child": + hook("subprocess.Popen", ("uname", ["uname", "-p"], None, None)) + else: + hook("open", (path, "r", os.O_RDONLY)) + assert refusals == [code] + + +@pytest.fixture +def coordinate_environment(diagnostic, monkeypatch): + """Exercise coordinate validation against an isolated invented environment.""" + values = { + "DIAG_CHECKOUT_SHA": "1" * 40, + "GITHUB_SHA": "1" * 40, + "DIAG_WORKFLOW_SHA": "2" * 40, + "DIAG_PR_HEAD_SHA": "3" * 40, + "DIAG_PR_BASE_SHA": "4" * 40, + "DIAG_MERGE_SHA": "5" * 40, + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_RUN_ID": "123456", + "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_REPOSITORY": "PolicyEngine/microcosm", + } + # coordinates() consumes only environ. Keep the real process environment, + # filesystem, dependency imports and bootstrap paths outside these cases. + monkeypatch.setattr(diagnostic, "os", SimpleNamespace(environ=values)) + return values + + +@pytest.mark.parametrize("event", ("pull_request", "push")) +def test_coordinates_preserve_all_distinct_supplied_identities( + diagnostic, coordinate_environment, event +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + original = dict(values) + assert diagnostic.coordinates() == { + "checkout_sha": "1" * 40, + "event_sha": "1" * 40, + "workflow_sha": "2" * 40, + "pr_head_sha": "3" * 40, + "pr_base_sha": "4" * 40, + "event_merge_sha": "5" * 40, + "github_run_id": "123456", + "github_run_attempt": "2", + "event": event, + "repository": "PolicyEngine/microcosm", + } + assert values == original + + +@pytest.mark.parametrize("missing", (False, True)) +def test_pull_request_coordinates_keep_absent_merge_metadata_empty( + diagnostic, coordinate_environment, missing +): + values = coordinate_environment + if missing: + del values["DIAG_MERGE_SHA"] + else: + values["DIAG_MERGE_SHA"] = "" + result = diagnostic.coordinates() + assert result["event"] == "pull_request" + assert result["event_merge_sha"] == "" + assert result["checkout_sha"] == result["event_sha"] == "1" * 40 + assert result["workflow_sha"] == "2" * 40 + assert result["pr_head_sha"] == "3" * 40 + assert result["pr_base_sha"] == "4" * 40 + assert values.get("DIAG_MERGE_SHA", "") == "" + + +@pytest.mark.parametrize("missing", (False, True)) +def test_push_coordinates_allow_absent_pull_request_metadata( + diagnostic, coordinate_environment, missing +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = "push" + for name in ("DIAG_PR_HEAD_SHA", "DIAG_PR_BASE_SHA", "DIAG_MERGE_SHA"): + if missing: + del values[name] + else: + values[name] = "" + result = diagnostic.coordinates() + assert result["event"] == "push" + assert result["pr_head_sha"] == result["pr_base_sha"] == "" + assert result["event_merge_sha"] == "" + assert result["checkout_sha"] == result["event_sha"] == "1" * 40 + assert result["workflow_sha"] == "2" * 40 + + +@pytest.mark.parametrize( + ("event", "name"), + ( + ("pull_request", "DIAG_CHECKOUT_SHA"), + ("pull_request", "GITHUB_SHA"), + ("pull_request", "DIAG_WORKFLOW_SHA"), + ("pull_request", "DIAG_PR_HEAD_SHA"), + ("pull_request", "DIAG_PR_BASE_SHA"), + ("push", "DIAG_CHECKOUT_SHA"), + ("push", "GITHUB_SHA"), + ("push", "DIAG_WORKFLOW_SHA"), + ), +) +def test_required_coordinate_hashes_still_refuse_when_absent( + diagnostic, coordinate_environment, event, name +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + del values[name] + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("event", "name"), + ( + ("pull_request", "DIAG_CHECKOUT_SHA"), + ("pull_request", "GITHUB_SHA"), + ("pull_request", "DIAG_WORKFLOW_SHA"), + ("pull_request", "DIAG_PR_HEAD_SHA"), + ("pull_request", "DIAG_PR_BASE_SHA"), + ("pull_request", "DIAG_MERGE_SHA"), + ("push", "DIAG_PR_HEAD_SHA"), + ("push", "DIAG_PR_BASE_SHA"), + ("push", "DIAG_MERGE_SHA"), + ), +) +def test_nonempty_coordinate_hashes_still_require_valid_format( + diagnostic, coordinate_environment, event, name +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + values[name] = "not-a-sha" + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + "value", ("null", "A" * 40, "f" * 39, "f" * 41, "f" * 40 + "\n") +) +def test_missing_merge_exception_does_not_accept_malformed_present_hashes( + diagnostic, coordinate_environment, value +): + coordinate_environment["DIAG_MERGE_SHA"] = value + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize("event", ("pull_request", "push")) +def test_checkout_must_match_event_even_without_merge_metadata( + diagnostic, coordinate_environment, event +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + values["DIAG_MERGE_SHA"] = "" + values["DIAG_CHECKOUT_SHA"] = "6" * 40 + with pytest.raises(diagnostic.RefusalError, match="^CHECKOUT$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("name", "value"), + ( + ("GITHUB_RUN_ID", None), + ("GITHUB_RUN_ID", ""), + ("GITHUB_RUN_ID", "not-a-run"), + ("GITHUB_RUN_ATTEMPT", None), + ("GITHUB_RUN_ATTEMPT", ""), + ("GITHUB_RUN_ATTEMPT", "not-an-attempt"), + ), +) +def test_coordinate_run_identity_remains_required( + diagnostic, coordinate_environment, name, value +): + if value is None: + del coordinate_environment[name] + else: + coordinate_environment[name] = value + with pytest.raises(diagnostic.RefusalError, match="^RUN_COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("name", "value", "code"), + ( + ("GITHUB_EVENT_NAME", None, "EVENT"), + ("GITHUB_EVENT_NAME", "", "EVENT"), + ("GITHUB_EVENT_NAME", "workflow_dispatch", "EVENT"), + ("GITHUB_REPOSITORY", None, "REPOSITORY"), + ("GITHUB_REPOSITORY", "invented/other", "REPOSITORY"), + ), +) +def test_coordinate_event_and_repository_boundaries_remain_required( + diagnostic, coordinate_environment, name, value, code +): + if value is None: + del coordinate_environment[name] + else: + coordinate_environment[name] = value + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$"): + diagnostic.coordinates() diff --git a/packages/microcosm-build/tests/test_spec_worker_identity_lock.py b/packages/microcosm-build/tests/test_spec_worker_identity_lock.py new file mode 100644 index 000000000..516f5398e --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_worker_identity_lock.py @@ -0,0 +1,83 @@ +"""Actual lock validation without worker, engine or identity-probe execution.""" + +import hashlib +import importlib.util +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture +def worker_identity(monkeypatch): + root = Path(__file__).resolve().parents[3] + source = root / ( + "packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py" + ) + spec = importlib.util.spec_from_file_location("worker_lock_controls", source) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + exec(compile(source.read_bytes(), str(source), "exec"), module.__dict__) + # Locate the real checked-in lock without importing the PUF worker. + monkeypatch.setattr(module, "_repository_root", lambda: root) + return module + + +def test_checked_in_lock_passes_actual_worker_validation(worker_identity): + root = Path(__file__).resolve().parents[3] + assert ( + worker_identity._approved_uv_lock_sha256() + == hashlib.sha256((root / "uv.lock").read_bytes()).hexdigest() + ) + + +def test_actual_worker_validation_refuses_an_incorrect_pin( + worker_identity, monkeypatch +): + monkeypatch.setattr(worker_identity, "APPROVED_UV_LOCK_SHA256", "0" * 64) + with pytest.raises(RuntimeError, match="unapproved uv.lock digest"): + worker_identity._approved_uv_lock_sha256() + + +@pytest.mark.parametrize( + "digest", + [ + # The superseded current lock must not become an implicit legacy alias. + "751d5ef5d25406bbae1798667f0e29890d4aad933d323c12912c7d45d8809bb9", + "0" * 64, + ], +) +def test_unapproved_explicit_lock_refuses_before_worker_probe( + worker_identity, monkeypatch, digest +): + def unexpected_probe(): + raise AssertionError("An unapproved lock reached the worker source probe") + + monkeypatch.setattr(worker_identity, "_worker_source_identity", unexpected_probe) + with pytest.raises(ValueError, match="worker lock is not approved"): + worker_identity._uncached_primary_qrf_worker_semantic_identity( + uv_lock_sha256=digest + ) + + +def test_wheel_without_checkout_uses_approved_lock(worker_identity, monkeypatch): + root = Path(__file__).resolve().parents[3] + expected = hashlib.sha256((root / "uv.lock").read_bytes()).hexdigest() + monkeypatch.setattr(worker_identity, "_repository_root", lambda: None) + assert worker_identity._approved_uv_lock_sha256() == expected + + +def test_explicit_legacy_campaign_remains_accepted_before_probe( + worker_identity, monkeypatch +): + class StopBeforeProbeError(Exception): + pass + + def stop_before_probe(): + raise StopBeforeProbeError + + monkeypatch.setattr(worker_identity, "_worker_source_identity", stop_before_probe) + with pytest.raises(StopBeforeProbeError): + worker_identity._uncached_primary_qrf_worker_semantic_identity( + uv_lock_sha256=worker_identity.LEGACY_CAMPAIGN_UV_LOCK_SHA256 + ) diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index 4c81e6173..7b1592e3b 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -158,6 +158,31 @@ def test_prepared_column_path_materializes_filtered_values(): ] +@pytest.mark.parametrize( + "left,right,expected", + [ + ([True, False, True], [True, False, False], [2.0, 0.0, 1.0]), + ([True, False, True], [2, 3, 4], [3.0, 3.0, 5.0]), + ([2, 3, 4], [True, False, True], [3.0, 3.0, 5.0]), + ([1, 2, 3], [4, 5, 6], [5.0, 7.0, 9.0]), + ([0.25, -0.5, 2.0], [0.5, 1.5, -1.0], [0.75, 1.0, 1.0]), + ], +) +def test_shared_expression_uses_arithmetic_for_boolean_terms(left, right, expected): + adapter = StubAdapter() + adapter.tables["person"]["left"] = np.asarray(left) + adapter.tables["person"]["right"] = np.asarray(right) + registry = _resolution_registry("sum") + result = materialize_target_bindings( + adapter, + registry, + {"sum": {"bindings": {"policyengine": {"value_expression": "left + right"}}}}, + period=2025, + ) + assert result.skipped == () + np.testing.assert_array_equal(adapter.tables["person"]["sum_measure"], expected) + + @pytest.mark.parametrize("fact_period", [2024, 2025]) @pytest.mark.parametrize("require_matching_fact_period", [None, False, True]) def test_existing_measure_respects_fact_guard_at_default_measurement_period( diff --git a/packages/microcosm-build/tests/test_uk_atomic_area_support.py b/packages/microcosm-build/tests/test_uk_atomic_area_support.py new file mode 100644 index 000000000..b25887a87 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_atomic_area_support.py @@ -0,0 +1,482 @@ +"""Invented array/identity controls only; no publisher files or country engine.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import atomic_geography as geo +from microcosm.build.uk_runtime.atomic_area_support import ( + IDENTITY_COLUMN, + SOURCES, + SYSTEMS, + assemble_uk_atomic_area_support, + uk_atomic_assignment_definition, +) +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE + + +def parts(system): + index = SYSTEMS.index(system) + regions = ( + [ + r + for r in FRS_REGION_TO_REGION_CODE + if r not in {"SCOTLAND", "NORTHERN_IRELAND"} + ] + if index == 0 + else ["SCOTLAND"] + if index == 1 + else ["NORTHERN_IRELAND"] + ) + rows = [] + region_order = list(FRS_REGION_TO_REGION_CODE) + for region in regions: + prefix = FRS_REGION_TO_REGION_CODE[region][0] + for offset in range(3): + serial = region_order.index(region) * 3 + offset + 1 + area = prefix + ("20" if index == 2 else "00") + f"{serial:06d}" + rows.append( + { + "oa_code": area, + "population": [2.0, 6.0, 1000.0][offset], + "households": [0.0, 30.0, 10.0][offset], + "constituency_code": prefix + + "14" + + f"{serial - offset + (offset == 2):06d}", + "region_code": FRS_REGION_TO_REGION_CODE[region], + "lsoa_code": area + if index == 2 + else prefix + "01" + f"{serial:06d}", + "msoa_code": prefix + "02" + f"{serial:06d}", + "local_authority_code": prefix + "06" + f"{serial:06d}", + "ward_code": prefix + "05" + f"{serial:06d}", + "itl3_code": "TL" + + "CDEFGHIJKLMN"[region_order.index(region)] + + "01", + } + ) + arrays = {column: np.asarray([r[column] for r in rows]) for column in rows[0]} + descriptions = {} + for column in arrays: + if column in {"population", "households"}: + descriptions[column] = { + "kind": "weight", + "source": "invented-" + system + "-" + column, + "basis": "invented persons" + if column == "population" + else "invented occupied households", + } + else: + descriptions[column] = { + "kind": "code", + "source": "invented-" + system + "-" + column, + "vintage": "2022_census" if index == 1 else "2021_census", + "relation": "exact" + if column in {"oa_code", "lsoa_code", "msoa_code", "region_code"} + else "best_fit", + } + descriptions["constituency_code"]["vintage"] = "2024_pcon" + if index == 2: + descriptions["constituency_code"]["relation"] = "official_tabulation" + return arrays, descriptions + + +def payloads(): + return { + system: assemble_uk_atomic_area_support( + system=system, arrays=arrays, column_metadata=metadata + ) + for system in SYSTEMS + for arrays, metadata in [parts(system)] + } + + +def key(source_id=101, path=()): + return household_draw_key( + source="invented-frs", + source_vintage="invented-2024-25", + source_household_id=source_id, + clone_path=path, + ) + + +def complete(households, payload): + spec = uk_atomic_assignment_definition(payload, seed=43) + supports = { + system: geo.decode_atomic_support(data) for system, data in payload.items() + } + result = pd.concat( + [households, geo.assign_atomic(households, spec, supports)], axis=1 + ) + result = pd.concat([result, geo.derive_geography(result, spec, supports)], axis=1) + return result, spec, supports + + +def test_normalization_preserves_all_rows_counts_mappings_and_source_labels(): + for system in SYSTEMS: + arrays, metadata = parts(system) + original = {k: v.copy() for k, v in arrays.items()} + data = assemble_uk_atomic_area_support( + system=system, arrays=arrays, column_metadata=metadata + ) + support = geo.decode_atomic_support(data) + order = np.argsort(arrays["oa_code"], kind="stable") + for column in arrays: + normalized = "area" if column == "oa_code" else column + np.testing.assert_array_equal( + support.arrays[normalized], arrays[column][order] + ) + assert dict(support.metadata["columns"][normalized]) == metadata[column] + np.testing.assert_array_equal(arrays[column], original[column]) + assert support.arrays["population"].dtype == np.dtype("int64") + assert support.arrays["households"].dtype == np.dtype("int64") + assert support.arrays["area"].flags.writeable is False + assert support.sha256 == geo.decode_atomic_support(data).sha256 + + +def test_normalized_bytes_are_stable_under_input_row_order_and_integer_count_storage(): + arrays, metadata = parts(SYSTEMS[0]) + expected = assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + reversed_arrays = {k: v[::-1].copy() for k, v in arrays.items()} + for column in ("population", "households"): + reversed_arrays[column] = reversed_arrays[column].astype(np.int64) + assert ( + assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=reversed_arrays, column_metadata=metadata + ) + == expected + ) + + +@pytest.mark.parametrize( + "defect", + [ + "nonintegral", + "float32", + "bool", + "negative", + "nan", + "overflow_total", + "zero_population", + "zero_region_households", + "duplicate_area", + "length", + "missing_column", + "object_code", + "foreign_nation", + "cross_ew_nation", + "blank_code", + "missing_region", + "bad_atomic_vintage", + "missing_source", + ], +) +def test_converter_refuses_bad_counts_codes_coverage_and_metadata(defect): + arrays, metadata = parts(SYSTEMS[0]) + if defect == "nonintegral": + arrays["households"][0] = 0.5 + elif defect == "float32": + arrays["households"] = arrays["households"].astype(np.float32) + elif defect == "bool": + arrays["households"] = arrays["households"] > 0 + elif defect == "negative": + arrays["households"][0] = -1 + elif defect == "nan": + arrays["population"][0] = np.nan + elif defect == "overflow_total": + arrays["population"][0] = float(2**53) + elif defect == "zero_population": + arrays["population"][0] = 0 + elif defect == "zero_region_households": + arrays["households"][:3] = 0 + elif defect == "duplicate_area": + arrays["oa_code"][1] = arrays["oa_code"][0] + elif defect == "length": + arrays["msoa_code"] = arrays["msoa_code"][:-1] + elif defect == "missing_column": + del arrays["population"] + elif defect == "object_code": + arrays["oa_code"] = arrays["oa_code"].astype(object) + elif defect == "foreign_nation": + arrays["ward_code"][0] = "S05000001" + elif defect == "cross_ew_nation": + arrays["ward_code"][0] = "W05000001" + elif defect == "blank_code": + arrays["ward_code"][0] = "" + elif defect == "missing_region": + arrays = {k: v[3:].copy() for k, v in arrays.items()} + elif defect == "bad_atomic_vintage": + metadata["oa_code"]["vintage"] = "2011_census" + elif defect == "missing_source": + metadata["population"]["source"] = "" + with pytest.raises(ValueError, match="UK atomic support:"): + assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + + +@pytest.mark.parametrize("defect", ["inferred_ni_constituency", "ni_lsoa_not_dz"]) +def test_ni_source_convention_refusals(defect): + arrays, metadata = parts(SYSTEMS[2]) + if defect == "inferred_ni_constituency": + metadata["constituency_code"]["relation"] = "inferred_modal" + else: + arrays["lsoa_code"][0] = "N01000001" + with pytest.raises(ValueError, match="UK atomic support: NI"): + assemble_uk_atomic_area_support( + system=SYSTEMS[2], arrays=arrays, column_metadata=metadata + ) + + +def test_three_system_assignment_keeps_observed_region_and_native_alias_roles(): + households = pd.DataFrame( + { + "household_id": np.arange(1, 5, dtype=np.int64), + "region": pd.array( + ["LONDON", "WALES", "SCOTLAND", "NORTHERN_IRELAND"], dtype="string" + ), + IDENTITY_COLUMN: pd.array( + [key(i) for i in (101, 102, 103, 104)], dtype="string" + ), + } + ) + result, spec, supports = complete(households, payloads()) + pd.testing.assert_series_equal(result["region"], households["region"]) + assert result["atomic_area_system"].tolist() == [ + SYSTEMS[0], + SYSTEMS[0], + SYSTEMS[1], + SYSTEMS[2], + ] + assert result["atomic_area_basis"].tolist() == ["assigned"] * 4 + assert result.loc[:2, "output_area_code"].equals(result.loc[:2, "atomic_area_code"]) + assert pd.isna(result.loc[3, "output_area_code"]) + assert result.loc[2, "data_zone_code"] == result.loc[2, "lsoa_code"] + assert result.loc[2, "intermediate_zone_code"] == result.loc[2, "msoa_code"] + assert result.loc[3, "data_zone_code"] == result.loc[3, "atomic_area_code"] + assert result.loc[3, "super_data_zone_code"] == result.loc[3, "msoa_code"] + assert result.loc[3, "district_electoral_area_code"] == result.loc[3, "ward_code"] + assert pd.isna(result.loc[0, "super_data_zone_code"]) + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + assert {s["source"] for s in spec["systems"]} == set(SOURCES.values()) + + +def test_source_metadata_changes_are_identified_and_declaration_refuses_forged_derived_field(): + payload = payloads() + original = payload[SYSTEMS[0]] + arrays, metadata = parts(SYSTEMS[0]) + metadata["population"]["source"] += "-revision-2" + changed = assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + assert changed != original + support = geo.decode_atomic_support(original) + forged_arrays = {k: v.copy() for k, v in support.arrays.items()} + forged_arrays["itl1_code"][0] = "TLN" + forged = geo.encode_atomic_support( + {k: dict(v) if k == "columns" else v for k, v in support.metadata.items()}, + forged_arrays, + ) + payload[SYSTEMS[0]] = forged + with pytest.raises(ValueError, match="canonical UK adapter output"): + uk_atomic_assignment_definition(payload, seed=43) + + +@pytest.mark.parametrize("defect", ["missing_system", "wrong_system", "bad_seed"]) +def test_declaration_refuses_mismatched_inputs(defect): + payload = payloads() + options = {"seed": 43} + if defect == "missing_system": + del payload[SYSTEMS[2]] + elif defect == "wrong_system": + payload[SYSTEMS[2]] = payload[SYSTEMS[1]] + elif defect == "bad_seed": + options["seed"] = True + with pytest.raises(ValueError): + uk_atomic_assignment_definition(payload, **options) + + +def test_declaration_cannot_substitute_an_income_or_counter_identity_column(): + with pytest.raises(TypeError, match="identity_column"): + uk_atomic_assignment_definition( + payloads(), seed=43, identity_column="household_weight" + ) + + +def test_distinct_post_clone_keys_assign_then_remain_stable_under_order_subset_and_growth(): + paths = [ + (("spi_support_channel", 0), ("cgt_incidence_clone", 0)), + (("spi_support_channel", 1), ("cgt_incidence_clone", 0)), + (("spi_support_channel", 1), ("cgt_incidence_clone", 1)), + ( + ("spi_support_channel", 1), + ("cgt_incidence_clone", 1), + ("cgt_band_donors", 1), + ), + ] + original = pd.DataFrame( + { + "household_id": np.asarray([11, 22, 33, 44], dtype=np.int64), + "region": pd.array(["LONDON"] * 4, dtype="string"), + IDENTITY_COLUMN: pd.array( + [key(101, path) for path in paths], dtype="string" + ), + } + ) + assert original[IDENTITY_COLUMN].nunique() == 4 + payload = payloads() + expected, spec, supports = complete(original, payload) + changed = original.iloc[[3, 1, 0]].copy() + changed["household_weight"] = [0.0, 2.0, 8.0] + extra = pd.DataFrame( + { + "household_id": [55], + "region": pd.array(["LONDON"], dtype="string"), + IDENTITY_COLUMN: pd.array( + [key(202, (("geographic_support", 1),))], dtype="string" + ), + "household_weight": [1.0], + }, + index=[8], + ) + actual, _, _ = complete(pd.concat([changed, extra]), payload) + for column in expected: + if column != "region": + pd.testing.assert_series_equal( + actual.loc[changed.index, column], expected.loc[changed.index, column] + ) + assert geo.validate_geography(actual, spec, supports)["outcome"] == "pass" + assigned_columns = ["atomic_area_code", "oa_code", "constituency_code"] + pruned = expected.iloc[[3, 1]].copy() + assert geo.validate_geography(pruned, spec, supports)["outcome"] == "pass" + pd.testing.assert_frame_equal( + pruned[assigned_columns], expected.loc[pruned.index, assigned_columns] + ) + with pytest.raises(ValueError, match="assignment output already exists"): + geo.assign_atomic(expected, spec, supports) + + +def test_country_declaration_retains_the_two_stage_sampling_law(): + households = pd.DataFrame( + { + "household_id": np.arange(1, 2001, dtype=np.int64), + "region": pd.array(["LONDON"] * 2000, dtype="string"), + IDENTITY_COLUMN: pd.array([key(i) for i in range(1, 2001)], dtype="string"), + } + ) + result, spec, _ = complete(households, payloads()) + assert all( + s["stages"] + == [ + {"level": "constituency_code", "weight": "households"}, + {"level": "area", "weight": "population"}, + ] + for s in spec["systems"] + ) + arrays, _ = parts(SYSTEMS[0]) + london = arrays["region_code"] == FRS_REGION_TO_REGION_CODE["LONDON"] + areas = arrays["oa_code"][london] + shares = result["atomic_area_code"].value_counts(normalize=True) + # Constituent household mass 30:10, then within-first-area population 2:6; + # the third area's population 1000 must not dominate the first-stage draw. + for area, expected in zip(areas, (0.1875, 0.5625, 0.25), strict=True): + assert abs(shares.get(area, 0) - expected) < 0.04 + + +def test_exact_source_ids_and_ordered_branches_are_preserved(): + first = key(2**53 + 1, (("spi_support_channel", 1), ("cgt_incidence_clone", 0))) + second = key(2**53 + 2, (("spi_support_channel", 1), ("cgt_incidence_clone", 0))) + assert first != second + assert str(2**53 + 1) in first + assert first == key( + np.int64(2**53 + 1), + (("spi_support_channel", np.int64(1)), ("cgt_incidence_clone", 0)), + ) + grown = [key(101, (("geographic_support", i),)) for i in range(6)] + assert grown[:3] == [key(101, (("geographic_support", i),)) for i in range(3)] + + +@pytest.mark.parametrize( + "defect", + [ + "float_id", + "bool_id", + "zero_id", + "oversize_id", + "no_vintage", + "mutable_path", + "counter_branch", + "income_branch", + "max_id_branch", + "out_of_order", + "duplicate_branch", + "float_ordinal", + "negative_ordinal", + "oversize_ordinal", + ], +) +def test_identity_refuses_inferred_or_inexact_conventions(defect): + args = { + "source": "invented-frs", + "source_vintage": "invented-2024-25", + "source_household_id": 101, + "clone_path": (), + } + if defect == "float_id": + args["source_household_id"] = 101.0 + elif defect == "bool_id": + args["source_household_id"] = True + elif defect == "zero_id": + args["source_household_id"] = 0 + elif defect == "oversize_id": + args["source_household_id"] = 2**63 + elif defect == "no_vintage": + args["source_vintage"] = "" + elif defect == "mutable_path": + args["clone_path"] = [] + elif defect in {"counter_branch", "income_branch", "max_id_branch"}: + args["clone_path"] = ( + ( + { + "counter_branch": "row_counter", + "income_branch": "income", + "max_id_branch": "sample_max_id", + }[defect], + 1, + ), + ) + elif defect == "out_of_order": + args["clone_path"] = (("cgt_incidence_clone", 1), ("spi_support_channel", 1)) + elif defect == "duplicate_branch": + args["clone_path"] = (("spi_support_channel", 0), ("spi_support_channel", 1)) + else: + ordinal = { + "float_ordinal": 1.5, + "negative_ordinal": -1, + "oversize_ordinal": 2**32, + }[defect] + args["clone_path"] = (("spi_support_channel", ordinal),) + with pytest.raises(ValueError, match="UK geography identity:"): + household_draw_key(**args) + + +def test_duplicate_or_unknown_household_identity_is_refused_by_shared_assignment(): + payload = payloads() + spec = uk_atomic_assignment_definition(payload, seed=43) + supports = {s: geo.decode_atomic_support(p) for s, p in payload.items()} + households = pd.DataFrame( + { + "household_id": [11, 22], + "region": ["LONDON", "LONDON"], + IDENTITY_COLUMN: [key(101), key(101)], + } + ) + with pytest.raises(ValueError, match="duplicate draw identity"): + geo.assign_atomic(households, spec, supports) + households.loc[1, IDENTITY_COLUMN] = key(102) + households.loc[1, "region"] = "UNKNOWN" + with pytest.raises(ValueError, match="exactly one"): + geo.assign_atomic(households, spec, supports) diff --git a/packages/microcosm-build/tests/test_uk_atomic_household_lineage.py b/packages/microcosm-build/tests/test_uk_atomic_household_lineage.py new file mode 100644 index 000000000..ace6106f1 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_atomic_household_lineage.py @@ -0,0 +1,588 @@ +"""Supplied lineage and real invented Graph/Frame controls; no FRS or engine.""" + +import json +from dataclasses import FrozenInstanceError, replace + +import numpy as np +import pandas as pd +import pytest +from test_uk_atomic_area_support import payloads + +from microcosm.build import atomic_geography +from microcosm.build.graph_atomic_geography import ( + atomic_geography_nodes, + register_atomic_geography_kernels, +) +from microcosm.build.uk_runtime.atomic_area_support import ( + IDENTITY_COLUMN, + SOURCES, + uk_atomic_assignment_definition, +) +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.atomic_household_lineage import ( + HouseholdExpansion, + HouseholdSelection, + project_atomic_household_keys, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) + + +def roots(ids=(1, 2)): + return pd.DataFrame( + { + "household_id": np.asarray(ids, dtype=np.int64), + "source": pd.array(["invented-frs"] * len(ids), dtype="string"), + "source_vintage": pd.array(["invented-2024-25"] * len(ids), dtype="string"), + "source_household_id": np.asarray(ids, dtype=np.int64), + } + ) + + +def expansion(branch, before, pairs, ordinals=None): + return HouseholdExpansion( + branch=branch, + before_ids=tuple(before), + after_ids=(*before, *(c for c, _ in pairs)), + parent_pairs=tuple(pairs), + child_ordinals=tuple((c, 1) for c, _ in pairs) + if ordinals is None + else ordinals, + ) + + +def chain(): + a = expansion("spi_support_channel", (1, 2), ((11, 1), (12, 2))) + b = expansion("cgt_incidence_clone", a.after_ids, ((21, 1), (31, 11))) + c = expansion("cgt_band_donors", b.after_ids, ((41, 31),)) + s = HouseholdSelection(c.after_ids, (1, 11, 21, 31, 41)) + d = expansion("geographic_support", s.after_ids, ((101, 1), (141, 41))) + return a, b, c, s, d + + +def project(source=None, steps=None, ids=None): + source = roots() if source is None else source + steps = chain() if steps is None else steps + ids = steps[-1].after_ids if ids is None else ids + return project_atomic_household_keys( + source, steps=steps, final_ids=np.asarray(ids, dtype=np.int64) + ) + + +def by_id(table): + return table.set_index("household_id")[IDENTITY_COLUMN].sort_index() + + +def test_explicit_full_chain_selection_and_clone_keys(): + result = project() + assert tuple(result.household_id) == chain()[-1].after_ids + assert list(result) == ["household_id", IDENTITY_COLUMN] + assert result.household_id.dtype == np.dtype("int64") + assert result[IDENTITY_COLUMN].dtype == pd.StringDtype(storage="python") + expected_paths = { + 1: (), + 11: (("spi_support_channel", 1),), + 21: (("cgt_incidence_clone", 1),), + 31: (("spi_support_channel", 1), ("cgt_incidence_clone", 1)), + 41: ( + ("spi_support_channel", 1), + ("cgt_incidence_clone", 1), + ("cgt_band_donors", 1), + ), + 101: (("geographic_support", 1),), + 141: ( + ("spi_support_channel", 1), + ("cgt_incidence_clone", 1), + ("cgt_band_donors", 1), + ("geographic_support", 1), + ), + } + actual = by_id(result) + for target, path in expected_paths.items(): + assert actual[target] == household_draw_key( + source="invented-frs", + source_vintage="invented-2024-25", + source_household_id=1, + clone_path=path, + ) + assert actual.is_unique + + +def test_axes_and_pair_order_do_not_supply_identity(): + steps = tuple( + replace( + x, + before_ids=x.before_ids[::-1], + after_ids=x.after_ids[::-1], + **( + { + "parent_pairs": x.parent_pairs[::-1], + "child_ordinals": x.child_ordinals[::-1], + } + if isinstance(x, HouseholdExpansion) + else {} + ), + ) + for x in chain() + ) + result = project(roots().iloc[::-1], steps, steps[-1].after_ids) + pd.testing.assert_series_equal(by_id(result), by_id(project())) + + +def test_exact_large_ids_and_growth_preserve_existing_keys(): + big = 2**53 + 7 + first = expansion("geographic_support", (big,), ((big + 2, big),)) + more = expansion( + "geographic_support", + (big, big + 1), + ((big + 2, big), (big + 3, big + 1)), + ((big + 2, 1), (big + 3, 9)), + ) + small = project(roots((big,)), (first,)) + large = project(roots((big, big + 1)), (more,)) + pd.testing.assert_series_equal(by_id(small), by_id(large).loc[by_id(small).index]) + assert by_id(large).is_unique + assert str(big) in by_id(small).iloc[0] + + +def test_source_and_explicit_ordinal_change_keys_without_numeric_inference(): + expected = by_id(project()) + source = roots() + source.loc[0, "source_vintage"] = "different-vintage" + changed = by_id(project(source)) + assert (changed != expected).all() + steps = chain() + changed = by_id( + project( + steps=(*steps[:-1], replace(steps[-1], child_ordinals=((101, 2), (141, 8)))) + ) + ) + assert (changed.loc[[101, 141]] != expected.loc[[101, 141]]).all() + pd.testing.assert_series_equal(changed.drop([101, 141]), expected.drop([101, 141])) + + +def test_noop_and_empty_selection_are_explicit_and_detached(): + original = roots() + saved = original.copy(deep=True) + no_steps = project(original, (), (2, 1)) + assert no_steps.household_id.tolist() == [2, 1] + empty = project(original, (HouseholdSelection((1, 2), ()),), ()) + assert empty.empty and empty.household_id.dtype == np.dtype("int64") + no_steps.loc[0, IDENTITY_COLUMN] = "changed descriptive output" + pd.testing.assert_frame_equal(original, saved) + with pytest.raises(FrozenInstanceError): + chain()[0].branch = "changed" + + +@pytest.mark.parametrize( + "defect", + [ + "duplicate_id", + "float_id", + "bool_id", + "zero_id", + "duplicate_root_identity", + "missing_source", + "blank_source", + "extra_column", + "duplicate_column", + "invalid_original", + "empty_roots", + ], +) +def test_roots_refuse_unqualified_or_ambiguous_descriptions(defect): + source = roots() + if defect == "duplicate_id": + source.loc[1, "household_id"] = 1 + elif defect == "float_id": + source["household_id"] = source.household_id.astype(float) + elif defect == "bool_id": + source["household_id"] = [True, False] + elif defect == "zero_id": + source.loc[0, "household_id"] = 0 + elif defect == "duplicate_root_identity": + source.loc[1, "source_household_id"] = 1 + elif defect == "missing_source": + source.loc[0, "source"] = pd.NA + elif defect == "blank_source": + source.loc[0, "source_vintage"] = " " + elif defect == "extra_column": + source["guess"] = 1 + elif defect == "duplicate_column": + source.columns = ["household_id", "source", "source", "source_household_id"] + elif defect == "invalid_original": + source["source_household_id"] = [1.0, 2.0] + elif defect == "empty_roots": + source = source.iloc[:0] + with pytest.raises((TypeError, ValueError)): + project(source) + + +@pytest.mark.parametrize( + "defect", + [ + "unknown_branch", + "reordered_branch", + "repeated_branch", + "mutable_steps", + "mutable_axis", + "duplicate_before", + "duplicate_after", + "unexplained_before", + "deleted_incumbent", + "unknown_parent", + "null_parent", + "same_step_parent", + "missing_parent", + "duplicate_parent_target", + "unexplained_child", + "missing_ordinal", + "extra_ordinal", + "duplicate_ordinal", + "negative_ordinal", + "bool_ordinal", + "float_child", + "overflow_child", + "selection_arrival", + "implicit_selection", + "final_extra", + "final_duplicate", + ], +) +def test_invalid_steps_and_final_axis_refuse(defect): + steps = list(chain()) + first = steps[0] + final = steps[-1].after_ids + if defect == "unknown_branch": + steps[0] = replace(first, branch="income_guess") + elif defect == "reordered_branch": + steps[0] = replace(first, branch="geographic_support") + elif defect == "repeated_branch": + steps[1] = replace(steps[1], branch=first.branch) + elif defect == "mutable_steps": + with pytest.raises((TypeError, ValueError)): + project(steps=steps) + return + elif defect == "mutable_axis": + steps[0] = replace(first, before_ids=[1, 2]) + elif defect == "duplicate_before": + steps[0] = replace(first, before_ids=(1, 1, 2)) + elif defect == "duplicate_after": + steps[0] = replace(first, after_ids=(*first.after_ids, 11)) + elif defect == "unexplained_before": + steps[0] = replace(first, before_ids=(1, 3)) + elif defect == "deleted_incumbent": + steps[0] = replace(first, after_ids=(1, 11, 12)) + elif defect == "unknown_parent": + steps[0] = replace(first, parent_pairs=((11, 3), (12, 2))) + elif defect == "null_parent": + steps[0] = replace(first, parent_pairs=((11, None), (12, 2))) + elif defect == "same_step_parent": + steps[0] = replace(first, parent_pairs=((11, 1), (12, 11))) + elif defect == "missing_parent": + steps[0] = replace(first, parent_pairs=((11, 1),)) + elif defect == "duplicate_parent_target": + steps[0] = replace(first, parent_pairs=((11, 1), (11, 2))) + elif defect == "unexplained_child": + steps[0] = replace(first, after_ids=(*first.after_ids, 13)) + elif defect == "missing_ordinal": + steps[0] = replace(first, child_ordinals=((11, 1),)) + elif defect == "extra_ordinal": + steps[0] = replace(first, child_ordinals=(*first.child_ordinals, (99, 1))) + elif defect == "duplicate_ordinal": + steps[0] = replace(first, parent_pairs=((11, 1), (12, 1))) + elif defect == "negative_ordinal": + steps[0] = replace(first, child_ordinals=((11, -1), (12, 1))) + elif defect == "bool_ordinal": + steps[0] = replace(first, child_ordinals=((11, True), (12, 1))) + elif defect == "float_child": + steps[0] = replace(first, parent_pairs=((11.0, 1), (12, 2))) + elif defect == "overflow_child": + steps[0] = replace( + first, + after_ids=(1, 2, 11, 2**63), + parent_pairs=((11, 1), (2**63, 2)), + child_ordinals=((11, 1), (2**63, 1)), + ) + elif defect == "selection_arrival": + steps[3] = replace(steps[3], after_ids=(*steps[3].after_ids, 99)) + elif defect == "implicit_selection": + del steps[3] + elif defect == "final_extra": + final = (*final, 99) + elif defect == "final_duplicate": + final = (*final, final[0]) + with pytest.raises((TypeError, ValueError)): + project(steps=tuple(steps), ids=final) + + +class _FrameKernel(KernelBase): + ref = "invented.uk.lineage.frame@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def __init__(self, frame): + self.frame = frame + + def run(self, context): + return KernelResult(frame=self.frame) + + +class _ExpandKernel(KernelBase): + ref = "invented.uk.lineage.expand@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND + ) + + def run(self, context): + return KernelResult( + expand={ + e: pd.Series( + [1, 2, 3], + index=pd.Index([11, 12, 13], name=e + "_id"), + dtype="int64", + ) + for e in ("person", "benunit", "household") + }, + weights=Weights(np.full(6, 0.5), WeightKind.IMPORTANCE), + ) + + +class _SelectKernel(KernelBase): + ref = "invented.uk.lineage.select@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.FILTER + ) + + def run(self, context): + ids = context.tables["person"].person_id + return KernelResult( + keep=pd.Series( + ids.isin([1, 3, 11, 13]).to_numpy(), + index=pd.Index(ids, name="person_id"), + ) + ) + + +def test_actual_expand_filter_receipts_feed_shared_atomic_graph_and_replay(tmp_path): + ids = np.arange(1, 4, dtype=np.int64) + person = pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": [21, 33, 44], + } + ) + benunit = pd.DataFrame({"benunit_id": ids, "family_marker": [4, 5, 6]}) + household = pd.DataFrame( + { + "household_id": ids, + "region": pd.array( + ["LONDON", "SCOTLAND", "NORTHERN_IRELAND"], dtype="string" + ), + } + ) + frame = Frame( + {"person": person, "benunit": benunit, "household": household}, + EntitySchema(group_entities=("benunit", "household")), + {"household": Weights(np.ones(3), WeightKind.DESIGN)}, + pd.Series(["invented"] * 3, name="stratum"), + metadata={"note": "invented UK lineage"}, + ) + columns = ( + Owned("person", "age", "int64"), + Owned("benunit", "family_marker", "int64"), + Owned("household", "region", "string"), + ) + graph = compile_graph( + Graph( + "uk", + (SourceRef("invented", "raw-bytes-v1"),), + ( + Node( + "root", + _FrameKernel.ref, + structural=StructuralDelta.CREATE, + sources=("invented",), + outputs=columns, + ), + Node( + "expand", + _ExpandKernel.ref, + base="root", + structural=StructuralDelta.EXPAND, + mass="conserve", + params={ + "expand_cells": (), + "expand_weight_entity": "household", + "expand_weight_kind": "importance", + }, + ), + Node( + "select", + _SelectKernel.ref, + base="expand", + structural=StructuralDelta.FILTER, + mass="free", + inputs=(Slice("person", ("age",)),), + ), + ), + ) + ) + registry = KernelRegistry() + for k in (_FrameKernel(frame), _ExpandKernel(), _SelectKernel()): + registry.register(k) + source = tmp_path / "invented.txt" + source.write_text("invented rows; no external survey") + store = ContentStore(tmp_path / "lineage-store") + cold = run_graph(graph, sources={"invented": source}, store=store, kernels=registry) + replay = run_graph( + graph, + sources={"invented": source}, + store=store, + kernels=registry, + resume="require", + ) + assert all(n.hit for n in replay.nodes.values()) + pair_rows = cold.nodes["expand"].receipt["expand"]["household"] + assert pair_rows == replay.nodes["expand"].receipt["expand"]["household"] + expanded = cold.population("expand") + selected = cold.population("select") + steps = ( + HouseholdExpansion( + "spi_support_channel", + tuple(ids), + tuple(expanded.table("household").household_id), + tuple(tuple(p) for p in pair_rows), + ((11, 1), (12, 1), (13, 1)), + ), + HouseholdSelection( + tuple(expanded.table("household").household_id), + tuple(selected.table("household").household_id), + ), + ) + keys = project( + roots((1, 2, 3)), steps, tuple(selected.table("household").household_id) + ) + selected_tables = {e: selected.table(e).copy(deep=True) for e in selected.entities} + selected_tables["household"][IDENTITY_COLUMN] = keys[IDENTITY_COLUMN].array.copy() + keyed = Frame( + selected_tables, + selected.schema, + {"household": selected.weights_for("household")}, + selected.strata, + metadata=selected.metadata, + mass_log=selected.mass_log, + ) + support_payloads = payloads() + definition = uk_atomic_assignment_definition(support_payloads, seed=43) + all_columns = (*columns, Owned("household", IDENTITY_COLUMN, "string")) + atomic_nodes = atomic_geography_nodes( + definition, all_columns, base="keyed", emit_validation_artifact=True + ) + assignment_graph = compile_graph( + Graph( + "uk", + ( + SourceRef("invented", "raw-bytes-v1"), + *(SourceRef(SOURCES[s], "raw-bytes-v1") for s in support_payloads), + ), + ( + Node( + "keyed", + _FrameKernel.ref, + structural=StructuralDelta.CREATE, + sources=("invented",), + outputs=all_columns, + ), + *atomic_nodes, + ), + ) + ) + source_paths = {"invented": source} + for system, payload in support_payloads.items(): + path = tmp_path / (system + ".npz") + path.write_bytes(payload) + source_paths[SOURCES[system]] = path + atomic_registry = KernelRegistry() + atomic_registry.register(_FrameKernel(keyed)) + register_atomic_geography_kernels(atomic_registry) + atomic_store = ContentStore(tmp_path / "atomic-store") + assigned = run_graph( + assignment_graph, + sources=source_paths, + store=atomic_store, + kernels=atomic_registry, + ) + required = run_graph( + assignment_graph, + sources=source_paths, + store=atomic_store, + kernels=atomic_registry, + resume="require", + ) + assert len(assigned.nodes) == 7 and all(n.hit for n in required.nodes.values()) + actual = assigned.population("keyed") + for entity in keyed.entities: + pd.testing.assert_frame_equal( + actual.table(entity)[list(keyed.table(entity))], + keyed.table(entity), + check_index_type=False, + ) + np.testing.assert_array_equal( + actual.weights_for("household").values, keyed.weights_for("household").values + ) + pd.testing.assert_series_equal(actual.strata, keyed.strata, check_index_type=False) + assert actual.metadata == keyed.metadata and actual.mass_log == keyed.mass_log + assert assigned.nodes["geography.gate"].receipt["outcome"] == "pass" + supports = { + k: atomic_geography.decode_atomic_support(v) + for k, v in support_payloads.items() + } + reordered = keyed.table("household").iloc[::-1] + expected = atomic_geography.assign_atomic(reordered, definition, supports) + for c in expected: + pd.testing.assert_series_equal( + expected[c].astype(pd.StringDtype(storage="python")).reset_index(drop=True), + actual.table("household")[c] + .iloc[::-1] + .astype(pd.StringDtype(storage="python")) + .reset_index(drop=True), + check_names=False, + ) + + (tmp_path / "acceptance.json").write_text( + json.dumps( + { + "scope": "invented actual shared CREATE/EXPAND/FILTER and atomic graph; pure descriptive lineage helper", + "lineage_nodes": len(cold.nodes), + "lineage_required_hits": sum(n.hit for n in replay.nodes.values()), + "atomic_nodes": len(assigned.nodes), + "atomic_required_hits": sum(n.hit for n in required.nodes.values()), + "root_households": frame.n("household"), + "expanded_households": expanded.n("household"), + "selected_households": selected.n("household"), + "source_admission": False, + "native_inputs": False, + }, + indent=2, + ) + + "\n" + ) diff --git a/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py b/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py new file mode 100644 index 000000000..90f57fdf8 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py @@ -0,0 +1,138 @@ +"""Normal-import contracts for the real UK runtime public package. + +A fresh no-country-resource guard supplies the cold-import check. In a larger +suite, already imported country modules are allowed; these tests never remove, +stub or replace modules to manufacture an isolated import environment. +""" + +import hashlib +import importlib +import json +import sys + +import pytest + +_PACKAGE = "microcosm.build.uk_runtime" +_UNRELATED_MODULES = ( + f"{_PACKAGE}.calibration_run", + f"{_PACKAGE}.national_calibration", + f"{_PACKAGE}.firm_generation", +) +# SHA256 of compact JSON for all 456 ordered entries from ec3b306f3f9742ff... +# This includes the existing duplicate ladder_vs_chronicle_household_dispersion. +_ORDERED_ALL_SHA256 = "282baaa8103b6a68ec916669dc9b96909d6faf0a9beecdbb8359df12f128f66c" + + +def _unrelated_modules() -> set[str]: + return { + name + for name in sys.modules + if any( + name == prefix or name.startswith(prefix + ".") + for prefix in _UNRELATED_MODULES + ) + } + + +def test_normal_package_and_pure_helper_imports_do_not_load_unrelated_modules(): + before = _unrelated_modules() + package = importlib.import_module(_PACKAGE) + helper = importlib.import_module(f"{_PACKAGE}.rowwise_geography") + + assert sys.modules[_PACKAGE] is package + assert package.rowwise_geography is helper + assert callable(helper.assign_household_geography) + assert _unrelated_modules() == before + + +def test_public_from_import_returns_and_caches_the_real_defining_object(): + from microcosm.build.uk_runtime import assign_household_geography + from microcosm.build.uk_runtime.rowwise_geography import ( + assign_household_geography as direct, + ) + + package = importlib.import_module(_PACKAGE) + assert assign_household_geography is direct + assert package.assign_household_geography is direct + assert vars(package)["assign_household_geography"] is direct + assert package.assign_household_geography is assign_household_geography + + +def test_direct_submodule_import_and_from_import_preserve_module_identity(): + # In the fresh guard this exercises Python's fallback before any direct + # content_identity import, without removing an already loaded module in CI. + from microcosm.build.uk_runtime import content_identity + + assert content_identity is sys.modules[f"{_PACKAGE}.content_identity"] + + import microcosm.build.uk_runtime.content_identity as content_direct + import microcosm.build.uk_runtime.rowwise_geography as direct + from microcosm.build.uk_runtime import rowwise_geography + + package = importlib.import_module(_PACKAGE) + assert content_identity is content_direct + assert content_direct is sys.modules[f"{_PACKAGE}.content_identity"] + assert package.content_identity is content_direct + assert "content_identity" not in package.__all__ + assert rowwise_geography is direct + assert direct is sys.modules[f"{_PACKAGE}.rowwise_geography"] + assert package.rowwise_geography is direct + assert "rowwise_geography" not in package.__all__ + + +def test_discovery_preserves_the_frozen_ordered_export_contract_without_resolution(): + package = importlib.import_module(_PACKAGE) + imports_before = _unrelated_modules() + aliases_before = { + name: vars(package)[name] for name in package.__all__ if name in vars(package) + } + + discovered = dir(package) + + assert len(package.__all__) == 456 + assert len(set(package.__all__)) == 455 + assert package.__all__.count("ladder_vs_chronicle_household_dispersion") == 2 + encoded = json.dumps(package.__all__, separators=(",", ":")).encode() + assert hashlib.sha256(encoded).hexdigest() == _ORDERED_ALL_SHA256 + assert set(package.__all__).issubset(discovered) + assert discovered == sorted(set(discovered)) + aliases_after = { + name: vars(package)[name] for name in package.__all__ if name in vars(package) + } + assert aliases_after.keys() == aliases_before.keys() + assert all(aliases_after[name] is value for name, value in aliases_before.items()) + assert _unrelated_modules() == imports_before + with pytest.raises(TypeError): + package._EXPORTS["invented_missing_export"] = ("invented", "missing") + + +def test_unknown_attribute_and_unknown_from_import_keep_normal_failures(): + package = importlib.import_module(_PACKAGE) + with pytest.raises( + AttributeError, match="has no attribute 'invented_missing_export'" + ): + _ = package.invented_missing_export + with pytest.raises( + ImportError, match="cannot import name 'invented_missing_export'" + ): + from microcosm.build.uk_runtime import invented_missing_export # noqa: F401 + + assert "invented_missing_export" not in vars(package) + + +def test_reload_clears_public_aliases_and_rebinds_the_same_defining_objects(): + package = importlib.import_module(_PACKAGE) + helper = importlib.import_module(f"{_PACKAGE}.rowwise_geography") + real_function = helper.assign_household_geography + assert package.assign_household_geography is real_function + assert "assign_household_geography" in vars(package) + imports_before = _unrelated_modules() + + assert importlib.reload(package) is package + + assert "assign_household_geography" not in vars(package) + assert package.rowwise_geography is helper + assert sys.modules[f"{_PACKAGE}.rowwise_geography"] is helper + assert package.assign_household_geography is real_function + assert vars(package)["assign_household_geography"] is real_function + assert _unrelated_modules() == imports_before diff --git a/packages/microcosm-build/tests/test_us_acs_housing_source.py b/packages/microcosm-build/tests/test_us_acs_housing_source.py new file mode 100644 index 000000000..eb5a461c9 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_housing_source.py @@ -0,0 +1,2634 @@ +"""Independent invented-source tests for the approved ACS housing-unit source bridge. + +Written against the approved v2 plan contract, not against the production +module: no byte of ``acs_housing_universe_source.py`` or of the packaged +``acs_2024_housing_universe.json`` was read while authoring this file, so a +failure here is evidence about the contract rather than a restatement of the +implementation. + +Every archive, member, header, identifier and row below is wholly invented. No +genuine ACS/Census archive, candidate, target or dictionary file is opened, +statted or referenced. Nothing here authenticates real data, and nothing here +makes a scientific claim: the fixtures exercise the closed source rules only. + +Scope is the source producer/readback pair. ``prepare_acs_housing_population`` +builds a Frame and is deliberately left to the production author's own +Frame/graph tests. + +Two reconciliation seams are collected in one place each, so integrating this +file with the finished module is a rename exercise and never a semantic one: + +* ``NAMES`` - column, constant, filename and definition-name spellings. +* ``EXPECTED_REASONS`` - exact refusal reason codes, asserted only once + ``REASON_CODES_AGREED`` is flipped to True after agreeing them with the + production author. Until then every refusal test still asserts the exception + type, message sanitisation, and that semantically distinct causes carry + distinct codes, which is the part that does not need agreement. +""" + +import csv +import hashlib +import io +import json +import os +import re +import signal +import stat +import struct +import tempfile +import zipfile +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +# The production import must surface as a loud collection error naming the +# module while it is absent. It must never skip: no engine is involved here. +from microcosm.build.us_runtime import acs_housing_universe_source as source +from microcosm.build.us_runtime.acs_housing_universe import ( + classify_acs_housing_universe, +) + +# -------------------------------------------------------------------------- +# Reconciliation seam 1: identifier spellings. +# -------------------------------------------------------------------------- + +NAMES = { + # Frame columns carrying the archive-member/row lineage of each row. + "member_column": "source_member", + "ordinal_column": "source_row_ordinal", + # The normalised two-digit state column exposed on the household frame. + # ``_state_series`` falls back to the raw ST/STATE header aliases, so only + # the dedicated naming test depends on this spelling. + "state_column": "ST", + # Files written into the source artifact directory. + "projection_filename": "projection.json", + "receipt_filename": "receipt.json", + # Named source definitions bound by the artifact receipt. + "vacancy_definition": "acs_2024_source_vacancy_v1", + "gq_definition": "acs_2024_gq_person_placeholder_design_v1", + # Module byte caps, monkeypatched tiny so no test allocates real GiBs. + "source_max_bytes": "ACS_HU_SOURCE_MAX_BYTES", + "receipt_max_bytes": "ACS_HU_RECEIPT_MAX_BYTES", + # The private archive pin tuple; entries are (role, filename, sha256, size). + "archive_pins": "_ARCHIVE_PINS", + "household_role": "household", + "person_role": "person", +} + +HOUSEHOLD_ARCHIVE = "csv_hus.zip" +PERSON_ARCHIVE = "csv_pus.zip" + +# -------------------------------------------------------------------------- +# Reconciliation seam 2: refusal reason codes. +# -------------------------------------------------------------------------- + +REASON_CODES_AGREED = True + +EXPECTED_REASONS: dict[str, str] = { + "artifact_forgery": "RECONSTRUCTION_MISMATCH", + "capture_digest": "SOURCE_SHA256", + "csv_record": "CSV_ROW_WIDTH", + "header_contract": "CSV_STATE_REQUIRED", + "join_integrity": "ORPHAN_PERSON", + "output_precondition": "OUTPUT_EXISTS", + "selection": "SELECTION_UNKNOWN", + "value_domain": "TEN", + "zip_preflight": "ZIP_DUPLICATE_MEMBER", +} + +# Coarse refusal families that must not collapse onto one reason code. +DISTINCT_FAMILIES = ( + "capture_digest", + "zip_preflight", + "csv_record", + "header_contract", + "value_domain", + "join_integrity", + "selection", + "artifact_forgery", + "output_precondition", +) + +# -------------------------------------------------------------------------- +# Invented source bytes. +# -------------------------------------------------------------------------- + +HOUSEHOLD_HEADER = ( + "RT", + "SERIALNO", + "DIVISION", + "PUMA", + "REGION", + "ST", + "ADJHSG", + "ADJINC", + "WGTP", + "NP", + "TYPEHUGQ", + "ACR", + "BDSP", + "TEN", + "VALP", + "HINCP", +) + +PERSON_HEADER = ( + "RT", + "SERIALNO", + "SPORDER", + "PUMA", + "ST", + "ADJINC", + "PWGTP", + "AGEP", + "SEX", + "WAGP", +) + +HOUSEHOLD_DEFAULTS = { + "RT": "H", + "DIVISION": "9", + "PUMA": "00100", + "REGION": "4", + "ST": "06", + "ADJHSG": "1000000", + "ADJINC": "1010000", + "ACR": "1", + "BDSP": "3", + "VALP": "410000", + "HINCP": "82000", +} + +PERSON_DEFAULTS = { + "RT": "P", + "PUMA": "00100", + "ST": "06", + "ADJINC": "1010000", + "AGEP": "41", + "SEX": "1", + "WAGP": "52000", +} + +# Projected observation rosters, per the closed plan. +HOUSEHOLD_ROSTER = ("SERIALNO", "TYPEHUGQ", "NP", "TEN", "WGTP", "PUMA") +PERSON_ROSTER = ("SERIALNO", "SPORDER", "PWGTP") + +# Fixed classifier output order, read from the pinned pure module. +CODE_COLUMNS = ( + "interview_scope", + "physical_unit", + "household_kind", + "tenure_subtype", + "occupied_hu", + "hu_tenure_class", + "unresolved_reasons", + "TEN_valid", +) + + +def household_row(serialno, typehugq, persons, ten, wgtp, **overrides): + """One invented household record as an ordered tuple of lexical tokens.""" + values = dict(HOUSEHOLD_DEFAULTS) + values.update(SERIALNO=serialno, TYPEHUGQ=typehugq, NP=persons, TEN=ten, WGTP=wgtp) + values.update(overrides) + return tuple(values[name] for name in HOUSEHOLD_HEADER) + + +def person_row(serialno, sporder, pwgtp, **overrides): + """One invented person record as an ordered tuple of lexical tokens.""" + values = dict(PERSON_DEFAULTS) + values.update(SERIALNO=serialno, SPORDER=sporder, PWGTP=pwgtp) + values.update(overrides) + return tuple(values[name] for name in PERSON_HEADER) + + +# Eight invented households covering every accepted source class exactly once, +# plus both WGTP boundaries and the whole observed TEN domain. +BASE_HOUSEHOLDS = ( + # occupied owner with a mortgage, two people + household_row("2024HU0000001", "1", "2", "1", "118", PUMA="00100", ST="06"), + # occupied renter paying rent, one person + household_row("2024HU0000002", "1", "1", "3", "76", PUMA="03701", ST="06"), + # occupied housing unit whose tenure is not available, three people + household_row("2024HU0000003", "1", "3", "", "1", PUMA="81003", ST="36"), + # vacant housing unit: no people, blank tenure, positive weight + household_row("2024HU0000004", "1", "0", "", "55", PUMA="00100", ST="11"), + # institutional group quarters placeholder + household_row("2024GQ0000005", "2", "1", "", "0", PUMA="03701", ST="36"), + # noninstitutional group quarters placeholder + household_row("2024GQ0000006", "3", "1", "", "0", PUMA="00100", ST="01"), + # occupied owner outright + household_row("2024HU0000007", "1", "1", "2", "200", PUMA="00100", ST="01"), + # occupied renter paying no rent, maximum housing weight + household_row("2024HU0000008", "1", "1", "4", "9999", PUMA="81003", ST="11"), +) + +BASE_PERSONS = ( + person_row("2024HU0000001", "01", "121", WAGP="52000"), + person_row("2024HU0000001", "02", "97", WAGP="0"), + person_row("2024HU0000002", "1", "80", WAGP="31000"), + person_row("2024HU0000003", "1", "12", WAGP="17000"), + person_row("2024HU0000003", "2", "11", WAGP="0"), + person_row("2024HU0000003", "3", "9", WAGP="0"), + person_row("2024GQ0000005", "01", "64", WAGP="0"), + person_row("2024GQ0000006", "1", "9999", WAGP="7000"), + person_row("2024HU0000007", "01", "205", WAGP="99000"), + person_row("2024HU0000008", "1", "1", WAGP="4000"), +) + +# The base population is split across two members per role so that member and +# ordinal lineage is exercised by every ordinary test rather than by one. +BASE_HOUSEHOLD_MEMBERS = ( + ("psam_husa.csv", BASE_HOUSEHOLDS[:5]), + ("psam_husb.csv", BASE_HOUSEHOLDS[5:]), +) +BASE_PERSON_MEMBERS = ( + ("psam_pusa.csv", BASE_PERSONS[:6]), + ("psam_pusb.csv", BASE_PERSONS[6:]), +) + +BASE_SERIALNOS = tuple( + row[HOUSEHOLD_HEADER.index("SERIALNO")] for row in BASE_HOUSEHOLDS +) + +# The plan fixes what each row contains, never the order rows are emitted in. +# Content assertions below are therefore keyed by identifier, and the ordering +# decision lives in exactly one place: this constant and the two tests that +# read it. "sorted_serialno" is the order the production module emits; +# "source" would be member-then-row-ordinal order. See the author report. +CANONICAL_ROW_ORDER = "sorted_serialno" + + +def expected_row_order(serialnos): + if CANONICAL_ROW_ORDER == "sorted_serialno": + return sorted(serialnos) + if CANONICAL_ROW_ORDER == "source": + return list(serialnos) + raise AssertionError(f"unknown row-order contract {CANONICAL_ROW_ORDER!r}") + + +# (occupied_hu, hu_tenure_class, tenure_subtype, unresolved_reasons, +# physical_unit, TEN_valid) recomputed by hand from the pinned classifier. +BASE_EXPECTED_CODES = { + "interview_scope": [0, 0, 0, 0, 0, 0, 0, 0], + "physical_unit": [1, 1, 1, 1, 2, 2, 1, 1], + "household_kind": [0, 0, 0, 0, 0, 0, 0, 0], + "tenure_subtype": [1, 3, 0, 0, 0, 0, 2, 4], + "occupied_hu": [1, 1, 1, 0, 2, 2, 1, 1], + "hu_tenure_class": [1, 2, 3, 0, 0, 0, 1, 2], + "unresolved_reasons": [0, 0, 2, 1, 0, 0, 0, 0], + "TEN_valid": [1, 1, 0, 0, 0, 0, 1, 1], +} + + +# -------------------------------------------------------------------------- +# Building invented archives. +# -------------------------------------------------------------------------- + + +EXPECTED_HOUSEHOLDS = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: { + name: row[HOUSEHOLD_HEADER.index(name)] for name in HOUSEHOLD_ROSTER + ("ST",) + } + for row in BASE_HOUSEHOLDS +} + +EXPECTED_PERSONS = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): row[PERSON_HEADER.index("PWGTP")] + for row in BASE_PERSONS +} + +EXPECTED_CODES = { + serialno: tuple(BASE_EXPECTED_CODES[name][index] for name in CODE_COLUMNS) + for index, serialno in enumerate(BASE_SERIALNOS) +} + +EXPECTED_HOUSEHOLD_MEMBER = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: member + for member, rows in BASE_HOUSEHOLD_MEMBERS + for row in rows +} + +EXPECTED_PERSON_MEMBER = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): member + for member, rows in BASE_PERSON_MEMBERS + for row in rows +} + + +def csv_bytes(header, rows, *, newline="\n", trailing_newline=True): + """Serialise invented tokens without any quoting the fixtures do not ask for.""" + buffer = io.StringIO(newline="") + writer = csv.writer(buffer, lineterminator=newline, quoting=csv.QUOTE_MINIMAL) + writer.writerow(header) + writer.writerows(rows) + text = buffer.getvalue() + if not trailing_newline and text.endswith(newline): + text = text[: -len(newline)] + return text.encode("utf-8") + + +@dataclass(frozen=True) +class Member: + """One planned archive entry, data or auxiliary.""" + + name: str + data: bytes + compress_type: int = zipfile.ZIP_DEFLATED + external_attr: int | None = None + create_system: int = 3 + + +def data_members(header, members, *, newline="\n", **kwargs): + return tuple( + Member(name, csv_bytes(header, rows, newline=newline, **kwargs)) + for name, rows in members + ) + + +def write_archive(path, members): + """Write invented members exactly as planned, preserving order and duplicates.""" + with zipfile.ZipFile(path, "w") as archive: + for member in members: + info = zipfile.ZipInfo(member.name, date_time=(2026, 1, 1, 0, 0, 0)) + info.compress_type = member.compress_type + info.create_system = member.create_system + info.external_attr = ( + member.external_attr + if member.external_attr is not None + else (0o644 << 16) + ) + archive.writestr(info, member.data) + return path + + +def flip_archive_byte(path, offset=None): + """Change one compressed byte in place: same size, different digest. + + A deflated archive holds no plaintext, so a token replacement here would + silently do nothing and the test would stop testing what it claims. + """ + payload = Path(path).read_bytes() + index = len(payload) // 2 if offset is None else offset + mutated = payload[:index] + bytes([payload[index] ^ 0xFF]) + payload[index + 1 :] + assert len(mutated) == len(payload) and mutated != payload + Path(path).write_bytes(mutated) + return payload + + +def digest_and_size(path): + payload = Path(path).read_bytes() + return hashlib.sha256(payload).hexdigest(), len(payload) + + +@dataclass +class Fixture: + """An invented source directory plus everything a test needs to drive it.""" + + tmp_path: Path + source_dir: Path + snapshot_root: Path + monkeypatch: pytest.MonkeyPatch + households: tuple = () + persons: tuple = () + secrets: list = field(default_factory=list) + _output_index: int = 0 + + @property + def household_zip(self): + return self.source_dir / HOUSEHOLD_ARCHIVE + + @property + def person_zip(self): + return self.source_dir / PERSON_ARCHIVE + + def pin(self): + """Point the private pin tuple at the invented archives as they are now.""" + pins = [] + for role, name in ( + (NAMES["household_role"], HOUSEHOLD_ARCHIVE), + (NAMES["person_role"], PERSON_ARCHIVE), + ): + path = self.source_dir / name + sha, size = digest_and_size(path) + pins.append((role, name, sha, size)) + self.monkeypatch.setattr(source, NAMES["archive_pins"], tuple(pins)) + return tuple(pins) + + def output_dir(self, label="artifact"): + self._output_index += 1 + return self.tmp_path / f"out-{self._output_index}-{label}" + + def produce(self, *, output_dir=None, serialnos=None, source_dir=None): + return source.produce_acs_housing_source( + self.source_dir if source_dir is None else source_dir, + snapshot_root=self.snapshot_root, + output_dir=self.output_dir() if output_dir is None else output_dir, + serialnos=serialnos, + ) + + def load(self, output_dir, *, serialnos=None, source_dir=None): + return source.load_acs_housing_source( + self.source_dir if source_dir is None else source_dir, + output_dir, + snapshot_root=self.snapshot_root, + serialnos=serialnos, + ) + + def produce_and_load(self, *, serialnos=None): + output_dir = self.output_dir() + produced = self.produce(output_dir=output_dir, serialnos=serialnos) + return produced, self.load(output_dir, serialnos=serialnos), output_dir + + +def build_fixture( + tmp_path, + monkeypatch, + *, + households=BASE_HOUSEHOLD_MEMBERS, + persons=BASE_PERSON_MEMBERS, + household_members=None, + person_members=None, + household_header=HOUSEHOLD_HEADER, + person_header=PERSON_HEADER, + newline="\n", + pin=True, +): + """Assemble an invented source directory; explicit members override the rows.""" + # Resolve first: on macOS pytest's tmp_path sits under /var, a symlink to + # /private/var, and the capture refuses symlinked path components. The + # deliberate symlink refusals below build their own links explicitly. + tmp_path = Path(tmp_path).resolve() + source_dir = Path(tmp_path) / "acs-source" + source_dir.mkdir(parents=True, exist_ok=True) + snapshot_root = Path(tmp_path) / "snapshots" + snapshot_root.mkdir(parents=True, exist_ok=True) + + if household_members is None: + household_members = data_members(household_header, households, newline=newline) + if person_members is None: + person_members = data_members(person_header, persons, newline=newline) + + write_archive(source_dir / HOUSEHOLD_ARCHIVE, household_members) + write_archive(source_dir / PERSON_ARCHIVE, person_members) + + fixture = Fixture( + tmp_path=Path(tmp_path), + source_dir=source_dir, + snapshot_root=snapshot_root, + monkeypatch=monkeypatch, + households=households, + persons=persons, + ) + fixture.secrets = sanitisation_secrets(fixture) + if pin: + fixture.pin() + return fixture + + +def sanitisation_secrets(fixture): + """Strings a sanitised refusal must never echo back to the caller.""" + secrets = [ + str(fixture.tmp_path), + str(fixture.source_dir), + str(fixture.snapshot_root), + ] + secrets.extend(BASE_SERIALNOS) + # Unprojected invented money tokens: no legitimate reason code names these, + # unlike the domain bounds (1..9999, 100..81003) a code may legitimately cite. + secrets.extend(["82000", "52000", "410000", "31000", "99000", "17000"]) + return secrets + + +@pytest.fixture(autouse=True) +def _invented_disk_budget(monkeypatch): + """Eight invented rows must not require a full-source disk reserve in CI.""" + monkeypatch.setattr( + source.shutil, "disk_usage", lambda _path: SimpleNamespace(free=64 * 1024**3) + ) + + +@pytest.fixture +def acs(tmp_path, monkeypatch): + """The plain accepted invented population, pinned and ready to produce.""" + return build_fixture(tmp_path, monkeypatch) + + +# -------------------------------------------------------------------------- +# Refusal helpers. +# -------------------------------------------------------------------------- + + +def refuses(fixture, call, *args, **kwargs): + """Assert the exact refusal type, capture the reason, and check sanitisation.""" + with pytest.raises(source.ACSHousingSourceError) as caught: + call(*args, **kwargs) + reason = str(caught.value) + assert type(caught.value) is source.ACSHousingSourceError + assert reason, "a refusal must carry a reason" + assert len(reason) <= 120, "a reason code must not be prose" + assert "\n" not in reason + for secret in fixture.secrets: + assert secret not in reason, "refusal leaked invented source content" + return reason + + +def flat_scalars(payload): + """Every scalar reachable in a decoded receipt, for shape-tolerant assertions.""" + found = [] + stack = [payload] + while stack: + item = stack.pop() + if isinstance(item, dict): + stack.extend(item.keys()) + stack.extend(item.values()) + elif isinstance(item, (list, tuple)): + stack.extend(item) + else: + found.append(item) + return found + + +def flat_strings(payload): + return {item for item in flat_scalars(payload) if isinstance(item, str)} + + +def flat_numbers(payload): + return { + item + for item in flat_scalars(payload) + if isinstance(item, int) and not isinstance(item, bool) + } + + +def state_series(households): + """Read the state column under whichever spelling the module exposes.""" + for name in (NAMES["state_column"], "ST", "STATE"): + if name in households.columns: + return households[name] + raise AssertionError("household frame exposes no state column") + + +def household_keys(households): + keys = lexical(households["SERIALNO"]) + assert len(set(keys)) == len(keys), "household identifiers must be unique" + return keys + + +def household_map(households, names=HOUSEHOLD_ROSTER + ("ST",)): + """Every projected household token, keyed by identifier rather than by row.""" + keys = household_keys(households) + columns = { + name: lexical(state_series(households) if name == "ST" else households[name]) + for name in names + } + return { + key: {name: columns[name][row] for name in names} + for row, key in enumerate(keys) + } + + +def person_map(persons): + """Person weight keyed by identifier and the raw SPORDER token.""" + entries = list( + zip( + lexical(persons["SERIALNO"]), + lexical(persons["SPORDER"]), + lexical(persons["PWGTP"]), + strict=True, + ) + ) + keyed = {(serialno, sporder): weight for serialno, sporder, weight in entries} + assert len(keyed) == len(entries), "person keys must be unique" + return keyed + + +def codes_map(result): + """The eight native codes as a tuple per household identifier.""" + keys = household_keys(result.households) + codes = result.codes + return { + key: tuple(int(codes[name].iloc[row]) for name in CODE_COLUMNS) + for row, key in enumerate(keys) + } + + +def lexical(series): + """A projected column must be lexical text with no NA conversion.""" + values = series.tolist() + assert all(isinstance(value, str) for value in values), ( + f"{series.name} is not lexical: {[type(v).__name__ for v in values]}" + ) + return values + + +def strict_arrays(households): + """Rebuild the classifier inputs from the projected lexical household tokens.""" + tenure_tokens = lexical(households["TEN"]) + tenure_valid = np.asarray([token != "" for token in tenure_tokens], dtype=bool) + frame = pd.DataFrame( + { + "TYPEHUGQ": np.asarray( + [int(token) for token in lexical(households["TYPEHUGQ"])], dtype="int64" + ), + "NP": np.asarray( + [int(token) for token in lexical(households["NP"])], dtype="int64" + ), + "TEN": np.asarray( + [int(token) if token != "" else 0 for token in tenure_tokens], + dtype="int64", + ), + }, + index=households.index, + ) + return frame, tenure_valid + + +def assert_lineage(frame, keys, expected_member): + """Find member/ordinal lineage without depending on either column spelling. + + Order-independent by construction: the member column must agree row by row + with the invented layout, and each member's ordinals must form one + consecutive run, whatever order the rows are emitted in. + """ + members = [expected_member[key] for key in keys] + member_column = next( + ( + name + for name in frame.columns + if [value if isinstance(value, str) else None for value in frame[name]] + == members + ), + None, + ) + assert member_column is not None, ( + "no column carries the archive member of each row; expected " + f"{sorted(set(members))}" + ) + + for name in frame.columns: + if name == member_column: + continue + try: + values = [int(value) for value in frame[name].tolist()] + except (TypeError, ValueError): + continue + if len(values) != len(members): + continue + runs = {} + for member, value in zip(members, values, strict=True): + runs.setdefault(member, []).append(value) + bases = {min(run) for run in runs.values()} + if bases <= {0} or bases <= {1}: + if all( + sorted(run) == list(range(min(run), min(run) + len(run))) + for run in runs.values() + ): + return member_column, name + raise AssertionError( + "no column carries a within-member row ordinal restarting at each member" + ) + + +# -------------------------------------------------------------------------- +# Accepted population: projection, classification and roundtrip. +# -------------------------------------------------------------------------- + + +def test_projected_households_are_lexical_and_complete(acs): + result = acs.produce() + households = result.households + assert len(households) == len(BASE_HOUSEHOLDS) + assert households.columns.is_unique + assert set(HOUSEHOLD_ROSTER) <= set(households.columns) + for name in HOUSEHOLD_ROSTER: + lexical(households[name]) + # Every projected token survives byte for byte: blank tenure stays an empty + # string rather than NaN, None or the dictionary's display character, and + # leading zeros on PUMA and ST are preserved. + assert household_map(households) == EXPECTED_HOUSEHOLDS + + +def test_projected_persons_keep_raw_sporder_tokens(acs): + persons = acs.produce().persons + assert len(persons) == len(BASE_PERSONS) + assert set(PERSON_ROSTER) <= set(persons.columns) + # "01" and "1" are both accepted and both retained exactly as written. + assert person_map(persons) == EXPECTED_PERSONS + + +def test_native_codes_are_uint8_in_the_pinned_order(acs): + result = acs.produce() + codes = result.codes + assert list(codes.columns) == list(CODE_COLUMNS) + for name in CODE_COLUMNS: + assert codes[name].dtype == np.dtype("uint8") + assert codes.index.equals(result.households.index) + assert codes_map(result) == EXPECTED_CODES + + +def test_codes_equal_the_pinned_pure_classifier_on_the_strict_arrays(acs): + """The source result must not invent its own semantics for the same rows.""" + result = acs.produce() + frame, tenure_valid = strict_arrays(result.households) + expected, _header = classify_acs_housing_universe(frame, tenure_valid=tenure_valid) + pd.testing.assert_frame_equal(result.codes, expected, check_like=False) + + +def test_np0_housing_unit_stays_occupancy_unknown_in_the_native_codes(acs): + """Source vacancy evidence must not be back-written into the pure classes.""" + codes = dict( + zip(CODE_COLUMNS, codes_map(acs.produce())["2024HU0000004"], strict=True) + ) + assert codes["occupied_hu"] == 0 + assert codes["hu_tenure_class"] == 0 + assert codes["unresolved_reasons"] == 1 + assert codes["physical_unit"] == 1 + assert codes["TEN_valid"] == 0 + + +def test_occupied_unit_with_unavailable_tenure_is_retained_not_recoded(acs): + codes = dict( + zip(CODE_COLUMNS, codes_map(acs.produce())["2024HU0000003"], strict=True) + ) + assert codes["occupied_hu"] == 1 + assert codes["hu_tenure_class"] == 3 + assert codes["tenure_subtype"] == 0 + assert codes["unresolved_reasons"] == 2 + + +def test_member_and_row_ordinal_lineage_is_accessible(acs): + result = acs.produce() + assert_lineage( + result.households, + household_keys(result.households), + EXPECTED_HOUSEHOLD_MEMBER, + ) + assert_lineage( + result.persons, + list(person_map(result.persons)), + EXPECTED_PERSON_MEMBER, + ) + + +def test_rows_are_emitted_in_the_agreed_canonical_order(acs): + """The plan leaves row order open; this is the single place it is decided.""" + assert household_keys(acs.produce().households) == expected_row_order( + BASE_SERIALNOS + ) + + +def test_row_order_does_not_depend_on_the_member_split(tmp_path, monkeypatch): + """Splitting the same invented rows differently must not reorder them.""" + one = single_member(tmp_path / "one", monkeypatch).produce() + three = build_fixture( + tmp_path / "three", + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[6:])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:3])), + Member("psam_husc.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:6])), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ).produce() + assert household_keys(three.households) == household_keys(one.households) + assert household_map(three.households) == household_map(one.households) + + +def test_source_artifact_holds_exactly_the_projection_and_receipt(acs): + output_dir = acs.output_dir() + result = acs.produce(output_dir=output_dir) + assert sorted(path.name for path in output_dir.iterdir()) == sorted( + (NAMES["projection_filename"], NAMES["receipt_filename"]) + ) + # The house convention appends a trailing newline to published JSON. + assert (output_dir / NAMES["projection_filename"]).read_bytes().rstrip( + b"\n" + ) == result.projection_json.rstrip(b"\n") + assert (output_dir / NAMES["receipt_filename"]).read_bytes().rstrip( + b"\n" + ) == result.receipt_json.rstrip(b"\n") + json.loads(result.projection_json) + json.loads(result.receipt_json) + + +def test_readback_returns_the_same_bytes_and_the_same_population(acs): + produced, loaded, _output_dir = acs.produce_and_load() + assert loaded.projection_json == produced.projection_json + assert loaded.receipt_json == produced.receipt_json + pd.testing.assert_frame_equal(loaded.households, produced.households) + pd.testing.assert_frame_equal(loaded.persons, produced.persons) + pd.testing.assert_frame_equal(loaded.codes, produced.codes) + + +def test_two_produces_from_identical_inputs_are_byte_identical(acs): + """Readback compares exact bytes, so both artifacts must be deterministic.""" + first = acs.produce(output_dir=acs.output_dir("first")) + second = acs.produce(output_dir=acs.output_dir("second")) + assert first.projection_json == second.projection_json + assert first.receipt_json == second.receipt_json + + +@pytest.mark.parametrize("attribute", ["households", "persons", "codes"]) +def test_returned_frames_are_owned_copies(acs, attribute): + result = acs.produce() + first = getattr(result, attribute) + assert getattr(result, attribute) is not first + before = first.copy(deep=True) + projection = result.projection_json + first.iloc[0, 0] = first.iloc[-1, 0] + first.rename(columns={first.columns[0]: "clobbered"}, inplace=True) + first.index = pd.RangeIndex(1000, 1000 + len(first)) + after = getattr(result, attribute) + pd.testing.assert_frame_equal(after, before) + assert result.projection_json == projection + + +def test_receipt_records_auditable_member_and_definition_evidence(acs): + result = acs.produce() + receipt = json.loads(result.receipt_json) + strings = flat_strings(receipt) + numbers = flat_numbers(receipt) + + assert NAMES["vacancy_definition"] in strings + assert NAMES["gq_definition"] in strings + assert {HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE} <= strings + assert {"psam_husa.csv", "psam_husb.csv", "psam_pusa.csv", "psam_pusb.csv"} <= ( + strings + ) + + for archive, names in ( + (acs.household_zip, ("psam_husa.csv", "psam_husb.csv")), + (acs.person_zip, ("psam_pusa.csv", "psam_pusb.csv")), + ): + with zipfile.ZipFile(archive) as opened: + for name in names: + info = opened.getinfo(name) + payload = opened.read(name) + assert hashlib.sha256(payload).hexdigest() in strings, ( + f"receipt omits the decompressed digest of {name}" + ) + assert info.CRC in numbers, f"receipt omits the CRC of {name}" + assert info.file_size in numbers + + # Counts an auditor needs: the whole population and the vacancy evidence. + assert len(BASE_HOUSEHOLDS) in numbers + assert len(BASE_PERSONS) in numbers + + +def test_receipt_binds_the_pinned_archive_digests(acs): + pins = acs.pin() + strings = flat_strings(json.loads(acs.produce().receipt_json)) + for _role, name, sha, _size in pins: + assert sha in strings, f"receipt omits the pinned digest of {name}" + + +# -------------------------------------------------------------------------- +# Population variants: one deliberate change against the accepted invented rows. +# -------------------------------------------------------------------------- + + +def with_household(index, **fields): + rows = list(BASE_HOUSEHOLDS) + values = dict(zip(HOUSEHOLD_HEADER, rows[index], strict=True)) + values.update(fields) + rows[index] = tuple(values[name] for name in HOUSEHOLD_HEADER) + return tuple(rows) + + +def with_person(index, **fields): + rows = list(BASE_PERSONS) + values = dict(zip(PERSON_HEADER, rows[index], strict=True)) + values.update(fields) + rows[index] = tuple(values[name] for name in PERSON_HEADER) + return tuple(rows) + + +def rename_serialno(old, new): + """Rename a key on both sides so the only change is the identifier token.""" + key = HOUSEHOLD_HEADER.index("SERIALNO") + households = tuple( + row[:key] + (new,) + row[key + 1 :] if row[key] == old else row + for row in BASE_HOUSEHOLDS + ) + key = PERSON_HEADER.index("SERIALNO") + persons = tuple( + row[:key] + (new,) + row[key + 1 :] if row[key] == old else row + for row in BASE_PERSONS + ) + return households, persons + + +def single_member(tmp_path, monkeypatch, *, households=None, persons=None, **kwargs): + """One member per role, so a variant's only difference is the row content.""" + return build_fixture( + tmp_path, + monkeypatch, + households=( + ("psam_husa.csv", BASE_HOUSEHOLDS if households is None else households), + ), + persons=(("psam_pusa.csv", BASE_PERSONS if persons is None else persons),), + **kwargs, + ) + + +# -------------------------------------------------------------------------- +# Key selection. +# -------------------------------------------------------------------------- + + +def test_selection_none_retains_the_whole_invented_source(acs): + result = acs.produce(serialnos=None) + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert len(result.persons) == len(BASE_PERSONS) + + +def test_selection_keeps_whole_households_and_no_other_person(acs): + result = acs.produce(serialnos=("2024HU0000001", "2024GQ0000005")) + assert set(household_keys(result.households)) == { + "2024HU0000001", + "2024GQ0000005", + } + assert person_map(result.persons) == { + ("2024HU0000001", "01"): "121", + ("2024HU0000001", "02"): "97", + ("2024GQ0000005", "01"): "64", + } + assert codes_map(result) == { + key: EXPECTED_CODES[key] for key in ("2024HU0000001", "2024GQ0000005") + } + + +def test_a_vacancy_only_selection_is_a_valid_source_projection(acs): + """Source-only preparation may validly retain no people at all.""" + result = acs.produce(serialnos=("2024HU0000004",)) + assert household_keys(result.households) == ["2024HU0000004"] + assert len(result.persons) == 0 + assert result.codes["occupied_hu"].tolist() == [0] + assert result.codes["unresolved_reasons"].tolist() == [1] + + +def test_selected_rows_ignore_the_caller_tuple_order(acs): + """The request is a set of keys; canonical order is the module's own.""" + forward = acs.produce( + output_dir=acs.output_dir("forward"), + serialnos=("2024HU0000002", "2024HU0000008"), + ) + reversed_request = acs.produce( + output_dir=acs.output_dir("reversed"), + serialnos=("2024HU0000008", "2024HU0000002"), + ) + assert forward.projection_json == reversed_request.projection_json + assert household_keys(forward.households) == expected_row_order( + ("2024HU0000002", "2024HU0000008") + ) + + +def test_selection_changes_the_projection_bytes(acs): + whole = acs.produce() + selected = acs.produce(serialnos=("2024HU0000001",)) + assert selected.projection_json != whole.projection_json + + +def test_readback_under_a_different_selection_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir, serialnos=("2024HU0000001",)) + refuses(acs, acs.load, output_dir, serialnos=("2024HU0000002",)) + refuses(acs, acs.load, output_dir, serialnos=None) + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param(("2024HU0000009",), id="unknown_key"), + pytest.param(("2024HU0000001", "2024HU0000009"), id="one_unknown_key"), + pytest.param(("2024HU0000001", "2024HU0000001"), id="duplicate_key"), + pytest.param((), id="empty_selection"), + pytest.param(("",), id="blank_key"), + pytest.param((" 2024HU0000001",), id="whitespace_padded_key"), + pytest.param(("2024hu0000001",), id="case_folded_key"), + ], +) +def test_invalid_selection_refuses(acs, serialnos): + refuses(acs, acs.produce, serialnos=serialnos) + + +# -------------------------------------------------------------------------- +# Complete-join integrity, checked before any exclusion or selection. +# -------------------------------------------------------------------------- + +JOIN_VARIANTS = [ + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000004", "1", "40"),), + id="person_attached_to_a_vacant_household", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + id="orphan_person_without_a_household", + ), + pytest.param( + BASE_HOUSEHOLDS + (BASE_HOUSEHOLDS[0],), + BASE_PERSONS, + id="duplicate_household_key", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000002", "1", "44"),), + id="duplicate_person_key", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000002", "01", "44"),), + id="duplicate_person_key_via_leading_zero_alias", + ), + pytest.param( + with_household(0, NP="3"), + BASE_PERSONS, + id="household_np_above_linked_person_count", + ), + pytest.param( + with_household(0, NP="1"), + BASE_PERSONS, + id="household_np_below_linked_person_count", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS[:6] + BASE_PERSONS[7:], + id="group_quarters_placeholder_without_its_person", + ), + pytest.param( + with_household(4, NP="2"), + BASE_PERSONS + (person_row("2024GQ0000005", "02", "51"),), + id="group_quarters_placeholder_with_two_people", + ), + pytest.param( + with_household(4, NP="0"), + BASE_PERSONS[:6] + BASE_PERSONS[7:], + id="group_quarters_placeholder_with_no_people", + ), +] + + +@pytest.mark.parametrize("households,persons", JOIN_VARIANTS) +def test_join_integrity_refusals(tmp_path, monkeypatch, households, persons): + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + refuses(fixture, fixture.produce) + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param(None, id="whole_source"), + pytest.param(("2024HU0000001",), id="unrelated_single_key"), + ], +) +def test_vacant_household_person_refuses_before_exclusion_or_selection( + tmp_path, monkeypatch, serialnos +): + """An NP0 household with a person must refuse even when nothing selects it. + + An implementation that dropped NP0 rows, or applied the key selection, + before validating would silently accept this invented contradiction. + """ + fixture = single_member( + tmp_path, + monkeypatch, + persons=BASE_PERSONS + (person_row("2024HU0000004", "1", "40"),), + ) + refuses(fixture, fixture.produce, serialnos=serialnos) + + +def test_orphan_person_refuses_even_when_selection_excludes_it(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, + monkeypatch, + persons=BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + ) + refuses(fixture, fixture.produce, serialnos=("2024HU0000002",)) + + +# -------------------------------------------------------------------------- +# Closed source-value domains. +# -------------------------------------------------------------------------- + +DOMAIN_VARIANTS = [ + # SERIALNO: exactly 13 ASCII characters, 2024HU/2024GQ plus seven digits, + # numeric suffix 1..9999999, prefix agreeing with TYPEHUGQ, no whitespace, + # no empty identifier and no normalisation of the accepted bytes. + pytest.param( + *rename_serialno("2024HU0000001", " 2024HU000001"), id="serialno_leading_space" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000001 "), id="serialno_trailing_space" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU 000001"), id="serialno_interior_space" + ), + pytest.param(*rename_serialno("2024HU0000001", ""), id="serialno_empty"), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000001"), id="serialno_too_short" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU00000010"), id="serialno_too_long" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2023HU0000001"), id="serialno_wrong_vintage" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024XX0000001"), + id="serialno_unknown_type_token", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024hu0000001"), + id="serialno_lowercase_type_token", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU0000000"), id="serialno_zero_suffix" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000000A"), + id="serialno_nondigit_suffix", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024GQ0000001"), + id="serialno_gq_prefix_on_housing_unit", + ), + pytest.param( + *rename_serialno("2024GQ0000005", "2024HU0000005"), + id="serialno_hu_prefix_on_group_quarters", + ), + # SPORDER: one or two ASCII digits, value 1..20, no sign/exponent/point. + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="0"), id="sporder_zero"), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="21"), id="sporder_above_domain" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="001"), id="sporder_three_digits" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER=" 1"), id="sporder_leading_space" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="1 "), id="sporder_trailing_space" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="+1"), id="sporder_signed"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="-1"), id="sporder_negative"), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="1.0"), id="sporder_decimal_point" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="1e0"), id="sporder_exponent"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER=""), id="sporder_empty"), + # TYPEHUGQ: 1, 2 or 3 only. + pytest.param(with_household(0, TYPEHUGQ="0"), BASE_PERSONS, id="typehugq_zero"), + pytest.param( + with_household(0, TYPEHUGQ="4"), BASE_PERSONS, id="typehugq_above_domain" + ), + pytest.param(with_household(0, TYPEHUGQ=""), BASE_PERSONS, id="typehugq_empty"), + pytest.param( + with_household(0, TYPEHUGQ="H"), BASE_PERSONS, id="typehugq_nonnumeric" + ), + # NP: published integer domain 0..20. + pytest.param(with_household(0, NP="21"), BASE_PERSONS, id="np_above_domain"), + pytest.param(with_household(0, NP="-1"), BASE_PERSONS, id="np_negative"), + pytest.param(with_household(0, NP=""), BASE_PERSONS, id="np_empty"), + pytest.param(with_household(0, NP=" 2"), BASE_PERSONS, id="np_leading_space"), + pytest.param(with_household(0, NP="2.0"), BASE_PERSONS, id="np_decimal_point"), + # WGTP: 1..9999 for housing units including vacant ones; zero only for the + # group-quarters placeholders. + pytest.param( + with_household(0, WGTP="0"), BASE_PERSONS, id="wgtp_zero_on_occupied_unit" + ), + pytest.param( + with_household(3, WGTP="0"), BASE_PERSONS, id="wgtp_zero_on_vacant_unit" + ), + pytest.param(with_household(0, WGTP="10000"), BASE_PERSONS, id="wgtp_above_domain"), + pytest.param(with_household(0, WGTP="-5"), BASE_PERSONS, id="wgtp_negative"), + pytest.param(with_household(0, WGTP=""), BASE_PERSONS, id="wgtp_empty"), + # PWGTP: 1..9999 for every person, group quarters included. + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP="0"), id="pwgtp_zero"), + pytest.param( + BASE_HOUSEHOLDS, + with_person(6, PWGTP="0"), + id="pwgtp_zero_on_group_quarters_person", + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, PWGTP="10000"), id="pwgtp_above_domain" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP="-1"), id="pwgtp_negative"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP=""), id="pwgtp_empty"), + # TEN: blank means missing; observed values are 1..4; group quarters and + # vacancy require the blank token. + pytest.param(with_household(0, TEN="5"), BASE_PERSONS, id="ten_above_domain"), + pytest.param(with_household(0, TEN="0"), BASE_PERSONS, id="ten_zero"), + pytest.param(with_household(0, TEN="b"), BASE_PERSONS, id="ten_display_character"), + pytest.param(with_household(0, TEN=" "), BASE_PERSONS, id="ten_single_space"), + pytest.param( + with_household(3, TEN="1"), BASE_PERSONS, id="ten_present_on_vacant_unit" + ), + pytest.param( + with_household(4, TEN="1"), BASE_PERSONS, id="ten_present_on_group_quarters" + ), + # PUMA: exactly five ASCII digits with integer value 100..81003. + pytest.param(with_household(0, PUMA="0010"), BASE_PERSONS, id="puma_four_digits"), + pytest.param(with_household(0, PUMA="000100"), BASE_PERSONS, id="puma_six_digits"), + pytest.param(with_household(0, PUMA="00099"), BASE_PERSONS, id="puma_below_domain"), + pytest.param(with_household(0, PUMA="81004"), BASE_PERSONS, id="puma_above_domain"), + pytest.param(with_household(0, PUMA="0010A"), BASE_PERSONS, id="puma_nondigit"), + pytest.param(with_household(0, PUMA=""), BASE_PERSONS, id="puma_empty"), + # ST: exactly two ASCII digits from the 50-state and DC subset. + pytest.param(with_household(0, ST="72"), BASE_PERSONS, id="state_puerto_rico"), + pytest.param(with_household(0, ST="99"), BASE_PERSONS, id="state_unassigned_code"), + pytest.param(with_household(0, ST="6"), BASE_PERSONS, id="state_single_digit"), + pytest.param(with_household(0, ST="006"), BASE_PERSONS, id="state_three_digits"), + pytest.param( + with_household(0, ST="CA"), BASE_PERSONS, id="state_postal_abbreviation" + ), + pytest.param(with_household(0, ST=""), BASE_PERSONS, id="state_empty"), +] + + +@pytest.mark.parametrize("households,persons", DOMAIN_VARIANTS) +def test_source_value_domain_refusals(tmp_path, monkeypatch, households, persons): + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + refuses(fixture, fixture.produce) + + +def test_sporder_leading_zero_is_accepted_and_the_raw_token_is_retained( + tmp_path, monkeypatch +): + """Leading-zero SPORDER normalises for the join without losing its bytes.""" + fixture = single_member(tmp_path, monkeypatch, persons=with_person(2, SPORDER="01")) + assert ("2024HU0000002", "01") in person_map(fixture.produce().persons) + + +def test_maximum_and_minimum_accepted_weights_are_retained(acs): + weights = household_map(acs.produce().households) + assert weights["2024HU0000003"]["WGTP"] == "1" + assert weights["2024HU0000008"]["WGTP"] == "9999" + + +# -------------------------------------------------------------------------- +# Strict lexical parsing and the ordered-header contract. +# -------------------------------------------------------------------------- + +BASE_HOUSEHOLD_CSV = csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS) +BASE_PERSON_CSV = csv_bytes(PERSON_HEADER, BASE_PERSONS) + + +def household_bytes_fixture(tmp_path, monkeypatch, payload, *, name="psam_husa.csv"): + return build_fixture( + tmp_path, + monkeypatch, + household_members=(Member(name, payload),), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + + +def person_bytes_fixture(tmp_path, monkeypatch, payload, *, name="psam_pusa.csv"): + return build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + person_members=(Member(name, payload),), + ) + + +def splice_line(payload, index, line): + """Insert an invented physical line at a record boundary.""" + lines = payload.split(b"\n") + lines.insert(index, line) + return b"\n".join(lines) + + +def drop_column(header, rows, name): + position = header.index(name) + trimmed = header[:position] + header[position + 1 :] + return trimmed, tuple(row[:position] + row[position + 1 :] for row in rows) + + +def rename_column(header, old, new): + return tuple(new if name == old else name for name in header) + + +WIDE_SUFFIX = tuple(f"AUX{index:02d}" for index in range(24)) +WIDE_HEADER = HOUSEHOLD_HEADER + WIDE_SUFFIX +WIDE_ROWS = tuple(row + ("0",) * len(WIDE_SUFFIX) for row in BASE_HOUSEHOLDS) + + +def oversized_record_csv(): + """One physical record above 1 MiB whose every field stays under 64 KiB.""" + padded = ("4" * 50_000,) * len(WIDE_SUFFIX) + rows = (WIDE_ROWS[0][: len(HOUSEHOLD_HEADER)] + padded,) + WIDE_ROWS[1:] + return csv_bytes(WIDE_HEADER, rows) + + +def oversized_field_csv(): + """One field token above 64 KiB in a column that is never projected.""" + return csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="7" * 65_537)) + + +HOUSEHOLD_PARSE_VARIANTS = [ + pytest.param(b"\xef\xbb\xbf" + BASE_HOUSEHOLD_CSV, id="utf8_byte_order_mark"), + pytest.param( + BASE_HOUSEHOLD_CSV.replace(b"2024HU0000002", b"2024HU\xff000002"), + id="invalid_utf8_byte", + ), + pytest.param( + BASE_HOUSEHOLD_CSV.replace(b"82000", b"820\x0000"), id="embedded_nul_in_field" + ), + pytest.param(splice_line(BASE_HOUSEHOLD_CSV, 3, b""), id="interior_blank_record"), + pytest.param(BASE_HOUSEHOLD_CSV + b"\n", id="trailing_blank_record"), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0][:-1]).encode() + + b"\n", + id="short_record", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0] + ("extra",)).encode() + + b"\n", + id="long_record", + ), + pytest.param( + csv_bytes(rename_column(HOUSEHOLD_HEADER, "REGION", "ST"), BASE_HOUSEHOLDS), + id="duplicate_header_name", + ), + pytest.param( + csv_bytes(rename_column(HOUSEHOLD_HEADER, "REGION", ""), BASE_HOUSEHOLDS), + id="blank_header_name", + ), + pytest.param( + csv_bytes( + rename_column(HOUSEHOLD_HEADER, "SERIALNO", " SERIALNO"), BASE_HOUSEHOLDS + ), + id="whitespace_padded_header_name", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "NP")), + id="missing_np_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "SERIALNO")), + id="missing_serialno_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "TEN")), + id="missing_ten_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "WGTP")), + id="missing_wgtp_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "PUMA")), + id="missing_puma_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "ST")), + id="neither_st_nor_state_column", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="82000\r82000")), + id="embedded_carriage_return_in_field", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="82000\n82000")), + id="embedded_line_feed_in_field", + ), + pytest.param(oversized_field_csv(), id="field_token_above_64_kib"), + pytest.param(oversized_record_csv(), id="physical_record_above_1_mib"), + pytest.param(BASE_HOUSEHOLD_CSV.replace(b",", b";"), id="semicolon_delimiter"), + pytest.param(b"", id="empty_member_without_a_header"), +] + + +@pytest.mark.parametrize("payload", HOUSEHOLD_PARSE_VARIANTS) +def test_household_parser_refusals(tmp_path, monkeypatch, payload): + fixture = household_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +PERSON_PARSE_VARIANTS = [ + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "SPORDER")), + id="missing_sporder_column", + ), + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "PWGTP")), + id="missing_pwgtp_column", + ), + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "SERIALNO")), + id="missing_person_serialno_column", + ), + pytest.param(splice_line(BASE_PERSON_CSV, 2, b""), id="person_blank_record"), + pytest.param( + csv_bytes(PERSON_HEADER, BASE_PERSONS[1:]) + + ",".join(BASE_PERSONS[0][:-2]).encode() + + b"\n", + id="person_short_record", + ), +] + + +@pytest.mark.parametrize("payload", PERSON_PARSE_VARIANTS) +def test_person_parser_refusals(tmp_path, monkeypatch, payload): + fixture = person_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +def test_cross_member_reordered_header_refuses(tmp_path, monkeypatch): + """Identical column sets in a different order are still a header mismatch.""" + reordered = list(HOUSEHOLD_HEADER) + first, second = reordered.index("REGION"), reordered.index("ADJHSG") + reordered[first], reordered[second] = reordered[second], reordered[first] + rows = tuple( + tuple(dict(zip(HOUSEHOLD_HEADER, row, strict=True))[name] for name in reordered) + for row in BASE_HOUSEHOLDS[5:] + ) + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member("psam_husb.csv", csv_bytes(tuple(reordered), rows)), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_cross_member_extra_column_refuses(tmp_path, monkeypatch): + extended = HOUSEHOLD_HEADER + ("FINCP",) + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member( + "psam_husb.csv", + csv_bytes(extended, tuple(row + ("0",) for row in BASE_HOUSEHOLDS[5:])), + ), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_header_only_member_is_allowed_and_contributes_no_rows(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, ())), + ), + person_members=( + Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())), + Member("psam_pusb.csv", BASE_PERSON_CSV), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert person_map(result.persons) == EXPECTED_PERSONS + assert_lineage( + result.households, + household_keys(result.households), + dict.fromkeys(BASE_SERIALNOS, "psam_husa.csv"), + ) + assert_lineage( + result.persons, + list(person_map(result.persons)), + dict.fromkeys(EXPECTED_PERSONS, "psam_pusb.csv"), + ) + # The empty members are still inventoried even though they carry no rows. + strings = flat_strings(json.loads(result.receipt_json)) + assert {"psam_husb.csv", "psam_pusa.csv"} <= strings + + +def test_an_entirely_header_only_source_is_not_a_population(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, ())),), + person_members=(Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())),), + ) + refuses(fixture, fixture.produce) + + +def test_state_only_household_header_is_supported(tmp_path, monkeypatch): + header = rename_column(HOUSEHOLD_HEADER, "ST", "STATE") + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(header, BASE_HOUSEHOLDS) + ) + result = fixture.produce() + assert household_map(result.households)["2024HU0000001"]["ST"] == "06" + # The header alias itself is preserved in the source evidence. + assert "STATE" in flat_strings(json.loads(result.receipt_json)) + + +def test_agreeing_st_and_state_columns_are_accepted(tmp_path, monkeypatch): + header = HOUSEHOLD_HEADER + ("STATE",) + state = HOUSEHOLD_HEADER.index("ST") + rows = tuple(row + (row[state],) for row in BASE_HOUSEHOLDS) + fixture = household_bytes_fixture(tmp_path, monkeypatch, csv_bytes(header, rows)) + result = fixture.produce() + assert household_map(result.households)["2024HU0000003"]["ST"] == "36" + + +def test_disagreeing_st_and_state_columns_refuse(tmp_path, monkeypatch): + header = HOUSEHOLD_HEADER + ("STATE",) + state = HOUSEHOLD_HEADER.index("ST") + rows = tuple( + row + ("36" if index == 0 else row[state],) + for index, row in enumerate(BASE_HOUSEHOLDS) + ) + fixture = household_bytes_fixture(tmp_path, monkeypatch, csv_bytes(header, rows)) + refuses(fixture, fixture.produce) + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_both_physical_line_terminators_project_identically( + tmp_path, monkeypatch, newline +): + """A genuine archive's terminator choice must not change the projection.""" + fixture = build_fixture(tmp_path, monkeypatch, newline=newline) + result = fixture.produce() + assert household_map(result.households) == EXPECTED_HOUSEHOLDS + assert person_map(result.persons) == EXPECTED_PERSONS + + +# -------------------------------------------------------------------------- +# Archive preflight, run on the central directory before any by-name lookup. +# -------------------------------------------------------------------------- + +SYMLINK_ATTR = (stat.S_IFLNK | 0o777) << 16 +DIRECTORY_ATTR = (stat.S_IFDIR | 0o755) << 16 | 0x10 + + +def household_archive_fixture(tmp_path, monkeypatch, members): + return build_fixture( + tmp_path, + monkeypatch, + household_members=members, + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + + +def aux_members(count, *, prefix="ACS2024_note"): + return tuple( + Member(f"{prefix}{index:03d}.txt", b"invented auxiliary text\n") + for index in range(count) + ) + + +ARCHIVE_VARIANTS = [ + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + ), + id="duplicate_member_name", + ), + pytest.param( + ( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member("PSAM_HUSA.CSV", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[5:])), + ), + id="case_folded_member_alias", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_hus/", b"", external_attr=DIRECTORY_ATTR), + ), + id="directory_entry", + ), + pytest.param( + (Member("pums/psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="nested_path_member", + ), + pytest.param( + (Member("/psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="absolute_path_member", + ), + pytest.param( + (Member("../psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="parent_traversal_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.txt", BASE_HOUSEHOLD_CSV), + ), + id="prefix_matching_non_csv_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member( + "psam_husb.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_BZIP2 + ), + ), + id="bzip2_compressed_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_link.csv", b"psam_husa.csv", external_attr=SYMLINK_ATTR), + ), + id="symlink_member", + ), + pytest.param( + (Member("psam_pusa.csv", BASE_PERSON_CSV),), + id="wrong_role_members_only", + ), + pytest.param( + (Member("ACS2024_readme.txt", b"invented auxiliary text\n"),), + id="no_applicable_csv_member", + ), + pytest.param((), id="empty_archive"), + pytest.param( + (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + aux_members(64), + id="member_count_above_cap", + ), +] + + +@pytest.mark.parametrize("members", ARCHIVE_VARIANTS) +def test_archive_preflight_refusals(tmp_path, monkeypatch, members): + fixture = household_archive_fixture(tmp_path, monkeypatch, members) + refuses(fixture, fixture.produce) + + +def test_member_count_at_the_cap_is_accepted(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + aux_members(63), + ) + assert len(fixture.produce().households) == len(BASE_HOUSEHOLDS) + + +def test_ordinary_auxiliary_members_are_inventory_only(tmp_path, monkeypatch): + """A non-matching ordinary file is recorded but never read as data.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("ACS2024_PUMS_README.txt", b"invented auxiliary text\n"), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert_lineage( + result.households, + household_keys(result.households), + dict.fromkeys(BASE_SERIALNOS, "psam_husa.csv"), + ) + assert "ACS2024_PUMS_README.txt" in flat_strings(json.loads(result.receipt_json)) + + +def test_a_non_matching_csv_member_is_auxiliary_not_data(tmp_path, monkeypatch): + """The role prefix, not the extension, decides what counts as data.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("ACS2024_shells.csv", csv_bytes(("A", "B"), (("1", "2"),))), + ), + ) + result = fixture.produce() + assert len(result.households) == len(BASE_HOUSEHOLDS) + assert "ACS2024_shells.csv" in flat_strings(json.loads(result.receipt_json)) + + +def test_stored_members_are_accepted(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member( + "psam_husa.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_STORED + ), + ), + ) + assert len(fixture.produce().households) == len(BASE_HOUSEHOLDS) + + +def test_several_members_per_role_join_in_name_order(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:3])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:6])), + Member("psam_husc.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[6:])), + ), + person_members=( + Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, BASE_PERSONS[:4])), + Member("psam_pusb.csv", csv_bytes(PERSON_HEADER, BASE_PERSONS[4:])), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + members = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: name + for name, rows in ( + ("psam_husa.csv", BASE_HOUSEHOLDS[:3]), + ("psam_husb.csv", BASE_HOUSEHOLDS[3:6]), + ("psam_husc.csv", BASE_HOUSEHOLDS[6:]), + ) + for row in rows + } + assert_lineage(result.households, household_keys(result.households), members) + # 2024HU0000003's three people straddle the two person members. + assert lexical(result.persons["SERIALNO"]).count("2024HU0000003") == 3 + assert person_map(result.persons) == EXPECTED_PERSONS + person_members = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): name + for name, rows in ( + ("psam_pusa.csv", BASE_PERSONS[:4]), + ("psam_pusb.csv", BASE_PERSONS[4:]), + ) + for row in rows + } + assert_lineage(result.persons, list(person_map(result.persons)), person_members) + + +@pytest.mark.parametrize("archive", [HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE]) +def test_missing_role_archive_refuses(acs, archive): + (acs.source_dir / archive).unlink() + refuses(acs, acs.produce) + + +def test_corrupted_member_payload_fails_the_recorded_crc(tmp_path, monkeypatch): + """A stored member whose bytes were swapped keeps its declared size and CRC.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member( + "psam_husa.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_STORED + ), + ), + ) + payload = fixture.household_zip.read_bytes() + assert b"82000" in payload, "the stored member is not plain in the archive" + fixture.household_zip.write_bytes(payload.replace(b"82000", b"82001", 1)) + fixture.pin() + refuses(fixture, fixture.produce) + + +# -------------------------------------------------------------------------- +# Private capture: paths, pins, originals and byte bounds. +# -------------------------------------------------------------------------- + + +@contextmanager +def bounded_time(seconds=30): + """Fail rather than hang if a non-regular original is opened blocking.""" + + def expire(signum, frame): + raise TimeoutError("the call did not return within the bounded test window") + + previous = signal.signal(signal.SIGALRM, expire) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def test_missing_snapshot_root_refuses(acs): + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=acs.tmp_path / "absent-root", + output_dir=acs.output_dir(), + ) + + +def test_snapshot_root_that_is_a_file_refuses(acs): + root = acs.tmp_path / "root-file" + root.write_bytes(b"invented\n") + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=root, + output_dir=acs.output_dir(), + ) + + +def test_symlinked_snapshot_root_refuses(acs): + link = acs.tmp_path / "root-link" + link.symlink_to(acs.snapshot_root, target_is_directory=True) + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=link, + output_dir=acs.output_dir(), + ) + + +def test_existing_output_directory_refuses(acs): + output_dir = acs.output_dir() + output_dir.mkdir() + refuses(acs, acs.produce, output_dir=output_dir) + + +def test_existing_output_file_refuses(acs): + output_dir = acs.output_dir() + output_dir.write_bytes(b"invented\n") + refuses(acs, acs.produce, output_dir=output_dir) + + +def test_output_directory_without_a_parent_refuses(acs): + refuses(acs, acs.produce, output_dir=acs.tmp_path / "absent" / "artifact") + + +def test_missing_source_directory_refuses(acs): + refuses(acs, acs.produce, source_dir=acs.tmp_path / "absent-source") + + +def test_source_directory_that_is_a_file_refuses(acs): + path = acs.tmp_path / "source-file" + path.write_bytes(b"invented\n") + refuses(acs, acs.produce, source_dir=path) + + +def test_symlinked_source_directory_refuses(acs): + link = acs.tmp_path / "source-link" + link.symlink_to(acs.source_dir, target_is_directory=True) + refuses(acs, acs.produce, source_dir=link) + + +def test_symlinked_original_archive_refuses(acs): + real = acs.tmp_path / "real-csv_hus.zip" + real.write_bytes(acs.household_zip.read_bytes()) + acs.household_zip.unlink() + acs.household_zip.symlink_to(real) + refuses(acs, acs.produce) + + +def test_fifo_original_archive_refuses(acs): + acs.household_zip.unlink() + os.mkfifo(acs.household_zip) + with bounded_time(): + refuses(acs, acs.produce) + + +def test_directory_in_place_of_an_original_archive_refuses(acs): + acs.household_zip.unlink() + acs.household_zip.mkdir() + refuses(acs, acs.produce) + + +def test_pinned_digest_mismatch_at_the_same_size_refuses(acs): + """Authentication is over the whole archive bytes, not over the projection.""" + pins = acs.pin() + flip_archive_byte(acs.household_zip) + acs.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + refuses(acs, acs.produce) + + +def test_pinned_size_mismatch_refuses(acs): + pins = acs.pin() + with acs.household_zip.open("ab") as handle: + handle.write(b"\n") + acs.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + refuses(acs, acs.produce) + + +def test_original_archives_are_never_modified(acs): + before = { + path: (path.read_bytes(), path.stat().st_mtime_ns) + for path in (acs.household_zip, acs.person_zip) + } + acs.produce() + for path, (payload, mtime) in before.items(): + assert path.read_bytes() == payload + assert path.stat().st_mtime_ns == mtime + + +def test_capture_writes_only_under_the_supplied_snapshot_root(acs): + assert not any(acs.snapshot_root.iterdir()) + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + children = list(acs.snapshot_root.iterdir()) + assert children, "the private capture did not land under the snapshot root" + for child in children: + assert child.is_dir() and not child.is_symlink() + assert stat.S_IMODE(child.stat().st_mode) == 0o700 + assert sorted(path.name for path in output_dir.iterdir()) == sorted( + (NAMES["projection_filename"], NAMES["receipt_filename"]) + ) + + +def test_capture_never_falls_back_to_system_temporary_storage(acs, monkeypatch): + absent = acs.tmp_path / "no-such-tmpdir" + monkeypatch.setenv("TMPDIR", str(absent)) + monkeypatch.setattr(tempfile, "tempdir", str(absent)) + result = acs.produce() + assert len(result.households) == len(BASE_HOUSEHOLDS) + assert not absent.exists() + + +def test_two_produces_share_one_snapshot_root_without_collision(acs): + first = acs.produce(output_dir=acs.output_dir("first")) + second = acs.produce(output_dir=acs.output_dir("second")) + assert first.projection_json == second.projection_json + assert len(list(acs.snapshot_root.iterdir())) >= 2 + + +def test_a_refused_produce_leaves_no_receipt_behind(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, monkeypatch, households=with_household(0, TEN="5") + ) + output_dir = fixture.output_dir() + refuses(fixture, fixture.produce, output_dir=output_dir) + assert not (output_dir / NAMES["receipt_filename"]).exists() + + +def byte_cap_names(): + return tuple(sorted(name for name in dir(source) if name.endswith("_MAX_BYTES"))) + + +def test_module_declares_its_own_byte_caps(): + assert byte_cap_names(), "the module declares no explicit byte caps" + + +@pytest.mark.parametrize("cap", [0, 1, 64]) +def test_tiny_byte_caps_refuse_before_any_payload_is_returned(acs, cap): + """Exercise the streaming bounds with mocked caps, never with real GiBs.""" + for name in byte_cap_names(): + acs.monkeypatch.setattr(source, name, cap) + refuses(acs, acs.produce) + + +def test_declared_cap_values_match_the_approved_plan(): + """Reconciliation: the plan fixes both ACS source caps by name and value.""" + assert getattr(source, NAMES["source_max_bytes"]) == 8 * 1024**3 + assert getattr(source, NAMES["receipt_max_bytes"]) == 1024**2 + + +# -------------------------------------------------------------------------- +# Artifact custody: readback reconstructs, it never trusts what it decoded. +# -------------------------------------------------------------------------- + + +def make_writable(path): + """The producer publishes read-only files; tampering needs write access.""" + path = Path(path) + path.chmod(stat.S_IMODE(path.stat().st_mode) | 0o600) + return path + + +def forge(output_dir, old, new): + """Rewrite the artifact so its own internal digests stay self-consistent.""" + projection = (output_dir / NAMES["projection_filename"]).read_bytes() + receipt = (output_dir / NAMES["receipt_filename"]).read_bytes() + assert old in projection, "the invented forgery target is not in the projection" + forged = projection.replace(old, new) + assert len(forged) == len(projection), "a forgery must not change the length" + old_digest = hashlib.sha256(projection).hexdigest().encode() + new_digest = hashlib.sha256(forged).hexdigest().encode() + make_writable(output_dir / NAMES["projection_filename"]).write_bytes(forged) + make_writable(output_dir / NAMES["receipt_filename"]).write_bytes( + receipt.replace(old_digest, new_digest) + ) + return old_digest in receipt + + +def test_rehashed_projection_forgery_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + forge(output_dir, b"2024HU0000002", b"2024HU0000012") + refuses(acs, acs.load, output_dir) + + +def test_receipt_binds_the_projection_digest(acs): + """Otherwise a forged projection would be self-consistent on its face.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + assert forge(output_dir, b"2024HU0000002", b"2024HU0000012") + + +def test_forging_a_weight_to_another_valid_value_refuses(acs): + """Custody is byte reconstruction, so an in-domain forgery refuses too.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + forge(output_dir, b"118", b"119") + refuses(acs, acs.load, output_dir) + + +def test_rewritten_receipt_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + receipt = json.loads((output_dir / NAMES["receipt_filename"]).read_bytes()) + forged = json.dumps(receipt, sort_keys=True).encode() + make_writable(output_dir / NAMES["receipt_filename"]).write_bytes(forged) + refuses(acs, acs.load, output_dir) + + +@pytest.mark.parametrize( + "filename", [NAMES["projection_filename"], NAMES["receipt_filename"]] +) +def test_truncated_or_missing_artifact_files_refuse(acs, filename): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + make_writable(output_dir / filename).write_bytes( + (output_dir / filename).read_bytes()[:20] + ) + refuses(acs, acs.load, output_dir) + + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + (output_dir / filename).unlink() + refuses(acs, acs.load, output_dir) + + +def test_missing_artifact_directory_refuses(acs): + refuses(acs, acs.load, acs.tmp_path / "absent-artifact") + + +def test_extra_file_in_the_artifact_directory_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + (output_dir / "notes.json").write_bytes(b"{}\n") + refuses(acs, acs.load, output_dir) + + +def test_parent_change_between_produce_and_readback_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + flip_archive_byte(acs.household_zip) + refuses(acs, acs.load, output_dir) + + +def test_reparented_readback_refuses_even_after_repinning(acs): + """Fresh authentic parents that no longer reconstruct the artifact refuse.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + rows = with_household(1, TEN="4") + write_archive( + acs.household_zip, + ( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, rows[:5])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, rows[5:])), + ), + ) + acs.pin() + refuses(acs, acs.load, output_dir) + + +def test_readback_against_swapped_role_archives_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + household = acs.household_zip.read_bytes() + acs.household_zip.write_bytes(acs.person_zip.read_bytes()) + acs.person_zip.write_bytes(household) + acs.pin() + refuses(acs, acs.load, output_dir) + + +def test_readback_from_an_unrelated_artifact_refuses(tmp_path, monkeypatch): + other = build_fixture( + tmp_path / "other", + monkeypatch, + households=(("psam_husa.csv", BASE_HOUSEHOLDS[:4]),), + persons=(("psam_pusa.csv", BASE_PERSONS[:6]),), + ) + output_dir = other.output_dir() + other.produce(output_dir=output_dir) + + mine = build_fixture(tmp_path / "mine", monkeypatch) + refuses(mine, mine.load, output_dir) + + +@pytest.mark.parametrize("attribute", ["households", "persons", "codes"]) +def test_readback_frames_are_owned_copies(acs, attribute): + _produced, loaded, _output_dir = acs.produce_and_load() + first = getattr(loaded, attribute) + assert getattr(loaded, attribute) is not first + before = first.copy(deep=True) + first.iloc[0, 0] = first.iloc[-1, 0] + pd.testing.assert_frame_equal(getattr(loaded, attribute), before) + + +# -------------------------------------------------------------------------- +# Refusal semantics: sanitised, and one code per cause. +# -------------------------------------------------------------------------- + + +def collect_family_reasons(tmp_path, monkeypatch): + """One refusal per coarse family, each from its own invented source.""" + reasons = {} + + def record(family, build, call): + fixture = build(tmp_path / family, monkeypatch) + fixture.pin() + reasons[family] = refuses(fixture, call, fixture) + + def digest_change(fixture): + pins = fixture.pin() + flip_archive_byte(fixture.household_zip) + fixture.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + fixture.produce() + + def forged_artifact(fixture): + output_dir = fixture.output_dir() + fixture.produce(output_dir=output_dir) + forge(output_dir, b"2024HU0000002", b"2024HU0000012") + fixture.load(output_dir) + + def existing_output(fixture): + output_dir = fixture.output_dir() + output_dir.mkdir() + fixture.produce(output_dir=output_dir) + + plain = build_fixture + + record("capture_digest", plain, digest_change) + record("artifact_forgery", plain, forged_artifact) + record("output_precondition", plain, existing_output) + record("selection", plain, lambda f: f.produce(serialnos=("2024HU0000009",))) + record( + "zip_preflight", + lambda base, patch: household_archive_fixture( + base, + patch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + ), + ), + lambda f: f.produce(), + ) + record( + "csv_record", + lambda base, patch: household_bytes_fixture( + base, + patch, + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0][:-1]).encode() + + b"\n", + ), + lambda f: f.produce(), + ) + record( + "header_contract", + lambda base, patch: household_bytes_fixture( + base, + patch, + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "ST")), + ), + lambda f: f.produce(), + ) + record( + "value_domain", + lambda base, patch: single_member( + base, patch, households=with_household(0, TEN="5") + ), + lambda f: f.produce(), + ) + record( + "join_integrity", + lambda base, patch: single_member( + base, + patch, + persons=BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + ), + lambda f: f.produce(), + ) + return reasons + + +@pytest.fixture +def family_reasons(tmp_path, monkeypatch): + return collect_family_reasons(tmp_path, monkeypatch) + + +def test_refusal_type_is_a_sanitised_value_error(): + assert issubclass(source.ACSHousingSourceError, ValueError) + + +def test_every_refusal_family_is_exercised(family_reasons): + assert set(family_reasons) == set(DISTINCT_FAMILIES) + + +def test_distinct_causes_carry_distinct_reason_codes(family_reasons): + """A single catch-all code would make every refusal above unfalsifiable.""" + collisions = {} + for family, reason in family_reasons.items(): + collisions.setdefault(reason, []).append(family) + shared = {reason: names for reason, names in collisions.items() if len(names) > 1} + assert not shared, f"distinct causes share a reason code: {shared}" + + +def test_no_refusal_reason_echoes_invented_source_content(family_reasons): + """refuses() checks each call; this states the whole-sweep contract once.""" + forbidden = set(BASE_SERIALNOS) | {"psam_husa.csv", "82000", "52000"} + for family, reason in family_reasons.items(): + for secret in forbidden: + assert secret not in reason, f"{family} leaked {secret!r}" + + +def test_reason_codes_match_the_agreed_table(family_reasons): + """Shape always; exact spellings once EXPECTED_REASONS is filled in. + + The house idiom in the committed sibling source modules is a bare + UPPER_SNAKE token, so that much is asserted unconditionally and nothing is + parked behind a skip. Only the agreement on which token names which cause + waits on the production author. + """ + for family, reason in family_reasons.items(): + assert re.fullmatch(r"[A-Z][A-Z0-9_]*", reason), ( + f"{family} reason {reason!r} is not a bare reason code" + ) + if REASON_CODES_AGREED: + assert family_reasons == EXPECTED_REASONS + else: + assert not EXPECTED_REASONS, ( + "EXPECTED_REASONS is populated; set REASON_CODES_AGREED = True" + ) + + +# -------------------------------------------------------------------------- +# Naming reconciliation: everything this file had to guess, in one place. +# -------------------------------------------------------------------------- + + +def test_module_exposes_the_agreed_public_surface(): + for name in ( + "ACSHousingSourceError", + "produce_acs_housing_source", + "load_acs_housing_source", + "prepare_acs_housing_population", + ): + assert hasattr(source, name), f"missing agreed API name {name}" + + +def test_archive_pins_are_consulted_at_call_time(acs): + """The whole suite rests on this seam, so assert it directly. + + Every fixture authenticates its invented archives by pointing the private + pin tuple at them. That only works if the module reads the attribute when + it captures, rather than copying it at import. + """ + real = acs.pin() + assert acs.produce().households is not None + + wrong = tuple((role, name, "f" * 64, size) for role, name, _sha, size in real) + acs.monkeypatch.setattr(source, NAMES["archive_pins"], wrong) + refuses(acs, acs.produce) + + +def test_private_archive_pins_are_a_sentinel_or_the_two_fixed_archives(): + """Naming seam, reconciled with the production author, not a defect. + + The brief described `_ARCHIVE_PINS` as a populated tuple of + (role, filename, sha256, size). The module instead defaults it to None and + loads the packaged manifest lazily, while tests still patch the same + private name - which is what + ``test_archive_pins_are_consulted_at_call_time`` proves still works. Both + readings are admitted here; the shape is checked whenever it is populated, + so a wrong shape cannot hide behind the sentinel. + """ + pins = getattr(source, NAMES["archive_pins"]) + if pins is None: + return + assert type(pins) is tuple and len(pins) == 2 + assert [entry[0] for entry in pins] == [ + NAMES["household_role"], + NAMES["person_role"], + ] + assert [entry[1] for entry in pins] == [HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE] + for _role, _name, sha, size in pins: + assert type(sha) is str and len(sha) == 64 and sha == sha.lower() + assert int(sha, 16) >= 0 + assert type(size) is int and size > 0 + + +def test_lineage_and_state_columns_use_the_agreed_names(acs): + households = acs.produce().households + assert NAMES["member_column"] in households.columns + assert NAMES["ordinal_column"] in households.columns + assert NAMES["state_column"] in households.columns + + +# -------------------------------------------------------------------------- +# Tail coverage: cases an obvious pass over the five dimensions omits. +# -------------------------------------------------------------------------- + + +def set_encrypted_flag(path): + """Mark every entry encrypted in place; zipfile itself will not write this.""" + raw = bytearray(Path(path).read_bytes()) + marked = 0 + for signature, offset in ((b"PK\x03\x04", 6), (b"PK\x01\x02", 8)): + index = 0 + while True: + index = raw.find(signature, index) + if index < 0: + break + flag = struct.unpack_from("= 2 + Path(path).write_bytes(bytes(raw)) + + +def raw_household_record(**overrides): + """A physical household line written by hand, bypassing the csv writer.""" + values = dict(zip(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[0], strict=True)) + values.update(overrides) + return ",".join(values[name] for name in HOUSEHOLD_HEADER).encode() + + +def appended_record(record): + return csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + record + b"\n" + + +def test_encrypted_member_refuses(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, monkeypatch, (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + ) + set_encrypted_flag(fixture.household_zip) + fixture.pin() + refuses(fixture, fixture.produce) + + +def test_lzma_member_refuses(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_LZMA), + ), + ) + refuses(fixture, fixture.produce) + + +def test_bytes_that_are_not_an_archive_refuse(acs): + acs.household_zip.write_bytes(b"this is invented, and it is not a zip archive\n") + acs.pin() + refuses(acs, acs.produce) + + +def test_person_archive_holding_household_members_refuses(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + person_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + ) + refuses(fixture, fixture.produce) + + +UTF16_MEMBER = BASE_HOUSEHOLD_CSV.decode().encode("utf-16") + +TAIL_PARSE_VARIANTS = [ + pytest.param(UTF16_MEMBER, id="utf16_encoded_member"), + pytest.param( + appended_record(raw_household_record(ST='"06"x')), + id="character_after_a_closing_quote", + ), + pytest.param( + appended_record(raw_household_record(HINCP='"82000')), + id="unterminated_quote", + ), + pytest.param( + splice_line(BASE_HOUSEHOLD_CSV, 3, b" "), id="whitespace_only_record" + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, TEN="NA")), + id="na_token_is_not_a_missing_tenure", + ), + pytest.param( + csv_bytes( + rename_column(HOUSEHOLD_HEADER, "SERIALNO", "serialno"), BASE_HOUSEHOLDS + ), + id="lowercase_roster_column_name", + ), +] + + +@pytest.mark.parametrize("payload", TAIL_PARSE_VARIANTS) +def test_tail_parser_refusals(tmp_path, monkeypatch, payload): + fixture = household_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +def test_household_rows_absent_while_person_rows_remain_refuses(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, ())),), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_a_final_record_without_a_trailing_newline_is_accepted(tmp_path, monkeypatch): + fixture = household_bytes_fixture( + tmp_path, + monkeypatch, + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, trailing_newline=False), + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_many_unprojected_columns_are_accepted(tmp_path, monkeypatch): + """Genuine PUMS members carry hundreds of columns this projection ignores.""" + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(WIDE_HEADER, WIDE_ROWS) + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_roster_columns_in_any_header_position_are_accepted(tmp_path, monkeypatch): + """The plan pins header order across members of a role, not within one.""" + reversed_header = tuple(reversed(HOUSEHOLD_HEADER)) + rows = tuple(tuple(reversed(row)) for row in BASE_HOUSEHOLDS) + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(reversed_header, rows) + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_maximum_serialno_suffix_is_accepted(tmp_path, monkeypatch): + households, persons = rename_serialno("2024HU0000001", "2024HU9999999") + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + keys = household_keys(fixture.produce().households) + assert "2024HU9999999" in keys and "2024HU0000001" not in keys + + +def test_np_at_the_published_upper_bound_is_accepted(tmp_path, monkeypatch): + households = (household_row("2024HU0000020", "1", "20", "1", "300"),) + persons = tuple( + person_row("2024HU0000020", str(order), str(order + 10)) + for order in range(1, 21) + ) + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + result = fixture.produce() + assert lexical(result.households["NP"]) == ["20"] + assert len(result.persons) == 20 + assert result.codes["occupied_hu"].tolist() == [1] + + +def test_an_all_group_quarters_source_is_retained(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, + monkeypatch, + households=BASE_HOUSEHOLDS[4:6], + persons=BASE_PERSONS[6:8], + ) + result = fixture.produce() + assert codes_map(result) == { + key: EXPECTED_CODES[key] for key in ("2024GQ0000005", "2024GQ0000006") + } + + +def test_a_vacancy_only_source_is_retained(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:4])), + ), + person_members=(Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())),), + ) + result = fixture.produce() + assert household_keys(result.households) == ["2024HU0000004"] + assert len(result.persons) == 0 + assert codes_map(result) == {"2024HU0000004": EXPECTED_CODES["2024HU0000004"]} + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param((1,), id="integer_element"), + pytest.param((None,), id="none_element"), + pytest.param((b"2024HU0000001",), id="bytes_element"), + ], +) +def test_non_string_selection_elements_refuse(acs, serialnos): + refuses(acs, acs.produce, serialnos=serialnos) + + +def test_an_explicit_full_selection_matches_the_none_selection(acs): + whole = acs.produce(output_dir=acs.output_dir("whole")) + explicit = acs.produce( + output_dir=acs.output_dir("explicit"), serialnos=BASE_SERIALNOS + ) + assert explicit.projection_json == whole.projection_json + + +def test_a_published_artifact_survives_a_later_refused_produce(acs): + output_dir = acs.output_dir("kept") + produced = acs.produce(output_dir=output_dir) + before = {path.name: path.read_bytes() for path in sorted(output_dir.iterdir())} + acs.household_zip.write_bytes(b"not an archive\n") + acs.pin() + refuses(acs, acs.produce, output_dir=acs.output_dir("doomed")) + assert {path.name: path.read_bytes() for path in sorted(output_dir.iterdir())} == ( + before + ) + assert before[NAMES["projection_filename"]].rstrip(b"\n") == ( + produced.projection_json.rstrip(b"\n") + ) + + +def test_readback_with_a_missing_snapshot_root_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + refuses( + acs, + source.load_acs_housing_source, + acs.source_dir, + output_dir, + snapshot_root=acs.tmp_path / "absent-root", + ) + + +def test_wrong_size_candidate_refuses_as_a_candidate_not_as_a_parent(acs): + """Readback size failures must name the candidate, not the pinned archives.""" + for filename, mutate in ( + (NAMES["projection_filename"], lambda raw: raw + b" "), + (NAMES["projection_filename"], lambda raw: raw[:20]), + (NAMES["receipt_filename"], lambda raw: raw + b" "), + (NAMES["receipt_filename"], lambda raw: raw[:20]), + ): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + path = output_dir / filename + make_writable(path).write_bytes(mutate(path.read_bytes())) + assert refuses(acs, acs.load, output_dir) == "RECONSTRUCTION_SIZE" + + +def test_parent_archive_size_failures_keep_their_own_reason_codes(acs): + """The candidate code must not be borrowed by, or borrow from, the parents.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + intact = acs.household_zip.read_bytes() + acs.household_zip.write_bytes(intact[:-1]) + assert refuses(acs, acs.load, output_dir) == "SOURCE_SIZE" + acs.household_zip.write_bytes(intact + b"\x00") + assert refuses(acs, acs.load, output_dir) == "FILE_TOO_LARGE" + acs.household_zip.write_bytes(intact) + assert acs.load(output_dir) is not None + + +def test_failure_record_write_failure_does_not_mask_the_original_refusal( + acs, monkeypatch +): + """A failed failure record must never replace the refusal it was recording.""" + actual_write = source._write + attempted = [] + + def refuse_the_failure_record(path, data, cap): + if Path(path).name == "failure.json": + attempted.append(Path(path).name) + raise OSError(28, "No space left on device") + return actual_write(path, data, cap) + + monkeypatch.setattr(source, "_write", refuse_the_failure_record) + corrupted = bytearray(acs.household_zip.read_bytes()) + corrupted[-1] ^= 0xFF + acs.household_zip.write_bytes(bytes(corrupted)) + assert refuses(acs, acs.produce) == "SOURCE_SHA256" + assert attempted == ["failure.json"] diff --git a/packages/microcosm-build/tests/test_us_acs_multispine.py b/packages/microcosm-build/tests/test_us_acs_multispine.py index afc38b72d..fbaa393b7 100644 --- a/packages/microcosm-build/tests/test_us_acs_multispine.py +++ b/packages/microcosm-build/tests/test_us_acs_multispine.py @@ -356,15 +356,25 @@ def _test_puma_ladder() -> UsPumaLadder: tract_overlap_puma=puma.copy(), tract_overlap_tract=np.asarray([6_001_000_100, 36_001_000_100], dtype=np.int64), tract_overlap_population=population.copy(), + joint_overlap_puma=puma.copy(), + joint_overlap_tract=np.asarray([6_001_000_100, 36_001_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([601, 3_601], dtype=np.int64), + joint_overlap_population=np.asarray([100, 200], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py b/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py index 31742ad61..47e423dcb 100644 --- a/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py +++ b/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py @@ -193,15 +193,25 @@ def test_main_wires_verified_sources_transfer_audit_export_and_summary( tract_overlap_puma=np.asarray([100_100], dtype=np.int64), tract_overlap_tract=np.asarray([1_001_000_100], dtype=np.int64), tract_overlap_population=np.asarray([100.0]), + joint_overlap_puma=np.asarray([100_100], dtype=np.int64), + joint_overlap_tract=np.asarray([1_001_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([101], dtype=np.int64), + joint_overlap_population=np.asarray([100], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py b/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py new file mode 100644 index 000000000..ea76a97ab --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py @@ -0,0 +1,588 @@ +"""Invented-only source authority and explicitly ungranted native consistency.""" + +import csv +import hashlib +import importlib.util +import io +import json +import os +import sys +from dataclasses import FrozenInstanceError, replace +from pathlib import Path +from types import SimpleNamespace + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import acs_housing_universe_source as custody +from microcosm.build.us_runtime import acs_person_coverage_authentication as owner +from microcosm.build.us_runtime import acs_person_coverage_columns as literal +from microcosm.build.us_runtime.acs_inputs import map_acs_native_inputs +from microcosm.build.us_runtime.acs_pums import AcsPumsSource, build_acs_pums_unit_frame + + +def _helper(filename): + spec = importlib.util.spec_from_file_location( + "coverage_helpers_" + filename, Path(__file__).with_name(filename + ".py") + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +_authority = _helper("test_us_acs_housing_source") +Member = _authority.Member +build_fixture = _authority.build_fixture +flip_archive_byte = _authority.flip_archive_byte +write_archive = _authority.write_archive +_native = _helper("test_us_acs_pums") +_household, _person = _native._household, _native._person + + +def _csv(rows, *, bom=False, ending="\n"): + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=list(rows[0]), quoting=csv.QUOTE_ALL, lineterminator=ending + ) + writer.writeheader() + writer.writerows(rows) + return (("\ufeff" if bom else "") + stream.getvalue()).encode() + + +@pytest.fixture +def invented(tmp_path, monkeypatch): + monkeypatch.setattr( + custody.shutil, "disk_usage", lambda _p: SimpleNamespace(free=64 * 1024**3) + ) + h = [_household("2024HU0000001", NP=2), _household("2024HU0000002", NP=1)] + p = [ + _person("2024HU0000001", 1, 20, AGEP=30, MIL="1", ESR="4"), + _person("2024HU0000001", 2, 25, AGEP=15, MIL="", ESR=""), + _person("2024HU0000002", 1, 20, AGEP=80, MIL="4", ESR="6"), + ] + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", _csv(h)),), + person_members=( + Member("psam_pusa.csv", _csv(p[:1])), + Member("psam_pusb.csv", _csv(p[1:])), + ), + ) + frame = map_acs_native_inputs( + build_acs_pums_unit_frame( + AcsPumsSource(fixture.household_zip, fixture.person_zip) + )[0] + ).frame + return fixture, frame, h, p + + +def _load(invented, **kwargs): + fixture, frame, _h, _p = invented + return owner.load_authenticated_acs_person_coverage( + fixture.source_dir, snapshot_root=fixture.snapshot_root, frame=frame, **kwargs + ) + + +def _rewrite( + invented, *, persons=None, households=None, members=None, bom=False, ending="\n" +): + fixture, _frame, h, p = invented + if households is not None: + write_archive( + fixture.household_zip, (Member("psam_husa.csv", _csv(households)),) + ) + if persons is not None or members is not None: + p = persons if persons is not None else p + write_archive( + fixture.person_zip, + members + if members is not None + else ( + Member("psam_pusa.csv", _csv(p[:1], bom=bom, ending=ending)), + Member("psam_pusb.csv", _csv(p[1:], bom=bom, ending=ending)), + ), + ) + fixture.pin() + + +def _refuses(code): + return pytest.raises(owner.ACSCoverageAuthenticationError, match="^" + code + "$") + + +def test_source_authentication_preserves_low_receipt_and_parent_without_population_grant( + invented, +): + fixture, frame, _h, _p = invented + before = custody.frame_content_sha256(frame) + original_tables = {e: frame.table(e).copy(deep=True) for e in frame.entities} + result = _load(invented) + table, receipt = literal.read_acs_person_coverage_columns( + AcsPumsSource(fixture.household_zip, fixture.person_zip), + person_keys=owner._native_roster(frame)[0], + chunksize=3, + ) + pd.testing.assert_frame_equal(result.table, table) + assert result.receipt["original_literal_receipt"] == receipt + assert receipt["source_authenticated"] is False + assert result.receipt["source_authenticated"] is True + assert result.native_binding.receipt["population_binding_authenticated"] is False + assert ( + result.native_binding.receipt["missing_authority"] + == "closed_prepared_acs_native_population" + ) + assert ( + result.native_binding.receipt["original_age"]["relation"] == "numeric_identity" + ) + for flag in ( + "domain_assignment_authenticated", + "period_harmonized", + "release_eligible", + "cross_survey_coverage_equivalence_established", + "population_binding_authenticated", + ): + assert result.receipt[flag] is False + assert custody.frame_content_sha256(frame) == before + for entity, original in original_tables.items(): + pd.testing.assert_frame_equal(frame.table(entity), original) + assert ( + owner.verify_acs_coverage_native_consistency(result, frame) + == result.native_binding + ) + assert [m["rows"] for m in result.receipt["members"]["person"]] == [1, 2] + assert all(len(m["sha256"]) == 64 for m in result.receipt["members"]["person"]) + + +def test_first_frame_hash_precedes_the_roster_snapshot(invented, monkeypatch): + """The returned binding must describe one state, including at hash entry.""" + _fixture, frame, _h, _p = invented + original_hash = custody.frame_content_sha256 + first = True + + def at_hash_entry(value): + nonlocal first + if first: + first = False + value.person.loc[:, "source_row_id"] += 100 + return original_hash(value) + + monkeypatch.setattr(custody, "frame_content_sha256", at_hash_entry) + result = _load(invented) + assert ( + owner.verify_acs_coverage_native_consistency(result, frame) + == result.native_binding + ) + + +def test_mutation_during_roster_snapshot_refuses_issuance(invented, monkeypatch): + original_roster = owner._native_roster + first = True + + def after_roster_read(value): + nonlocal first + result = original_roster(value) + if first: + first = False + value.person.loc[:, "source_row_id"] += 100 + return result + + monkeypatch.setattr(owner, "_native_roster", after_roster_read) + with _refuses("NATIVE_FRAME_CHANGED"): + _load(invented) + + +def test_closed_default_manifest_has_no_caller_pin_or_prepared_authority( + invented, monkeypatch +): + fixture, frame, _h, _p = invented + pins = custody._ARCHIVE_PINS + monkeypatch.setattr(custody, "_ARCHIVE_PINS", None) + calls = [] + + def default_manifest(*args, **kwargs): + assert not args and not kwargs + calls.append(True) + return SimpleNamespace( + artifacts=tuple( + SimpleNamespace(role=r, filename=n, sha256=d, size_bytes=s) + for r, n, d, s in pins + ) + ) + + monkeypatch.setattr(custody, "load_acs_source_manifest", default_manifest) + result = _load(invented) + assert len(calls) == 2 + with pytest.raises(TypeError): + _load(invented, manifest={"sha256": "0" * 64}) + for constructor, payload in ( + (owner.AuthenticatedACSPersonCoverage, result.payload), + (owner.UngrantedACSNativeBinding, result.native_binding.receipt_json), + ): + with _refuses("SOURCE_CONSTRUCTOR|BINDING_CONSTRUCTOR"): + constructor(payload) + for forged in ( + frame.person, + {"frame": frame, "frame_sha256": custody.frame_content_sha256(frame)}, + "0" * 64, + ): + with _refuses("NATIVE_FRAME_REQUIRED"): + owner.load_authenticated_acs_person_coverage( + fixture.source_dir, snapshot_root=fixture.snapshot_root, frame=forged + ) + + +@pytest.mark.parametrize( + "field,value,code", + [ + ("SPORDER", 3, "NATIVE_ALIASES"), + ("A_LINENO", 3, "NATIVE_ALIASES"), + ("source_person_id", "2", "NATIVE_ALIASES"), + ("source_household_id", 2, "NATIVE_ALIASES"), + ("source_year", 2023, "NATIVE_VINTAGE"), + ("person_household_id", 2, "NATIVE_ALIASES"), + ("source_row_id", 1, "NATIVE_IDS"), + ], +) +def test_native_alias_and_membership_changes_with_same_person_ids_refuse( + invented, field, value, code +): + invented[1].person.loc[0, field] = value + with _refuses(code): + _load(invented) + + +@pytest.mark.parametrize( + "change", ["serial", "age", "membership", "weight", "source_row"] +) +def test_live_binding_refuses_same_ids_changed_native_parent(invented, change): + _fixture, frame, _h, _p = invented + result = _load(invented) + if change == "serial": + h = frame.table("household") + h["SERIALNO"] = h.SERIALNO.iloc[::-1].to_numpy() + elif change == "age": + frame.person.loc[2, "age"] = 82 + elif change == "membership": + for c in ("person_household_id", "source_household_id"): + frame.person.loc[[1, 2], c] = frame.person.loc[[2, 1], c].to_numpy() + elif change == "weight": + frame.table("household")["extra_cell"] = ( + 123 # all parent cells bind, including additions + ) + else: + frame.person.loc[0, "source_row_id"] = 100 + with _refuses( + "NATIVE_FRAME_CHANGED|NATIVE_HOUSEHOLD_COUNT|NATIVE_CONSISTENCY_REFUSED" + ): + owner.verify_acs_coverage_native_consistency(result, frame) + + +@pytest.mark.parametrize( + "change", + [ + "missing", + "extra", + "duplicate", + "household_np", + "household_missing", + "wrong_key", + "wrong_vintage", + ], +) +def test_source_complete_household_mismatch_refuses(invented, change): + _fixture, _frame, h, p = invented + if change == "missing": + p = p[:2] + elif change == "extra": + p += [{**p[1], "SPORDER": 3}] + elif change == "duplicate": + p[1] = p[0] + elif change == "household_np": + h[0]["NP"] = 1 + elif change == "household_missing": + h = h[:1] + elif change == "wrong_vintage": + p[0]["SERIALNO"] = "2023HU0000001" + else: + p[0]["SPORDER"] = 3 + _rewrite(invented, persons=p, households=h) + with _refuses("SOURCE_PERSON_ROSTER|SOURCE_HOUSEHOLD_ROSTER|SOURCE_DUPLICATE_KEY"): + _load(invented) + + +@pytest.mark.parametrize( + "age,mil,esr", + [ + ("", "", ""), + ("NA", "NA", "1.0"), + ("30 ", " 1", "4 "), + ("30", '1,"quoted"\r\n\t', "4\r\n\t"), + ("30\t", "1", "4"), + ("100", "0", "7"), + ], +) +@pytest.mark.parametrize("ending", ["\n", "\r", "\r\n"]) +def test_bom_multiline_tabs_and_malformed_literals_survive_envelope( + invented, age, mil, esr, ending +): + _fixture, frame, _h, p = invented + p[0].update(AGEP=age, MIL=mil, ESR=esr) + _rewrite(invented, persons=p, bom=True, ending=ending) + result = _load(invented) + assert result.table.loc[0, ["AGEP", "MIL", "ESR"]].tolist() == [age, mil, esr] + assert not result.table.isna().any().any() + assert frame.person.loc[0, "age"] == 30 + if age != "30": + assert result.table.loc[0, "MIL_state"] == "age_unresolved" + assert result.native_binding.receipt["original_age"]["relation"] == "unproven" + + +@pytest.mark.parametrize( + "column,value", [("AGEP", 82), ("A_AGE", 82), ("age", 82), ("age", float("nan"))] +) +def test_model_age_never_fills_or_authenticates_original_relation( + invented, column, value +): + frame = invented[1] + if isinstance(value, float): + frame.person[column] = frame.person[column].astype("float64") + frame.person.loc[2, column] = value + result = _load(invented) + assert result.table.loc[2, "AGEP"] == "80" + assert result.native_binding.receipt["original_age"]["mismatching_rows"] == 1 + assert result.native_binding.receipt["population_binding_authenticated"] is False + + +def test_candidate_reconstructed_before_comparison_and_self_rehash_cannot_authorize( + invented, monkeypatch +): + result = _load(invented) + candidate = invented[0].tmp_path / "candidate.coverage" + candidate.write_bytes(result.payload) + assert _load(invented, candidate_path=candidate).payload == result.payload + header_raw, body = result._parts() + header = json.loads(header_raw) + tampered = body.replace(b'"30","1","4"', b'"30","2","4"', 1) + assert tampered != body + header["body_sha256"] = hashlib.sha256(tampered).hexdigest() + raw = owner._json(header) + candidate.write_bytes(owner._MAGIC + len(raw).to_bytes(4, "big") + raw + tampered) + with _refuses("CANDIDATE_MISMATCH"): + _load(invented, candidate_path=candidate) + flip_archive_byte(invented[0].person_zip) + original_copy = custody._copy + + def guarded_copy(source, destination, *args, **kwargs): + assert source != candidate, ( + "candidate was accessed before source reconstruction" + ) + return original_copy(source, destination, *args, **kwargs) + + monkeypatch.setattr(custody, "_copy", guarded_copy) + with _refuses("SOURCE_RECONSTRUCTION_REFUSED"): + _load(invented, candidate_path=candidate) + + +@pytest.mark.parametrize("replacement", [None, "", "different"]) +def test_candidate_null_empty_and_field_contract_tampering_refuse( + invented, replacement +): + result = _load(invented) + header, body = result._parts() + rows = [json.loads(row) for row in body.splitlines()] + rows[1][3] = replacement + if replacement == "": + metadata = json.loads(header) + metadata["field_contract"]["fields"]["MIL"]["minimum_age"] = 16 + else: + metadata = json.loads(header) + changed_body = b"".join(owner._json(row) + b"\n" for row in rows) + metadata.update( + body_sha256=hashlib.sha256(changed_body).hexdigest(), + body_bytes=len(changed_body), + ) + changed_header = owner._json(metadata) + candidate = invented[0].tmp_path / "candidate.coverage" + candidate.write_bytes( + owner._MAGIC + + len(changed_header).to_bytes(4, "big") + + changed_header + + changed_body + ) + with _refuses("CANDIDATE_MISMATCH|SOURCE_RECONSTRUCTION_REFUSED"): + _load(invented, candidate_path=candidate) + + +def test_views_and_closed_objects_cannot_mutate_owned_evidence(invented): + result = _load(invented) + before = result.payload + view = result.table + view.loc[0, "MIL"] = "2" + result.receipt["field_contract"].clear() + result.native_binding.receipt["population_binding_authenticated"] = True + assert result.payload == before + assert result.table.loc[0, "MIL"] == "1" + assert result.native_binding.receipt["population_binding_authenticated"] is False + with pytest.raises(FrozenInstanceError): + result.payload = b"forged" + with _refuses("SOURCE_CONSTRUCTOR"): + replace(result, payload=b"forged") + + +@pytest.mark.parametrize( + "kind", ["duplicate", "case_duplicate", "unsafe", "symlink", "prefix_non_csv"] +) +def test_entire_zip_directory_is_checked_before_literal_reader( + invented, monkeypatch, kind +): + p = invented[3] + extra = { + "duplicate": Member("psam_pusa.csv", _csv(p)), + "case_duplicate": Member("PSAM_PUSA.CSV", _csv(p)), + "unsafe": Member("../unrelated.txt", b"ignored"), + "symlink": Member("unrelated.txt", b"target", external_attr=0o120777 << 16), + "prefix_non_csv": Member("psam_pus.txt", b"ignored"), + }[kind] + if kind == "duplicate": + with pytest.warns(UserWarning, match="Duplicate name"): + _rewrite(invented, members=(Member("psam_pusa.csv", _csv(p)), extra)) + else: + _rewrite(invented, members=(Member("psam_pusa.csv", _csv(p)), extra)) + monkeypatch.setattr( + literal, + "read_acs_person_coverage_columns", + lambda *a, **k: pytest.fail("reader entered"), + ) + with _refuses( + "ZIP_DUPLICATE_MEMBER|ZIP_MEMBER_PATH|ZIP_MEMBER_TYPE|ZIP_PREFIX_NONCSV" + ): + _load(invented) + + +@pytest.mark.parametrize( + "target,constant,value,code", + [ + (custody, "_MEMBER_COUNT_MAX", 1, "ZIP_MEMBER_COUNT"), + (custody, "_MEMBER_MAX", 100, "ZIP_MEMBER_SIZE"), + (custody, "_EXPANDED_MAX", 100, "ZIP_EXPANDED_SIZE"), + (owner, "MAX_CSV_HEADER_BYTES", 30, "CSV_RECORD_BYTES"), + (owner, "MAX_RECORD_BYTES", 64, "CSV_RECORD_BYTES"), + (owner, "MAX_TOKEN_BYTES", 10, "CSV_TOKEN_BYTES"), + (owner, "MAX_BODY_BYTES", 100, "SELECTED_BODY_BUDGET"), + (owner, "MAX_HEADER_BYTES", 100, "CANONICAL_SIZE"), + (literal, "MAX_ROWS", 1, "SOURCE_ROWS"), + (literal, "MAX_SELECTED_ROWS", 2, "NATIVE_ROWS"), + ], +) +def test_small_limits_refuse_before_relevant_allocation( + invented, monkeypatch, target, constant, value, code +): + monkeypatch.setattr(target, constant, value) + with _refuses(code): + _load(invented) + + +@pytest.mark.parametrize("kind", ["corrupt", "growth", "symlink", "fifo"]) +def test_exact_capture_refusals_are_static_and_hide_cause_chains(invented, kind): + path = invented[0].person_zip + if kind == "corrupt": + flip_archive_byte(path) + elif kind == "growth": + with path.open("ab") as stream: + stream.write(b"extra") + else: + saved = path.with_suffix(".saved") + path.rename(saved) + if kind == "symlink": + path.symlink_to(saved) + else: + os.mkfifo(path) + # Keep directory roster exact; saved bytes live outside the source dir. + saved.rename(invented[0].tmp_path / "saved.zip") + with _refuses("SOURCE_RECONSTRUCTION_REFUSED") as exc: + _load(invented) + assert exc.value.__cause__ is None + assert exc.value.__suppress_context__ + + +@pytest.mark.parametrize( + "race", ["private_bytes", "original_replace", "pins", "producer", "frame"] +) +def test_races_refuse_before_issuance(invented, monkeypatch, race): + original = literal.read_acs_person_coverage_columns + + def raced(source, **kwargs): + result = original(source, **kwargs) + if race == "private_bytes": + source.person_zip.chmod(0o600) + flip_archive_byte(source.person_zip) + elif race == "original_replace": + # Replace during _copy instead, below. + pass + elif race == "pins": + monkeypatch.setattr(custody, "_ARCHIVE_PINS", ()) + elif race == "producer": + monkeypatch.setattr(owner, "MAX_TOKEN_BYTES", owner.MAX_TOKEN_BYTES - 1) + else: + invented[1].person.loc[0, "age"] = 31 + return result + + if race == "original_replace": + old_identity = custody._identity + changed = False + + def replaced_identity(value): + nonlocal changed + if not changed: + changed = True + path = invented[0].household_zip + raw = path.read_bytes() + path.unlink() + path.write_bytes(raw) + return old_identity(value) + + monkeypatch.setattr(custody, "_identity", replaced_identity) + monkeypatch.setattr(literal, "read_acs_person_coverage_columns", raced) + with _refuses( + "SNAPSHOT_CHANGED|SOURCE_RECONSTRUCTION_REFUSED|PRODUCER_CHANGED|NATIVE_FRAME_CHANGED" + ): + _load(invented) + + +@pytest.mark.parametrize("ending", [b"\n", b"\r", b"\r\n", b""]) +def test_raw_record_byte_boundary_precedes_decoding_and_preserves_endings( + monkeypatch, ending +): + header = b"A,B\n" + body = b'"x\r\n\ty",z' + ending + monkeypatch.setattr(owner, "MAX_RECORD_BYTES", len(body)) + assert list(owner._records(io.BytesIO(header + body))) == [header, body] + monkeypatch.setattr(owner, "MAX_RECORD_BYTES", len(body) - 1) + with _refuses("CSV_RECORD_BYTES"): + list(owner._records(io.BytesIO(header + body))) + + +@pytest.mark.parametrize( + "value", + [ + "", + None, + "\r\n\t", + '😀"\\\b\f', + ["", None, "é"], + {"b": True, "a": [1, -2, False]}, + ], +) +def test_canonical_size_is_checked_before_string_encoding(monkeypatch, value): + expected = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + assert owner._json(value, len(expected)) == expected + + def forbidden_encoder(*a, **k): + pytest.fail("JSON encoder allocated before preflight refusal") + + monkeypatch.setattr(owner.json, "JSONEncoder", forbidden_encoder) + with _refuses("CANONICAL_SIZE"): + owner._json(value, len(expected) - 1) diff --git a/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py b/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py new file mode 100644 index 000000000..5ac09804c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py @@ -0,0 +1,496 @@ +"""Literal coverage projections from invented, multi-member ACS archives.""" + +import csv +import hashlib +import io +import json +from zipfile import ZipFile + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import acs_person_coverage_columns as module +from microcosm.build.us_runtime import acs_pums + + +def _row(serial="2024HU0000001", order="1", age="30", mil="4", esr="6"): + return dict(zip(module.READ_COLUMNS, (serial, order, age, mil, esr), strict=True)) + + +def _archive(tmp_path, members): + path = tmp_path / "persons.zip" + with ZipFile(path, "w") as archive: + for name, rows in members.items(): + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=module.READ_COLUMNS, lineterminator="\n" + ) + writer.writeheader() + writer.writerows(rows) + archive.writestr(name, stream.getvalue()) + return acs_pums.AcsPumsSource( + tmp_path / "unused-households.zip", path, max_households=1 + ) + + +def _keys(rows): + return pd.DataFrame(rows).loc[:, list(module.KEYS)] + + +def test_private_scanner_exhausts_members_despite_consumer_return_value(tmp_path): + first = _row(order="02", age="16", mil="", esr="1") + second = _row(order="01", age="17", mil="4", esr=" 6") + third = _row("2024GQ0000001", age="", mil='x,"\t', esr="") + source = _archive( + tmp_path, + {"psam_pusb.csv": [second, third], "psam_pusa.csv": [first]}, + ) + received = [] + + def consume(cells): + received.append(cells) + return True # No return value can turn a prefix into a successful scan. + + members, rows = module._scan_acs_person_coverage(source, consume) + assert members == ["psam_pusa.csv", "psam_pusb.csv"] + assert rows == 3 + assert received == [ + ("2024HU0000001", "02", "16", "", "1"), + ("2024HU0000001", "01", "17", "4", " 6"), + ("2024GQ0000001", "1", "", 'x,"\t', ""), + ] + + +@pytest.mark.parametrize( + "tail,match", + [ + ("2024HU0000009,1,30,4,6,EXTRA\n", "record width"), + ("2024HU0000009,1,30,4,6\x00\n", "forbidden control"), + ('2024HU0000009,1,30,"unfinished,6\n', "invalid literal CSV"), + ], +) +def test_private_scanner_refuses_invalid_tail_after_delivering_expected_rows( + tmp_path, tail, match +): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR\n2024HU0000001,1,30,4,6\n" + tail, + ) + received = [] + with pytest.raises(ValueError, match=match): + module._scan_acs_person_coverage(source, received.append) + assert received == [("2024HU0000001", "1", "30", "4", "6")] + + +def test_exact_household_roster_across_members_preserves_blanks_and_request_order( + tmp_path, +): + rows = [ + _row(mil="1", esr="4"), + _row(order="2", age="10", mil="", esr=""), + _row("2024GQ0000001", age="16", mil="", esr="1"), + ] + unrelated = _row("2024HU0000009") + source = _archive( + tmp_path, + { + "nested/psam_pusb.csv": [rows[0], unrelated], + "psam_pusa.csv": [rows[2], rows[1]], + }, + ) + requested = _keys([rows[1], rows[2], rows[0]]) + requested["SPORDER"] = requested.SPORDER.astype("int64") + result, receipt = module.read_acs_person_coverage_columns( + source, person_keys=requested, chunksize=1 + ) + assert result.MIL.tolist() == ["", "", "1"] + assert result.ESR.tolist() == ["", "1", "4"] + assert result.MIL_state.tolist() == [ + "outside_age_universe", + "outside_age_universe", + "observed_code", + ] + assert result.ESR_state.tolist() == [ + "outside_age_universe", + "observed_code", + "observed_code", + ] + assert result.SERIALNO.tolist() == requested.SERIALNO.tolist() + assert result.SPORDER.tolist() == requested.SPORDER.tolist() + assert not result.isna().any().any() + assert receipt["source_rows_streamed"] == 4 + assert receipt["selected_person_rows"] == 3 + assert receipt["source_authenticated"] is False + assert receipt["release_eligible"] is False + assert receipt["legacy_max_households_applied"] is False + assert receipt["coverage_status"] == "literal_source_fields_only" + second, other = module.read_acs_person_coverage_columns( + source, person_keys=requested, chunksize=3 + ) + pd.testing.assert_frame_equal(result, second) + assert other == receipt + + +@pytest.mark.parametrize( + "age,mil,esr,mil_state,esr_state", + [ + ("17", "2", "2", "observed_code", "observed_code"), + ("45", "3", "5", "observed_code", "observed_code"), + ("30", "", "", "missing_in_universe", "missing_in_universe"), + ("30", "0", "7", "unlabelled_code", "unlabelled_code"), + ("30", "NA", "1.0", "unlabelled_code", "unlabelled_code"), + ("30", " 1", "4 ", "unlabelled_code", "unlabelled_code"), + ("15", "1", "4", "value_below_age_universe", "value_below_age_universe"), + ("", "1", "4", "age_unresolved", "age_unresolved"), + ("100", "4", "6", "age_unresolved", "age_unresolved"), + ], +) +def test_unresolved_states_preserve_literal_source_tokens( + tmp_path, age, mil, esr, mil_state, esr_state +): + row = _row(age=age, mil=mil, esr=esr) + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + table, _ = module.read_acs_person_coverage_columns(source, person_keys=_keys([row])) + assert table.loc[0, ["AGEP", "MIL", "ESR"]].tolist() == [age, mil, esr] + assert table.MIL_state.tolist() == [mil_state] + assert table.ESR_state.tolist() == [esr_state] + + +@pytest.mark.parametrize( + "mutation,match", + [ + ("extra", "extra selected-household"), + ("missing", "native roster differ"), + ("duplicate", "keys repeat"), + ("wrong_order", "native roster differ"), + ], +) +def test_partial_household_or_conflicting_native_roster_refuses( + tmp_path, mutation, match +): + rows = [_row(), _row(order="2")] + observed = rows.copy() + if mutation == "extra": + observed += [_row(order="3")] + elif mutation == "missing": + observed.pop() + elif mutation == "duplicate": + observed[1] = rows[0] + else: + observed[1] = _row(order="3") + source = _archive(tmp_path, {"psam_pusa.csv": observed}) + with pytest.raises(ValueError, match=match): + module.read_acs_person_coverage_columns( + source, person_keys=_keys(rows), chunksize=1 + ) + + +@pytest.mark.parametrize( + "serial,order", + [ + ("2023HU0000001", "1"), + ("2024HU0000001", "0"), + ("2024HU0000001", "21"), + ("2024HU0000001", "1.0"), + ("2024HU0000001\x00other", "1"), + ("2024HU0000001", "1\x00other"), + (None, "1"), + ("2024HU0000001", True), + ], +) +def test_invalid_native_keys_refuse_before_file_read(tmp_path, serial, order): + source = acs_pums.AcsPumsSource( + tmp_path / "absent-h.zip", tmp_path / "absent-p.zip" + ) + with pytest.raises(ValueError, match="native"): + module.read_acs_person_coverage_columns( + source, person_keys=pd.DataFrame({"SERIALNO": [serial], "SPORDER": [order]}) + ) + + +@pytest.mark.parametrize( + "mutation", ["missing_column", "duplicate_column", "duplicate_member"] +) +def test_ambiguous_source_schema_refuses(tmp_path, mutation): + source = _archive(tmp_path, {"psam_pusa.csv": [_row()]}) + with ZipFile(source.person_zip, "r") as archive: + raw = archive.read("psam_pusa.csv") + if mutation == "missing_column": + raw = raw.replace(b"MIL", b"OTHER") + elif mutation == "duplicate_column": + raw = raw.replace(b"ESR", b"MIL") + with ZipFile(source.person_zip, "w") as archive: + archive.writestr("psam_pusa.csv", raw) + if mutation == "duplicate_member": + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr("psam_pusa.csv", raw) + with pytest.raises(ValueError, match="missing or duplicated"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +def test_explicit_row_bound_refuses_without_truncation(tmp_path, monkeypatch): + row = _row() + source = _archive(tmp_path, {"psam_pusa.csv": [row, _row("2024HU0000002")]}) + monkeypatch.setattr(module, "MAX_ROWS", 1) + with pytest.raises(ValueError, match="row bound"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([row]), chunksize=1 + ) + + +def test_selected_row_ceiling_refuses_before_opening_the_source(tmp_path, monkeypatch): + source = acs_pums.AcsPumsSource( + tmp_path / "absent-h.zip", tmp_path / "absent-p.zip" + ) + monkeypatch.setattr(module, "MAX_SELECTED_ROWS", 1) + with pytest.raises(ValueError, match="selected person count is outside the bound"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row(), _row(order="2")]) + ) + + +def test_contract_is_owned_and_separate_from_legacy_native_roster(): + contract = module.coverage_field_contract() + assert contract["fields"]["MIL"]["minimum_age"] == 17 + assert contract["fields"]["ESR"]["minimum_age"] == 16 + contract["fields"]["MIL"]["codes"].clear() + assert len(module.coverage_field_contract()["fields"]["MIL"]["codes"]) == 4 + assert not {"MIL", "ESR"}.intersection(acs_pums._PERSON_REQUIRED) + + +def _raw_archive(tmp_path, raw): + source = acs_pums.AcsPumsSource( + tmp_path / "unused-households.zip", tmp_path / "literal-persons.zip" + ) + with ZipFile(source.person_zip, "w") as archive: + archive.writestr("psam_pusa.csv", raw) + return source + + +@pytest.mark.parametrize("field", module.READ_COLUMNS) +def test_nul_suffix_refuses_before_truncating_value_or_identity(tmp_path, field): + row = _row(mil="1") + row[field] += "\x00garbage" + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize( + "control", + [chr(code) for code in range(32) if code not in (9, 10, 13)] + + [chr(code) for code in range(127, 160)], +) +def test_non_csv_control_characters_refuse(tmp_path, control): + source = _archive(tmp_path, {"psam_pusa.csv": [_row(mil="1" + control)]}) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("location", ["header", "ignored_column", "unselected_row"]) +def test_control_validation_precedes_column_and_household_selection(tmp_path, location): + header = "SERIALNO,SPORDER,AGEP,MIL,ESR,OTHER\n" + body = "2024HU0000001,1,30,1,6,ok\n" + if location == "header": + header = header.replace("OTHER", "OTHER\x00suffix") + elif location == "ignored_column": + body = body.replace("ok", "ok\x00suffix") + else: + body += "2024HU0000009,1,30,1\x00suffix,6,ok\n" + source = _raw_archive(tmp_path, header + body) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("serial", ["2024HU0000001", "2024HU0000009"]) +@pytest.mark.parametrize( + "tail", + ["1,30,4", "1,30,4,6", "1,30,4,6,ok,extra", "1,30,4,6,ok,"], + ids=["missing_required", "missing_unused", "too_many", "trailing_delimiter"], +) +def test_full_record_width_refuses_even_outside_projection(tmp_path, serial, tail): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR,OTHER\n" + serial + "," + tail + "\n", + ) + with pytest.raises(ValueError, match="record width"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize( + "body", + ["\n", '2024HU0000001,1,30,4,"unterminated\n', '2024HU0000001,1,30,"1"garbage,6\n'], +) +def test_blank_records_and_malformed_quoting_refuse(tmp_path, body): + source = _raw_archive(tmp_path, "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + body) + with pytest.raises(ValueError, match="CSV|record width"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("line_end", ["\n", "\r", "\r\n"]) +def test_quoted_delimiters_newlines_tabs_and_digest_preserve_tokens(tmp_path, line_end): + stream = io.StringIO(newline="") + writer = csv.writer(stream, quoting=csv.QUOTE_ALL, lineterminator=line_end) + writer.writerow(["OTHER", "ESR", "MIL", "AGEP", "SPORDER", "SERIALNO"]) + writer.writerow( + [ + 'ignored, "quoted"\r\nfield', + "4\r\n\t", + '1,"quoted"\n\r', + "30", + "01", + "2024HU0000001", + ] + ) + source = _raw_archive(tmp_path, "\ufeff" + stream.getvalue()) + table, receipt = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]), chunksize=1 + ) + expected = [ + { + "SERIALNO": "2024HU0000001", + "SPORDER": 1, + "AGEP": "30", + "MIL": '1,"quoted"\n\r', + "ESR": "4\r\n\t", + "MIL_state": "unlabelled_code", + "ESR_state": "unlabelled_code", + } + ] + assert table.to_dict("records") == expected + assert receipt["source_rows_streamed"] == 1 + + def digest(records): + return hashlib.sha256( + json.dumps( + records, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + + assert receipt["projection_sha256"] == digest(expected) + for token in ("4\n\t", "4\r\t", r"4\r\n\t", "4"): + assert receipt["projection_sha256"] != digest([{**expected[0], "ESR": token}]) + + +@pytest.mark.parametrize("field", ["AGEP", "MIL", "ESR"]) +@pytest.mark.parametrize("control", ["\t", "\r", "\n", "\r\n"]) +def test_accepted_csv_whitespace_is_literal_and_never_an_observed_code( + tmp_path, field, control +): + row = _row(mil="1", esr="4") + row[field] += control + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=module.READ_COLUMNS, quoting=csv.QUOTE_ALL + ) + writer.writeheader() + writer.writerow(row) + source = _raw_archive(tmp_path, stream.getvalue()) + table, _ = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]) + ) + assert table.loc[0, field] == row[field] + if field == "AGEP": + assert ( + table.MIL_state.tolist() == table.ESR_state.tolist() == ["age_unresolved"] + ) + else: + assert table[field + "_state"].tolist() == ["unlabelled_code"] + + +@pytest.mark.parametrize("field", module.KEYS) +@pytest.mark.parametrize("suffix", ["\t", "\n", "\r\n", " other", ",other"]) +def test_literal_identity_suffix_cannot_join_canonical_requested_key( + tmp_path, field, suffix +): + row = _row() + row[field] += suffix + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + with pytest.raises(ValueError, match="missing|native"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +def test_duplicate_canonical_person_order_in_later_member_refuses(tmp_path): + source = _archive( + tmp_path, + {"psam_pusa.csv": [_row(order="01")], "psam_pusb.csv": [_row(order="1")]}, + ) + with pytest.raises(ValueError, match="keys repeat"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row(), _row(order="2")]), chunksize=1 + ) + + +def test_invalid_utf8_refuses(tmp_path): + source = _raw_archive( + tmp_path, b"SERIALNO,SPORDER,AGEP,MIL,ESR\n2024HU0000001,1,30,1\xff,6\n" + ) + with pytest.raises(ValueError): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("multiline", [False, True]) +def test_record_character_bound_includes_all_quoted_physical_lines( + tmp_path, monkeypatch, multiline +): + monkeypatch.setattr(module, "MAX_CSV_RECORD_CHARS", 100, raising=False) + suffix = ("x\n" if multiline else "xx") * 60 + source = _archive(tmp_path, {"psam_pusa.csv": [_row(mil=suffix)]}) + with pytest.raises(ValueError, match="record character bound"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("line_end", ["", "\n", "\r\n"]) +def test_record_character_bound_accepts_exact_limit_and_resets_per_record( + tmp_path, monkeypatch, line_end +): + monkeypatch.setattr(module, "MAX_CSV_RECORD_CHARS", 100) + prefix, suffix = '2024HU0000001,1,30,"', '",6' + line_end + token = "x\n" + "x" * (100 - len(prefix) - len(suffix) - 2) + body = prefix + token + suffix + assert len(body) == 100 + source = _raw_archive(tmp_path, "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + body) + table, receipt = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]) + ) + assert table.MIL.tolist() == [token] + assert receipt["source_rows_streamed"] == 1 + + +def test_stream_opens_member_once_and_stops_at_refusal_without_full_validation( + tmp_path, monkeypatch +): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + + "2024HU0000001,1,30,4,6\n" * 2 + + "2024HU0000009,1,30,4,6\n" * 20_000, + ) + opened, consumed = [], [] + + class GuardedZipFile(ZipFile): + def open(self, name, *args, **kwargs): + opened.append(name) + member = super().open(name, *args, **kwargs) + original_read, original_read1 = member.read, member.read1 + + def bounded(method, size=-1): + assert size >= 0, "unbounded archive read" + result = method(size) + consumed.append(len(result)) + return result + + member.read = lambda size=-1: bounded(original_read, size) + member.read1 = lambda size=-1: bounded(original_read1, size) + return member + + def read(self, *args, **kwargs): + pytest.fail("whole-member validation read") + + monkeypatch.setattr(module, "ZipFile", GuardedZipFile) + with pytest.raises(ValueError, match="extra selected-household"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]), chunksize=1 + ) + assert opened == ["psam_pusa.csv"] + assert sum(consumed) < 32_768 diff --git a/packages/microcosm-build/tests/test_us_acs_population_catalogue.py b/packages/microcosm-build/tests/test_us_acs_population_catalogue.py new file mode 100644 index 000000000..ab98b77dc --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_population_catalogue.py @@ -0,0 +1,966 @@ +"""Actual pinned invented archives; never a native Frame or fake source issuer.""" + +import _csv +import ast +import copy +import csv +import gc +import hashlib +import inspect +import io +import json +import pickle +import sys +import weakref +import zipfile +from pathlib import Path +from types import CodeType, FunctionType, SimpleNamespace + +import pytest + +from microcosm.build.us_runtime import acs_population_catalogue as owner +from microcosm.frame import Frame + + +def csv_bytes(rows): + stream = io.StringIO(newline="") + writer = csv.DictWriter(stream, list(rows[0]), lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + return stream.getvalue().encode() + + +@pytest.fixture +def invented(tmp_path, monkeypatch): + source, snapshots = tmp_path / "source", tmp_path / "snapshots" + source.mkdir() + snapshots.mkdir() + households = [ + dict( + SERIALNO="2024HU0000001", + TYPEHUGQ="1", + NP="2", + WGTP="0010", + TEN="1", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024GQ0000001", + TYPEHUGQ="2", + NP="1", + WGTP="0", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024HU0000002", + TYPEHUGQ="1", + NP="0", + WGTP="20", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024GQ0000002", + TYPEHUGQ="3", + NP="1", + WGTP="0", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024HU0000003", + TYPEHUGQ="1", + NP="1", + WGTP="30", + TEN="3", + ST="01", + PUMA="00100", + ), + ] + people = [ + dict( + SERIALNO="2024HU0000001", + SPORDER="01", + PWGTP="0011", + AGEP="30", + MIL="1", + ESR="4", + ), + dict( + SERIALNO="2024GQ0000001", + SPORDER="01", + PWGTP="0077", + AGEP="40", + MIL="4", + ESR="6", + ), + dict( + SERIALNO="2024HU0000001", + SPORDER="02", + PWGTP="12", + AGEP="15", + MIL="", + ESR="", + ), + dict( + SERIALNO="2024GQ0000002", + SPORDER="1", + PWGTP="78", + AGEP="50", + MIL="4", + ESR="6", + ), + dict( + SERIALNO="2024HU0000003", SPORDER="1", PWGTP="31", AGEP="99", MIL="", ESR="" + ), + ] + + def write(): + for _role, filename, prefix, rows in ( + ("household", "csv_hus.zip", "psam_hus", households), + ("person", "csv_pus.zip", "psam_pus", people), + ): + with zipfile.ZipFile(source / filename, "w") as archive: + empty = ( + b"SERIALNO,SPORDER,PWGTP,AGEP,MIL,ESR\n" + if _role == "person" + else b"SERIALNO,TYPEHUGQ,NP,WGTP,TEN,ST,PUMA\n" + ) + archive.writestr( + prefix + "a.csv", csv_bytes(rows[:2]) if rows[:2] else empty + ) + archive.writestr( + prefix + "b.csv", csv_bytes(rows[2:]) if rows[2:] else empty + ) + pins = tuple( + ( + role, + name, + hashlib.sha256((source / name).read_bytes()).hexdigest(), + (source / name).stat().st_size, + ) + for role, name in (("household", "csv_hus.zip"), ("person", "csv_pus.zip")) + ) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", pins) + + monkeypatch.setattr( + owner.housing.shutil, + "disk_usage", + lambda _path: SimpleNamespace(free=64 * 1024**3), + ) + monkeypatch.setattr(owner, "_BATCH_PEOPLE", 2) + write() + return SimpleNamespace( + source=source, + snapshots=snapshots, + households=households, + people=people, + write=write, + root=tmp_path, + ) + + +def issue(fixture, **kwargs): + return owner.issue_acs_source_catalogue( + fixture.source, snapshot_root=fixture.snapshots, **kwargs + ) + + +def test_real_complete_catalogue_without_population_or_classification(invented): + forbidden = [] + + def trace(frame, event, arg): + module = frame.f_globals.get("__name__", "") + if event == "call" and ( + frame.f_code is Frame.__init__.__code__ + or module.startswith("microunit") + or frame.f_code.co_name + in { + "build_acs_pums_unit_frame", + "prepare_acs_housing_population", + "classify_household", + "classify_households", + } + ): + forbidden.append(frame.f_code.co_name) + raise AssertionError("Population construction/classification occurred") + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = issue(invented) + rows = result.households + receipt = result.receipt + finally: + sys.setprofile(previous) + assert not forbidden + assert result.validate() is result + assert owner.verify_acs_source_catalogue(result) is result + assert len(rows) == 4 and len(result.exclusion_ledger) == 1 + assert {r.key.native_id for r in rows} == { + r["SERIALNO"] for r in invented.households if r["NP"] != "0" + } + row = rows[0] + assert row.wgtp == "0010" + assert [(p.sporder, p.age, p.esr, p.mil, p.pwgtp) for p in row.persons] == [ + ("01", "30", "4", "1", "0011"), + ("02", "15", "", "", "12"), + ] + assert row.persons[1].esr_state == "outside_age_universe" + assert rows[-1].persons[0].esr_state == "missing_in_universe" + assert rows[1].persons[0].pwgtp == "0077" + assert result.lineage[row.key.native_id]["persons"] == ( + ("01", "psam_pusa.csv", 1), + ("02", "psam_pusb.csv", 1), + ) + assert receipt["counts"]["people"] == 5 + assert receipt["counts"]["literal_batches"] == 3 + assert ( + receipt["source_authenticated"] + and not receipt["population_binding_authenticated"] + ) + assert ( + not receipt["domain_assignment_authenticated"] + and not receipt["selection_performed"] + ) + assert not receipt["execution"]["full_person_scan_per_literal_batch"] + assert receipt["execution"]["literal_full_scans"] == 1 + assert issue(invented, candidate=result.to_bytes()).to_bytes() == result.to_bytes() + (invented.root / "catalogue.receipt.json").write_bytes(result.to_bytes()) + held = owner._lookup(result) + (invented.root / "invented-records.json").write_bytes( + owner.native.coverage._json( + {"households": held.records, "vacancy_ledger": held.vacancies} + ) + ) + + +def test_independent_views_cannot_rewrite_issued_rows(invented): + result = issue(invented) + original = result.to_bytes() + row = result.households[0] + object.__setattr__(row, "wgtp", "9999") + object.__setattr__(row.persons[0], "esr", "6") + result.receipt["source_authenticated"] = False + result.lineage.clear() + assert result.households[0].wgtp == "0010" + assert result.households[0].persons[0].esr == "4" + assert result.to_bytes() == original + + +def expected_records(): + """Explicit invented source-order records, independent of collector logic.""" + return ( + ( + "2024HU0000001", + "1", + "2", + "0010", + "psam_husa.csv", + 1, + ( + ( + "01", + "30", + "4", + "observed_code", + "1", + "observed_code", + "0011", + "psam_pusa.csv", + 1, + ), + ( + "02", + "15", + "", + "outside_age_universe", + "", + "outside_age_universe", + "12", + "psam_pusb.csv", + 1, + ), + ), + ), + ( + "2024GQ0000001", + "2", + "1", + "0", + "psam_husa.csv", + 2, + ( + ( + "01", + "40", + "6", + "observed_code", + "4", + "observed_code", + "0077", + "psam_pusa.csv", + 2, + ), + ), + ), + ("2024HU0000002", "1", "0", "20", "psam_husb.csv", 1, ()), + ( + "2024GQ0000002", + "3", + "1", + "0", + "psam_husb.csv", + 2, + ( + ( + "1", + "50", + "6", + "observed_code", + "4", + "observed_code", + "78", + "psam_pusb.csv", + 2, + ), + ), + ), + ( + "2024HU0000003", + "1", + "1", + "30", + "psam_husb.csv", + 3, + ( + ( + "1", + "99", + "", + "missing_in_universe", + "", + "missing_in_universe", + "31", + "psam_pusb.csv", + 3, + ), + ), + ), + ) + + +@pytest.mark.parametrize("batch_limit", [2, 3, 5]) +def test_explicit_canonical_records_and_bytes_ignore_batch_boundaries( + invented, monkeypatch, batch_limit +): + monkeypatch.setattr(owner, "_BATCH_PEOPLE", batch_limit) + result = issue(invented) + held, expected = owner._lookup(result), expected_records() + assert held.records == (expected[0], expected[1], expected[3], expected[4]) + assert held.vacancies == (expected[2],) + raw = b"".join( + json.dumps(row, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + + b"\n" + for row in expected + ) + assert result.receipt["counts"]["canonical_record_bytes"] == len(raw) + assert ( + result.receipt["counts"]["canonical_record_sha256"] + == hashlib.sha256(raw).hexdigest() + ) + + +@pytest.mark.parametrize("batch_limit", [2, 5]) +def test_catalogue_literal_member_scan_count_is_independent_of_batches( + invented, monkeypatch, batch_limit +): + monkeypatch.setattr(owner, "_BATCH_PEOPLE", batch_limit) + scanner = owner.literal._scan_acs_person_coverage.__code__ + active, scans, literal_opens, all_person_opens = set(), [], [], [] + + def trace(frame, event, arg): + if frame.f_code is scanner: + if event == "call": + scans.append(True) + active.add(id(frame)) + elif event == "return": + active.remove(id(frame)) + if event == "call" and frame.f_code is zipfile.ZipFile.open.__code__: + name = frame.f_locals["name"] + name = name.filename if isinstance(name, zipfile.ZipInfo) else name + if name.startswith("psam_pus"): + all_person_opens.append(name) + if active: + literal_opens.append(name) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = issue(invented) + finally: + sys.setprofile(previous) + assert scans == [True] and not active + assert literal_opens == ["psam_pusa.csv", "psam_pusb.csv"] + assert all_person_opens == literal_opens * 3 + assert result.receipt["counts"]["literal_batches"] == (3 if batch_limit == 2 else 1) + + +@pytest.mark.parametrize( + "mutation,code", + [ + ("duplicate", "DUPLICATE_LITERAL_PERSON"), + ("extra", "UNEXPECTED_LITERAL_PERSON"), + ("orphan", "UNEXPECTED_LITERAL_PERSON"), + ("missing", "GLOBAL_PERSON_KEYS"), + ("key_suffix", "LITERAL_PERSON_KEY"), + ], +) +def test_collector_reconciles_every_literal_slot_after_housing_validation( + invented, mutation, code +): + # Exercise this private reconciliation separately from the unchanged + # housing/preflight checks. A mismatched helper input issues no authority. + original = issue(invented) + projection = json.loads(owner._lookup(original).projection_path.read_bytes()) + if mutation == "duplicate": + invented.people.append(dict(invented.people[0], SPORDER="1")) + elif mutation == "extra": + invented.people.append(dict(invented.people[0], SPORDER="3")) + elif mutation == "orphan": + invented.people[-1]["SERIALNO"] = "2024HU0000099" + elif mutation == "missing": + invented.people.pop() + else: + invented.people[-1]["SPORDER"] = "1 " + invented.write() + paths = { + "household": invented.source / "csv_hus.zip", + "person": invented.source / "csv_pus.zip", + } + with pytest.raises(owner.ACSSourceCatalogueError, match=f"^{code}$"): + owner._collect(projection, paths) + + +def test_prospective_byte_charge_refuses_before_retaining_the_next_person( + invented, monkeypatch +): + original = issue(invented) + held = owner._lookup(original) + projection = json.loads(held.projection_path.read_bytes()) + expected = expected_records() + + def encoded(value): + return json.dumps(value, separators=(",", ":"), allow_nan=False).encode() + + overhead = sum(len(encoded((*row[:6], ()))) + 1 for row in expected) + monkeypatch.setattr( + owner, "_CATALOGUE_BYTES", overhead + len(encoded(expected[0][6][0])) + ) + retained = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_globals.get("__name__") == owner.__name__ + and frame.f_code.co_qualname == "_collect..consume_row" + ): + retained.append( + sum(person is not None for person in frame.f_locals["people"]) + ) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises( + owner.native.coverage.ACSCoverageAuthenticationError, + match="^CANONICAL_SIZE$", + ): + owner._collect(projection, dict(held.paths)) + finally: + sys.setprofile(previous) + assert retained == [1, 1] + + +def test_source_order_and_original_person_order_survive_reverse_zip_insertion(invented): + # Keep serials unsorted and make the split household's raw person order + # descending; canonical order remains source-member then source-row order. + invented.people[0]["SPORDER"], invented.people[2]["SPORDER"] = "02", "01" + invented.write() + first = issue(invented) + before = owner._lookup(first) + for name in ("csv_hus.zip", "csv_pus.zip"): + path = invented.source / name + with zipfile.ZipFile(path) as archive: + members = [ + (member.filename, archive.read(member)) for member in archive.infolist() + ] + with zipfile.ZipFile(path, "w") as archive: + for member, raw in reversed(members): + archive.writestr(member, raw) + pins = tuple( + ( + role, + name, + hashlib.sha256((invented.source / name).read_bytes()).hexdigest(), + (invented.source / name).stat().st_size, + ) + for role, name in (("household", "csv_hus.zip"), ("person", "csv_pus.zip")) + ) + # New physical ZIPs have their own invented pins; compare source records, + # not archive/producer receipts from different physical artifacts. + owner.housing._ARCHIVE_PINS = pins + second = issue(invented) + after = owner._lookup(second) + assert after.records == before.records and after.vacancies == before.vacancies + assert [p[0] for p in after.records[0][6]] == ["02", "01"] + + +@pytest.mark.parametrize("kind", ["new", "copy", "deepcopy", "pickle", "payload"]) +def test_foreign_changed_or_copied_authority_refuses(invented, kind): + result = issue(invented) + if kind == "new": + other = object.__new__(owner.AuthenticatedACSSourceCatalogue) + object.__setattr__(other, "payload", result.payload) + elif kind == "payload": + other = result + raw = json.loads(result.payload) + raw["counts"]["people"] = 16 + object.__setattr__(other, "payload", owner.native.coverage._json(raw)) + else: + try: + other = { + "copy": copy.copy, + "deepcopy": copy.deepcopy, + "pickle": lambda x: pickle.loads(pickle.dumps(x)), + }[kind](result) + except (TypeError, owner.ACSSourceCatalogueError): + return + with pytest.raises( + owner.ACSSourceCatalogueError, match="^ISSUANCE_(NOT_OWNED|CHANGED)$" + ): + other.validate() + + +def test_independent_reissue_and_weak_cleanup(invented): + one, two = issue(invented), issue(invented) + assert one.to_bytes() == two.to_bytes() and one is not two + identity, reference = id(one), weakref.ref(one) + del one + gc.collect() + assert reference() is None and identity not in owner._ISSUED + assert two.validate() is two + + +@pytest.mark.parametrize("kind", ["duplicate", "orphan", "missing", "late_extra", "np"]) +def test_global_complete_roster_refuses_across_members_and_batches(invented, kind): + if kind == "duplicate": + invented.people[-1] = dict(invented.people[0]) + elif kind == "orphan": + invented.people[-1]["SERIALNO"] = "2024HU0000099" + elif kind == "missing": + invented.people.pop() + elif kind == "late_extra": + extra = dict(invented.people[0], SPORDER="3") + invented.people.append(extra) + else: + invented.households[0]["NP"] = "3" + invented.write() + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +@pytest.mark.parametrize( + "cap,value", + [ + ("_ARCHIVE_BYTES", 1), + ("_EXPANDED_BYTES", 1), + ("_MEMBERS", 1), + ("_SOURCE_ROWS", 1), + ("_BATCH_PEOPLE", 1), + ("_CATALOGUE_BYTES", 1), + ("_RECEIPT_BYTES", 1), + ], +) +def test_prospective_small_caps_refuse(invented, monkeypatch, cap, value): + monkeypatch.setattr(owner, cap, value) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +@pytest.mark.parametrize("escaped_literals", [False, True]) +def test_byte_ceiling_exact_boundary_is_record_based( + invented, monkeypatch, escaped_literals +): + if escaped_literals: + invented.people[0]["MIL"] = 'x,"\\\t' + invented.people[0]["ESR"] = "é" + invented.write() + size = issue(invented).receipt["counts"]["canonical_record_bytes"] + monkeypatch.setattr(owner, "_CATALOGUE_BYTES", size) + assert issue(invented).receipt["counts"]["canonical_record_bytes"] == size + monkeypatch.setattr(owner, "_CATALOGUE_BYTES", size - 1) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +def test_candidate_is_reconstructed_and_cannot_authorize_changed_values(invented): + result = issue(invented) + candidate = result.to_bytes() + invented.people[0]["ESR"] = "6" + invented.write() + with pytest.raises(owner.ACSSourceCatalogueError, match="^CANDIDATE_MISMATCH$"): + issue(invented, candidate=candidate) + assert issue(invented).households[0].persons[0].esr == "6" + with pytest.raises( + owner.ACSSourceCatalogueError, match="^SOURCE_AUTHORITY_CHANGED$" + ): + result.validate() + + +@pytest.mark.parametrize("target", ["original", "projection", "producer"]) +def test_live_source_projection_and_producer_changes_refuse( + invented, monkeypatch, target +): + result = issue(invented) + if target == "original": + path = invented.source / "csv_pus.zip" + raw = path.read_bytes() + path.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + elif target == "projection": + owned = owner._lookup(result) + raw = owned.projection_path.read_bytes() + owned.projection_path.chmod(0o600) + owned.projection_path.write_bytes(b" " + raw[1:]) + else: + monkeypatch.setattr(owner, "_BATCH_PEOPLE", 3) + with pytest.raises(owner.ACSSourceCatalogueError): + result.validate() + + +def test_new_owner_static_surface_excludes_population_selection_and_classification(): + tree = ast.parse(Path(owner.__file__).read_text()) + calls = { + node.func.id if isinstance(node.func, ast.Name) else node.func.attr + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, (ast.Name, ast.Attribute)) + } + assert not calls & { + "Frame", + "build_acs_pums_unit_frame", + "prepare_acs_housing_population", + "assign_us_unit_structure", + "classify_household", + "classify_households", + "issue_acs_native_coverage", + "sample", + "choice", + } + assert {"_reconstruct", "_scan_acs_person_coverage", "_live_code"} <= calls + assert "read_acs_person_coverage_columns" not in calls + + +@pytest.mark.parametrize("mutation", ["source", "limit"]) +def test_mutations_at_literal_return_refuse(invented, mutation): + target = owner.literal._scan_acs_person_coverage + fired = [] + + def timing(frame, event, arg): + if frame.f_code is target.__code__ and event == "return" and not fired: + fired.append(True) + if mutation == "source": + path = invented.source / "csv_pus.zip" + raw = path.read_bytes() + path.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + else: + owner._BATCH_PEOPLE = 3 + + before, previous = owner._BATCH_PEOPLE, sys.getprofile() + sys.setprofile(timing) + try: + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + finally: + sys.setprofile(previous) + owner._BATCH_PEOPLE = before + assert fired + + +def test_returned_housing_capsule_cannot_replace_actual_projected_anchors(invented): + target, fired = owner.housing._reconstruct, [] + + def timing(frame, event, arg): + if frame.f_code is target.__code__ and event == "return" and not fired: + fired.append(True) + changed = json.loads(arg.projection_json) + changed["households"][0][changed["household_columns"].index("WGTP")] = ( + "9999" + ) + object.__setattr__(arg, "projection_json", owner.housing._json(changed)) + + previous = sys.getprofile() + sys.setprofile(timing) + try: + result = issue(invented) + finally: + sys.setprofile(previous) + assert fired and result.households[0].wgtp == "0010" + + +@pytest.mark.parametrize("aliases", ["csv", "native", "both"]) +@pytest.mark.parametrize("moment", ["entry", "before_call"]) +def test_actual_csv_alias_timing_cannot_supply_catalogue_values( + invented, aliases, moment +): + target = owner.literal._literal_csv_records + source, first = inspect.getsourcelines(target) + call_line = next( + first + i for i, line in enumerate(source) if "reader = csv_reader(" in line + ) + original, native, fired, calls = csv.reader, _csv.reader, [], [] + + def restore(): + csv.reader, _csv.reader = original, native + + def replacement(*args, **kwargs): + calls.append(True) + restore() + return original(*args, **kwargs) + + def timing(frame, event, arg): + if frame.f_code is target.__code__: + trigger = (moment == "entry" and event == "call") or ( + moment == "before_call" + and event == "line" + and frame.f_lineno == call_line + ) + if trigger and not fired: + fired.append(True) + if aliases in ("csv", "both"): + csv.reader = replacement + if aliases in ("native", "both"): + _csv.reader = replacement + if event == "return": + restore() + return timing + + previous = sys.gettrace() + sys.settrace(timing) + try: + if moment == "entry": + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + else: + result = issue(invented) + finally: + sys.settrace(previous) + restore() + assert fired and not calls + if moment == "before_call": + assert result.households[0].persons[0].esr == "4" + + +def test_public_constructor_has_no_authority(invented): + raw = issue(invented).to_bytes() + with pytest.raises(owner.ACSSourceCatalogueError, match="^ISSUANCE_CONSTRUCTOR$"): + owner.AuthenticatedACSSourceCatalogue(raw) + + +def test_full_source_can_exceed_unchanged_selected_reader_cap(invented, monkeypatch): + monkeypatch.setattr(owner.literal, "MAX_SELECTED_ROWS", 2) + result = issue(invented) + assert result.receipt["counts"]["people"] == 5 + assert result.receipt["counts"]["literal_batches"] == 3 + + +def test_vacancy_only_catalogue_has_no_literal_batch(invented): + invented.households[:] = [dict(invented.households[2])] + invented.people.clear() + invented.write() + result = issue(invented) + assert result.households == () + assert len(result.exclusion_ledger) == 1 + assert result.exclusion_ledger[0].persons == () + assert result.receipt["counts"]["literal_batches"] == 0 + assert result.receipt["execution"]["literal_full_scans"] == 0 + + +def test_reordered_literal_column_declaration_refuses_before_capture( + invented, monkeypatch +): + monkeypatch.setattr( + owner.literal, "READ_COLUMNS", ("SERIALNO", "SPORDER", "ESR", "MIL", "AGEP") + ) + with pytest.raises( + owner.ACSSourceCatalogueError, match="^LITERAL_CONTRACT_CHANGED$" + ): + issue(invented) + assert list(invented.snapshots.iterdir()) == [] + + +@pytest.mark.parametrize( + "interface", + ["validate", "to_bytes", "receipt", "households", "exclusion_ledger", "lineage"], +) +@pytest.mark.parametrize("moment", ["source_checks", "final_producer"]) +def test_final_source_registry_seal_all_borrows( + invented, monkeypatch, interface, moment +): + result = issue(invented) + pins = owner.housing._ARCHIVE_PINS + changed = tuple((r, n, "0" * 64, s) for r, n, d, s in pins) + fired, producers = [], [] + + def timing(frame, event, arg): + if event != "return": + return + if frame.f_code is owner._producer.__code__: + producers.append(True) + if not fired and ( + moment == "source_checks" + and frame.f_code is owner._source_checks.__code__ + or moment == "final_producer" + and frame.f_code is owner._producer.__code__ + and len(producers) == 2 + ): + fired.append(True) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", changed) + + previous = sys.getprofile() + try: + sys.setprofile(timing) + with pytest.raises( + owner.ACSSourceCatalogueError, match="SOURCE_AUTHORITY_CHANGED" + ): + value = getattr(result, interface) + if callable(value): + value() + finally: + sys.setprofile(previous) + assert fired + + +@pytest.mark.parametrize("semantic", [False, True]) +def test_import_time_bytes_and_matching_live_code_remain_authority( + invented, monkeypatch, semantic +): + source = Path(owner.__file__).read_text() + if semantic: + before = " weight,\n tuple(" + assert source.count(before) == 1 + source = source.replace(before, ' "9999",\n tuple(') + else: + source += "\n# invented drift after the real module import\n" + replacement = invented.root / "changed-catalogue.py" + replacement.write_text(source) + monkeypatch.setattr(owner, "__file__", str(replacement)) + if semantic: + code = next( + c + for c in compile( + source, str(replacement), "exec", dont_inherit=True + ).co_consts + if isinstance(c, CodeType) and c.co_name == "_household" + ) + monkeypatch.setattr(owner._household, "__code__", code) + with pytest.raises(owner.ACSSourceCatalogueError, match="PRODUCER_CODE_CHANGED"): + issue(invented) + + +@pytest.mark.parametrize("change", ["bytes", "generated_constructor"]) +def test_raw_domain_import_authority_precedes_first_issuance( + invented, monkeypatch, change +): + if change == "bytes": + replacement = invented.root / "changed-raw-types.py" + replacement.write_bytes( + Path(owner.domains.__file__).read_bytes() + b"\n# drift\n" + ) + monkeypatch.setattr(owner.domains, "__file__", str(replacement)) + else: + # Native live-code traversal explicitly skips generated code; + # the local import-time seal must still bind those actual constructors. + code = next( + c + for c in compile( + "def __init__(self, *args, **kwargs): pass", "", "exec" + ).co_consts + if isinstance(c, CodeType) + ) + monkeypatch.setattr( + owner.domains.AcsHousehold, + "__init__", + FunctionType(code, vars(owner.domains)), + ) + with pytest.raises(owner.ACSSourceCatalogueError, match="PRODUCER_CODE_CHANGED"): + issue(invented) + + +@pytest.mark.parametrize("issuance", [False, True]) +@pytest.mark.parametrize("change", ["pins", "live_code"]) +def test_final_producer_return_seal_issuance_and_borrow( + invented, monkeypatch, issuance, change +): + result = None if issuance else issue(invented) + fired, producers = [], [] + original = owner._household + + def changed(record): + return original(record) + + def timing(frame, event, arg): + if event == "return" and frame.f_code is owner._producer.__code__: + producers.append(True) + if len(producers) == 2: + fired.append(True) + if change == "pins": + monkeypatch.setattr( + owner.housing, + "_ARCHIVE_PINS", + tuple( + (r, n, "0" * 64, s) + for r, n, d, s in owner.housing._ARCHIVE_PINS + ), + ) + else: + monkeypatch.setattr(owner, "_household", changed) + + previous = sys.getprofile() + try: + sys.setprofile(timing) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) if issuance else result.households + finally: + sys.setprofile(previous) + assert fired + + +@pytest.mark.parametrize( + "shape", ["outer_list", "inner_list", "string_subclass", "bool_size"] +) +def test_source_authority_requires_deeply_immutable_primitive_pins( + invented, monkeypatch, shape +): + original = owner.housing._ARCHIVE_PINS + if shape == "outer_list": + changed = list(original) + elif shape == "inner_list": + changed = tuple(list(pin) for pin in original) + elif shape == "string_subclass": + + class MutableString(str): + pass + + changed = ((MutableString(original[0][0]), *original[0][1:]), original[1]) + else: + changed = ((*original[0][:3], True), original[1]) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", changed) + with pytest.raises(owner.ACSSourceCatalogueError, match="SOURCE_AUTHORITY_TYPE"): + issue(invented) + assert list(invented.snapshots.iterdir()) == [] diff --git a/packages/microcosm-build/tests/test_us_acs_pums.py b/packages/microcosm-build/tests/test_us_acs_pums.py index 48b656d96..d663d21ac 100644 --- a/packages/microcosm-build/tests/test_us_acs_pums.py +++ b/packages/microcosm-build/tests/test_us_acs_pums.py @@ -8,13 +8,13 @@ import pytest from microcosm.build.serialization_dtypes import CANONICAL_STRING_DTYPE +from microcosm.build.us_runtime import acs_pums from microcosm.build.us_runtime.acs_pums import ( ACS_2024_1YR_SPINE, AcsPumsSource, build_acs_pums_unit_frame, load_acs_pums_tables, ) -from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -126,6 +126,15 @@ def _source(tmp_path: Path) -> AcsPumsSource: return AcsPumsSource(household_zip=household_zip, person_zip=person_zip) +def _axis_household(kinds: list[int]) -> pd.DataFrame: + return pd.DataFrame( + { + "SERIALNO": [f"2024HU000000{index}" for index in range(len(kinds))], + "TYPEHUGQ": kinds, + } + ) + + def _asec_shaped_frame() -> Frame: """Return one ASEC-like row sharing only structural and lineage fields.""" @@ -264,6 +273,8 @@ def test_built_acs_lineage_assembles_with_asec_without_measured_coercion( measured_wages = acs.table("person")["WAGP"].copy() raw_serials = acs.table("household")["SERIALNO"].copy() + from microcosm.build.us_runtime.spine_assembly import assemble_spines + assembled = assemble_spines( {"asec": _asec_shaped_frame(), "acs": acs}, household_mass_shares={"asec": 0.5, "acs": 0.5}, @@ -595,3 +606,167 @@ def test_load_acs_pums_tables_requires_native_mapping_columns( load_acs_pums_tables( AcsPumsSource(household_zip=household_zip, person_zip=person_zip) ) + + +def test_load_acs_pums_tables_rejects_group_quarters_with_positive_weight( + tmp_path: Path, +) -> None: + source = _source(tmp_path) + _write_csv_zip( + source.household_zip, + { + "psam_husa.csv": [ + _household( + "2024GQ0000001", + WGTP=7, + TYPEHUGQ=2, + TEN=None, + TAXAMT=None, + ) + ] + }, + ) + + with pytest.raises(ValueError, match="TYPEHUGQ/WGTP disagree"): + load_acs_pums_tables(source, chunksize=1) + + +def test_load_acs_pums_tables_rejects_housing_unit_with_zero_weight( + tmp_path: Path, +) -> None: + source = _source(tmp_path) + _write_csv_zip( + source.household_zip, + {"psam_husa.csv": [_household("2024HU0000001", WGTP=0)]}, + ) + + with pytest.raises(ValueError, match="TYPEHUGQ/WGTP disagree"): + load_acs_pums_tables(source, chunksize=1) + + +def test_build_acs_pums_unit_frame_reports_household_axis_composition( + tmp_path: Path, +) -> None: + pytest.importorskip("microunit") # sanctioned tax-unit constructor (us extra) + household_zip = tmp_path / "csv_hus.zip" + person_zip = tmp_path / "csv_pus.zip" + _write_csv_zip( + household_zip, + { + "psam_husa.csv": [ + _household("2024HU0000001", WGTP=10, NP=2), + _household("2024HU0000002", WGTP=20), + _household( + "2024GQ0000001", + WGTP=0, + TYPEHUGQ=2, + TEN=None, + TAXAMT=None, + ), + _household( + "2024GQ0000002", + WGTP=0, + TYPEHUGQ=3, + TEN=None, + TAXAMT=None, + ), + _household("2024HU0000003", WGTP=40, NP=0), + ] + }, + ) + _write_csv_zip( + person_zip, + { + "psam_pusa.csv": [ + _person("2024HU0000001", 1, 20, PWGTP=11), + _person("2024HU0000001", 2, 25, MAR=5, PWGTP=12), + _person("2024HU0000002", 1, 20, PWGTP=13), + _person("2024GQ0000001", 1, 37, MAR=5, PWGTP=99), + _person("2024GQ0000002", 1, 38, MAR=5, PWGTP=55), + ] + }, + ) + + frame, metadata = build_acs_pums_unit_frame( + AcsPumsSource(household_zip=household_zip, person_zip=person_zip) + ) + + composition = metadata["household_axis_composition"] + assert composition == { + "occupied_housing_unit_rows": 2, + "occupied_housing_unit_design_weight_total": 30.0, + "institutional_gq_person_rows": 1, + "institutional_gq_person_design_weight_total": 99.0, + "noninstitutional_gq_person_rows": 1, + "noninstitutional_gq_person_design_weight_total": 55.0, + } + # The dropped vacant housing unit is outside the described axis entirely. + assert metadata["vacant_household_rows_dropped"] == 1 + # Units conserve, and design mass conserves separately from units. + assert ( + composition["occupied_housing_unit_rows"] + + composition["institutional_gq_person_rows"] + + composition["noninstitutional_gq_person_rows"] + == metadata["household_rows"] + == frame.n("household") + == 4 + ) + assert composition["occupied_housing_unit_design_weight_total"] + composition[ + "institutional_gq_person_design_weight_total" + ] + composition["noninstitutional_gq_person_design_weight_total"] == pytest.approx( + metadata["weighted_household_population"] + ) + + +# Reached through the module, not imported by name, so this file still imports +# against a build without the helper: the four checks below then fail on their +# own instead of erroring the whole module at collection. +def test_household_axis_composition_separates_units_and_design_mass() -> None: + household = _axis_household([1, 1, 2, 3]) + weights = Weights( + np.asarray([10.0, 20.0, 99.0, 55.0]), + WeightKind.DESIGN, + ) + + composition = acs_pums._household_axis_composition(household, weights) + + assert composition == { + "occupied_housing_unit_rows": 2, + "occupied_housing_unit_design_weight_total": 30.0, + "institutional_gq_person_rows": 1, + "institutional_gq_person_design_weight_total": 99.0, + "noninstitutional_gq_person_rows": 1, + "noninstitutional_gq_person_design_weight_total": 55.0, + } + assert sum( + value for key, value in composition.items() if key.endswith("_rows") + ) == len(household) + assert sum( + value + for key, value in composition.items() + if key.endswith("_design_weight_total") + ) == pytest.approx(weights.total) + + +def test_household_axis_composition_requires_typehugq() -> None: + household = _axis_household([1]).drop(columns=["TYPEHUGQ"]) + weights = Weights(np.asarray([10.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="requires the TYPEHUGQ column"): + acs_pums._household_axis_composition(household, weights) + + +def test_household_axis_composition_refuses_unpartitioned_rows() -> None: + household = _axis_household([1, 4]) + weights = Weights(np.asarray([10.0, 20.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="does not partition the loaded rows"): + acs_pums._household_axis_composition(household, weights) + + +def test_household_axis_composition_refuses_misaligned_weights() -> None: + household = _axis_household([1, 2]) + weights = Weights(np.asarray([10.0, 99.0, 20.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="one DESIGN weight per loaded"): + acs_pums._household_axis_composition(household, weights) diff --git a/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py b/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py new file mode 100644 index 000000000..6d108fe7a --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py @@ -0,0 +1,457 @@ +"""Invented controls for the real ACS owner's source-only compilation cache.""" + +from __future__ import annotations +import __future__ + +import ast +import builtins +import sys +from contextlib import contextmanager +from pathlib import Path +from types import CodeType, FunctionType, ModuleType + +import pytest + +from microcosm.build.us_runtime import acs_native_coverage_binding as native + + +@pytest.fixture(autouse=True) +def empty_compile_cache(): + native._clear_compile_cache() + yield + native._clear_compile_cache() + + +@contextmanager +def _observed_calls(): + """Observe real compile/read calls without replacing attested functions.""" + assert sys.getprofile() is None + counts = {"compile": 0, "read": 0} + helper_code = native._compile_source.__code__ + read_code = Path.read_bytes.__code__ + + def observe(frame, event, arg): + if ( + event == "c_call" + and frame.f_code is helper_code + and arg is builtins.compile + ): + counts["compile"] += 1 + elif event == "call" and frame.f_code is read_code: + counts["read"] += 1 + + sys.setprofile(observe) + try: + yield counts + finally: + sys.setprofile(None) + + +def _module(tmp_path, monkeypatch, source, *, name="invented_acs_compile_cache"): + path = tmp_path / f"{name}.py" + path.write_bytes(source) + module = ModuleType(name) + module.__file__ = str(path) + module.__package__ = "" + monkeypatch.setitem(sys.modules, name, module) + exec(builtins.compile(source, str(path), "exec", dont_inherit=True), vars(module)) + return module, path + + +def _index(code): + result = {} + + def visit(item): + result[item.co_qualname] = item + for value in item.co_consts: + if isinstance(value, CodeType): + visit(value) + + visit(code) + return result + + +@pytest.mark.parametrize("optimize", [0, 1, 2]) +def test_plain_helper_preserves_compiled_code_and_qualified_name_order(optimize): + source = """# coding: utf-8 +from __future__ import annotations +label = "café" +def outer(value: Missing): + "docstring" + assert value is not None + def nested(): + return [item for item in range(value)] + def nested(): + return lambda: {item: item for item in range(value)} + return nested +class Example: + @property + def item(self): + return label + @item.setter + def item(self, value): + self._item = value +""".encode() + filename = "invented/café/../module.py" + expected = builtins.compile( + source, filename, "exec", dont_inherit=True, optimize=optimize + ) + with _observed_calls() as calls: + first = native._compile_source(source, filename, optimize=optimize) + second = native._compile_source(source, filename, optimize=optimize) + assert type(native._compile_source) is FunctionType + assert first is second + assert first == expected + assert first.co_filename == filename + assert _index(first) == _index(expected) + assert calls["compile"] == 1 + + +def test_complete_inputs_separate_cache_entries(): + source = b"42" + options = [ + ("literal/../source.py", "exec", 0, 0), + ("source.py", "exec", 0, 0), + ("source.py", "eval", 0, 0), + ("source.py", "exec", __future__.annotations.compiler_flag, 0), + ("source.py", "exec", 0, 1), + ("source.py", "exec", 0, 2), + ] + with _observed_calls() as calls: + for filename, mode, flags, optimize in options: + expected = builtins.compile( + source, + filename, + mode, + flags=flags, + dont_inherit=True, + optimize=optimize, + ) + first = native._compile_source( + source, filename, mode, flags=flags, optimize=optimize + ) + assert first == expected + assert first.co_filename == filename + assert ( + native._compile_source( + source, filename, mode, flags=flags, optimize=optimize + ) + is first + ) + assert calls["compile"] == len(options) + assert len(native._COMPILE_CACHE) == len(options) + with _observed_calls() as calls: + effective = native._compile_source( + source, "source.py", optimize=sys.flags.optimize + ) + assert native._compile_source(source, "source.py", optimize=-1) is effective + assert calls["compile"] == 0 + + +def test_same_literal_path_tracks_a_b_a_source_bytes(): + first_source, second_source = b"value = 1", b"value = 2" + assert len(first_source) == len(second_source) + with _observed_calls() as calls: + first = native._compile_source(first_source, "unchanged.py") + second = native._compile_source(second_source, "unchanged.py") + again = native._compile_source(first_source, "unchanged.py") + assert first is again + assert second != first + assert calls["compile"] == 2 + + +def test_changed_compiler_is_called_each_time_and_original_entry_remains(monkeypatch): + source = b"value = 1" + original = native._compile_source(source, "compiler.py") + replacements = [] + + def replacement(raw, filename, mode, **options): + replacements.append(raw) + return builtins.compile( + f"value = {len(replacements) + 1}".encode(), filename, mode, **options + ) + + with monkeypatch.context() as context: + context.setattr(native, "compile", replacement, raising=False) + first = native._compile_source(source, "compiler.py") + second = native._compile_source(source, "compiler.py") + assert replacements == [source, source] + assert first != second + assert first != original + assert len(native._COMPILE_CACHE) == 1 + assert native._compile_source(source, "compiler.py") is original + + +@pytest.mark.parametrize( + "options", [{"dont_inherit": False}, {"flags": ast.PyCF_ONLY_AST}] +) +def test_context_dependent_or_mutable_compiler_outputs_are_not_retained(options): + with _observed_calls() as calls: + first = native._compile_source(b"value = 1", "bypass.py", **options) + second = native._compile_source(b"value = 1", "bypass.py", **options) + assert first is not second + assert calls["compile"] == 2 + assert not native._COMPILE_CACHE + + +def test_failures_are_not_memoized(): + with _observed_calls() as calls: + for _ in range(2): + with pytest.raises(SyntaxError): + native._compile_source(b"def invalid(:", "repair.py") + assert not native._COMPILE_CACHE + fixed = native._compile_source(b"value = 1", "repair.py") + assert calls["compile"] == 3 + assert type(fixed) is CodeType + assert len(native._COMPILE_CACHE) == 1 + + +def test_fifo_entry_limit_and_clear(monkeypatch): + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_ENTRIES", 2) + source = b"value = 1" + first = native._compile_source(source, "first.py") + second = native._compile_source(source, "second.py") + assert native._compile_source(source, "first.py") is first + native._compile_source(source, "third.py") + assert len(native._COMPILE_CACHE) == 2 + assert native._compile_source(source, "second.py") is second + with _observed_calls() as calls: + assert native._compile_source(source, "first.py") is not first + assert calls["compile"] == 1 + native._clear_compile_cache() + assert not native._COMPILE_CACHE + with _observed_calls() as calls: + native._compile_source(source, "first.py") + assert calls["compile"] == 1 + + +def test_source_byte_budget_counts_distinct_filename_keys(monkeypatch): + source = b"value = 1" + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_SOURCE_BYTES", len(source) * 2) + first = native._compile_source(source, "first.py") + native._compile_source(source, "second.py") + native._compile_source(source, "third.py") + assert sum(len(key[0]) for key in native._COMPILE_CACHE) == len(source) * 2 + with _observed_calls() as calls: + assert native._compile_source(source, "first.py") is not first + assert calls["compile"] == 1 + + +@pytest.mark.parametrize( + "limit_name", ["_COMPILE_CACHE_MAX_ENTRY_BYTES", "_COMPILE_CACHE_MAX_SOURCE_BYTES"] +) +def test_oversize_source_is_compiled_without_cache(limit_name, monkeypatch): + source = b"value = 1" + monkeypatch.setattr(native, limit_name, len(source) - 1) + with _observed_calls() as calls: + first = native._compile_source(source, "oversize.py") + second = native._compile_source(source, "oversize.py") + assert first == second + assert first is not second + assert calls["compile"] == 2 + assert not native._COMPILE_CACHE + + +def test_live_check_reads_source_twice_even_after_warming(tmp_path, monkeypatch): + module, _ = _module(tmp_path, monkeypatch, b"def value():\n return 1\n") + with _observed_calls() as cold: + first_index = {} + native._live_code(module, first_index) + with _observed_calls() as warm: + second_index = {} + native._live_code(module, second_index) + assert cold == {"compile": 1, "read": 2} + assert warm == {"compile": 0, "read": 2} + assert first_index == second_index + assert first_index is not second_index + first_index[module.__file__]["value"] = None + native._live_code(module, {}) + assert second_index[module.__file__]["value"] is not None + + +def test_real_producer_keeps_read_counts_and_full_evidence_on_warm_cache(): + # Settle unrelated import/manifest memoization, then compare only compile + # cache cold versus warm with the real producer and unchanged source files. + native._producer() + native._clear_compile_cache() + with _observed_calls() as cold: + first = native._producer() + with _observed_calls() as warm: + second = native._producer() + assert cold["compile"] > 0 + assert warm["compile"] == 0 + assert cold["read"] > 0 + assert warm["read"] == cold["read"] + assert first == second + assert len(native._COMPILE_CACHE) <= native._COMPILE_CACHE_MAX_ENTRIES + assert ( + sum(len(key[0]) for key in native._COMPILE_CACHE) + <= native._COMPILE_CACHE_MAX_SOURCE_BYTES + ) + + +@pytest.mark.parametrize("drift", ["code", "globals", "declaration"]) +def test_warm_cache_does_not_hide_loaded_function_drift(drift, tmp_path, monkeypatch): + module, _ = _module(tmp_path, monkeypatch, b"def value():\n return 1\n") + native._live_code(module, {}) + if drift == "code": + module.value.__code__ = module.value.__code__.replace(co_consts=(None, 2)) + elif drift == "globals": + module.value = FunctionType(module.value.__code__, dict(vars(module))) + else: + del module.value + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + + +def test_warm_cache_preserves_imported_alias_identity(tmp_path, monkeypatch): + origin, _ = _module( + tmp_path, + monkeypatch, + b"def value():\n return 1\n", + name="invented_acs_origin", + ) + consumer, _ = _module( + tmp_path, + monkeypatch, + b"from invented_acs_origin import value\n", + name="invented_acs_consumer", + ) + native._live_code(consumer, {}) + consumer.value = FunctionType(origin.value.__code__, vars(origin)) + assert consumer.value.__code__ is origin.value.__code__ + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(consumer, {}) + + +def test_warm_cache_checks_distinct_functions_with_shared_code_and_closures( + tmp_path, monkeypatch +): + source = b"""def make(): + def target(): + return 1 + def entry(): + return target() + return entry +first = make() +second = make() +""" + module, _ = _module(tmp_path, monkeypatch, source) + native._live_code(module, {}) + assert module.first.__code__ is module.second.__code__ + first_target = module.first.__closure__[0].cell_contents + second_target = module.second.__closure__[0].cell_contents + assert first_target is not second_target + second_target.__code__ = second_target.__code__.replace(co_consts=(None, 2)) + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + + +def test_second_source_read_can_invalidate_a_warm_entry(tmp_path, monkeypatch): + original = b"def value():\n return 1\n" + changed = b"def value():\n return 2\n" + module, path = _module(tmp_path, monkeypatch, original) + native._live_code(module, {}) + read_bytes = Path.read_bytes + calls = 0 + + def change_on_second_read(self): + nonlocal calls + if self == path: + calls += 1 + if calls == 2: + path.write_bytes(changed) + return read_bytes(self) + + monkeypatch.setattr(Path, "read_bytes", change_on_second_read) + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + assert calls == 2 + + +def test_malformed_cache_entry_is_recompiled_and_replaced(): + source = b"value = 1" + first = native._compile_source(source, "malformed.py") + key = next(iter(native._COMPILE_CACHE)) + native._COMPILE_CACHE[key] = None + with _observed_calls() as calls: + repaired = native._compile_source(source, "malformed.py") + again = native._compile_source(source, "malformed.py") + assert repaired == first + assert repaired is not first + assert again is repaired + assert calls["compile"] == 1 + + +def test_nested_compilation_keeps_current_fifo_bound(monkeypatch): + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_ENTRIES", 1) + assert sys.getprofile() is None + nested = [] + helper_code = native._compile_source.__code__ + + def after_compile(frame, event, arg): + if ( + not nested + and event == "c_return" + and frame.f_code is helper_code + and arg is builtins.compile + ): + nested.append(True) + native._compile_source(b"nested = 2", "nested.py") + + sys.setprofile(after_compile) + try: + outer = native._compile_source(b"outer = 1", "outer.py") + finally: + sys.setprofile(None) + assert nested == [True] + assert outer == builtins.compile( + b"outer = 1", "outer.py", "exec", dont_inherit=True + ) + assert len(native._COMPILE_CACHE) == 1 + assert next(iter(native._COMPILE_CACHE))[1] == "outer.py" + + +def test_swapped_valid_entries_are_recompiled_for_their_complete_inputs(): + first_source, second_source = b"value = 1", b"value = 2" + first = native._compile_source(first_source, "first.py") + second = native._compile_source(second_source, "second.py") + first_key, second_key = tuple(native._COMPILE_CACHE) + native._COMPILE_CACHE[first_key], native._COMPILE_CACHE[second_key] = ( + native._COMPILE_CACHE[second_key], + native._COMPILE_CACHE[first_key], + ) + with _observed_calls() as calls: + repaired_first = native._compile_source(first_source, "first.py") + repaired_second = native._compile_source(second_source, "second.py") + assert native._compile_source(first_source, "first.py") is repaired_first + assert native._compile_source(second_source, "second.py") is repaired_second + assert calls["compile"] == 2 + assert repaired_first == first and repaired_first is not first + assert repaired_second == second and repaired_second is not second + assert repaired_first.co_filename == "first.py" + assert repaired_second.co_filename == "second.py" + + +def test_python_compiler_captured_as_initial_compiler_still_bypasses(monkeypatch): + # This is the exact captured/current state after a pre-import replacement; + # leave all issued-owner modules loaded and exercise the real cache helper. + calls = [] + + def replacement(source, filename, mode, **options): + calls.append(source) + return builtins.compile( + f"value = {len(calls)}".encode(), filename, mode, **options + ) + + # Textual labels alone must not classify a Python replacement as builtin. + replacement.__module__ = "builtins" + replacement.__name__ = "compile" + replacement.__self__ = builtins + monkeypatch.setattr(native, "_COMPILE_CACHE_COMPILER", replacement) + monkeypatch.setattr(native, "compile", replacement, raising=False) + first = native._compile_source(b"value = 0", "captured.py") + second = native._compile_source(b"value = 0", "captured.py") + assert calls == [b"value = 0", b"value = 0"] + assert first != second + assert not native._COMPILE_CACHE diff --git a/packages/microcosm-build/tests/test_us_acs_transfer.py b/packages/microcosm-build/tests/test_us_acs_transfer.py index e9e615bec..42d03205a 100644 --- a/packages/microcosm-build/tests/test_us_acs_transfer.py +++ b/packages/microcosm-build/tests/test_us_acs_transfer.py @@ -1467,9 +1467,8 @@ def test_pregnancy_draws_once_per_eligible_source_person_and_fans_to_clones( ) person = result.frame.person - eligible = ( - person["is_female"].astype(bool) - & person["age"].between(15, 44, inclusive="both") + eligible = person["is_female"].astype(bool) & person["age"].between( + 15, 44, inclusive="both" ) assert person.loc[eligible, "is_pregnant"].all() assert not person.loc[~eligible, "is_pregnant"].any() @@ -1490,7 +1489,9 @@ def test_pregnancy_draws_once_per_eligible_source_person_and_fans_to_clones( .first() .sum() ) - record = next(item for item in result.imputed_inputs if item.column == "is_pregnant") + record = next( + item for item in result.imputed_inputs if item.column == "is_pregnant" + ) receipt = record.structural_receipt assert receipt is not None assert sum(pattern.recipient_rows for pattern in record.patterns) == ( @@ -1576,12 +1577,8 @@ def test_pregnancy_partial_clone_fanout_receipt_categories_are_disjoint( record = result.imputed_inputs[0] receipt = record.structural_receipt assert receipt is not None - assert receipt["preexisting_value_fanout_rows"] == int( - (missing & eligible).sum() - ) - assert receipt["ineligible_rows_assigned_false"] == int( - (missing & ~eligible).sum() - ) + assert receipt["preexisting_value_fanout_rows"] == int((missing & eligible).sum()) + assert receipt["ineligible_rows_assigned_false"] == int((missing & ~eligible).sum()) assert ( receipt["preexisting_value_fanout_rows"] + receipt["ineligible_rows_assigned_false"] @@ -2468,7 +2465,12 @@ def test_strict_leaf_audit_reports_missing_us_extra( from microcosm.frame.adapters import policyengine_us as adapter_module class _MissingMetadataIndex: - def __init__(self) -> None: + # Mirrors PolicyEngineUSVariableMetadataIndex's keyword-only surface. + # The audit path constructs it twice: puf_support._formula_owned_engine + # passes include_consumers=False before the strict-leaf branch + # constructs it bare, and both must raise the absent-extra ImportError + # rather than a signature TypeError. + def __init__(self, *, include_consumers: bool = True) -> None: raise ImportError("policyengine-us is absent") monkeypatch.setattr( diff --git a/packages/microcosm-build/tests/test_us_asec_2024_native_population.py b/packages/microcosm-build/tests/test_us_asec_2024_native_population.py new file mode 100644 index 000000000..79b14b6ea --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_2024_native_population.py @@ -0,0 +1,692 @@ +"""Real closed loaders over invented source bytes; no genuine source admission.""" + +import csv +import hashlib +import importlib.util +import json +import sys +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.frame_checkpoint import ( + load_frame_checkpoint, + write_frame_checkpoint, +) +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.us_runtime import asec_2024_native_population as native +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_current_money_source as money +from microcosm.build.us_runtime import asec_household_observations as household_owner +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import native_household_origin as origin +from microcosm.frame import Frame, WeightKind, Weights + + +def _helpers(): + spec = importlib.util.spec_from_file_location( + "native_2024_invented_fixture", + Path(__file__).with_name("test_us_asec_coverage_authentication.py"), + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _rebuild_observations( + tmp_path, + monkeypatch, + persons, + *, + extra_household, + legacy_weights, + missing_money=None, + current_predictor_money=None, +): + """Build larger invented real checkpoints and the actual observation attachment.""" + parent_path = tmp_path / "invented-original-v4.h5" + household_path = tmp_path / "invented-household-attachment.h5" + loaded = load_frame_checkpoint(parent_path) + tables = {e: loaded.frame.table(e).copy(deep=True) for e in loaded.frame.entities} + strata = loaded.frame.strata.copy(deep=True) + if extra_household: + extra = tables["person"].iloc[-2:].copy(deep=True) + extra["person_id"] = np.array([107, 108], dtype=np.int64) + extra["PERIDNUM"] = pd.array([str(i).zfill(22) for i in (7, 8)], dtype="string") + extra["source_household_id"] = 8 + extra["PH_SEQ"] = 4 + for entity in loaded.frame.schema.group_entities: + extra[f"person_{entity}_id"] = 4 + group = tables[entity].iloc[-1:].copy(deep=True) + group[f"{entity}_id"] = 4 + tables[entity] = pd.concat([tables[entity], group], ignore_index=True) + tables["person"] = pd.concat([tables["person"], extra]) + strata = pd.concat([strata, strata.iloc[-2:]]) + cohort_path = tmp_path / "invented-cohort-2024.h5" + cohort = pd.read_hdf(cohort_path, key="household") + extra_cohort = cohort.iloc[-1:].copy(deep=True) + extra_cohort["H_SEQ"] = 8 + pd.concat([cohort, extra_cohort], ignore_index=True).to_hdf( + cohort_path, key="household", format="fixed", mode="w" + ) + + def add_people(rows): + extra_rows = [r.copy() for r in rows[1:]] + for row in extra_rows: + row[0] = str(int(row[0]) + 2).zfill(22) + row[1] = "8" + rows.extend(extra_rows) + + _helpers()._rewrite(persons[2024], add_people) + if missing_money is not None: + year, field = missing_money + assert (year, field) in ((2024, "ANN_VAL"), (2022, "WSAL_VAL")) + person = tables["person"] + position = np.flatnonzero(person.source_year.to_numpy() == year)[0] + # Alter only invented source construction, before identities, attachments + # and normal issuance. Never mutate an already authenticated parent. + person.iloc[position, person.columns.get_loc(field)] = np.nan + if current_predictor_money is not None: + # Alter only invented original source construction before checkpoint + # identities, observation attachments, private pins and fresh issuance. + person = tables["person"] + positions = np.flatnonzero(person.source_year.to_numpy() == 2024) + assert set(current_predictor_money) == { + "WSAL_VAL", + "SEMP_VAL", + "INT_VAL", + "DIV_VAL", + "CAP_VAL", + } + for field, observations in current_predictor_money.items(): + observations = np.asarray(observations) + assert observations.dtype == np.dtype("float64") + assert ( + observations.shape == (len(positions),) + and np.isfinite(observations).all() + ) + person.iloc[positions, person.columns.get_loc(field)] = observations + values = ( + legacy_weights + if legacy_weights is not None + else ([1, 2, 3, 9] if extra_household else [1, 2, 3]) + ) + frame = Frame( + tables, + loaded.frame.schema, + {"household": Weights(np.array(values), WeightKind.DESIGN)}, + strata, + ) + metadata = loaded.metadata + for key in ("identity", "source_construction_identity"): + metadata[key] = frame_identity(frame).to_payload() + sources = [] + for year in (2022, 2023, 2024): + path = tmp_path / f"invented-cohort-{year}.h5" + sources.append( + household_owner.AsecHouseholdObservationSource( + year, path, hashlib.sha256(path.read_bytes()).hexdigest() + ) + ) + metadata["source_receipt"]["sources"] = [ + {"year": source.year, "path": str(source.path), "sha256": source.sha256} + for source in sources + ] + write_frame_checkpoint(parent_path, frame, metadata=metadata) + parent_sha = hashlib.sha256(parent_path.read_bytes()).hexdigest() + attachment = household_owner.with_asec_household_observations( + frame, + checkpoint_metadata=metadata, + checkpoint_sha256=parent_sha, + sources=sources, + ) + write_frame_checkpoint( + household_path, + attachment.frame, + metadata={ + "schema_version": 1, + "artifact_kind": "microcosm.asec_household_observations_source", + "parent_checkpoint_sha256": parent_sha, + "household_observations": attachment.receipt, + }, + ) + monkeypatch.setattr( + money, + "_SOURCE_PINS", + ( + parent_sha, + hashlib.sha256(household_path.read_bytes()).hexdigest(), + tuple((s.year, s.sha256) for s in sources), + ), + ) + _helpers()._pin(persons, monkeypatch) + + +def _fixture( + tmp_path, + monkeypatch, + *, + household_rows=None, + extra_household=False, + legacy_weights=None, + missing_money=None, + current_predictor_money=None, + **changes, +): + helpers = _helpers() + _, persons = helpers._fixtures(tmp_path, monkeypatch, **changes) + if ( + extra_household + or legacy_weights is not None + or missing_money is not None + or current_predictor_money is not None + ): + _rebuild_observations( + tmp_path, + monkeypatch, + persons, + extra_household=extra_household, + legacy_weights=legacy_weights, + missing_money=missing_money, + current_predictor_money=current_predictor_money, + ) + # Both real owners must pin the same final invented original member bytes. + monkeypatch.setattr(restoration, "_MEMBER_PINS", coverage._MEMBER_PINS) + parent = tmp_path / "invented-original-v4.h5" + household = tmp_path / "invented-household-attachment.h5" + output = tmp_path / "restored" + restoration.restore_asec_person_income_source( + parent, household, member_paths=persons, output_dir=output + ) + member = tmp_path / "hhpub25.csv" + rows = ( + household_rows + if household_rows is not None + else [ + ["00007", "1", "000255212", "6", "1", "2"], + ["00008", "2", "", "0", "0", "0"], + ] + ) + if extra_household and household_rows is None: + rows[1] = ["00008", "1", "00000000", "6", "1", "2"] + with member.open("w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow( + ["H_SEQ", "H_HHTYPE", "HSUP_WGT", "HRHTYPE", "H_LIVQRT", "H_NUMPER"] + ) + writer.writerows(rows) + data = member.read_bytes() + monkeypatch.setattr( + origin, + "_ASEC_MEMBER_PINS", + ( + origin.AsecNativeMemberPin( + income_year=2024, + survey_year=2025, + canonical_member_id="invented/2025/household/hhpub25.csv", + member_name="hhpub25.csv", + archive_sha256=origin.ASEC_EDUCATION_ASSISTANCE_ARCHIVES[ + 2024 + ].zip_sha256, + member_sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + rows=len(rows), + ), + ), + ) + return dict( + parent_path=parent, + household_attachment_path=household, + person_income_attachment_path=output / restoration.CHECKPOINT_FILENAME, + person_member_paths=persons, + household_member_path=member, + ) + + +def test_real_three_cohort_loader_selects_2024_original_anchors(tmp_path, monkeypatch): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + frame, receipt = result.frame, result.receipt + assert frame.person.source_year.tolist() == [2024, 2024] + assert frame.person.person_id.tolist() == [105, 106] + assert frame.person.A_AGE.tolist() == [55, 14] + assert frame.person.asec_PTOTVAL.tolist() == [-9999, 0] + assert frame.weights_for("household").kind is WeightKind.DESIGN + assert frame.weights_for("household").values.tolist() == [2552.12] + assert dict(frame.metadata) == {} and frame.mass_log == () + assert json.loads(result.context)["entities"]["person"]["rows"] == 2 + assert receipt["parent_custody"]["cohorts"] == [2022, 2023, 2024] + assert receipt["parent_custody"]["population_cohorts"] == [2024] + assert receipt["households"][0]["HSUP_WGT"] == "000255212" + assert receipt["households"][0]["fraction"] == [63803, 25] + assert receipt["households"][0]["native_key"] == [2024, 7] + assert not receipt["release_eligible"] + assert not receipt["selection"]["inclusion_probabilities_known"] + assert {r["A_LINENO"] for r in receipt["persons"]} == {1, 2} + for entity in frame.schema.group_entities: + assert len(frame.table(entity)) == 1 + # Real parent remains separate, complete and sealed after all child borrows. + state = native._ISSUED[id(result)][2] + assert state.parent.frame.person.source_year.tolist() == [ + 2022, + 2022, + 2023, + 2023, + 2024, + 2024, + ] + assert state.parent.frame.weights_for("household").values.tolist() == [1, 2, 3] + state.parent.validate() + + +def test_exact_selection_candidate_reconstruction_and_value_equal_copy_refusal( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + candidate = result.to_bytes() + reconstructed = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),), candidate=candidate + ) + assert reconstructed == result and reconstructed is not result + reconstructed.validate() + for copy in ( + replace(result), + native.AuthenticatedAsec2024NativePopulation(candidate), + ): + assert copy == result + with pytest.raises(native.AsecNativePopulationError, match="UNISSUED"): + copy.validate() + with pytest.raises(native.AsecNativePopulationError, match="CANDIDATE_MISMATCH"): + native.load_authenticated_asec_2024_native_population(**paths, candidate=b"{}") + + +@pytest.mark.parametrize( + "selection", + [ + (), + ((2023, 7),), + ((2024, 7), (2024, 7)), + ((2024, True),), + [(2024, 7)], + ((2024, 8),), + ], +) +def test_non_exact_or_absent_selection_refuses(tmp_path, monkeypatch, selection): + paths = _fixture(tmp_path, monkeypatch) + with pytest.raises(native.AsecNativePopulationError, match="SELECTION"): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=selection + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("H_NUMPER", "1"), + ("H_NUMPER", ""), + ("H_SEQ", "9"), + ("HSUP_WGT", "2552.12"), + ("HSUP_WGT", ""), + ("HSUP_WGT", "-1"), + ("HSUP_WGT", "0"), + ("H_HHTYPE", "2"), + ], +) +def test_source_household_roster_or_unusable_anchor_refuses( + tmp_path, monkeypatch, field, value +): + names = ["H_SEQ", "H_HHTYPE", "HSUP_WGT", "HRHTYPE", "H_LIVQRT", "H_NUMPER"] + row = ["00007", "1", "255212", "6", "1", "2"] + row[names.index(field)] = value + paths = _fixture(tmp_path, monkeypatch, household_rows=[row]) + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population(**paths) + + +def test_absent_interview_household_not_silently_lost(tmp_path, monkeypatch): + paths = _fixture( + tmp_path, + monkeypatch, + household_rows=[ + ["7", "1", "100", "6", "1", "2"], + ["8", "1", "200", "6", "1", "1"], + ], + ) + with pytest.raises( + native.AsecNativePopulationError, match="HOUSEHOLD_MEMBER_COUNT" + ): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "age", + "unrelated", + "group", + "weight", + "strata", + "strata_flags", + "table_flags", + "attrs", + "columns", + "metadata", + "membership", + ], +) +def test_full_receiving_frame_mutation_refuses(tmp_path, monkeypatch, mutation): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + frame = result.frame + if mutation == "age": + frame.person.loc[:, "A_AGE"] += 1 + elif mutation == "unrelated": + frame.person.loc[:, "unrelated_raw_observation"] += 1 + elif mutation == "group": + frame.table("household").loc[:, "state_fips"] += 1 + elif mutation == "weight": + vector = frame.weights_for("household").values + vector.setflags(write=True) + vector[0] += 1 + vector.setflags(write=False) + elif mutation == "strata": + frame.strata.iloc[0] = "other" + elif mutation == "strata_flags": + frame.strata.flags.allows_duplicate_labels = False + elif mutation == "table_flags": + frame.table("household").flags.allows_duplicate_labels = False + elif mutation == "attrs": + frame.person.attrs["changed"] = True + elif mutation == "columns": + frame.person.columns.name = "changed" + elif mutation == "metadata": + frame._metadata = {"changed": True} + elif mutation == "membership": + frame.person.iloc[0, frame.person.columns.get_loc("person_tax_unit_id")] = 999 + with pytest.raises(native.AsecNativePopulationError): + result.validate() + + +@pytest.mark.parametrize( + "which", + [ + "parent_path", + "household_attachment_path", + "person_income_attachment_path", + "household_member_path", + "older_person", + ], +) +def test_changed_original_source_refuses_existing_borrow(tmp_path, monkeypatch, which): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + path = ( + paths["person_member_paths"][2022] if which == "older_person" else paths[which] + ) + with path.open("ab") as handle: + handle.write(b"changed") + with pytest.raises(native.AsecNativePopulationError, match="SOURCE_FILE"): + result.validate() + + +def test_capsule_payload_rehash_and_live_owner_changes_refuse(tmp_path, monkeypatch): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + state = native._ISSUED[id(result)][2] + payload = state.anchors.payload + b" " + object.__setattr__(state.anchors, "payload", payload) + object.__setattr__( + state.anchors, "_issued_sha256", hashlib.sha256(payload).hexdigest() + ) + with pytest.raises(native.AsecNativePopulationError): + result.validate() + original = restoration._implementation + monkeypatch.setattr(restoration, "_implementation", lambda: original()) + with pytest.raises(native.AsecNativePopulationError, match="PRODUCER_CODE_CHANGED"): + native.load_authenticated_asec_2024_native_population(**paths) + + +@pytest.mark.parametrize("legacy", [[100, 200, 17], [0.001, 10_000, 0.0001]]) +def test_original_anchor_ignores_legacy_pooled_weights(tmp_path, monkeypatch, legacy): + paths = _fixture(tmp_path, monkeypatch, legacy_weights=legacy) + result = native.load_authenticated_asec_2024_native_population(**paths) + assert result.frame.weights_for("household").values.tolist() == [2552.12] + assert result.receipt["households"][0]["fraction"] == [63803, 25] + assert ( + native._ISSUED[id(result)][2] + .parent.frame.weights_for("household") + .values.tolist() + == legacy + ) + + +def test_mixed_zero_complete_households_and_selected_zero_refusal( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch, extra_household=True) + result = native.load_authenticated_asec_2024_native_population(**paths) + assert result.frame.person.person_id.tolist() == [105, 106, 107, 108] + assert result.frame.person.A_AGE.tolist() == [55, 14, 55, 14] + assert result.frame.weights_for("household").values.tolist() == [2552.12, 0] + assert [r["fraction"] for r in result.receipt["households"]] == [ + [63803, 25], + [0, 1], + ] + selected = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + assert selected.frame.person.person_id.tolist() == [105, 106] + for entity in selected.frame.schema.group_entities: + assert selected.frame.table(entity)[f"{entity}_id"].tolist() == [3] + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 8),) + ) + + +def test_late_mutation_during_final_file_verification_refuses(tmp_path, monkeypatch): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + frame = result.frame + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + frame.person.loc[:, "A_AGE"] += 1 + return trace + + sys.settrace(trace) + try: + with pytest.raises(native.AsecNativePopulationError, match="FRAME_CHANGED"): + result.validate() + finally: + sys.settrace(None) + assert fired + + +def test_real_loaders_run_without_prepared_population_or_money_execution( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch) + called = set() + + def trace(stack, event, argument): + if event == "call": + module = stack.f_globals.get("__name__", "") + name = stack.f_code.co_name + assert module not in ( + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.engine", + ) + assert name not in ( + "classify_asec_money", + "prepare_asec_current_money_population", + ) + if module.startswith("microcosm.build.us_runtime."): + called.add(name) + + sys.setprofile(trace) + try: + native.load_authenticated_asec_2024_native_population(**paths) + finally: + sys.setprofile(None) + assert { + "load_authenticated_restored_current_money_source", + "authenticate_asec_coverage", + "load_authenticated_asec_household_weights", + "load_authenticated_asec_household_coverage_fields", + } <= called + + +@pytest.mark.parametrize( + "mutation", ["missing", "duplicate", "substitute", "split", "age"] +) +def test_original_person_mutation_refuses_reconstruction( + tmp_path, monkeypatch, mutation +): + paths = _fixture(tmp_path, monkeypatch) + + def change(rows): + if mutation == "missing": + rows.pop() + elif mutation == "duplicate": + rows[2] = rows[1].copy() + elif mutation == "substitute": + rows[1][0] = "9" * 22 + elif mutation == "split": + rows[1][1] = "8" + elif mutation == "age": + rows[1][3] = "54" + + _helpers()._rewrite(paths["person_member_paths"][2024], change) + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population(**paths) + + +def test_external_state_and_rehashed_population_cannot_be_rewritten( + tmp_path, monkeypatch +): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + with pytest.raises(AttributeError): + object.__setattr__(state, "frame_identity", "a" * 64) + changed = json.loads(result.payload) + changed["frame_sha256"] = "a" * 64 + object.__setattr__(result, "payload", native._encode(changed)) + with pytest.raises(native.AsecNativePopulationError, match="UNISSUED_OR_CHANGED"): + result.validate() + + +@pytest.mark.parametrize("owner", ["anchors", "fields", "coverage"]) +def test_equal_unissued_source_evidence_refuses(tmp_path, monkeypatch, owner): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + evidence = getattr(state, owner) + copy = object.__new__(type(evidence)) + # Copy/deserialization cannot inherit the reader's external issuance seal. + for name in evidence.__slots__: + if name != "__weakref__": + object.__setattr__(copy, name, getattr(evidence, name)) + if owner != "coverage": + assert copy == evidence + verifier = { + "anchors": native.anchor_owner.verify_asec_household_weights_source, + "fields": native.field_owner.verify_asec_household_coverage_fields, + "coverage": lambda value: native.coverage_owner.verify_asec_coverage_parent( + value, state.parent + ), + }[owner] + with pytest.raises(ValueError): + verifier(copy) + result.validate() + + +@pytest.mark.parametrize( + "interface", ["validate", "frame", "context", "receipt", "to_bytes"] +) +def test_late_population_payload_mutation_refuses_every_interface( + tmp_path, monkeypatch, interface +): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + object.__setattr__(result, "payload", b'{"release_eligible":true}') + return trace + + sys.settrace(trace) + try: + with pytest.raises(native.AsecNativePopulationError): + value = getattr(result, interface) + if callable(value): + value() + finally: + sys.settrace(None) + assert fired + + +@pytest.mark.parametrize("owner", ["anchors", "fields", "coverage", "parent"]) +def test_late_attached_evidence_mutation_refuses(tmp_path, monkeypatch, owner): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + if owner == "parent": + object.__setattr__( + state.parent.source, "evidence", state.parent.source.evidence + b" " + ) + elif owner == "coverage": + object.__setattr__(state.coverage, "_body", state.coverage._body + b"x") + else: + source = getattr(state, owner) + object.__setattr__(source, "payload", source.payload + b" ") + + return_trace = trace + + def tracing(stack, event, argument): + return_trace(stack, event, argument) + return tracing + + sys.settrace(tracing) + try: + with pytest.raises(native.AsecNativePopulationError): + result.to_bytes() + finally: + sys.settrace(None) + assert fired diff --git a/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py b/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py new file mode 100644 index 000000000..d22eed814 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py @@ -0,0 +1,352 @@ +"""Immutable-leaf reuse of the ASEC records identity plus actual issuer controls. + +Supplied records exercise private value helpers only; they are never treated +as an authenticated source catalogue. The last controls use real issuance over +the maintained invented preparation fixture. +""" + +import sys + +import pytest +from test_us_survey_population_preparation import fixture + +from microcosm.build.us_runtime import asec_population_catalogue as owner +from microcosm.build.us_runtime import survey_population_domains as domains +from microcosm.build.us_runtime import survey_population_preparation as preparation + + +def _key(native_id="1"): + return domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, native_id) + + +def _records(): + key = _key() + person = domains.AsecPerson("2024000000000000000001", "1", "37", "1", "1", key) + household = domains.AsecHousehold(key, "1", "1", "1", "1", "1234567", (person,)) + ledger = owner.UnrepresentedAsecHousehold(_key("2"), "2", "9", "1", "0", "7654321") + return (household,), (ledger,) + + +def _calls(action): + calls = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is owner._records_identity.__code__: + calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = action() + finally: + sys.setprofile(previous) + return result, len(calls) + + +def _issued(households, ledger): + (identity, memo), calls = _calls( + lambda: owner._records_identity(households, ledger, memo=True) + ) + assert calls == 1 + return identity, memo + + +def _memoized(households, ledger, memo, identity): + return _calls( + lambda: owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + ) + + +def test_same_issued_leaves_reuse_exact_identity_without_another_encode(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + assert identity == owner._records_identity(households, ledger) + assert type(memo) is owner._RecordsMemo + assert memo.households is households and memo.ledger is ledger + assert memo.identity == identity + assert memo.household_leaves == (owner._household_leaves(households[0]),) + assert memo.ledger_leaves == (owner._ledger_leaves(ledger[0]),) + result, calls = _calls( + lambda: [ + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + for _ in range(3) + ] + ) + assert result == [identity] * 3 and calls == 0 + + +def test_equal_replacement_roots_miss_without_changing_identity_or_memo(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + replacement = (tuple(list(households)), tuple(list(ledger))) + assert replacement == (households, ledger) + assert replacement[0] is not households and replacement[1] is not ledger + result, calls = _memoized(*replacement, memo, identity) + assert result == identity and calls == 1 + assert memo.households is households and memo.ledger is ledger + + +@pytest.mark.parametrize( + "target", + ["household", "person", "household_key", "person_key", "ledger", "reason"], +) +def test_in_place_leaf_change_misses_and_changes_identity(target): + households, ledger = _records() + identity, memo = _issued(households, ledger) + household, person = households[0], households[0].persons[0] + if target == "household": + object.__setattr__(household, "hsup_wgt", "0000000") + elif target == "person": + object.__setattr__(person, "age", "38") + elif target == "household_key": + object.__setattr__(household.key, "native_id", "9") + elif target == "person_key": + object.__setattr__(person, "household_key", _key("9")) + elif target == "ledger": + object.__setattr__(ledger[0], "h_numper", "1") + else: + object.__setattr__(ledger[0], "reason", "other") + result, calls = _memoized(households, ledger, memo, identity) + assert calls == 1 and result != identity + assert result == owner._records_identity(households, ledger) + + +@pytest.mark.parametrize("target", ["persons", "key"]) +def test_replaced_container_misses_even_when_equal(target): + households, ledger = _records() + identity, memo = _issued(households, ledger) + household = households[0] + original = getattr(household, target) + replacement = ( + tuple(list(original)) if target == "persons" else _key(original.native_id) + ) + assert replacement == original and replacement is not original + object.__setattr__(household, target, replacement) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 1 + + +def test_persons_replaced_by_a_list_takes_the_refusing_path(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + object.__setattr__(households[0], "persons", list(households[0].persons)) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + + +def test_deleted_leaf_misses_instead_of_raising_inside_the_memo(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + object.__delattr__(households[0].persons[0], "age") + with pytest.raises(AttributeError): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + + +@pytest.mark.parametrize( + "kind", + [ + "str_subclass_literal", + "str_subclass_person", + "str_subclass_reason", + "int_subclass_year", + "str_subclass_native_id", + ], +) +def test_subclass_leaves_yield_no_memo_and_keep_the_full_identity_path(kind): + class StrSubclass(str): + pass + + class IntSubclass(int): + pass + + households, ledger = _records() + household, person = households[0], households[0].persons[0] + if kind == "str_subclass_literal": + object.__setattr__(household, "h_hhtype", StrSubclass("1")) + elif kind == "str_subclass_person": + object.__setattr__(person, "prpertyp", StrSubclass("1")) + elif kind == "str_subclass_reason": + object.__setattr__(ledger[0], "reason", StrSubclass("x")) + elif kind == "int_subclass_year": + object.__setattr__(household.key, "source_year", IntSubclass(2024)) + else: + object.__setattr__(ledger[0].key, "native_id", StrSubclass("2")) + identity, memo = _issued(households, ledger) + assert memo is None + assert identity == owner._records_identity(households, ledger) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 1 + + +def test_memo_requires_agreement_with_the_owner_identity(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + wrong = (*identity[:3], identity[3] + 1) + result, calls = _memoized(households, ledger, memo, wrong) + assert result == identity and calls == 1 + drifted = memo._replace(identity=wrong) + result, calls = _memoized(households, ledger, drifted, identity) + assert result == identity and calls == 1 + plain = tuple(memo) + assert plain == memo + result, calls = _memoized(households, ledger, plain, identity) + assert result == identity and calls == 1 + + +def test_swapped_memo_entry_misses(): + households, ledger = _records() + identity, _ = _issued(households, ledger) + other_households, other_ledger = _records() + object.__setattr__(other_households[0], "hsup_wgt", "0000001") + other_identity, other_memo = _issued(other_households, other_ledger) + assert other_identity != identity + result, calls = _memoized(households, ledger, other_memo, identity) + assert result == identity and calls == 1 + result, calls = _memoized(households, ledger, other_memo, other_identity) + assert result == identity and calls == 1 + + +def test_crafted_memo_over_lists_still_refuses(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + rows = list(households) + crafted = owner._RecordsMemo( + rows, ledger, memo.household_leaves, memo.ledger_leaves, identity + ) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + rows, ledger, crafted, expected_identity=identity + ) + + +def test_wrong_source_member_or_foreign_types_yield_no_memo(): + households, ledger = _records() + foreign = domains.HouseholdKey(domains.Source.ACS, 2024, 2025, "1") + object.__setattr__(ledger[0], "key", foreign) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_SOURCE$"): + owner._records_identity(households, ledger, memo=True) + assert owner._eligible_leaves(ledger[0], "unrepresented") is None + assert owner._eligible_leaves(households[0], "unrepresented") is None + assert owner._eligible_leaves(ledger[0], "household") is None + + +def test_source_member_value_tampering_misses_and_the_full_path_reflects_it(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + member = domains.Source.ASEC + original = member._value_ + object.__setattr__(member, "_value_", "tampered") + try: + assert member.value == "tampered" + result, calls = _memoized(households, ledger, memo, identity) + assert calls == 1 and result != identity + assert result == owner._records_identity(households, ledger) + finally: + object.__setattr__(member, "_value_", original) + assert member.value == original + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 0 + + +@pytest.mark.parametrize("target", ["household", "person", "key", "ledger"]) +def test_class_reassignment_misses_and_the_full_path_refuses(target): + class SwappedHousehold(domains.AsecHousehold): + __slots__ = () + + class SwappedPerson(domains.AsecPerson): + __slots__ = () + + class SwappedKey(domains.HouseholdKey): + __slots__ = () + + class SwappedLedger(owner.UnrepresentedAsecHousehold): + __slots__ = () + + households, ledger = _records() + identity, memo = _issued(households, ledger) + victim, swapped = { + "household": (households[0], SwappedHousehold), + "person": (households[0].persons[0], SwappedPerson), + "key": (households[0].key, SwappedKey), + "ledger": (ledger[0], SwappedLedger), + }[target] + original = type(victim) + object.__setattr__(victim, "__class__", swapped) + try: + assert type(victim) is swapped + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + finally: + object.__setattr__(victim, "__class__", original) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 0 + + +@pytest.fixture(scope="module") +def actual_preparation(tmp_path_factory): + with pytest.MonkeyPatch.context() as monkeypatch: + arguments = fixture(tmp_path_factory.mktemp("asec-records-memo"), monkeypatch) + result = preparation.prepare_authenticated_survey_population(**arguments) + state = preparation._ISSUED[id(result)][2] + catalogue = state.catalogues[1] + catalogue_state = owner._ISSUED[id(catalogue)][2] + assert type(catalogue_state.records_memo) is owner._RecordsMemo + assert catalogue_state.records_memo.identity == catalogue_state.records_identity + yield result, catalogue, catalogue_state + result.validate() + catalogue.validate() + + +def test_actual_borrows_reuse_the_issued_identity(actual_preparation): + result, catalogue, _ = actual_preparation + _, calls = _calls(catalogue.validate) + assert calls == 0 + _, calls = _calls(result.checked_view) + assert calls == 0 + + +def test_actual_household_leaf_change_refuses_every_borrow(actual_preparation): + result, catalogue, catalogue_state = actual_preparation + row = catalogue_state.households[0] + original = row.hsup_wgt + object.__setattr__(row, "hsup_wgt", original + "0") + try: + with pytest.raises(owner.AsecSourceCatalogueError, match="RECORDS_CHANGED"): + catalogue.validate() + with pytest.raises( + preparation.SurveyPopulationPreparationError, + match="NESTED_EVIDENCE_CHANGED", + ): + result.checked_view() + finally: + object.__setattr__(row, "hsup_wgt", original) + _, calls = _calls(catalogue.validate) + assert calls == 0 + + +def test_actual_person_key_change_refuses_every_borrow(actual_preparation): + result, catalogue, catalogue_state = actual_preparation + person = catalogue_state.households[0].persons[0] + original = person.household_key + object.__setattr__(person, "household_key", _key(original.native_id + "0")) + try: + with pytest.raises(owner.AsecSourceCatalogueError, match="RECORDS_CHANGED"): + catalogue.validate() + with pytest.raises( + preparation.SurveyPopulationPreparationError, + match="NESTED_EVIDENCE_CHANGED", + ): + result.checked_view() + finally: + object.__setattr__(person, "household_key", original) + result.validate() diff --git a/packages/microcosm-build/tests/test_us_asec_checkpoint.py b/packages/microcosm-build/tests/test_us_asec_checkpoint.py index aa2a1338a..3496eb470 100644 --- a/packages/microcosm-build/tests/test_us_asec_checkpoint.py +++ b/packages/microcosm-build/tests/test_us_asec_checkpoint.py @@ -7,6 +7,7 @@ import pandas as pd import pytest +import microcosm.build.us_runtime.asec_checkpoint as checkpoint_module from microcosm.build.frame_checkpoint import write_frame_checkpoint from microcosm.build.outer_stage_runtime import ( OUTER_STAGE_CONTEXT_SCHEMA_VERSION, @@ -26,6 +27,396 @@ ) from microcosm.frame import US_SCHEMA, EntitySchema, Frame, WeightKind, Weights + +def _v4_frame() -> Frame: + frame = _raw_us_frame() + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + frame.table("person")[column] = np.asarray([1, 2], dtype=np.int64) + return frame + + +def _v4_binding(frame: Frame) -> dict: + metadata = _raw_binding(frame) + metadata["schema_version"] = 4 + metadata["source_receipt"]["sources"].append( + dict(metadata["source_receipt"]["sources"][0], year=2023) + ) + pins = checkpoint_module.ASEC_EDUCATION_ASSISTANCE_ARCHIVES + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + metadata["raw_source_mappings"][column] = { + "column": column, + "entity": "person", + "operation": "exact_source_join", + "join_keys": ["source_year", "PERIDNUM"], + "source_pins": [ + { + "income_year": year, + "locator": pins[year].zip_url, + "member": pins[year].member, + "sha256": pins[year].zip_sha256, + "member_sha256": pins[year].member_sha256, + } + for year in (2022, 2023) + ], + # Audit describes the full official member, not this two-row subset. + "audit": { + str(year): { + "rows": pins[year].rows, + "yes_rows": 1, + "no_rows": pins[year].rows - 1, + "weighted_yes_share": 0.25, + } + for year in (2022, 2023) + }, + } + return metadata + + +def test_v4_roundtrip_requires_explicit_codec(tmp_path: Path) -> None: + frame = _v4_frame() + path = tmp_path / "v4.h5" + _write_checkpoint(path, frame, metadata=_v4_binding(frame)) + loaded, binding = checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + assert binding["schema_version"] == 4 + assert loaded.table("person")["NOW_MCAID"].tolist() == [1, 2] + with pytest.raises(ValueError, match="unsupported raw-stage schema version"): + load_asec_raw_stage_checkpoint(path) + path3 = tmp_path / "v3.h5" + legacy = _raw_us_frame() + _write_checkpoint(path3, legacy, metadata=_raw_binding(legacy)) + with pytest.raises(ValueError, match="unsupported raw-stage schema version"): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path3) + + +def test_restored_raw_observations_carry_to_input_leaves_without_engine() -> None: + from microcosm.build.us_runtime.cps_carried import derive_us_cps_carried_inputs + + frame = _raw_us_frame() + person = frame.table("person") + person["NOW_GRP"] = [1, 2] + person["NOW_MRK"] = [2, 1] + person["OI_VAL"] = [0.0, 0.0] + person["OI_OFF"] = [0, 0] + sidecar = person[["source_year", "PERIDNUM"]].copy() + sidecar["PH_SEQ"] = [101, 102] + sidecar["P_SEQ"] = [1, 1] + sidecar["A_LINENO"] = [1, 1] + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + sidecar[column] = [1, 2] + result = derive_us_cps_carried_inputs(frame, reported_coverage_source=sidecar) + assert "NOW_MCAID" not in frame.table("person") + assert result.table("person")[ + "has_medicaid_health_coverage_at_interview" + ].tolist() == [True, False] + assert result.table("person")["has_esi"].tolist() == [True, False] + assert result.table("person")[ + "has_marketplace_health_coverage_at_interview" + ].tolist() == [False, True] + assert result.weights_for("household").kind is WeightKind.DESIGN + pd.testing.assert_frame_equal( + result.table("person"), derive_us_cps_carried_inputs(result).table("person") + ) + + +@pytest.mark.parametrize( + "with_mass_history,forward_metadata", + [(False, False), (True, False), (True, True)], + ids=["empty_log", "declared_mass_change", "metadata_forwarding"], +) +def test_v4_restoration_producer_authenticates_source_and_writes_new_bundle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + with_mass_history: bool, + forward_metadata: bool, +) -> None: + import hashlib + from dataclasses import replace + + from microcosm.build.us_runtime import asec_raw_stage_v4 as restoration + from microcosm.build.us_runtime import reported_coverage_source + from microcosm.build.us_runtime.asec_raw_stage_v4 import restore_asec_raw_stage_v4 + from microcosm.frame import MassChange + + legacy = _raw_us_frame() + if with_mass_history: + original = legacy + weights = original.weights_for("household") + legacy = original.with_weights( + "household", + Weights(weights.values * 2.0, weights.kind), + mass=MassChange( + factor=2.0, reason="Invented raw-source design-weight rescaling" + ), + ) + assert original.mass_log == () + assert len(legacy.mass_log) == 1 + record = legacy.mass_log[0] + assert record.entity == "household" + assert record.old_total == float(weights.values.sum()) + assert record.new_total == 2.0 * record.old_total + assert record.declared_factor == 2.0 + assert record.reason == "Invented raw-source design-weight rescaling" + expected_mass_log = legacy.mass_log + assert bool(expected_mass_log) is with_mass_history + binding = _raw_binding(legacy) + binding["source_receipt"]["sources"].append( + dict(binding["source_receipt"]["sources"][0], year=2023) + ) + input_path = tmp_path / "v3.h5" + _write_checkpoint(input_path, legacy, metadata=binding) + loaded_input, _ = checkpoint_module.load_asec_raw_stage_checkpoint(input_path) + assert loaded_input.mass_log == expected_mass_log + np.testing.assert_array_equal( + loaded_input.weights_for("household").values, + legacy.weights_for("household").values, + ) + input_sha = hashlib.sha256(input_path.read_bytes()).hexdigest() + pins = {} + paths = {} + for year, key in ( + (2022, "0000000000000000000001"), + (2023, "0000000000000000000002"), + ): + path = tmp_path / f"person-{year}.csv" + row = { + "PERIDNUM": key, + "PH_SEQ": 1, + "P_SEQ": 1, + "A_LINENO": 1, + "A_FNLWGT": 100.0, + } + row.update( + { + column: 1 if year == 2022 else 2 + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS + } + ) + pd.DataFrame([row]).to_csv(path, index=False) + pins[year] = replace( + checkpoint_module.ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year], + rows=1, + member=path.name, + member_size_bytes=path.stat().st_size, + member_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), + ) + paths[year] = path + monkeypatch.setattr(checkpoint_module, "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", pins) + monkeypatch.setattr( + reported_coverage_source, "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", pins + ) + forwarded = [] + if forward_metadata: + # The checkpoint codec does not persist Frame.metadata. Supply a + # descriptive value only after the real v3 loader has validated its + # input, then observe the actual converter at the real writer boundary. + supplied_metadata = {"forwarding_probe": {"purpose": "invented observation"}} + real_load = checkpoint_module.load_asec_raw_stage_checkpoint + real_write = restoration.write_frame_checkpoint + + def load_with_descriptive_metadata(path): + loaded, binding = real_load(path) + assert not loaded.metadata + return ( + Frame( + {entity: loaded.table(entity) for entity in loaded.entities}, + loaded.schema, + { + entity: loaded.weights_for(entity) + for entity in loaded.weighted_entities + }, + loaded.strata, + mass_log=loaded.mass_log, + metadata=supplied_metadata, + ), + binding, + ) + + def observe_real_write(path, frame, **kwargs): + assert set(frame.metadata) == {"forwarding_probe"} + assert ( + dict(frame.metadata["forwarding_probe"]) + == supplied_metadata["forwarding_probe"] + ) + assert frame.mass_log == expected_mass_log + forwarded.append(True) + return real_write(path, frame, **kwargs) + + monkeypatch.setattr( + checkpoint_module, + "load_asec_raw_stage_checkpoint", + load_with_descriptive_metadata, + ) + monkeypatch.setattr(restoration, "write_frame_checkpoint", observe_real_write) + output_dir = tmp_path / "restored" + receipt = restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=output_dir, + ) + assert forwarded == ([True] if forward_metadata else []) + assert receipt["input_sha256"] == input_sha + output = output_dir / "asec_raw_stage.checkpoint.h5" + assert receipt["output_sha256"] == hashlib.sha256(output.read_bytes()).hexdigest() + assert hashlib.sha256(input_path.read_bytes()).hexdigest() == input_sha + restored, metadata = checkpoint_module.load_asec_raw_stage_checkpoint_v4(output) + assert restored.mass_log == expected_mass_log + # The codec does not reconstruct the separately supplied Frame metadata. + assert not restored.metadata + assert restored.weights_for("household").kind is WeightKind.DESIGN + np.testing.assert_array_equal( + restored.weights_for("household").values, + loaded_input.weights_for("household").values, + ) + assert restored.table("person")["NOW_MCAID"].tolist() == [1, 2] + assert metadata["source_receipt"] == binding["source_receipt"] + assert json.loads((output_dir / "restoration.receipt.json").read_text()) == receipt + with pytest.raises(FileExistsError): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=output_dir, + ) + with pytest.raises(ValueError, match="SHA-256"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256="f" * 64, + coverage_paths=paths, + output_dir=tmp_path / "bad", + ) + with pytest.raises(ValueError, match="local coverage paths"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths={2022: paths[2022]}, + output_dir=tmp_path / "missing", + ) + paths[2022].write_text("tampered") + with pytest.raises(ValueError, match="byte length mismatch"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=tmp_path / "tampered", + ) + assert not (tmp_path / "bad").exists() + assert not (tmp_path / "missing").exists() + assert not (tmp_path / "tampered").exists() + + +@pytest.mark.parametrize( + "mutation", + ( + "pin_sha", + "pin_member", + "pin_year_missing", + "pin_year_duplicate", + "audit_year", + "audit_rows", + "audit_counts", + "audit_nan", + "audit_bool", + "receipt_year", + "missing_mapping", + "float_version", + ), +) +def test_v4_refuses_inconsistent_attestation(tmp_path: Path, mutation: str) -> None: + frame = _v4_frame() + metadata = _v4_binding(frame) + mapping = metadata["raw_source_mappings"]["NOW_MCAID"] + if mutation == "pin_sha": + mapping["source_pins"][0]["sha256"] = "f" * 64 + elif mutation == "pin_member": + mapping["source_pins"][0]["member"] = "other.csv" + elif mutation == "pin_year_missing": + mapping["source_pins"].pop() + elif mutation == "pin_year_duplicate": + mapping["source_pins"].append(dict(mapping["source_pins"][0])) + elif mutation == "audit_year": + del mapping["audit"]["2023"] + elif mutation == "audit_rows": + mapping["audit"]["2022"]["rows"] = 2 + elif mutation == "audit_counts": + mapping["audit"]["2022"]["no_rows"] = 0 + elif mutation == "audit_nan": + mapping["audit"]["2022"]["weighted_yes_share"] = float("nan") + elif mutation == "audit_bool": + mapping["audit"]["2022"]["yes_rows"] = True + elif mutation == "receipt_year": + metadata["source_receipt"]["sources"].pop() + elif mutation == "missing_mapping": + del metadata["raw_source_mappings"]["NOW_MCAID"] + elif mutation == "float_version": + metadata["schema_version"] = 4.0 + path = tmp_path / "invalid.h5" + # Exercise the same metadata validators directly for NaN because canonical + # checkpoint serialization correctly refuses it before the loader does. + if mutation == "audit_nan": + with pytest.raises(ValueError, match="audit counts/share"): + checkpoint_module._validate_coverage_v4_binding(frame, metadata, path=path) + else: + _write_checkpoint(path, frame, metadata=metadata) + with pytest.raises(ValueError): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +@pytest.mark.parametrize("value", (0, 3, 1.5, float("inf"), float("nan"), True)) +def test_v4_refuses_bad_measured_coverage(tmp_path: Path, value: object) -> None: + frame = _v4_frame() + frame.table("person")["NOW_MCAID"] = [value, value] + path = tmp_path / "bad-code.h5" + _write_checkpoint(path, frame, metadata=_v4_binding(frame)) + with pytest.raises(ValueError, match="NOW_MCAID must be complete integer recodes"): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +@pytest.mark.parametrize( + "mutation", ("missing_column", "peridnum", "duplicate", "weights") +) +def test_v4_refuses_wrong_source_boundary(tmp_path: Path, mutation: str) -> None: + frame = _v4_frame() + person = frame.table("person") + if mutation == "missing_column": + person.drop(columns="NOW_IHSFLG", inplace=True) + elif mutation == "peridnum": + person["PERIDNUM"] = ["1", "2"] + elif mutation == "duplicate": + person["source_year"] = [2022, 2022] + person["PERIDNUM"] = ["0" * 22, "0" * 22] + elif mutation == "weights": + frame = Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + {"household": Weights(np.asarray([2.0, 3.0]), WeightKind.CALIBRATED)}, + frame.strata, + ) + metadata = _v4_binding(frame) + if mutation == "duplicate": + metadata["source_receipt"]["sources"].pop() + path = tmp_path / "boundary.h5" + _write_checkpoint(path, frame, metadata=metadata) + with pytest.raises(ValueError): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +def test_v4_duplicate_guard_compares_normalized_identity() -> None: + frame = _v4_frame() + person = frame.table("person") + person["source_year"] = pd.Series([2022, "2022"], dtype=object) + person["PERIDNUM"] = pd.Series(["0" * 22, b"0" * 22], dtype=object) + metadata = _v4_binding(frame) + metadata["source_receipt"]["sources"].pop() + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + metadata["raw_source_mappings"][column]["source_pins"].pop() + del metadata["raw_source_mappings"][column]["audit"]["2023"] + with pytest.raises(ValueError, match="repeats a source-year/PERIDNUM"): + checkpoint_module._validate_coverage_v4_binding( + frame, metadata, path=Path("fixture.h5") + ) + + _OUTER_STAGE_ARTIFACT_KIND = "populace_outer_stage_frame" diff --git a/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py b/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py new file mode 100644 index 000000000..22ee44b3c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py @@ -0,0 +1,944 @@ +"""Invented closed source-parent fixtures only; no genuine preparation.""" + +import _csv +import csv +import hashlib +import importlib.util +import json +import os +import struct +import subprocess +import sys +import textwrap +import traceback +from dataclasses import FrozenInstanceError, replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.frame_checkpoint import ( + load_frame_checkpoint, + write_frame_checkpoint, +) +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_current_money_source as money +from microcosm.build.us_runtime import asec_person_coverage_source as literal + + +def _helper(name): + spec = importlib.util.spec_from_file_location( + "coverage_invented_" + name, Path(__file__).with_name(name + ".py") + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pin(paths, monkeypatch): + pins = [] + for year, member, archive, _, _, _ in coverage._MEMBER_PINS: + value = paths[year].read_bytes() + with paths[year].open(newline="") as handle: + rows = list(csv.reader(handle)) + pins.append( + (year, member, archive, coverage._sha(value), len(rows) - 1, len(value)) + ) + monkeypatch.setattr(coverage, "_MEMBER_PINS", tuple(pins)) + + +def _rewrite(path, change): + with path.open(newline="") as handle: + rows = list(csv.reader(handle)) + change(rows) + with path.open("w", newline="") as handle: + csv.writer(handle).writerows(rows) + + +def _changed_parent(parent, attachment, monkeypatch, changes): + for path in (parent, attachment): + loaded = load_frame_checkpoint(path) + for column, values in changes.items(): + loaded.frame.person[column] = values + identity = frame_identity(loaded.frame) + for key in ("identity", "source_construction_identity"): + if key in loaded.metadata: + loaded.metadata[key] = identity.to_payload() + if path == parent: + parent_identity = identity + else: + digest = coverage._sha(parent.read_bytes()) + loaded.metadata["parent_checkpoint_sha256"] = digest + loaded.metadata["household_observations"]["input_checkpoint_sha256"] = ( + digest + ) + loaded.metadata["household_observations"][ + "input_structural_identity_sha256" + ] = parent_identity.sha256 + write_frame_checkpoint(path, loaded.frame, metadata=loaded.metadata) + monkeypatch.setattr( + money, + "_SOURCE_PINS", + ( + coverage._sha(parent.read_bytes()), + coverage._sha(attachment.read_bytes()), + money._SOURCE_PINS[2], + ), + ) + + +def _fixtures( + tmp_path, + monkeypatch, + tokens=("1", "2", "3", "", "-4", "NA"), + *, + native_ids=None, + person_changes=None, + raw_ages=None, +): + parent, attachment, paths = _helper( + "test_us_asec_person_income_source" + ).invented_sources(tmp_path, monkeypatch, missing=False) + if native_ids is not None: + _helper("test_us_asec_demographic_source")._repoint_published_households( + parent, attachment, monkeypatch, native_ids + ) + if person_changes: + _changed_parent(parent, attachment, monkeypatch, person_changes) + source = money.load_authenticated_current_money_source(parent, attachment) + for path in paths.values(): + + def change(rows): + rows[0].append("PRPERTYP") + for row in rows[1:]: + position = int(row[0]) - 1 + row.append(tokens[position % len(tokens)]) + if native_ids is not None: + row[1] = str(native_ids[position]) + if raw_ages is not None: + row[3] = str(raw_ages[position]) + if person_changes and "PERIDNUM" in person_changes: + row[0] = person_changes["PERIDNUM"][position] + + _rewrite(path, change) + _pin(paths, monkeypatch) + return source, paths + + +def _read(source, paths, **kwargs): + return coverage.authenticate_asec_coverage(source, member_paths=paths, **kwargs) + + +def test_closed_complete_source_exact_parent_and_defensive_views(tmp_path, monkeypatch): + source, paths = _fixtures(tmp_path, monkeypatch) + before = money._frame_signature(source.frame) + tables = {e: source.frame.table(e).copy(deep=True) for e in source.frame.entities} + result = _read(source, paths) + coverage.verify_asec_coverage_parent(result, source) + view = result.table() + assert view.PRPERTYP.tolist() == ["1", "2", "3", "", "-4", "NA"] + assert view.PRPERTYP_state.tolist() == ["observed_code"] * 3 + [ + "blank_unresolved", + "unlabelled_in_range", + "malformed_token", + ] + assert view.PERIDNUM.tolist() == list(source.scope.person_native_keys) + assert view.person_household_id.tolist() == [1, 1, 2, 2, 3, 3] + raw, original_receipt = literal.read_asec_person_coverage_source( + paths, person_roster=source.frame.person + ) + receipt = result.receipt + assert receipt["literal_reader_receipt"] == original_receipt + # Native key cells retain their exact strings; the artifact view declares + # one non-nullable vocabulary independently of the parent's string storage. + pd.testing.assert_frame_equal(view[list(raw)], raw.astype({"PERIDNUM": "string"})) + assert receipt["source_authenticated"] is True + assert receipt["named_parent_binding_authenticated"] is True + for name in ( + "release_eligible", + "domain_authority", + "period_harmonized", + "cross_survey_coverage_equivalence_established", + "original_design_weight_semantics_established", + "archive_bytes_read", + ): + assert receipt[name] is False + assert receipt["literal_reader_receipt"]["source_authenticated"] is False + assert ( + receipt["literal_reader_receipt"]["population_binding_authenticated"] is False + ) + assert receipt["age_relation"]["compared_rows"] == 6 + assert ( + receipt["age_relation"]["relation"] == "original_csv_A_AGE_equals_parent_A_AGE" + ) + assert receipt["household_partitions"]["households"] == 3 + assert [p["rows"] for p in receipt["sources"]] == [2, 2, 2] + assert str(tmp_path) not in json.dumps(receipt) + assert "0000000000000000000001" not in json.dumps(receipt) + pin = result.content_sha256 + view.loc[0, "PRPERTYP"] = "changed" + view.index = pd.Index([99, 1, 2, 3, 4, 5]) + receipt["parent"]["frame_sha256"] = "changed" + assert result.content_sha256 == pin and result.table().PRPERTYP.iloc[0] == "1" + assert money._frame_signature(source.frame) == before + for entity, expected in tables.items(): + pd.testing.assert_frame_equal( + source.frame.table(entity), expected, check_exact=True + ) + with pytest.raises(FrozenInstanceError): + result._body = b"changed" + with pytest.raises(coverage.AsecCoverageAuthenticationError, match="CONSTRUCTOR"): + coverage.AuthenticatedAsecCoverage(result._header, result._body) + with pytest.raises(coverage.AsecCoverageAuthenticationError, match="CONSTRUCTOR"): + replace(result) + + +@pytest.mark.parametrize( + "token,state", + [ + ("", "blank_unresolved"), + ("\0", "malformed_token"), + ("1\0x", "malformed_token"), + (" 1", "malformed_token"), + ("1.0", "malformed_token"), + ("NA", "malformed_token"), + ('a"b', "malformed_token"), + ("a,b\r\n\t😀", "malformed_token"), + ("-4", "unlabelled_in_range"), + ("4", "out_of_range"), + ("3", "observed_code"), + ], +) +def test_literal_utf8_roundtrip_without_coercion(tmp_path, monkeypatch, token, state): + source, paths = _fixtures(tmp_path, monkeypatch, (token,)) + result = _read(source, paths) + assert result.table().PRPERTYP.tolist() == [token] * 6 + assert result.table().PRPERTYP_state.tolist() == [state] * 6 + assert not result.table().PRPERTYP.isna().any() + + +@pytest.mark.parametrize("kind", ["frame", "receipt", "hash"]) +def test_no_plain_parent_can_issue_authority(tmp_path, monkeypatch, kind): + source, paths = _fixtures(tmp_path, monkeypatch) + candidate = { + "frame": source.frame, + "receipt": json.loads(source.source.identity), + "hash": money._frame_signature(source.frame), + }[kind] + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="AUTHENTICATED_PARENT" + ): + _read(candidate, paths) + with pytest.raises(TypeError): + coverage.authenticate_asec_coverage(source, member_paths=paths, member_pins=()) + + +@pytest.mark.parametrize( + "column,value", + [ + ("A_AGE", "82"), + ("A_AGE", ""), + ("A_AGE", "55.0"), + ("A_AGE", "\0"), + ("PH_SEQ", "8"), + ("A_LINENO", "8"), + ("PERIDNUM", "9" * 22), + ], +) +def test_authenticated_raw_coordinate_conflicts_refuse( + tmp_path, monkeypatch, column, value +): + source, paths = _fixtures(tmp_path, monkeypatch) + _rewrite( + paths[2022], lambda rows: rows[1].__setitem__(rows[0].index(column), value) + ) + _pin(paths, monkeypatch) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="SOURCE_REFUSAL" + ): + _read(source, paths) + + +@pytest.mark.parametrize( + "column", + [ + "A_AGE", + "PERIDNUM", + "person_household_id", + "unrelated_raw_observation", + "WSAL_VAL", + ], +) +def test_same_ids_mutated_parent_cannot_reuse_binding(tmp_path, monkeypatch, column): + source, paths = _fixtures(tmp_path, monkeypatch) + result = _read(source, paths) + source.frame.person.loc[:, column] = "9" * 22 if column == "PERIDNUM" else 1 + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="PARENT_REFUSAL" + ): + coverage.verify_asec_coverage_parent(result, source) + + +def test_distinct_authenticated_parent_with_same_ids_cannot_reuse_artifact( + tmp_path, monkeypatch +): + first = tmp_path / "first" + second = tmp_path / "second" + first.mkdir() + second.mkdir() + source, paths = _fixtures(first, monkeypatch) + result = _read(source, paths) + other, other_paths = _fixtures( + second, + monkeypatch, + person_changes={"unrelated_raw_observation": np.arange(6, dtype=np.int64) + 1}, + ) + assert other.scope.person_ids == source.scope.person_ids + # Restore the producer's fixture pins so this specifically checks parent + # identity rather than the independent source-pin implementation guard. + _pin(paths, monkeypatch) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="PARENT_CHANGED" + ): + coverage.verify_asec_coverage_parent(result, other) + assert set(other_paths) == {2022, 2023, 2024} + + +def test_merged_native_households_refuse_even_with_matching_per_row_coordinates( + tmp_path, monkeypatch +): + # The current-money parent already refuses this shape, before coverage + # issuance. Keep that layering and also test the wrapper's own fence. + with pytest.raises(ValueError, match="SOURCE_CONTRACT_REFUSAL"): + _fixtures(tmp_path, monkeypatch, native_ids=[7, 8, 7, 7, 7, 7]) + person = pd.DataFrame( + { + "source_year": [2022, 2022], + "source_household_id": [7, 8], + "person_household_id": [1, 1], + }, + dtype="int64", + ) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_MERGE" + ): + coverage._partitions(person) + + +@pytest.mark.parametrize("parent_age,raw_age", [(80, 80), (82, 80), (0, "")]) +def test_raw_age_identity_never_substitutes_model_age( + tmp_path, monkeypatch, parent_age, raw_age +): + source, paths = _fixtures( + tmp_path, + monkeypatch, + person_changes={ + "A_AGE": np.array([parent_age, 14] * 3, dtype=np.int64), + }, + raw_ages=[raw_age, 14] * 3, + ) + before = money._frame_signature(source.frame) + if parent_age == raw_age: + result = _read(source, paths) + assert result.table().A_AGE.tolist() == [80, 14] * 3 + assert result.receipt["age_relation"]["model_age_used"] is False + else: + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="SOURCE_REFUSAL" + ): + _read(source, paths) + assert money._frame_signature(source.frame) == before + + +def test_parent_authority_refuses_model_age_before_source_issuance( + tmp_path, monkeypatch +): + # Canonical model age is already forbidden by the raw parent boundary. + # Keep that refusal; do not loosen the parent merely to exercise coverage. + with pytest.raises(ValueError, match="SOURCE_CONTRACT_REFUSAL"): + _fixtures( + tmp_path, + monkeypatch, + person_changes={ + "A_AGE": np.array([80, 14] * 3, dtype=np.int64), + "age": [82.0, 14.0] * 3, + }, + raw_ages=[80, 14] * 3, + ) + + +def test_same_native_person_keys_across_cohorts_remain_distinct(tmp_path, monkeypatch): + keys = [str(i).zfill(22) for i in (1, 2)] * 3 + source, paths = _fixtures(tmp_path, monkeypatch, person_changes={"PERIDNUM": keys}) + result = _read(source, paths) + assert result.table().PERIDNUM.tolist() == keys + assert result.table().PRPERTYP.tolist() == ["1", "2", "3", "", "-4", "NA"] + + +def test_partition_bijection_refuses_splits_and_cross_cohort_merges(): + person = pd.DataFrame( + { + "source_year": [2022, 2022], + "source_household_id": [7, 7], + "person_household_id": [1, 2], + }, + dtype="int64", + ) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_SPLIT" + ): + coverage._partitions(person) + person.source_year = [2022, 2023] + person.person_household_id = [1, 1] + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_MERGE" + ): + coverage._partitions(person) + + +@pytest.mark.parametrize( + "kind", + [ + "missing_cohort", + "duplicate_cohort", + "duplicate_row", + "missing_row", + "swapped_cohort", + ], +) +def test_complete_cohort_authority_refuses_incomplete_or_duplicate_inputs( + tmp_path, monkeypatch, kind +): + source, paths = _fixtures(tmp_path, monkeypatch) + if kind == "missing_cohort": + paths.pop(2023) + elif kind == "duplicate_cohort": + pins = coverage._MEMBER_PINS + monkeypatch.setattr(coverage, "_MEMBER_PINS", (pins[0], pins[0], pins[2])) + elif kind == "swapped_cohort": + paths[2022], paths[2023] = paths[2023], paths[2022] + else: + _rewrite( + paths[2022], + lambda rows: ( + rows.__setitem__(1, rows[2].copy()) + if kind == "duplicate_row" + else rows.pop() + ), + ) + _pin(paths, monkeypatch) + with pytest.raises(coverage.AsecCoverageAuthenticationError): + _read(source, paths) + + +def test_candidate_self_rehash_is_not_authority_and_reconstruction_runs_first( + tmp_path, monkeypatch +): + source, paths = _fixtures(tmp_path, monkeypatch) + result = _read(source, paths) + candidate = tmp_path / "candidate.bin" + candidate.write_bytes(result.to_bytes()) + assert ( + _read(source, paths, candidate_path=candidate).to_bytes() == result.to_bytes() + ) + body = result._body.replace(b"observed_code", b"invented_code", 1) + header = json.loads(result._header) + header["body_sha256"] = coverage._sha(body) + header = coverage._json(header) + payload = coverage.MAGIC + struct.pack(" UsPumaLadder: tract_overlap_puma=puma.copy(), tract_overlap_tract=np.asarray([1_001_000_100, 1_003_000_100], dtype=np.int64), tract_overlap_population=population.copy(), + joint_overlap_puma=puma.copy(), + joint_overlap_tract=np.asarray([1_001_000_100, 1_003_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([101, 102], dtype=np.int64), + joint_overlap_population=np.asarray([40, 60], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_common_frame_export_contract.py b/packages/microcosm-build/tests/test_us_common_frame_export_contract.py new file mode 100644 index 000000000..349e78527 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_common_frame_export_contract.py @@ -0,0 +1,405 @@ +"""Invented supplied-parent comparisons; no calibration or source issuer imitation.""" + +import json + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import frame_checkpoint +from microcosm.build.us_runtime import common_frame_export_contract as contract +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + +def _parent(): + person = pd.DataFrame( + { + "person_id": np.arange(1, 7, dtype=np.int64), + "person_household_id": np.array([10, 10, 20, 20, 30, 30], dtype=np.int64), + "person_tax_unit_id": np.array( + [100, 100, 200, 201, 300, 300], dtype=np.int64 + ), + "money": np.array([-0.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + "hours": pd.Series([10, pd.NA, 30, 40, pd.NA, 60], dtype="Int32"), + "reported": pd.Series( + [True, pd.NA, False, True, pd.NA, False], dtype="boolean" + ), + "source_person_id": np.arange(101, 107, dtype=np.int64), + } + ) + household = pd.DataFrame( + { + "household_id": np.array([10, 20, 30], dtype=np.int64), + "origin": pd.Series(["asec", "acs", "acs"], dtype="string"), + "assigned_block": pd.Series(["001", "002", "003"], dtype="string"), + "county": pd.Series(["A", "A", "B"], dtype="string"), + "clone": np.array([0, 1, 0], dtype=np.int8), + } + ) + tax = pd.DataFrame( + { + "tax_unit_id": np.array([100, 200, 201, 300], dtype=np.int64), + "deduction": np.array([1.0, 2.0, 3.0, 4.0]), + } + ) + return Frame( + {"person": person, "household": household, "tax_unit": tax}, + EntitySchema(group_entities=("household", "tax_unit")), + {"household": Weights(np.array([1.0, 2.0, 3.0]), WeightKind.IMPORTANCE)}, + pd.Series(["a", "a", "b", "b", "b", "b"], name="stratum"), + metadata={"construction": "one-common-parent"}, + ) + + +def _weighted(parent, person_mask, weights): + selected = parent.select(np.asarray(person_mask, dtype=np.bool_)) + return Frame( + {e: selected.table(e).copy(deep=True) for e in selected.entities}, + selected.schema, + { + "household": Weights( + np.asarray(weights, dtype=np.float64), WeightKind.CALIBRATED + ) + }, + selected.strata.copy(), + metadata=selected.metadata, + ) + + +def _arguments(parent): + return { + "parent_reference": "supplied-parent-example-not-an-issued-graph-key", + "ordered_household_ids": parent.table("household") + .household_id.to_numpy() + .copy(), + "calibrated_weights": np.array([0.0, 9.0, 10.0]), + "calibration_specification": b'{"targets":"invented","scope":"national"}', + } + + +@pytest.mark.parametrize("view", ["full", "pruned", "local"]) +def test_one_parent_full_pruned_and_local_preserve_all_inputs(view): + parent = _parent() + args = _arguments(parent) + if view == "full": + candidate = _weighted(parent, [True] * 6, [0, 9, 10]) + args["prune_zero_weight"] = False + elif view == "pruned": + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + else: + candidate = _weighted(parent, [False, False, False, False, True, True], [10]) + args["scope_household_ids"] = np.array([30], dtype=np.int64) + binding = contract.verify_retained_frame_export(parent, candidate, **args) + report = json.loads(binding) + assert report["actual_graph_calibration_ancestry_verified"] is False + assert report["release_eligible"] is False + assert report["retained_entity_ids"]["person"]["rows"] == candidate.n("person") + if view != "full": + # A complete survey origin may legitimately have zero calibrated weight. + assert candidate.table("household").origin.tolist() == ["acs"] * candidate.n( + "household" + ) + assert ( + contract.verify_retained_frame_export( + parent, candidate, expected_binding=binding, **args + ) + == binding + ) + + +def test_actual_checkpoint_readback_compares_retained_values_and_masks(tmp_path): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + before = contract.verify_retained_frame_export(parent, candidate, **args) + path = tmp_path / "invented-export.h5" + frame_checkpoint.write_frame_checkpoint( + path, candidate, metadata={"comparison": before.decode()} + ) + loaded = frame_checkpoint.load_frame_checkpoint( + path, frame_metadata=parent.metadata + ) + assert loaded.metadata["comparison"] == before.decode() + assert ( + contract.verify_retained_frame_export( + parent, + loaded.frame, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + == before + ) + loaded.frame.person.loc[loaded.frame.person.index[0], "money"] += 1 + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.money" + ): + contract.verify_retained_frame_export( + parent, + loaded.frame, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + + +@pytest.mark.parametrize( + "change", + [ + "value", + "mask", + "membership", + "id", + "geography", + "lineage", + "weight", + "strata", + "column", + ], +) +def test_retained_input_changes_refuse(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + if change == "value": + candidate.person.loc[2, "money"] += 1 + elif change == "mask": + candidate.person.loc[4, "hours"] = 0 + elif change == "membership": + candidate.person.loc[2, "person_household_id"] = 30 + elif change == "id": + candidate.person.loc[2, "person_id"] = 99 + elif change == "geography": + candidate.table("household").loc[0, "assigned_block"] = "004" + elif change == "lineage": + candidate.person.loc[2, "source_person_id"] = 999 + elif change == "weight": + candidate = _weighted(parent, [False, False, True, True, True, True], [8, 10]) + elif change == "strata": + candidate.strata.iloc[0] = "changed" + else: + candidate.person.drop(columns="money", inplace=True) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_|COLUMN_ROSTER" + ): + contract.verify_retained_frame_export(parent, candidate, **_arguments(parent)) + + +def test_partial_household_selection_refuses_even_with_valid_frame_links(): + parent = _parent() + candidate = _weighted(parent, [False, False, True, False, True, True], [9, 10]) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_ENTITY_IDS:person" + ): + contract.verify_retained_frame_export(parent, candidate, **_arguments(parent)) + + +@pytest.mark.parametrize( + "change", + ["order", "dtype", "nan", "negative", "shape", "scope_order", "scope_unknown"], +) +def test_weight_and_scope_axes_are_explicit(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + if change == "order": + args["ordered_household_ids"] = args["ordered_household_ids"][::-1] + elif change == "dtype": + args["ordered_household_ids"] = args["ordered_household_ids"].astype(float) + elif change == "nan": + args["calibrated_weights"][0] = np.nan + elif change == "negative": + args["calibrated_weights"][0] = -1 + elif change == "shape": + args["calibrated_weights"] = args["calibrated_weights"][:2] + elif change == "scope_order": + args["scope_household_ids"] = np.array([30, 20], dtype=np.int64) + else: + args["scope_household_ids"] = np.array([99], dtype=np.int64) + with pytest.raises(contract.RetainedFrameExportError): + contract.verify_retained_frame_export(parent, candidate, **args) + + +@pytest.mark.parametrize( + "change", ["specification", "parent_reference", "excluded_weight", "scope", "prune"] +) +def test_readback_binding_covers_even_inputs_outside_retained_rows(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + args["scope_household_ids"] = np.array([20, 30], dtype=np.int64) + before = contract.verify_retained_frame_export(parent, candidate, **args) + if change == "specification": + args["calibration_specification"] = b'{"targets":"different"}' + elif change == "parent_reference": + args["parent_reference"] = "another-unverified-reference" + elif change == "excluded_weight": + args["calibrated_weights"][0] = 100 + elif change == "scope": + args["scope_household_ids"] = None + else: + args["prune_zero_weight"] = False + with pytest.raises(contract.RetainedFrameExportError, match="EXPORT_BINDING"): + contract.verify_retained_frame_export( + parent, candidate, expected_binding=before, **args + ) + + +@pytest.mark.parametrize("scope", ["empty", "all_zero"]) +def test_empty_analysis_is_an_explicit_unsupported_frame_contract(scope): + parent = _parent() + candidate = _weighted(parent, [True] * 6, [0, 9, 10]) + args = _arguments(parent) + if scope == "empty": + args["scope_household_ids"] = np.array([], dtype=np.int64) + else: + args["calibrated_weights"] = np.zeros(3, dtype=np.float64) + with pytest.raises(contract.RetainedFrameExportError, match="EMPTY_EXPORT_SUPPORT"): + contract.verify_retained_frame_export(parent, candidate, **args) + + +def test_uint64_stable_ids_above_int64_are_not_coerced(): + ids = np.array([2**63 + 1, 2**63 + 7], dtype=np.uint64) + parent = Frame( + { + "person": pd.DataFrame( + { + "person_id": np.array([1, 2], dtype=np.int32), + "person_household_id": ids, + } + ), + "household": pd.DataFrame({"household_id": ids}), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.ones(2), WeightKind.IMPORTANCE)}, + ) + candidate = _weighted(parent, [True, True], [2, 3]) + args = { + "parent_reference": "large-ids", + "ordered_household_ids": ids, + "calibrated_weights": np.array([2.0, 3.0]), + "calibration_specification": b"{}", + } + report = json.loads( + contract.verify_retained_frame_export(parent, candidate, **args) + ) + assert report["complete_ordered_household_ids"]["dtype"] == ids.dtype.str + with pytest.raises(contract.RetainedFrameExportError, match="HOUSEHOLD_ID_VECTOR"): + contract.verify_retained_frame_export( + parent, candidate, **{**args, "ordered_household_ids": ids.astype(float)} + ) + + +def test_readback_normalization_is_closed_and_never_changes_knownness(): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + before = contract.verify_retained_frame_export(parent, candidate, **args) + values = candidate.person["hours"].array + assert values._mask[2] + values._data[2] ^= np.int32(1) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.hours" + ): + contract.verify_retained_frame_export( + parent, candidate, expected_binding=before, **args + ) + assert ( + contract.verify_retained_frame_export( + parent, + candidate, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + == before + ) + with pytest.raises(contract.RetainedFrameExportError, match="COMPARISON_MODE"): + contract.verify_retained_frame_export( + parent, candidate, comparison="approximate", **args + ) + values._mask[2] = False + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.hours" + ): + contract.verify_retained_frame_export( + parent, candidate, comparison="frame-checkpoint-readback", **args + ) + + +@pytest.mark.parametrize("prune_zero_weight", [True, False]) +@pytest.mark.parametrize("scope", ["all", "zero_weight_household"]) +def test_zero_weight_export_support_refuses(prune_zero_weight, scope): + parent = _parent() + # Keep a valid candidate so the export contract itself checks the supplied + # weight/scope inputs before comparing retained rows or weight storage. + candidate = _weighted(parent, [True] * 6, [1, 1, 1]) + args = _arguments(parent) + args["calibrated_weights"] = np.zeros(3, dtype=np.float64) + if scope == "zero_weight_household": + args["calibrated_weights"][1:] = [9, 10] + args["scope_household_ids"] = args["ordered_household_ids"][:1].copy() + with pytest.raises(contract.RetainedFrameExportError, match="EMPTY_EXPORT_SUPPORT"): + contract.verify_retained_frame_export( + parent, candidate, prune_zero_weight=prune_zero_weight, **args + ) + + +@pytest.mark.parametrize("prune_zero_weight", [True, False]) +def test_positive_calibrated_scope_can_have_zero_parent_weight(prune_zero_weight): + baseline = _parent() + parent = Frame( + {e: baseline.table(e).copy(deep=True) for e in baseline.entities}, + baseline.schema, + { + "household": Weights(np.array([0.0, 2.0, 3.0]), WeightKind.IMPORTANCE), + "tax_unit": Weights(np.array([7.0, 8.0, 9.0, 10.0]), WeightKind.DESIGN), + }, + baseline.strata.copy(), + metadata=baseline.metadata, + ) + before_tables = {e: parent.table(e).copy(deep=True) for e in parent.entities} + before_strata = parent.strata.copy(deep=True) + before_metadata, before_mass_log = parent.metadata, parent.mass_log + before_weights = { + e: (parent.weights_for(e).values.copy(), parent.weights_for(e).kind) + for e in parent.weighted_entities + } + retained = baseline.select(np.array([True, True, False, False, False, False])) + + def candidate(tax_weight): + return Frame( + {e: retained.table(e).copy(deep=True) for e in retained.entities}, + retained.schema, + { + "household": Weights(np.array([4.0]), WeightKind.CALIBRATED), + "tax_unit": Weights(np.array([tax_weight]), WeightKind.DESIGN), + }, + retained.strata.copy(), + metadata=retained.metadata, + ) + + args = _arguments(parent) + args["calibrated_weights"] = np.array([4.0, 0.0, 0.0]) + args["scope_household_ids"] = np.array([10], dtype=np.int64) + report = json.loads( + contract.verify_retained_frame_export( + parent, candidate(7.0), prune_zero_weight=prune_zero_weight, **args + ) + ) + assert report["actual_graph_calibration_ancestry_verified"] is False + assert report["release_eligible"] is False + assert report["complete_ordered_household_ids"]["rows"] == 3 + assert report["retained_entity_ids"]["household"]["rows"] == 1 + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_WEIGHTS:tax_unit" + ): + contract.verify_retained_frame_export( + parent, candidate(8.0), prune_zero_weight=prune_zero_weight, **args + ) + for entity, table in before_tables.items(): + pd.testing.assert_frame_equal(parent.table(entity), table, check_exact=True) + pd.testing.assert_series_equal(parent.strata, before_strata, check_exact=True) + assert parent.metadata == before_metadata and parent.mass_log == before_mass_log + for entity, (values, kind) in before_weights.items(): + assert parent.weights_for(entity).kind is kind + assert parent.weights_for(entity).values.tobytes() == values.tobytes() diff --git a/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py b/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py new file mode 100644 index 000000000..00d1c2269 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py @@ -0,0 +1,310 @@ +"""Actual retained ACS source owners over privately pinned invented bytes.""" + +import copy +import io + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_acs_income_anchor_source as owner + + +@pytest.mark.parametrize( + "field,token,age,status,known", + [ + ("INTP", "-10000", "30", "observed", True), + ("INTP", "-4", "30", "observed", True), + ("INTP", "0", "30", "observed", True), + ("RETP", "0", "30", "observed", True), + ("RETP", "999999", "30", "observed", True), + ("INTP", "-3", "30", "outside_published_domain", False), + ("RETP", "-4", "30", "outside_published_domain", False), + ("INTP", "1000000", "30", "outside_published_domain", False), + ("INTP", "", "14", "outside_universe_blank", False), + ("INTP", "", "15", "missing_source_amount", False), + ("RETP", "4", "14", "outside_universe_observation", False), + ("INTP", "4.0", "30", "malformed_source_amount", False), + ("INTP", " 4", "30", "malformed_source_amount", False), + ("INTP", "NA", "30", "malformed_source_amount", False), + ], +) +def test_published_anchor_domain_is_distinct_from_storage_and_universe( + field, token, age, status, known +): + value = owner.parse_anchor( + token, age=age, adjustment="1000133", allocation="1", field=field + ) + assert value["status"] == status and value["known"] is known + assert value["allocation_status"] == "allocated" + if known: + expected = np.float64(int(token)) * (np.float64(1000133) / 1_000_000.0) + assert np.float64(value["amount"]).view("uint64") == expected.view("uint64") + else: + assert value["amount"] is None + + +@pytest.mark.parametrize( + "flag,status", + [ + ("0", "not_allocated"), + ("1", "allocated"), + ("", "allocation_missing"), + ("x", "allocation_unrecognized"), + ], +) +def test_allocation_does_not_decide_amount_knownness(flag, status): + value = owner.parse_anchor( + "4", age="15", adjustment="1000133", allocation=flag, field="RETP" + ) + assert value["known"] and value["allocation_status"] == status + + +@pytest.mark.parametrize("adjustment", ["", "0", "-1", "1.1", "inf"]) +def test_unknown_or_invalid_adjustment_never_supplies_an_amount(adjustment): + value = owner.parse_anchor( + "4", age="15", adjustment=adjustment, allocation="0", field="INTP" + ) + assert not value["known"] and value["amount"] is None + assert value["status"] == "invalid_adjustment" + + +def test_column_numeric_identity_keeps_scalar_mapper_bits_and_unknowns(): + tokens = ["-800", "0", "bad", "", "4.0", "-0.0", "NA", "999999", "1000133"] + expected = np.array( + [ + pd.to_numeric(pd.Series([v], dtype=object), errors="coerce").iloc[0] + for v in tokens + ], + dtype=np.float64, + ) + actual = owner._native_numbers(tokens) + assert np.array_equal(np.isnan(actual), np.isnan(expected)) + assert np.array_equal( + actual[~np.isnan(actual)].view("uint64"), + expected[~np.isnan(expected)].view("uint64"), + ) + + +def _literal_rows(): + return [ + { + "SERIALNO": "2024HU0000001", + "SPORDER": "1", + "INTP": "-800", + "RETP": "444", + "ADJINC": "1000133", + "AGEP": "30", + "FINTP": "0", + "FRETP": "1", + }, + { + "SERIALNO": "2024HU0000001", + "SPORDER": "2", + "INTP": "0", + "RETP": "", + "ADJINC": "1000133", + "AGEP": "15", + "FINTP": "1", + "FRETP": "", + }, + { + "SERIALNO": "2024HU0000002", + "SPORDER": "1", + "INTP": "bad", + "RETP": "4", + "ADJINC": "1000133", + "AGEP": "80", + "FINTP": "", + "FRETP": "x", + }, + *( + { + "SERIALNO": serial, + "SPORDER": "1", + "INTP": "0", + "RETP": "0", + "ADJINC": "1000000", + "AGEP": age, + "FINTP": "0", + "FRETP": "0", + } + for serial, age in (("2024GQ0000001", "40"), ("2024GQ0000002", "50")) + ), + ] + + +def _scan(rows, *, wanted=None): + import csv + + text = io.StringIO(newline="") + writer = csv.DictWriter(text, fieldnames=owner.COLUMNS) + writer.writeheader() + writer.writerows(rows) + selected = {} + wanted = ( + wanted + if wanted is not None + else {(r["SERIALNO"], int(r["SPORDER"])) for r in rows} + ) + count = owner._scan( + io.BytesIO(text.getvalue().encode()), wanted, selected, maximum=100 + ) + return count, selected + + +def test_literal_scan_is_keyed_and_keeps_every_original_token(): + count, values = _scan(_literal_rows()[::-1]) + assert count == 5 + assert values["2024HU0000001", 2]["RETP"] == "" + assert values["2024HU0000002", 1]["INTP"] == "bad" + + +@pytest.mark.parametrize("defect", ["duplicate", "foreign_bad_key", "short_row"]) +def test_literal_reader_refuses_duplicate_selected_and_invalid_tail(defect): + rows = _literal_rows() + if defect == "duplicate": + rows.append(rows[0]) + elif defect == "foreign_bad_key": + rows.append({**rows[0], "SERIALNO": "bad"}) + else: + stream = io.BytesIO((",".join(owner.COLUMNS) + "\n1,2\n").encode()) + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + owner._scan(stream, set(), {}, maximum=100) + return + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + _scan(rows) + + +def _arguments(tmp_path, monkeypatch): + import test_us_survey_population_preparation as fixture + + original = fixture._person + lookup = {(r["SERIALNO"], int(r["SPORDER"])): r for r in _literal_rows()} + + def person(*args, **kwargs): + row = original(*args, **kwargs) + key = (row["SERIALNO"], int(row["SPORDER"])) + row.update( + { + k: v + for k, v in lookup.get(key, {}).items() + if k not in ("SERIALNO", "SPORDER") + } + ) + row.setdefault("FINTP", "0") + row.setdefault("FRETP", "0") + return row + + monkeypatch.setattr(fixture, "_person", person) + return fixture.fixture(tmp_path, monkeypatch, zero=False) + + +def test_actual_qualifier_binds_originals_and_adjusted_bits(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original = prepared.checked_view().frame + before = owner.preparation._frame_identity(original) + qualified = owner.qualify_current_acs_income_anchors(prepared) + assert qualified.evidence["source_admission_issued"] is False + assert qualified.evidence["decomposition_performed"] is False + assert len(qualified.anchors) == 5 + first = qualified.anchors.set_index(["SERIALNO", "SPORDER"]) + assert first.loc[("2024HU0000001", "1"), "property_income_amount"] < 0 + assert first.loc[("2024HU0000001", "2"), "property_income_amount"] == 0 + assert ( + first.loc[("2024HU0000001", "2"), "retirement_income_status"] + == "missing_source_amount" + ) + assert ( + first.loc[("2024HU0000002", "1"), "property_income_status"] + == "malformed_source_amount" + ) + assert pd.isna(first.loc[("2024HU0000002", "1"), "property_income_amount"]) + assert owner.preparation._frame_identity(original) == before + fresh = owner.qualify_current_acs_income_anchors(prepared) + assert owner.income_anchor_seal(fresh) == owner.income_anchor_seal(qualified) + with pytest.raises(ValueError): + owner.qualify_current_acs_income_anchors(copy.copy(prepared)) + + +@pytest.mark.parametrize( + "defect", ["row_order", "person_id", "raw_amount", "adjusted_bit"] +) +def test_exact_retained_comparison_refuses_source_or_amount_mutation( + tmp_path, monkeypatch, defect +): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + entry = prepared._checked() + document = owner.json.loads(entry[1]) + origins = owner._origins(entry[2], document) + _, selected = _scan(_literal_rows()) + raw = owner._raw_table(origins, selected) + native = owner.preparation._copy_source(entry[2].source_frames[0]) + if defect == "row_order": + origins = origins.iloc[::-1].copy() + elif defect == "person_id": + native.person.loc[native.person.index[0], "person_id"] += 100 + elif defect == "raw_amount": + native.person.loc[native.person.index[0], "RETP"] = 888 + else: + column = "acs_interest_dividend_rental_income" + native.person.loc[native.person.index[0], column] = np.nextafter( + native.person[column].iloc[0], np.inf + ) + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + owner._compare_retained(raw, origins, native) + + +def test_final_source_io_mutation_cannot_return_qualified_value(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original = prepared.checked_view().frame + capture = owner._capture_person + mutated = [] + + def mutate_after_io(*args): + value = capture(*args) + original.person.loc[original.person.index[0], "age"] += 1 + mutated.append(True) + return value + + monkeypatch.setattr(owner, "_capture_person", mutate_after_io) + with pytest.raises(ValueError): + owner.qualify_current_acs_income_anchors(prepared) + assert mutated == [True] + + +def test_final_source_fence_seals_exact_returned_amount_bits(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original_seal = owner.income_anchor_seal + changed = [] + + def change_after_seal(qualified): + seal = original_seal(qualified) + if not changed: + changed.append(True) + table = qualified.anchors + index = table.index[ + table.SERIALNO.eq("2024HU0000001") & table.SPORDER.eq("1") + ][0] + table.loc[index, "property_income_amount"] = np.nextafter( + table.loc[index, "property_income_amount"], np.inf + ) + # Portable JSON's precision is not an exact physical mutation seal. + assert ( + qualified.projection + == table.reset_index().to_json(orient="table", index=False).encode() + ) + return seal + + monkeypatch.setattr(owner, "income_anchor_seal", change_after_seal) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_acs_income_anchors(prepared) + assert changed == [True] diff --git a/packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py b/packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py new file mode 100644 index 000000000..f250f65c2 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py @@ -0,0 +1,44 @@ +"""Published allocation meanings over source-pure invented projections.""" + +import copy + +import pytest +from test_us_current_asec_income_routing import _pure_rows, _set_amount + +from microcosm.build.us_runtime import current_asec_income_routing_source as owner + + +@pytest.mark.parametrize( + "earn_code,farm_code,farm_status,expected", + [ + (0, 4, "in_printed_range", "allocation_code_meaning_unpublished"), + (4, 4, "in_printed_range", "publisher_allocated"), + (4, 0, "in_printed_range", "publisher_allocated"), + (4, None, "missing", "publisher_allocated"), + (0, 0, "in_printed_range", "published_flags_all_zero_with_unflagged_fields"), + (0, None, "missing", "allocation_flag_not_populated"), + (0, None, "outside_printed_range", "unresolved_allocation_provenance"), + ], +) +def test_farm_allocation_requires_a_published_code_meaning( + earn_code, farm_code, farm_status, expected +): + raw, ages = _pure_rows(1, [40.0], ERN_YN=["1"], FRMOTR=["1"], FRSE_YN=["1"]) + _set_amount(raw, "FRSE_VAL", [100.0]) + raw["allocations"]["I_ERNYN"] = ([earn_code], ["in_printed_range"]) + raw["allocations"]["I_FRMYN"] = ([farm_code], [farm_status]) + before = copy.deepcopy(raw["allocations"]) + output = owner.project_income_routing(raw, ages) + assert output.farm_allocation_origin.iloc[0] == expected + assert output.farm_known_amount.iloc[0] == 100.0 + assert raw["allocations"] == before + assert "meaning of its codes" in owner.AMBIGUOUS_FLAG_COVERAGE["I_FRMYN"] + + +@pytest.mark.parametrize( + "name,code", [("I_RNTVAL", 4), ("I_DSTSC", 9), ("I_DSTYNCOMP", 11)] +) +def test_documented_nonzero_flags_still_establish_allocation(name, code): + assert code in owner.ALLOCATION_ENTRIES[name][5] + flags = {name: ([code], ["in_printed_range"])} + assert owner._allocation_origin(flags, unflagged=False)[0] == "publisher_allocated" diff --git a/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py b/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py new file mode 100644 index 000000000..d3c820736 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py @@ -0,0 +1,339 @@ +"""Child-support source semantics and invented actual-owner boundaries.""" + +import copy +import hashlib +import shutil + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_child_support_source as owner + + +def row(**changes): + return { + **{name: "0" for name in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "CSP_VAL": "1200", + "CSP_YN": "1", + "CHSP_VAL": "2000", + "CHSP_YN": "1", + "CHELSEW_YN": "1", + **changes, + } + + +def project(**changes): + return owner.project_child_support_literals(pd.DataFrame([row(**changes)])).iloc[0] + + +@pytest.mark.parametrize( + "receipt,amount,status,known", + [ + ("1", "1200", "known_receipt", True), + ("1", "0", "ambiguous_recipient_zero", False), + ("2", "0", "known_nonreceipt", True), + ("2", "1200", "contradictory_no_nonzero", False), + ("0", "0", "niu", False), + ("", "1200", "missing_receipt_literal", False), + ], +) +def test_received_support_uses_receipt_but_does_not_complete_recipient_zeros( + receipt, amount, status, known +): + value = project(CSP_YN=receipt, CSP_VAL=amount) + assert value.CSP_VAL_reporting_status == status + assert bool(value.CSP_VAL_amount_known) is known + + +@pytest.mark.parametrize("obligation", ["0", "1", "2", ""]) +def test_paid_zero_is_niu_under_every_obligation_answer(obligation): + value = project(CHSP_VAL="0", CHSP_YN=obligation) + assert value.CHSP_VAL_published_amount == 0 + assert value.CHSP_VAL_reporting_status == "declared_niu_amount" + assert not value.CHSP_VAL_amount_known and pd.isna(value.CHSP_VAL_amount) + assert not value.voluntary_payment_absence_known + assert not value.obligation_implies_payment + + +@pytest.mark.parametrize( + "elsewhere,obligation,status", + [ + ("1", "1", "observed_positive_payment"), + ("1", "2", "payment_outside_published_obligation_universe"), + ("1", "0", "payment_outside_published_obligation_universe"), + ("2", "1", "unresolved_child_elsewhere_route"), + ("0", "1", "unresolved_child_elsewhere_route"), + ("", "1", "unresolved_child_elsewhere_literal"), + ("1", "", "missing_obligation_literal"), + ("1", "x", "invalid_obligation_literal"), + ], +) +def test_positive_paid_cells_remain_visible_when_routing_is_unresolved( + elsewhere, obligation, status +): + value = project(CHELSEW_YN=elsewhere, CHSP_YN=obligation) + assert value.CHSP_VAL_reporting_status == status + assert value.CHSP_VAL_published_amount == 2000 + assert bool(value.CHSP_VAL_amount_known) is (status == "observed_positive_payment") + + +@pytest.mark.parametrize("field", ["CSP_VAL", "CHSP_VAL"]) +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing"), + ("-1", "malformed"), + ("1.5", "malformed"), + ("NA", "malformed"), + ], +) +def test_amount_missing_malformed_and_negative_literals_remain_distinct( + field, token, status +): + value = project(**{field: token}) + assert value[field + "_literal"] == token + assert value[field + "_literal_status"] == status + assert not value[field + "_amount_known"] + + +def test_under15_observations_do_not_become_analytical_zeros(): + value = project( + A_AGE="14", CSP_VAL="0", CSP_YN="0", CHSP_VAL="0", CHSP_YN="0", CHELSEW_YN="0" + ) + for name in owner.AMOUNT_FIELDS: + assert value[name + "_reporting_status"] == "outside_reporting_universe" + assert not value[name + "_amount_known"] + + +def test_allocated_values_remain_known_and_missing_flags_remain_unknown(): + value = project(I_CHSPVAL="4", I_CSPVAL="", TCHSP_VAL="1") + assert value.CHSP_VAL_amount == 2000 and value.CHSP_VAL_amount_known + assert value.paid_allocation_origin == "publisher_allocated" + assert value.received_allocation_origin == "allocation_flag_not_populated" + assert value.I_CSPVAL_literal_status == "missing" + assert value.TCHSP_VAL_code == 1 + + +def test_pinned_domains_and_universe_discrepancy_are_preserved(): + entries = owner.amount_entries() + assert entries["CHSP_VAL"]["domain"]["zero_semantics"] == "niu" + assert entries["CHSP_VAL"]["domain"]["valid_dollar_range_inclusive"]["minimum"] == 1 + assert owner.RESPONSE_ENTRIES["CHSP_YN"][4] == "CHELSEW_YN" + assert owner.ALLOCATION_ENTRIES["I_CHSPYN"][4] == "CHELSEW_YN = 1" + assert "bare" in owner.OBLIGATION_UNIVERSE_NOTE + + +def test_nullable_seal_covers_values_masks_and_hidden_backing(): + raw = pd.DataFrame([row(CHSP_VAL="0")]) + values = owner.CurrentAsecChildSupportValues( + owner.project_child_support_literals(raw), raw, {} + ) + original = owner.child_support_values_seal(values) + assert owner.child_support_values_seal(copy.deepcopy(values)) == original + for change in ("bit", "mask", "hidden"): + other = copy.deepcopy(values) + array = ( + other.person.CSP_VAL_amount.array + if change == "bit" + else other.person.CHSP_VAL_amount.array + ) + if change == "bit": + array._data[0] = np.nextafter(array._data[0], np.inf) + elif change == "mask": + array._mask[0] = False + else: + array._data[0] = 17.0 + assert owner.child_support_values_seal(other) != original + + +def child_support_arguments(tmp_path, monkeypatch): + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + literals = { + 105: row(I_CHSPVAL="4"), + 106: row( + A_AGE="14", + CSP_VAL="0", + CSP_YN="0", + CHSP_VAL="0", + CHSP_YN="0", + CHELSEW_YN="0", + ), + 107: row(A_AGE="70", CSP_VAL="0", CSP_YN="2", CHSP_VAL="0", CHSP_YN="2"), + 108: row(CSP_VAL="0", CHSP_VAL="0"), + } + changes = {} + for name in (*owner.AMOUNT_FIELDS, "A_AGE"): + values = people[name].to_numpy(copy=True) + for person_id, literal in literals.items(): + values[ids == person_id] = int(literal[name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in (*owner.AMOUNT_FIELDS, "A_AGE"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + for name in ( + *owner.RESPONSE_ENTRIES, + *owner.ALLOCATION_ENTRIES, + *owner.TOPCODE_ENTRIES, + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row())[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "child-support-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(tmp_path, monkeypatch): + return owner.routing.source.prepare_authenticated_survey_population( + **child_support_arguments(tmp_path, monkeypatch) + ) + + +def test_actual_owner_keeps_obligation_distinct_and_refuses_copied_owner( + tmp_path, monkeypatch +): + parent = prepared(tmp_path, monkeypatch) + result = owner.qualify_current_asec_child_support(parent) + values = result.person.set_index("native_person_id") + assert values.loc[105, "CHSP_VAL_amount"] == 2000 + assert values.loc[105, "CSP_VAL_amount"] == 1200 + assert values.loc[107, "CSP_VAL_amount"] == 0 + assert not values.loc[107, "CHSP_VAL_amount_known"] + assert not values.loc[108, "CHSP_VAL_amount_known"] + assert not values.loc[108, "CSP_VAL_amount_known"] + assert not result.evidence["paid_niu_completed_with_zero"] + assert owner.child_support_values_seal( + owner.qualify_current_asec_child_support(parent) + ) == owner.child_support_values_seal(result) + with pytest.raises(ValueError): + owner.qualify_current_asec_child_support(copy.copy(parent)) + + # The actual qualifier checks the retained MoneyDomain before source capture. + check, checked = owner._domain_agreement, [] + + def rejected_domain(ready): + check(ready) + checked.append(True) + raise ValueError("INVENTED_DOMAIN_REFUSAL") + + def forbidden_capture(*args): + pytest.fail("source capture preceded live domain agreement") + + with monkeypatch.context() as patch: + patch.setattr(owner, "_domain_agreement", rejected_domain) + patch.setattr(owner, "_capture_member", forbidden_capture) + with pytest.raises(ValueError, match="INVENTED_DOMAIN_REFUSAL"): + owner.qualify_current_asec_child_support(parent) + assert checked == [True] + + +@pytest.mark.parametrize("field", ["PERIDNUM", "A_AGE", "CSP_VAL", "CHSP_VAL"]) +def test_actual_owner_requires_original_source_identity_and_amounts( + tmp_path, monkeypatch, field +): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if field == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(field)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_child_support(parent) + + +def test_actual_owner_requalifies_after_capture_io(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + calls = [] + + def changed(*args): + raw = capture(*args) + table = parent._checked()[2].frame.person + table.iloc[0, table.columns.get_loc("age")] += 1 + calls.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_child_support(parent) + assert calls + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "literal", "evidence"] +) +def test_actual_owner_final_seal_covers_complete_returned_values( + tmp_path, monkeypatch, mutation +): + parent = prepared(tmp_path, monkeypatch) + seal, calls = owner.child_support_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + if mutation == "amount_bit": + array = value.person.CSP_VAL_amount.array + i = np.flatnonzero(~array._mask)[0] + array._data[i] = np.nextafter(array._data[i], np.inf) + elif mutation == "masked_backing": + array = value.person.CHSP_VAL_amount.array + array._data[np.flatnonzero(array._mask)[0]] = 17.0 + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("CHSP_YN") + ] = "2" + else: + value.evidence["paid_niu_completed_with_zero"] = True + return before + + monkeypatch.setattr(owner, "child_support_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_child_support(parent) diff --git a/packages/microcosm-build/tests/test_us_current_asec_demographics.py b/packages/microcosm-build/tests/test_us_current_asec_demographics.py new file mode 100644 index 000000000..c61c62b15 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_demographics.py @@ -0,0 +1,191 @@ +"""Actual source readers over constructed, privately pinned invented source bytes.""" + +import csv +import hashlib +import json +import shutil +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_survey_population_preparation import fixture + +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import current_asec_demographics as owner +from microcosm.build.us_runtime import native_household_origin as origin + + +def _csv(path, headers, rows): + with path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(headers) + writer.writerows(rows) + data = path.read_bytes() + return SimpleNamespace( + size_bytes=len(data), + rows=len(rows), + member_sha256=hashlib.sha256(data).hexdigest(), + ) + + +@pytest.mark.parametrize( + "literal,code,status", + ( + ("06", 6, "in_printed_range"), + ("36", 36, "in_printed_range"), + ("", None, "missing"), + ("NA", None, "malformed"), + (" 6", None, "malformed"), + ("6.0", None, "malformed"), + ("57", None, "outside_printed_range"), + ("00", None, "outside_printed_range"), + ), +) +def test_household_state_reader_preserves_literal_and_knownness( + tmp_path, literal, code, status +): + path = tmp_path / "invented.csv" + pin = _csv(path, ("H_SEQ", "GESTFIPS", "unread"), [("00007", literal, "anything")]) + value = owner._read_state_capture(path, pin)[7] + assert value == { + "H_SEQ": "00007", + "GESTFIPS": literal, + "state_code": code, + "status": status, + "member_row_1based": 1, + } + + +@pytest.mark.parametrize("defect", ("duplicate", "digest", "size", "rows", "header")) +def test_state_capture_structure_and_exact_bytes_refuse(tmp_path, defect): + path = tmp_path / "invented.csv" + headers, rows = ("H_SEQ", "GESTFIPS"), [("00007", "06")] + if defect == "duplicate": + rows.append(("7", "36")) + if defect == "header": + headers = ("H_SEQ", "other") + pin = _csv(path, headers, rows) + if defect == "digest": + pin.member_sha256 = "f" * 64 + elif defect == "size": + pin.size_bytes += 1 + elif defect == "rows": + pin.rows += 1 + with pytest.raises(ValueError, match="CURRENT_ASEC_DEMOGRAPHICS_"): + owner._read_state_capture(path, pin) + + +def _demographic_arguments(tmp_path, monkeypatch, *, unknown=False, zero=True): + """Extend invented source members before actual preparation/issuance. + + Rebuild the real person-income attachment after member bytes change. Only + fixture registry pins are changed; no source issuer or ready() is replaced. + """ + arguments = fixture(tmp_path, monkeypatch, zero=zero) + source = arguments["source_dir"] / "asec" + members, pins = {}, [] + for year, member, archive, *_ in coverage._MEMBER_PINS: + path = source / f"pppub{year - 1999}.csv" + table = pd.read_csv(path, dtype=str, keep_default_na=False) + lines = table.A_LINENO.map(int) + table["A_SEX"] = lines.map({1: "1", 2: "2"}) + table["AXSEX"] = lines.map({1: "0", 2: "4"}) + table["A_EXPRRP"] = lines.map({1: "1", 2: "5"}) + table["P_SEQ"] = table.A_LINENO + if unknown and year == 2024: + table.loc[table.index[0], "AXSEX"] = "1" + assert table.notna().all().all() + # Reverse the actual source order, so receiving-row alignment cannot pass. + table.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(table), + len(data), + ) + ) + members[year] = path + for module in (coverage, restoration, owner.demographic): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "demographic-restored-money" + restoration.restore_asec_person_income_source( + source / "parent.h5", + source / "household-attachment.h5", + member_paths=members, + output_dir=output, + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, source / "person-income-attachment.h5" + ) + path = source / "hhpub25.csv" + table = pd.read_csv(path, dtype=str, keep_default_na=False) + table["GESTFIPS"] = table.H_SEQ.map({"00007": "06", "00008": "36"}) + if unknown: + table.loc[table.H_SEQ.eq("00008"), "GESTFIPS"] = "" + assert table.notna().all().all() + table.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + monkeypatch.setattr( + origin, + "_ASEC_MEMBER_PINS", + tuple( + replace( + pin, + member_sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + for pin in origin._ASEC_MEMBER_PINS + ), + ) + return arguments + + +@pytest.mark.parametrize("unknown", (False, True)) +def test_actual_current_native_parent_binds_demographics_by_origin( + tmp_path, monkeypatch, unknown +): + arguments = _demographic_arguments(tmp_path, monkeypatch, unknown=unknown) + prepared = owner.preparation_owner.prepare_authenticated_survey_population( + **arguments + ) + original = prepared.checked_view().frame + retained = {e: original.table(e).copy(deep=True) for e in original.entities} + result = owner.qualify_current_asec_demographics(prepared) + receipt = json.loads(result.receipt) + assert receipt["sex_observation_year"] == receipt["state_observation_year"] == 2025 + assert receipt["income_year"] == 2024 + assert not receipt["state_is_income_year_residence_claim"] + assert not receipt["source_admission_issued"] and not receipt["release_eligible"] + h = result.household.set_index("H_SEQ_integer") + assert h.loc[7, "GESTFIPS"] == "06" and h.loc[7, "state_fips"] == 6 + if unknown: + assert h.loc[8, "GESTFIPS"] == "" and pd.isna(h.loc[8, "state_fips"]) + assert receipt["state_unknown_households"] == 1 + assert receipt["sex_unknown_persons"] == 1 + unbound = result.person.asec_AXSEX.eq(1) + assert result.person.loc[unbound, "is_female"].isna().all() + assert not result.person.loc[unbound, "sex_known"].any() + else: + assert h.loc[8, "state_fips"] == 36 + assert ( + receipt["state_unknown_households"] == receipt["sex_unknown_persons"] == 0 + ) + allocated = result.person.asec_AXSEX.eq(4) + assert allocated.any() and result.person.loc[allocated, "is_female"].all() + assert result.person.loc[allocated, "sex_origin"].eq("census_allocated").all() + for entity, expected in retained.items(): + pd.testing.assert_frame_equal( + expected, original.table(entity), check_exact=True + ) + assert owner.qualify_current_asec_demographics(prepared).receipt == result.receipt + # Value transport is deliberately not an issued authority. Mutating it must + # not affect a fresh live projection or its source-bound receipt. + result.household.loc[result.household.index[0], "state_fips"] = 99 + fresh = owner.qualify_current_asec_demographics(prepared) + assert not fresh.household.state_fips.eq(99).any() diff --git a/packages/microcosm-build/tests/test_us_current_asec_dividend_source.py b/packages/microcosm-build/tests/test_us_current_asec_dividend_source.py new file mode 100644 index 000000000..70483f62f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_dividend_source.py @@ -0,0 +1,497 @@ +"""Invented dividend literals, survivor routes and actual retained source owners.""" + +import copy +import hashlib +import json +import shutil +from fractions import Fraction +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_dividend_source as owner + + +def row(**changes): + return { + **{name: "0" for name in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "DIV_YN": "1", + "DIV_VAL": "140", + "SUR_YN": "2", + **changes, + } + + +def project(**changes): + return owner.project_dividend_literals(pd.DataFrame([row(**changes)])).iloc[0] + + +@pytest.mark.parametrize( + "changes,status,amount", + [ + ({}, "known_receipt", 140), + ({"DIV_VAL": "0", "DIV_YN": "2"}, "known_nonreceipt", 0), + ({"DIV_VAL": "0"}, "ambiguous_recipient_zero", None), + ({"DIV_VAL": "0", "DIV_YN": "0"}, "niu", None), + ({"DIV_YN": "0"}, "contradictory_niu_nonzero", None), + ({"DIV_YN": "2"}, "contradictory_no_nonzero", None), + ({"DIV_YN": ""}, "missing_receipt_literal", None), + ({"DIV_YN": "x"}, "unrecognized_receipt_literal", None), + ({"DIV_VAL": ""}, "missing_amount", None), + ({"DIV_VAL": "-1"}, "invalid_amount_literal", None), + ({"DIV_VAL": "1.0"}, "invalid_amount_literal", None), + ( + {"A_AGE": "14", "DIV_VAL": "0", "DIV_YN": "0"}, + "outside_reporting_universe", + None, + ), + ({"A_AGE": "14"}, "contradictory_outside_reporting_universe", None), + ], +) +def test_dividend_receipt_unknownness_never_infers_age_only_or_recipient_zeros( + changes, status, amount +): + value = project(**changes) + assert value.DIV_VAL_reporting_status == status + assert bool(value.DIV_VAL_amount_known) is (amount is not None) + assert ( + pd.isna(value.DIV_VAL_amount) + if amount is None + else value.DIV_VAL_amount == amount + ) + assert value.DIV_VAL_literal == changes.get("DIV_VAL", "140") + + +@pytest.mark.parametrize( + "token,header,reference,agreement", + [ + ("0", "in_printed_range", "in_printed_range", "header_reference_agree"), + ("1", "in_printed_range", "in_printed_range", "header_reference_agree"), + ("5", "outside_printed_range", "in_printed_range", "header_reference_conflict"), + ("9", "outside_printed_range", "in_printed_range", "header_reference_conflict"), + ("", "missing", "missing", "literal_unresolved"), + ("x", "malformed", "malformed", "literal_unresolved"), + ("01", "malformed", "malformed", "literal_unresolved"), + ], +) +def test_dividend_receipt_allocation_preserves_both_conflicting_codebook_axes( + token, header, reference, agreement +): + value = project(I_DIVYN=token) + assert value.I_DIVYN_literal == token + assert value.I_DIVYN_header_range_status == header + assert value.I_DIVYN_referenced_values_status == reference + assert value.I_DIVYN_codebook_status == agreement + if reference == "in_printed_range": + assert value.I_DIVYN_code == int(token) + assert value.I_DIVYN_literal_status == "well_formed" + assert value.DIV_VAL_amount == 140 and value.DIV_VAL_amount_known + + +@pytest.mark.parametrize( + "changes,status,clear", + [ + ({}, "known_nonreceipt", True), + ({"SUR_YN": "1", "SUR_SC1": "1"}, "known_other_survivor_sources", True), + ({"SUR_YN": "1", "SUR_SC2": "9"}, "known_other_survivor_sources", True), + ({"SUR_YN": "1", "SUR_SC2": "8"}, "possible_estate_or_trust_route", False), + ( + {"SUR_YN": "1", "SUR_SC1": "8", "SUR_SC2": ""}, + "possible_estate_or_trust_route", + False, + ), + ({"SUR_YN": "1", "SUR_SC1": "10"}, "source_type_unspecified", None), + ({"SUR_YN": "1"}, "unreported_source_slots", None), + ( + {"SUR_YN": "1", "SUR_SC1": "1", "SUR_SC2": ""}, + "unresolved_source_literal", + None, + ), + ({"SUR_YN": "1", "SUR_SC1": "11"}, "unresolved_source_literal", None), + ({"SUR_YN": "1", "SUR_SC1": "x"}, "unresolved_source_literal", None), + ({"SUR_YN": "", "SUR_SC1": "8"}, "missing_receipt_literal", None), + ({"SUR_YN": "x"}, "unrecognized_receipt_literal", None), + ({"SUR_SC1": "8"}, "unresolved_nonreceipt_source_route", None), + ({"SUR_SC1": ""}, "unresolved_nonreceipt_source_route", None), + ({"SUR_YN": "0"}, "niu", None), + ({"SUR_YN": "0", "SUR_SC1": "1"}, "unresolved_niu_route", None), + ({"A_AGE": "14", "SUR_YN": "0"}, "outside_reporting_universe", None), + ({"A_AGE": "14", "SUR_YN": ""}, "unresolved_outside_reporting_universe", None), + ( + {"A_AGE": "14", "SUR_YN": "0", "SUR_SC1": ""}, + "unresolved_outside_reporting_universe", + None, + ), + ], +) +def test_survivor_clearance_requires_resolved_receipt_and_sources( + changes, status, clear +): + value = project(**changes) + assert value.survivor_property_route_status == status + assert ( + pd.isna(value.survivor_property_route_clear) + if clear is None + else bool(value.survivor_property_route_clear) is clear + ) + assert value.DIV_VAL_published_amount == 140 # Routes never change donor values. + if "8" in (changes.get("SUR_SC1"), changes.get("SUR_SC2")): + assert value.survivor_estate_or_trust_code_present + if changes.get("SUR_SC1") == "10": + assert value.SUR_SC1_literal_status == "in_printed_range" + assert value.survivor_unspecified_source_code_present + + +@pytest.mark.parametrize( + "field,token", + [ + ("I_DIVVAL", "9"), + ("I_DIVVAL", ""), + ("I_DIVVAL", "x"), + ("I_DIVYN", "5"), + ("TDIV_VAL", "1"), + ("TDIV_VAL", ""), + ("TRNT_VAL", "1"), + ("TRNT_VAL", ""), + ], +) +def test_source_allocation_topcode_flags_are_never_donor_filters(field, token): + value = project(**{field: token}) + assert value.DIV_VAL_amount == 140 and value.DIV_VAL_amount_known + assert bool(value.survivor_property_route_clear) + assert value[field + "_literal"] == token + + +def test_source_roster_and_primary_pin_have_no_survivor_amount_or_tax_outputs(): + assert owner.amount_entries()["DIV_VAL"][:4] == (6, 478, 45, "6C-24") + assert owner.ALLOCATION_ENTRIES["I_DIVYN"][5:] == ("0:1", "See I_ANNVAL") + assert owner.SURVIVOR_CODES[8] == "regular payments from estates or trusts" + assert owner.SURVIVOR_CODES[10] == "other or don't know" + assert not { + "SUR_VAL1", + "SUR_VAL2", + "SRVS_VAL", + "OI_VAL", + "RNT_VAL", + "CAP_VAL", + } & set(owner.READ_COLUMNS) + + +def test_cached_domain_entry_does_not_expose_a_shared_mutable_mapping(): + original = owner.amount_entries() + changed = owner.amount_entries() + changed["DIV_VAL"] = (99, *changed["DIV_VAL"][1:]) + changed["invented"] = () + assert owner.amount_entries() == original + assert type(owner._amount_entry()) is tuple + assert project().DIV_VAL_published_amount == 140 + + +@pytest.mark.parametrize( + "change", + [ + "duplicate_field", + "absent_vintage", + "duplicate_vintage", + "negative_code", + "negative_dollars", + "zero_semantics", + "missing_code", + "minimum", + "dictionary", + ], +) +def test_supported_domain_and_unique_current_vintage_are_required(change): + data = json.loads( + owner.resources.files(owner.__package__) + .joinpath(owner.routing.DOMAINS_RESOURCE) + .read_bytes() + ) + field = next(f for f in data["fields"] if f["name"] == "DIV_VAL") + current = next(v for v in field["vintages"] if v["income_year"] == 2024) + assert owner._decode_amount_entry(data) == owner.amount_entries()["DIV_VAL"] + if change == "duplicate_field": + data["fields"].append(copy.deepcopy(field)) + elif change == "absent_vintage": + field["vintages"].remove(current) + elif change == "duplicate_vintage": + field["vintages"].append(copy.deepcopy(current)) + elif change == "negative_code": + field["domain"]["declared_negative_nonmoney_codes"] = [-1] + elif change == "negative_dollars": + field["domain"]["negative_dollars_permitted"] = True + elif change == "zero_semantics": + field["domain"]["zero_semantics"] = "valid_zero_dollars" + elif change == "missing_code": + field["domain"]["declared_other_missing_codes"] = [999999] + elif change == "minimum": + field["domain"]["encoded_range_inclusive"]["minimum"] = -1 + else: + current["dictionary_spelling"] = "WRONG" + with pytest.raises(ValueError, match="ASEC_DIVIDEND_SOURCE_(DOMAIN_|DICTIONARY_)"): + owner._decode_amount_entry(data) + + +@pytest.mark.parametrize( + "change", ["missing", "duplicate", "entity", "minimum", "maximum", "zero_semantics"] +) +def test_live_ready_money_domain_must_match_the_pinned_source_contract(change): + domain = SimpleNamespace( + name="DIV_VAL", + entity="person", + minimum=0, + maximum=999999, + zero_semantics="none_or_niu_not_distinguishable_from_amount_alone", + ) + fields = [domain] + ready = SimpleNamespace( + bindings=SimpleNamespace(spec=SimpleNamespace(fields=fields)) + ) + owner._domain_agreement(ready) + if change == "missing": + fields.clear() + elif change == "duplicate": + fields.append(copy.copy(domain)) + else: + setattr( + domain, change, "wrong" if change in ("entity", "zero_semantics") else 1 + ) + with pytest.raises(ValueError, match="MONEY_DOMAIN_"): + owner._domain_agreement(ready) + + +@pytest.mark.parametrize("change", ["amount_bit", "validity", "literal", "missing"]) +def test_current_money_total_requires_exact_bits_and_literal_validity(change): + field = SimpleNamespace(amounts=np.array([140.0, 0.0]), validity=np.array([1, 0])) + ready = SimpleNamespace(field=lambda name: field) + positions, literals = np.array([0, 1]), ["140", ""] + assert owner._compare_total(ready, positions, literals) is field + if change == "amount_bit": + field.amounts[0] = np.nextafter(140.0, np.inf) + elif change == "validity": + field.validity[1] = 1 + elif change == "literal": + literals[0] = "-1" + else: + literals[1] = "0" + with pytest.raises(ValueError, match="TOTAL_"): + owner._compare_total(ready, positions, literals) + + +def test_physical_seal_covers_nullable_boolean_and_money_hidden_backing(): + raw = pd.DataFrame([row(DIV_VAL="0", SUR_YN="1")]) + values = owner.CurrentAsecDividendValues( + owner.project_dividend_literals(raw), raw, {"descriptive": True} + ) + original = owner.dividend_values_seal(values) + assert owner.dividend_values_seal(copy.deepcopy(values)) == original + for name in ("DIV_VAL_amount", "survivor_property_route_clear"): + changed = copy.deepcopy(values) + array = changed.person[name].array + assert array._mask[0] + array._data[0] = not array._data[0] if name.endswith("clear") else 99.0 + assert owner.dividend_values_seal(changed) != original + + +def dividend_arguments(tmp_path, monkeypatch, *, fraction=Fraction(1)): + """Amend complete invented literals and retained money before any issuance.""" + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch, fraction=fraction, zero=False) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + literals = { + 105: row(I_DIVYN="5"), + 106: row(A_AGE="14", DIV_VAL="0", DIV_YN="0", SUR_YN="0"), + 107: row(A_AGE="70", DIV_VAL="500", SUR_YN="1", SUR_SC2="8", TDIV_VAL="1"), + 108: row(DIV_VAL="0", SUR_YN="1", SUR_SC1="10"), + } + changes = {} + for name in ("DIV_VAL", "A_AGE"): + values = people[name].to_numpy(copy=True) + for person_id, literal in literals.items(): + values[ids == person_id] = int(literal[name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in ("DIV_VAL", "A_AGE"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + for name in ( + *owner.RECEIPT_ENTRIES, + *owner.SURVIVOR_ENTRIES, + *owner.ALLOCATION_ENTRIES, + *owner.TOPCODE_ENTRIES, + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row())[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "dividend-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(tmp_path, monkeypatch, *, fraction=Fraction(1)): + return owner.routing.source.prepare_authenticated_survey_population( + **dividend_arguments(tmp_path, monkeypatch, fraction=fraction) + ) + + +def test_actual_owner_qualified_source_values_and_requalification( + tmp_path, monkeypatch +): + parent = prepared(tmp_path, monkeypatch) + result = owner.qualify_current_asec_dividend(parent) + values = result.person.set_index("native_person_id") + assert values.loc[105, "DIV_VAL_amount"] == 140 + assert values.loc[105, "I_DIVYN_codebook_status"] == "header_reference_conflict" + assert values.loc[106, "DIV_VAL_reporting_status"] == "outside_reporting_universe" + assert values.loc[107, "DIV_VAL_amount"] == 500 + assert not bool(values.loc[107, "survivor_property_route_clear"]) + assert pd.isna(values.loc[108, "survivor_property_route_clear"]) + assert not values.loc[108, "DIV_VAL_amount_known"] + assert { + "DIV_VAL_parent_statuses", + "DIV_VAL_parent_validity", + "DIV_VAL_parent_zero_origin", + } <= set(values) + assert owner.dividend_values_seal( + owner.qualify_current_asec_dividend(parent) + ) == owner.dividend_values_seal(result) + assert ( + not result.evidence["source_admission_issued"] + and not result.evidence["survivor_amounts_read"] + ) + assert result.evidence == json.loads(json.dumps(result.evidence)) + with pytest.raises(ValueError): + owner.qualify_current_asec_dividend(copy.copy(parent)) + + +@pytest.mark.parametrize( + "coordinate", ["PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "DIV_VAL"] +) +def test_actual_owner_refuses_original_coordinate_or_money_change( + tmp_path, monkeypatch, coordinate +): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if coordinate == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(coordinate)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_dividend(parent) + + +def test_actual_owner_compares_unselected_current_year_amounts(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch, fraction=Fraction(1, 2)) + values = owner.qualify_current_asec_dividend(parent) + omitted = set(values.asec_literals.index) - set(values.person.native_person_id) + assert omitted + key = values.asec_literals.loc[next(iter(omitted)), "PERIDNUM"] + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + raw.loc[key, "DIV_VAL"] = "997" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError, match="TOTAL_BITS"): + owner.qualify_current_asec_dividend(parent) + + +def test_actual_owner_requalifies_after_capture_io(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + calls = [] + + def changed(*args): + raw = capture(*args) + table = parent._checked()[2].frame.person + table.iloc[0, table.columns.get_loc("age")] += 1 + calls.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_dividend(parent) + assert calls + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "route_mask", "literal", "evidence"] +) +def test_final_physical_seal_covers_complete_returned_values( + tmp_path, monkeypatch, mutation +): + parent = prepared(tmp_path, monkeypatch) + seal, calls = owner.dividend_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + if mutation == "amount_bit": + array = value.person.DIV_VAL_amount.array + i = np.flatnonzero(~array._mask)[0] + array._data[i] = np.nextafter(array._data[i], np.inf) + elif mutation == "masked_backing": + array = value.person.DIV_VAL_amount.array + array._data[np.flatnonzero(array._mask)[0]] = 17.0 + elif mutation == "route_mask": + array = value.person.survivor_property_route_clear.array + array._mask[np.flatnonzero(array._mask)[0]] = False + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("SUR_SC1") + ] = "8" + else: + value.evidence["survivor_amounts_read"] = True + return before + + monkeypatch.setattr(owner, "dividend_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_dividend(parent) diff --git a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py new file mode 100644 index 000000000..73b68e66e --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py @@ -0,0 +1,1169 @@ +"""Actual source readers over constructed, privately pinned invented bytes. + +No native microdata, no country engine and no full PUF donor fixture: the +current ASEC income routing qualifier only needs the bounded survey preparation +fixture plus invented member literals. +""" + +import ast +import hashlib +import json +import shutil +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from test_us_asec_coverage_authentication import _changed_parent +from test_us_survey_population_preparation import fixture + +from microcosm.build.frame_checkpoint import load_frame_checkpoint +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import current_asec_income_routing_source as owner + +# Invented current-year people. Ages 55/14 come from the shared fixture; the +# older adult exercises the printed age-58 distribution route. +CURRENT_PEOPLE = (105, 106, 107, 108) +AGES = {105: 55, 106: 14, 107: 70, 108: 14} +AMOUNTS = { + # PNSN ANN DSTV1 DSTV1Y DSTV2 DSTV2Y RNT FRSE OI + 105: (12000.0, -1.0, 0.0, 7000.0, 0.0, 0.0, -400.0, 0.0, 3000.0), + 106: (0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0), + 107: (9000.0, 0.0, 5000.0, 0.0, 0.0, 0.0, 500.0, -9000.0, 250.0), + 108: (0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 700.0), +} +AMOUNT_ORDER = ( + "PNSN_VAL", + "ANN_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "RNT_VAL", + "FRSE_VAL", + "OI_VAL", +) +LITERALS = { + 105: { + "PEN_YN": "1", + "ANN_YN": "0", + "DST_YN": "0", + "DST_YN_YNG": "1", + "DST_SC1_YNG": "4", + "RNT_YN": "1", + "FRSE_YN": "1", + "ERN_YN": "1", + "OI_YN": "1", + "OI_OFF": "20", + }, + 106: {}, + 107: { + "PEN_YN": "2", + "ANN_YN": "1", + "DST_YN": "1", + "DST_SC1": "9", + "RNT_YN": "0", + "FRSE_YN": "1", + "ERN_YN": "1", + "OI_YN": "1", + "OI_OFF": "0", + }, + 108: {"OI_YN": "0", "ERN_YN": "", "FRMOTR": ""}, +} +DEFAULT_LITERAL = { + **{name: "0" for name in owner.RECEIPT_ENTRIES}, + **{name: "0" for name in owner.ACCOUNT_ENTRIES}, + "OI_OFF": "0", + **{name: "0" for name in owner.ALLOCATION_ENTRIES}, +} + + +def _member_literals(person_id): + return {**DEFAULT_LITERAL, **LITERALS.get(person_id, {})} + + +def routing_arguments(tmp_path, monkeypatch, *, reverse=True, ages=None): + """Extend the invented member before any owner issues a preparation. + + Only fixture registry pins change. No source issuer, ready() or native + loader is replaced, and the person-income attachment is rebuilt from the + changed member bytes exactly as production would. + """ + arguments = fixture(tmp_path, monkeypatch) + asec = arguments["source_dir"] / "asec" + parent_path, attachment = asec / "parent.h5", asec / "household-attachment.h5" + person = load_frame_checkpoint(parent_path).frame.person + ids = person.person_id.to_numpy() + ages = AGES if ages is None else ages + changes = {} + for offset, field in enumerate(AMOUNT_ORDER): + data = person[field].to_numpy(dtype="float64", copy=True) + for person_id in CURRENT_PEOPLE: + positions = np.flatnonzero(ids == person_id) + assert len(positions) == 1 + data[positions[0]] = AMOUNTS[person_id][offset] + changes[field] = data + age_column = person.A_AGE.to_numpy(copy=True) + for person_id in CURRENT_PEOPLE: + age_column[np.flatnonzero(ids == person_id)[0]] = ages[person_id] + changes["A_AGE"] = age_column + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in coverage._MEMBER_PINS: + path = asec / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for field in (*AMOUNT_ORDER, "A_AGE"): + raw[field] = [str(int(updated.loc[key, field])) for key in raw.PERIDNUM] + keys = [int(key) for key in raw.PERIDNUM] + current = dict(zip(keys, updated.loc[raw.PERIDNUM].person_id, strict=True)) + for name in DEFAULT_LITERAL: + raw[name] = [_member_literals(int(current[key]))[name] for key in keys] + assert raw.notna().all().all() + ordered = raw.iloc[::-1] if reverse else raw + ordered.to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(raw), + len(data), + ) + ) + paths[year] = path + for module in (coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "routing-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, asec / "person-income-attachment.h5" + ) + return arguments + + +def _qualified(tmp_path, monkeypatch, **kwargs): + arguments = routing_arguments(tmp_path, monkeypatch, **kwargs) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + return prepared, owner.qualify_current_asec_income_routing(prepared) + + +def test_actual_invented_member_joins_the_real_preparation_by_native_keys( + tmp_path, monkeypatch +): + prepared, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert set(person.index) == set(CURRENT_PEOPLE) + # Borrowed money, native identity and member literal agree row by row. + assert person.loc[105, "pension_annuity_pension_source_total"] == 12000.0 + assert person.loc[105, "pension_annuity_pension_reporting_status"] == ( + "known_receipt" + ) + assert person.loc[105, "pension_annuity_pension_known_amount"] == 12000.0 + assert person.source_age.to_dict() == {k: float(v) for k, v in AGES.items()} + literals = qualified.asec_literals.set_index("PERIDNUM") + for person_id in CURRENT_PEOPLE: + key = str(person_id - 100).zfill(22) + assert int(literals.loc[key, "PNSN_VAL"]) == AMOUNTS[person_id][0] + assert int(literals.loc[key, "A_AGE"]) == AGES[person_id] + assert person.amount_validity_PNSN_VAL.eq(1).all() + assert ( + hashlib.sha256(qualified.person.to_json(orient="table").encode()).hexdigest() + == qualified.evidence["projection_sha256"] + ) + assert not qualified.evidence["source_admission_issued"] + assert not qualified.evidence["release_eligible"] + # A host records this receipt, so it must round-trip as JSON unchanged. + receipt = json.dumps(qualified.evidence, sort_keys=True) + assert json.loads(receipt) == qualified.evidence + assert qualified.evidence["joined_person_years"] == [2024] + assert qualified.evidence["acs_channel_rows"] > 0 + assert not qualified.evidence["acs_components_modeled"] + assert set(qualified.evidence["families"]) == set(owner.FAMILIES) + assert ( + qualified.evidence["dictionary"]["amount_entries_source"]["sha256"] + == owner.money.RESOURCE_PINS[0] + ) + assert qualified.evidence["declared_niu_normalized_to_zero"] == ["ANN_VAL"] + printed = qualified.evidence["dictionary"] + assert set(printed["printed_universe_questions"]) == { + "DST_VAL1", + "DST_SC2_YNG", + "I_DSTVAL1COMP", + "DST_YN", + } + assert set(printed["printed_scope_and_code_questions"]) == { + "RNT_YN", + "RNT_VAL", + "FRSE_VAL", + "PNSN_VAL", + "OI_YN", + "DST_SC1", + "FRSE_YN", + } + assert set(printed["ambiguous_allocation_flag_coverage"]) == set( + owner.AMBIGUOUS_FLAG_COVERAGE + ) + prepared.checked_view() + + +def test_pension_and_annuity_totals_stay_separate_and_unsplit(tmp_path, monkeypatch): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + # PNSN_VAL is the printed combined total over all pension sources. Neither + # a private share nor a taxable share is derived from it. + assert person.loc[105, "pension_annuity_combined_total_scope"] == ( + owner.PENSION_TOTAL_SCOPE + ) + assert not person.pension_annuity_private_share_applied.any() + assert not person.pension_annuity_taxable_amount_known.any() + assert not person.pension_annuity_pension_total_has_published_flag.any() + # ANN_VAL's printed -1 is a NIU code, never a one dollar annuity loss. + assert np.isnan(person.loc[105, "pension_annuity_annuity_source_total"]) + assert person.loc[105, "pension_annuity_annuity_amount_kind"] == "declared_niu" + assert person.loc[105, "pension_annuity_annuity_reporting_status"] == "niu" + # ANN_VAL is the one entry here whose printed zero is valid dollars (its NIU + # is the separate -1 code), so a yes answer with a zero amount resolves. + assert person.loc[107, "pension_annuity_annuity_reporting_status"] == ( + "known_recipient_zero" + ) + assert person.loc[107, "pension_annuity_annuity_known_amount"] == 0.0 + # PNSN_VAL prints "0 = none or niu", so the same pattern would not resolve + # there; the module reads that distinction from the pinned domains artifact. + assert owner._zero_is_dollars("ANN_VAL") + assert not owner._zero_is_dollars("PNSN_VAL") + # A no answer against a positive total is a retained contradiction. + assert person.loc[107, "pension_annuity_pension_reporting_status"] == ( + "contradictory_no_nonzero" + ) + assert person.loc[107, "pension_annuity_pension_source_total"] == 9000.0 + assert np.isnan(person.loc[107, "pension_annuity_pension_known_amount"]) + + +def test_distribution_routing_preserves_slots_ages_and_unresolved_accounts( + tmp_path, monkeypatch +): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert person.loc[105, "retirement_distribution_route"] == "under_age58" + assert person.loc[107, "retirement_distribution_route"] == "age58_and_over" + # Regular IRA is account code 4; the slot amount is not a taxable amount. + assert person.loc[105, "retirement_distribution_slot1_young_account_code"] == 4 + assert person.loc[105, "retirement_distribution_slot1_young_account_label"] == ( + "Regular IRA" + ) + assert person.loc[105, "retirement_distribution_regular_ira_amount"] == 7000.0 + assert person.loc[105, "retirement_distribution_regular_ira_slots"] == 1 + assert person.loc[105, "retirement_distribution_source_total"] == 7000.0 + assert not person.retirement_distribution_taxable_amount_known.any() + assert bool(person.loc[107, "retirement_distribution_route_has_published_flag"]) + assert not bool(person.loc[105, "retirement_distribution_route_has_published_flag"]) + # An account literal outside the printed range leaves the composition and + # the regular IRA share unresolved rather than defaulting either way. + assert person.loc[107, "retirement_distribution_slot1_account_literal"] == "9" + assert person.loc[107, "retirement_distribution_slot1_account_literal_status"] == ( + "outside_printed_range" + ) + assert person.loc[107, "retirement_distribution_slot1_slot_status"] == ( + "unresolved_slot_account" + ) + assert pd.isna(person.loc[107, "retirement_distribution_slot1_account_code"]) + assert np.isnan(person.loc[107, "retirement_distribution_regular_ira_amount"]) + assert pd.isna(person.loc[107, "retirement_distribution_regular_ira_slots"]) + # The slot total is still the retained literal sum for the applicable route. + assert person.loc[107, "retirement_distribution_source_total"] == 5000.0 + # The printed distribution universes name only the age 58 split, so + # coverage below age 15 is a source question, not a resolved answer. + assert pd.isna(person.loc[106, "retirement_distribution_source_reporting_universe"]) + assert person.loc[106, "retirement_distribution_reporting_status"] == ( + "unresolved_reporting_universe" + ) + + +def test_signed_property_and_farm_totals_keep_losses_and_net_zero_receipt( + tmp_path, monkeypatch +): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert person.loc[105, "net_property_source_total"] == -400.0 + assert person.loc[105, "net_property_known_amount"] == -400.0 + assert person.loc[105, "net_property_reporting_status"] == "known_receipt" + assert bool(person.loc[105, "net_property_is_net_loss"]) + # The receipt question is wider than the amount question, so the total is + # not independently labelled rental and no component split is claimed. + assert person.loc[105, "net_property_receipt_scope"] == ( + owner.NET_PROPERTY_RECEIPT_SCOPE + ) + assert person.loc[105, "net_property_amount_scope"] == ( + owner.NET_PROPERTY_AMOUNT_SCOPE + ) + assert not person.net_property_component_split_known.any() + # A NIU receipt literal against a positive amount stays a contradiction. + assert person.loc[107, "net_property_reporting_status"] == ( + "contradictory_niu_nonzero" + ) + assert person.loc[107, "net_property_source_total"] == 500.0 + assert np.isnan(person.loc[107, "net_property_known_amount"]) + # Farm receipt with a net zero stays distinct from absence, NIU and missing, + # but FRSE_VAL prints "0 = none or niu" just as the gross entries do, so the + # amount is not completed to a known zero. + assert person.loc[105, "farm_reporting_status"] == "receipt_with_net_zero" + assert np.isnan(person.loc[105, "farm_known_amount"]) + assert person.loc[105, "farm_source_total"] == 0.0 + assert "receipt_with_net_zero" not in owner.KNOWN_AMOUNT_STATUSES + assert person.loc[107, "farm_source_total"] == -9000.0 + assert person.loc[107, "farm_reporting_status"] == "known_receipt" + assert bool(person.loc[107, "farm_is_net_loss"]) + assert not person.farm_is_nonfarm_self_employment.any() + # The farm universe comes from ERN_YN/FRMOTR evidence, never from age. + assert bool(person.loc[105, "farm_source_reporting_universe"]) + assert pd.isna(person.loc[108, "farm_source_reporting_universe"]) + assert person.loc[108, "farm_reporting_status"] == "unresolved_reporting_universe" + + +def test_other_income_keeps_reported_alimony_apart_from_residual_rules( + tmp_path, monkeypatch +): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert person.loc[105, "other_income_category_code"] == 20 + assert person.loc[105, "other_income_category_label"] == "alimony" + assert person.loc[105, "other_income_routing_status"] == "reported_category" + assert bool(person.loc[105, "other_income_is_reported_alimony"]) + assert person.loc[105, "other_income_source_total"] == 3000.0 + # A receipt without a category is retained as an unresolved pattern; no + # residual rule assigns it to alimony or to miscellaneous income. + assert person.loc[107, "other_income_routing_status"] == "receipt_without_category" + assert not bool(person.loc[107, "other_income_is_reported_alimony"]) + assert not person.other_income_residual_rule_applied.any() + # A NIU receipt with a positive amount is a contradiction, not a nonfiler. + assert person.loc[108, "other_income_reporting_status"] == ( + "contradictory_outside_reporting_universe" + ) + assert person.loc[108, "other_income_source_total"] == 700.0 + assert np.isnan(person.loc[108, "other_income_known_amount"]) + + +def test_row_order_of_the_source_member_cannot_change_the_projection( + tmp_path, monkeypatch +): + roots = {} + for name in ("forward", "reverse"): + roots[name] = tmp_path / name + roots[name].mkdir() + with pytest.MonkeyPatch.context() as forward: + _, ordered = _qualified(roots["forward"], forward, reverse=False) + with pytest.MonkeyPatch.context() as backward: + _, reversed_ = _qualified(roots["reverse"], backward, reverse=True) + pd.testing.assert_frame_equal(ordered.person, reversed_.person, check_exact=True) + assert ( + ordered.evidence["projection_sha256"] == reversed_.evidence["projection_sha256"] + ) + # The member digest legitimately differs; the projection must not. + assert ( + ordered.evidence["source_member_sha256"] + != reversed_.evidence["source_member_sha256"] + ) + + +def test_repeated_qualification_is_stable_and_transport_is_not_authority( + tmp_path, monkeypatch +): + prepared, qualified = _qualified(tmp_path, monkeypatch) + again = owner.qualify_current_asec_income_routing(prepared) + assert ( + again.evidence["projection_sha256"] == qualified.evidence["projection_sha256"] + ) + # Mutating the returned transport cannot reach a fresh live projection. + qualified.person.loc[qualified.person.index[0], "farm_source_total"] = 99.0 + qualified.asec_literals.loc[qualified.asec_literals.index[0], "OI_OFF"] = "99" + fresh = owner.qualify_current_asec_income_routing(prepared) + assert not fresh.person.farm_source_total.eq(99.0).any() + assert not fresh.asec_literals.OI_OFF.eq("99").any() + pd.testing.assert_frame_equal(fresh.person, again.person, check_exact=True) + + +@pytest.mark.parametrize("defect", ("digest", "member", "rows", "bytes")) +def test_changed_member_bytes_or_pin_refuse_before_any_projection( + tmp_path, monkeypatch, defect +): + arguments = routing_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + pins = list(coverage._MEMBER_PINS) + if defect == "bytes": + member = next(p[1] for p in pins if p[0] == 2024) + path = arguments["source_dir"] / "asec" / member + path.write_bytes(path.read_bytes().replace(b"12000", b"12001", 1)) + else: + index = next(i for i, p in enumerate(pins) if p[0] == 2024) + year, name, archive, digest, rows, size = pins[index] + pins[index] = { + "digest": (year, name, archive, "f" * 64, rows, size), + "member": (year, "pppub99.csv", archive, digest, rows, size), + "rows": (year, name, archive, digest, rows + 1, size), + }[defect] + monkeypatch.setattr(coverage, "_MEMBER_PINS", tuple(pins)) + with pytest.raises(ValueError): + owner.qualify_current_asec_income_routing(prepared) + + +@pytest.mark.parametrize("swap", ("coordinate", "amount")) +def test_borrowed_member_must_match_the_parent_row_for_row(tmp_path, monkeypatch, swap): + arguments = routing_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + real = owner._read_capture + + def tampered(path, *, rows): + frame = real(path, rows=rows) + column = "A_LINENO" if swap == "coordinate" else "PNSN_VAL" + values = frame[column].to_numpy(copy=True) + frame[column] = np.concatenate([values[1:], values[:1]]) + return frame + + monkeypatch.setattr(owner, "_read_capture", tampered) + expected = ( + "PARENT_COORDINATE_IDENTITY" + if swap == "coordinate" + else "CURRENT_AMOUNT_SOURCE_IDENTITY" + ) + with pytest.raises(ValueError, match=expected): + owner.qualify_current_asec_income_routing(prepared) + + +def test_a_tampered_preparation_refuses_before_any_projection(tmp_path, monkeypatch): + """A replaced checked method is refused by the owner, not worked around.""" + arguments = routing_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + real = type(prepared)._checked + calls = [] + + def late(self): + entry = real(self) + calls.append(entry) + return entry if len(calls) == 1 else (entry[0], b"{}", entry[2]) + + monkeypatch.setattr(type(prepared), "_checked", late) + with pytest.raises(ValueError, match="FINAL_AUTHORITY_CHANGED|SOURCE_CHANGED"): + owner.qualify_current_asec_income_routing(prepared) + + +def test_the_final_requalification_cannot_be_replaced_or_skipped(tmp_path, monkeypatch): + """The owner refuses a substituted final requalification, so this qualifier + cannot be made to return a projection that skipped it.""" + prepared, qualified = _qualified(tmp_path, monkeypatch) + real = owner.source._pure_final + seen = [] + + def spy(state): + seen.append(state) + return real(state) + + # Scoped so undoing the substitution leaves the fixture pins in place. + with pytest.MonkeyPatch.context() as replaced: + replaced.setattr(owner.source, "_pure_final", spy) + with pytest.raises(ValueError, match="FINAL_AUTHORITY_CHANGED"): + owner.qualify_current_asec_income_routing(prepared) + # Unreplaced, the same preparation still yields the identical projection. + again = owner.qualify_current_asec_income_routing(prepared) + assert ( + again.evidence["projection_sha256"] == qualified.evidence["projection_sha256"] + ) + assert ( + hashlib.sha256(again.person.to_json(orient="table").encode()).hexdigest() + == again.evidence["projection_sha256"] + ) + + +def test_the_qualifier_imports_only_the_accepted_source_neighbourhood(): + """Source-only portability: no graph, fit, engine or modelled stage import.""" + path = Path(owner.__file__) + tree = ast.parse(path.read_text()) + absolute, relative = set(), set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + absolute.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.level: + relative.update( + alias.name for alias in node.names + ) if node.module is None else relative.add(node.module) + else: + absolute.add(node.module.split(".")[0]) + assert absolute <= { + "__future__", + "csv", + "hashlib", + "json", + "re", + "tempfile", + "dataclasses", + "functools", + "importlib", + "pathlib", + "typing", + "numpy", + "pandas", + } + assert relative == { + "asec_coverage_authentication", + "asec_current_money", + "source_csv_builtin", + "survey_population_preparation", + "support_provenance", + } + # The legacy modelled assumptions are deliberately not reachable from here. + text = path.read_text() + for forbidden in ("microcosm.graph", "policyengine", "0.590", "0.59"): + assert forbidden not in text + + +def _member_row(**overrides): + values = { + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + **{name: "0" for name in owner.AMOUNT_FIELDS}, + **DEFAULT_LITERAL, + **overrides, + } + return [values[name] for name in owner.READ_COLUMNS] + + +def _write_member(path, rows): + lines = [",".join(owner.READ_COLUMNS)] + lines.extend(",".join(row) for row in rows) + path.write_text("\n".join(lines) + "\n") + return path + + +@pytest.mark.parametrize( + "universe,token,kind,net,dollars,label", + [ + (True, "1", "nonzero", False, False, "known_receipt"), + (True, "1", "zero", False, False, "ambiguous_recipient_zero"), + # A signed net measure's recipient zero stays distinct, and stays + # unknown: RNT_VAL and FRSE_VAL print the same "0 = none or niu" label + # the gross entries print. + (True, "1", "zero", True, False, "receipt_with_net_zero"), + # Only an entry whose printed zero is valid dollars (ANN_VAL) resolves. + (True, "1", "zero", False, True, "known_recipient_zero"), + (True, "1", "nonzero", True, False, "known_receipt"), + (True, "2", "zero", False, False, "known_nonreceipt"), + (True, "2", "nonzero", False, False, "contradictory_no_nonzero"), + (True, "0", "zero", False, False, "niu"), + (True, "0", "nonzero", False, False, "contradictory_niu_nonzero"), + (True, "0", "declared_niu", False, True, "niu"), + (True, "1", "declared_niu", False, True, "contradictory_declared_niu_amount"), + (True, "1", "missing", False, False, "missing_amount"), + (True, "", "zero", False, False, "missing_receipt_literal"), + (True, "9", "zero", False, False, "unrecognized_receipt_literal"), + (True, "01", "zero", False, False, "unrecognized_receipt_literal"), + (False, "0", "zero", False, False, "outside_reporting_universe"), + (False, "0", "missing", False, False, "outside_reporting_universe"), + ( + False, + "1", + "nonzero", + False, + False, + "contradictory_outside_reporting_universe", + ), + ( + False, + "2", + "zero", + False, + False, + "contradictory_outside_reporting_universe", + ), + (None, "1", "nonzero", False, False, "unresolved_reporting_universe"), + (None, "0", "zero", True, False, "unresolved_reporting_universe"), + ], +) +def test_receipt_and_amount_knownness_stay_separate( + universe, token, kind, net, dollars, label +): + receipt = owner.literal_code(token, owner.receipt_codes("PEN_YN"), width=1) + assert ( + owner.receipt_status( + universe, receipt, kind, net_measure=net, zero_is_dollars=dollars + ) + == label + ) + # Only three statuses establish a dollar reading; nothing else completes. + assert (label in owner.KNOWN_AMOUNT_STATUSES) == ( + label in ("known_receipt", "known_recipient_zero", "known_nonreceipt") + ) + + +def test_only_ann_val_prints_a_zero_that_reads_as_valid_dollars(): + entries = owner.printed_amount_entries() + dollars = { + name + for name, entry in entries.items() + if entry.zero_semantics == owner.DOLLAR_ZERO_SEMANTICS + } + assert dollars == {"ANN_VAL"} + for name in set(owner.AMOUNT_FIELDS) - dollars: + assert ( + entries[name].zero_semantics + == "none_or_niu_not_distinguishable_from_amount_alone" + ) + assert owner._zero_is_dollars("ANN_VAL") + assert not owner._zero_is_dollars("RNT_VAL") + assert not owner._zero_is_dollars("FRSE_VAL") + + +@pytest.mark.parametrize( + "status,value,kind,dollars", + [ + (owner.money.CodebookStatus.AMOUNT_NONZERO, 120.0, "nonzero", 120.0), + (owner.money.CodebookStatus.AMOUNT_NONZERO, -900.0, "nonzero", -900.0), + (owner.money.CodebookStatus.ZERO_NONE_OR_NIU, 0.0, "zero", 0.0), + (owner.money.CodebookStatus.ZERO_DOLLARS_AS_CODED, 0.0, "zero", 0.0), + (owner.money.CodebookStatus.DECLARED_NIU, 0.0, "declared_niu", None), + (owner.money.CodebookStatus.MISSING_NULL, np.nan, "missing", None), + ], +) +def test_amount_meaning_comes_from_the_parent_status_axis(status, value, kind, dollars): + read_kind, read_dollars = owner.amount_state(value, status) + assert read_kind == kind + if dollars is None: + assert np.isnan(read_dollars) + else: + assert read_dollars == dollars + # A normalized NIU zero must never be read back as a zero dollar amount. + if status == owner.money.CodebookStatus.DECLARED_NIU: + assert read_kind != "zero" + + +@pytest.mark.parametrize( + "token,width,code,status", + [ + ("4", 1, 4, "in_printed_range"), + ("20", 2, 20, "in_printed_range"), + ("", 1, None, "missing"), + ("01", 1, None, "malformed"), + (" 4", 1, None, "malformed"), + ("4.0", 1, None, "malformed"), + ("NA", 1, None, "malformed"), + ("9", 1, None, "outside_printed_range"), + ("21", 2, None, "outside_printed_range"), + ], +) +def test_routing_literals_stay_unknown_rather_than_recoded(token, width, code, status): + allowed = owner.ACCOUNT_CODES if width == 1 else owner.OTHER_INCOME_CATEGORIES + assert owner.literal_code(token, allowed, width=width) == (code, status) + + +@pytest.mark.parametrize( + "defect,match", + [ + ("duplicate_key", "DUPLICATE_OR_INVALID_COORDINATE"), + ("duplicate_coordinate", "DUPLICATE_OR_INVALID_COORDINATE"), + ("short_key", "PERSON_KEY"), + ("bad_age", "COORDINATE:A_AGE"), + ("unsigned_negative", "AMOUNT_TOKEN:PNSN_VAL"), + ("wide_amount", "AMOUNT_TOKEN:OI_VAL"), + ("oversized_token", "TOKEN_BOUND:OI_OFF"), + ("missing_column", "HEADER"), + ("row_count", "ROW_COUNT"), + ], +) +def test_bounded_member_reader_refuses_structural_defects(tmp_path, defect, match): + path = tmp_path / "invented.csv" + rows = [_member_row()] + if defect == "duplicate_key": + rows.append(_member_row(PH_SEQ="2")) + elif defect == "duplicate_coordinate": + rows.append(_member_row(PERIDNUM="0000000000000000000002")) + elif defect == "short_key": + rows = [_member_row(PERIDNUM="1")] + elif defect == "bad_age": + rows = [_member_row(A_AGE="123")] + elif defect == "unsigned_negative": + rows = [_member_row(PNSN_VAL="-5")] + elif defect == "wide_amount": + rows = [_member_row(OI_VAL="1234567")] + elif defect == "oversized_token": + rows = [_member_row(OI_OFF="9" * 65)] + _write_member(path, rows) + if defect == "missing_column": + text = path.read_text().split("\n") + keep = [i for i, c in enumerate(owner.READ_COLUMNS) if c != "OI_OFF"] + path.write_text( + "\n".join(",".join(line.split(",")[i] for i in keep) for line in text[:-1]) + + "\n" + ) + expected = 2 if defect.startswith("duplicate") else 1 + if defect == "row_count": + expected = 2 + with pytest.raises(ValueError, match=match): + owner._read_capture(path, rows=expected) + + +def test_bounded_member_reader_accepts_signed_and_missing_amount_literals(tmp_path): + path = _write_member( + tmp_path / "invented.csv", + [_member_row(RNT_VAL="-9999", FRSE_VAL="-9999999", ANN_VAL="-1", OI_VAL="")], + ) + frame = owner._read_capture(path, rows=1) + assert frame.iloc[0].RNT_VAL == "-9999" + assert frame.iloc[0].FRSE_VAL == "-9999999" + assert frame.iloc[0].ANN_VAL == "-1" + assert frame.iloc[0].OI_VAL == "" + + +def _field(name, amounts, statuses, validity): + return owner.money.MoneyField( + name, + np.array(amounts, dtype=" 0 and a_age ≥ 58"} + assert owner.ACCOUNT_ENTRIES["DST_SC1_YNG"][4] == ("DST_YN_YNG = 1 and a_age < 58") + assert owner.ACCOUNT_ENTRIES["DST_SC2_YNG"][4] == ("DST_VAL_YNG > 0 and a_age < 58") + assert owner.RECEIPT_ENTRIES["OI_YN"][5] == "none or niu" + assert owner.receipt_codes("OI_YN")[0] == "none or niu" + for name in ("PEN_YN", "ANN_YN", "DST_YN", "DST_YN_YNG", "RNT_YN", "ERN_YN"): + assert owner.receipt_codes(name)[0] == "niu" + assert owner.receipt_codes("FRSE_YN")[0] == "Niu" + assert owner.RECEIPT_ENTRIES["FRSE_YN"].universe_as_printed == ( + "ERN_YN=1 or FRMOTR=1" + ) + for name in ("PEN_YN", "ANN_YN", "RNT_YN", "OI_YN"): + assert owner.RECEIPT_ENTRIES[name].universe_as_printed == ( + "All Persons aged 15+" + ) + + +def _pure_rows(n, ages, **literals): + codebook = owner.money.CodebookStatus + amounts = {name: np.zeros(n) for name in owner.AMOUNT_FIELDS} + statuses = { + name: np.full(n, int(codebook.ZERO_NONE_OR_NIU), "u1") + for name in owner.AMOUNT_FIELDS + } + statuses["ANN_VAL"] = np.full(n, int(codebook.ZERO_DOLLARS_AS_CODED), "u1") + raw = { + "amounts": amounts, + "statuses": statuses, + "allocations": { + name: ([0] * n, ["in_printed_range"] * n) + for name in owner.ALLOCATION_ENTRIES + }, + } + for name in (*owner.RECEIPT_ENTRIES, *owner.ACCOUNT_ENTRIES, "OI_OFF"): + raw[name] = ["0"] * n + raw.update(literals) + return raw, np.asarray(ages, dtype="float64") + + +def _set_amount(raw, field, values): + codebook = owner.money.CodebookStatus + raw["amounts"][field] = np.asarray(values, dtype="float64") + raw["statuses"][field] = np.array( + [ + int(codebook.AMOUNT_NONZERO) + if v + else int( + codebook.ZERO_DOLLARS_AS_CODED + if field == "ANN_VAL" + else codebook.ZERO_NONE_OR_NIU + ) + for v in values + ], + dtype="u1", + ) + + +def test_offroute_distribution_dollars_block_a_known_absence(): + raw, ages = _pure_rows(1, [70.0], DST_YN=["2"], DST_SC1_YNG=["4"]) + _set_amount(raw, "DST_VAL1_YNG", [8000.0]) + out = owner.project_income_routing(raw, ages) + assert out.retirement_distribution_route.iloc[0] == "age58_and_over" + assert bool(out.retirement_distribution_offroute_nonzero.iloc[0]) + # The applicable literals alone would read as a known zero; the retained + # off-route dollars contradict that, so nothing is resolved. + assert out.retirement_distribution_reporting_status.iloc[0] == ( + "contradictory_offroute_evidence" + ) + assert np.isnan(out.retirement_distribution_known_amount.iloc[0]) + assert out.retirement_distribution_slot1_young_amount.iloc[0] == 8000.0 + + +def test_an_ambiguous_slot_zero_leaves_the_total_and_the_ira_share_unknown(): + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["4"], DST_SC2=["1"]) + _set_amount(raw, "DST_VAL1", [5000.0]) + out = owner.project_income_routing(raw, ages) + # Slot 2 declares a 401k account whose amount is a "none or niu" zero. + assert out.retirement_distribution_slot2_slot_status.iloc[0] == ( + "ambiguous_slot_zero" + ) + assert bool(out.retirement_distribution_slot_zero_ambiguity.iloc[0]) + assert out.retirement_distribution_source_total.iloc[0] == 5000.0 + assert out.retirement_distribution_reporting_status.iloc[0] == ( + "unresolved_slot_composition" + ) + assert np.isnan(out.retirement_distribution_known_amount.iloc[0]) + assert np.isnan(out.retirement_distribution_regular_ira_amount.iloc[0]) + assert pd.isna(out.retirement_distribution_regular_ira_slots.iloc[0]) + # A regular IRA slot whose own amount is that ambiguous zero is not a known + # zero distribution from that account either. + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["4"], DST_SC2=["1"]) + _set_amount(raw, "DST_VAL2", [5000.0]) + ambiguous = owner.project_income_routing(raw, ages) + assert ambiguous.retirement_distribution_slot1_slot_status.iloc[0] == ( + "ambiguous_slot_zero" + ) + assert np.isnan(ambiguous.retirement_distribution_regular_ira_amount.iloc[0]) + # A contradictory NIU slot carrying dollars is likewise unresolved. + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["0"]) + _set_amount(raw, "DST_VAL1", [5000.0]) + contradictory = owner.project_income_routing(raw, ages) + assert contradictory.retirement_distribution_slot1_slot_status.iloc[0] == ( + "contradictory_niu_slot_amount" + ) + assert np.isnan(contradictory.retirement_distribution_regular_ira_amount.iloc[0]) + assert pd.isna(contradictory.retirement_distribution_regular_ira_slots.iloc[0]) + # Two fully resolved slots do give a known composition. + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["4"], DST_SC2=["0"]) + _set_amount(raw, "DST_VAL1", [5000.0]) + resolved = owner.project_income_routing(raw, ages) + assert resolved.retirement_distribution_reporting_status.iloc[0] == "known_receipt" + assert resolved.retirement_distribution_known_amount.iloc[0] == 5000.0 + assert resolved.retirement_distribution_regular_ira_amount.iloc[0] == 5000.0 + assert resolved.retirement_distribution_regular_ira_slots.iloc[0] == 1 + + +@pytest.mark.parametrize( + "age,ern,frmotr,routing", + [ + (14, "1", "0", "outside_reporting_universe_routing"), + (40, "", "", "reported_category"), + ], +) +def test_other_income_routing_follows_the_printed_receipt_universe( + age, ern, frmotr, routing +): + raw, ages = _pure_rows( + 1, + [float(age)], + OI_YN=["1"], + OI_OFF=["20"], + ERN_YN=[ern], + FRMOTR=[frmotr], + ) + _set_amount(raw, "OI_VAL", [3000.0]) + out = owner.project_income_routing(raw, ages) + assert out.other_income_routing_status.iloc[0] == routing + assert bool(out.other_income_is_reported_alimony.iloc[0]) == ( + routing == "reported_category" + ) + # The farm universe on the same rows comes from ERN_YN/FRMOTR, not from age. + assert pd.isna(out.farm_source_reporting_universe.iloc[0]) == (ern == "") + + +def test_allocation_origins_do_not_assert_publisher_confirmed_non_allocation(): + raw, ages = _pure_rows(1, [40.0]) + zeroed = owner.project_income_routing(raw, ages) + # Every flag here prints a conditional universe that is not evaluated, so an + # all-zero reading is reported as exactly that, never as "no allocation". + assert zeroed.net_property_allocation_origin.iloc[0] == "published_flags_all_zero" + assert zeroed.pension_annuity_allocation_origin.iloc[0] == ( + "published_flags_all_zero_with_unflagged_fields" + ) + for column in [c for c in zeroed.columns if c.endswith("_allocation_origin")]: + assert "no_allocation" not in zeroed[column].iloc[0] + raw, ages = _pure_rows(1, [40.0]) + raw["allocations"]["I_RNTVAL"] = ([4], ["in_printed_range"]) + assert ( + owner.project_income_routing(raw, ages).net_property_allocation_origin.iloc[0] + == "publisher_allocated" + ) + raw, ages = _pure_rows(1, [40.0]) + raw["allocations"]["I_RNTYN"] = ([None], ["missing"]) + assert ( + owner.project_income_routing(raw, ages).net_property_allocation_origin.iloc[0] + == "allocation_flag_not_populated" + ) + raw, ages = _pure_rows(1, [40.0]) + raw["allocations"]["I_RNTYN"] = ([None], ["outside_printed_range"]) + assert ( + owner.project_income_routing(raw, ages).net_property_allocation_origin.iloc[0] + == "unresolved_allocation_provenance" + ) + + +@pytest.mark.parametrize("field", ("OI_YN", "DST_SC1", "OI_OFF")) +def test_the_public_projection_validates_its_routing_token_arrays(field): + raw, ages = _pure_rows(2, [40.0, 40.0]) + raw[field] = ["0"] + with pytest.raises(ValueError, match="ROUTING_TOKEN_CONTRACT:" + field): + owner.project_income_routing(raw, ages) + raw[field] = ["0", 0] + with pytest.raises(ValueError, match="ROUTING_TOKEN_CONTRACT:" + field): + owner.project_income_routing(raw, ages) + + +def test_the_literal_transport_carries_its_own_digest(tmp_path, monkeypatch): + _, qualified = _qualified(tmp_path, monkeypatch) + assert ( + hashlib.sha256( + qualified.asec_literals.to_json(orient="table").encode() + ).hexdigest() + == qualified.evidence["literals_sha256"] + ) + assert ( + qualified.evidence["literals_sha256"] != qualified.evidence["projection_sha256"] + ) + # A host comparing the documented digests detects a corrupted literal frame. + index = qualified.asec_literals.index[0] + qualified.asec_literals.loc[index, "PNSN_VAL"] = "999999" + assert ( + hashlib.sha256( + qualified.asec_literals.to_json(orient="table").encode() + ).hexdigest() + != qualified.evidence["literals_sha256"] + ) diff --git a/packages/microcosm-build/tests/test_us_current_asec_interest_source.py b/packages/microcosm-build/tests/test_us_current_asec_interest_source.py new file mode 100644 index 000000000..bf8e34334 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_interest_source.py @@ -0,0 +1,409 @@ +"""Invented literal and real-source-owner controls for the interest extension.""" + +import copy +import hashlib +import json +import shutil +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_interest_source as owner + + +def row(**changes): + return { + **{c: "0" for c in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "INT_VAL": "140", + "INT_YN": "1", + "TRDINT_VAL": "100", + "RINT_YN": "1", + "RINT_SC1": "4", + "RINT_VAL1": "40", + **changes, + } + + +def project(**changes): + return owner.project_interest_literals(pd.DataFrame([row(**changes)])).iloc[0] + + +def test_component_totals_remain_observed_and_discrepancy_is_not_balanced(): + value = project( + INT_VAL="500", TRDINT_VAL="0", RINT_VAL1="400", RINT_SC2="7", RINT_VAL2="200" + ) + assert value.INT_VAL_amount == 500 + assert value.TRDINT_VAL_amount == 0 and value.TRDINT_VAL_amount_known + assert value.TRDINT_VAL_reporting_status == "observed_zero_component" + assert value.RINT_VAL1_amount == 400 and value.RINT_VAL2_amount == 200 + assert value.published_components_sum == 600 + assert value.combined_minus_published_components == -100 + assert value.component_discrepancy_known and value.all_component_amounts_observed + + +def test_an_unreported_account_slot_is_not_an_observed_zero(): + value = project() + assert value.RINT_VAL2_published_amount == 0 + assert value.RINT_VAL2_reporting_status == "unreported_account_slot" + assert not value.RINT_VAL2_amount_known and pd.isna(value.RINT_VAL2_amount) + assert not value.RINT_SC2_account_known + assert ( + value.component_discrepancy_known + and value.combined_minus_published_components == 0 + ) + assert not value.all_component_amounts_observed + + +@pytest.mark.parametrize( + "changes,field,status,known", + [ + ({"INT_VAL": "0"}, "INT_VAL", "ambiguous_recipient_zero", False), + ({"RINT_VAL1": "0"}, "RINT_VAL1", "ambiguous_recipient_zero", False), + ({"TRDINT_VAL": ""}, "TRDINT_VAL", "missing_amount", False), + ({"TRDINT_VAL": "-1"}, "TRDINT_VAL", "invalid_amount_literal", False), + ({"RINT_VAL1": "-1"}, "RINT_VAL1", "invalid_amount_literal", False), + ({"RINT_VAL1": "4.0"}, "RINT_VAL1", "invalid_amount_literal", False), + ({"RINT_VAL1": "NA"}, "RINT_VAL1", "invalid_amount_literal", False), + ({"RINT_SC1": ""}, "RINT_VAL1", "missing_account_literal", False), + ({"RINT_SC1": "9"}, "RINT_VAL1", "invalid_account_literal", False), + ({"RINT_SC1": "0"}, "RINT_VAL1", "amount_without_account", False), + ({"RINT_YN": ""}, "RINT_VAL1", "missing_receipt_literal", False), + ({"RINT_YN": "2"}, "RINT_VAL1", "account_without_receipt", False), + ({"INT_YN": "2"}, "TRDINT_VAL", "contradictory_no_nonzero", False), + ({"INT_YN": "0", "TRDINT_VAL": "0"}, "TRDINT_VAL", "niu", False), + ({"INT_YN": "2", "TRDINT_VAL": "0"}, "TRDINT_VAL", "known_nonreceipt", True), + ], +) +def test_receipt_account_and_amount_conflicts_remain_explicit( + changes, field, status, known +): + value = project(**changes) + assert value[field + "_reporting_status"] == status + assert bool(value[field + "_amount_known"]) is known + if not known: + assert pd.isna(value[field + "_amount"]) + + +def test_under15_universe_is_checked_without_analytical_zeros(): + value = project( + A_AGE="14", + INT_VAL="0", + INT_YN="0", + TRDINT_VAL="0", + RINT_YN="0", + RINT_SC1="0", + RINT_VAL1="0", + ) + for name in owner.AMOUNT_FIELDS: + assert value[name + "_reporting_status"] == "outside_reporting_universe" + assert not value[name + "_amount_known"] + assert value.published_components_sum == 0 # Published-cell diagnostic only. + + +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing"), + ("1", "outside_printed_range"), + ("11", "in_printed_range"), + ("15", "in_printed_range"), + ("x", "malformed"), + ], +) +def test_composite_allocation_uses_printed_codes_not_header_interval(token, status): + value = project(I_INTVAL=token) + assert value.I_INTVAL_literal_status == status + assert ( + value.RINT_VAL1_amount_known + ) # Allocation never changes observation validity. + if token in ("11", "15"): + assert value.allocation_origin == "publisher_allocated" + + +def test_missing_component_or_flag_never_becomes_false_or_zero(): + value = project(RINT_VAL2="", TRINT_VAL2="", I_RINTVAL2="") + assert pd.isna(value.published_components_sum) + assert not value.component_discrepancy_known + assert value.TRINT_VAL2_literal_status == "missing" + assert value.allocation_origin == "allocation_flag_not_populated" + + +def test_shared_account_codes_and_pinned_total_domain_are_reused(): + assert owner.amount_entries()["INT_VAL"][4] == 999999 + assert owner.amount_entries()["TRDINT_VAL"][4] == 99999 + value = project(RINT_SC1="3") + assert value.RINT_SC1_label == owner.routing.ACCOUNT_CODES[3] == "Roth IRA" + assert value.RINT_SC1_account_known + value = project(INT_YN="0", RINT_YN="0") + assert value.INT_YN_label == value.RINT_YN_label == "niu" + + +def test_nullable_float_seal_preserves_masks_hidden_values_and_exact_bits(): + raw = pd.DataFrame([row()]) + values = owner.CurrentAsecInterestValues( + owner.project_interest_literals(raw), raw, {"descriptive": True} + ) + original = owner.interest_values_seal(values) + assert owner.interest_values_seal(copy.deepcopy(values)) == original + for change in ("bit", "hidden", "mask"): + altered = copy.deepcopy(values) + if change == "bit": + array = altered.person.INT_VAL_amount.array + array._data[0] = np.nextafter(array._data[0], np.inf) + else: + array = altered.person.RINT_VAL2_amount.array + assert array._mask[0] + if change == "hidden": + array._data[0] = 17.0 + else: + array._mask[0] = False + assert owner.interest_values_seal(altered) != original + + +@pytest.mark.parametrize("change", ["bit", "validity", "literal"]) +def test_retained_combined_total_requires_exact_amount_and_validity(change): + field = SimpleNamespace(amounts=np.array([140.0]), validity=np.array([1])) + ready = SimpleNamespace(field=lambda name: field) + positions, literals = np.array([0]), ["140"] + assert owner._compare_total(ready, positions, literals) is field + if change == "bit": + field.amounts[0] = np.nextafter(140.0, np.inf) + elif change == "validity": + field.validity[0] = 0 + else: + literals[0] = "-1" + with pytest.raises(ValueError, match="TOTAL_"): + owner._compare_total(ready, positions, literals) + + +def interest_arguments(tmp_path, monkeypatch): + """Amend original invented bytes before any preparation is issued.""" + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + amounts, ages = people.INT_VAL.to_numpy(copy=True), people.A_AGE.to_numpy(copy=True) + literals = { + 105: row(), + 106: row( + A_AGE="14", + INT_VAL="0", + INT_YN="0", + TRDINT_VAL="0", + RINT_YN="0", + RINT_SC1="0", + RINT_VAL1="0", + ), + 107: row( + A_AGE="70", + INT_VAL="500", + TRDINT_VAL="0", + RINT_VAL1="400", + RINT_SC2="7", + RINT_VAL2="200", + I_INTVAL="11", + TRINT_VAL1="1", + ), + 108: row( + INT_VAL="0", + INT_YN="2", + TRDINT_VAL="0", + RINT_YN="2", + RINT_SC1="0", + RINT_VAL1="0", + ), + } + for person_id, values in literals.items(): + amounts[ids == person_id], ages[ids == person_id] = ( + int(values["INT_VAL"]), + int(values["A_AGE"]), + ) + _changed_parent( + parent_path, attachment, monkeypatch, {"INT_VAL": amounts, "A_AGE": ages} + ) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + raw["INT_VAL"] = [str(int(updated.loc[key, "INT_VAL"])) for key in raw.PERIDNUM] + raw["A_AGE"] = [str(int(updated.loc[key, "A_AGE"])) for key in raw.PERIDNUM] + for name in ( + set(owner.READ_COLUMNS) + - set(owner.routing.COORDINATE_COLUMNS) + - {"INT_VAL"} + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row())[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "interest-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(tmp_path, monkeypatch): + return owner.routing.source.prepare_authenticated_survey_population( + **interest_arguments(tmp_path, monkeypatch) + ) + + +def test_actual_owner_join_keeps_source_total_and_requalifies(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + result = owner.qualify_current_asec_interest(parent) + values = result.person.set_index("native_person_id") + assert values.loc[107, "INT_VAL_amount"] == 500 + assert values.loc[107, "combined_minus_published_components"] == -100 + assert values.loc[107, "TRINT_VAL1_code"] == 1 + assert values.loc[105, "RINT_SC1_label"] == "Regular IRA" + assert values.loc[106, "INT_VAL_reporting_status"] == "outside_reporting_universe" + assert values.loc[108, "RINT_VAL2_reporting_status"] == "known_nonreceipt" + assert owner.interest_values_seal( + owner.qualify_current_asec_interest(parent) + ) == owner.interest_values_seal(result) + assert ( + not result.evidence["tax_treatment_assigned"] + and not result.evidence["source_admission_issued"] + ) + assert result.evidence == json.loads(json.dumps(result.evidence)) + with pytest.raises(ValueError): + owner.qualify_current_asec_interest(copy.copy(parent)) + + # The actual qualifier checks the retained MoneyDomain before source capture. + check, checked = owner._domain_agreement, [] + + def rejected_domain(ready): + check(ready) + checked.append(True) + raise ValueError("INVENTED_DOMAIN_REFUSAL") + + def forbidden_capture(*args): + pytest.fail("source capture preceded live domain agreement") + + with monkeypatch.context() as patch: + patch.setattr(owner, "_domain_agreement", rejected_domain) + patch.setattr(owner, "_capture_member", forbidden_capture) + with pytest.raises(ValueError, match="INVENTED_DOMAIN_REFUSAL"): + owner.qualify_current_asec_interest(parent) + assert checked == [True] + + +@pytest.mark.parametrize("coordinate", ["PH_SEQ", "A_LINENO", "A_AGE", "PERIDNUM"]) +def test_actual_owner_refuses_changed_source_coordinates( + tmp_path, monkeypatch, coordinate +): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if coordinate == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(coordinate)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_interest(parent) + + +def test_actual_owner_refuses_mutation_after_capture_io(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + called = [] + + def changed(*args): + raw = capture(*args) + parent._checked()[2].frame.person.iloc[ + 0, parent._checked()[2].frame.person.columns.get_loc("age") + ] += 1 + called.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_interest(parent) + assert called + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "literal", "evidence"] +) +def test_final_physical_seal_covers_complete_returned_values( + tmp_path, monkeypatch, mutation +): + parent = prepared(tmp_path, monkeypatch) + seal, calls = owner.interest_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + if mutation == "amount_bit": + column = value.person["INT_VAL_amount"].array + i = np.flatnonzero(~column._mask)[0] + column._data[i] = np.nextafter(column._data[i], np.inf) + elif mutation == "masked_backing": + column = value.person["RINT_VAL2_amount"].array + i = np.flatnonzero(column._mask)[0] + column._data[i] = 17.0 + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("RINT_SC1") + ] = "7" + else: + value.evidence["tax_treatment_assigned"] = True + return before + + monkeypatch.setattr(owner, "interest_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_interest(parent) + + +def test_default_reader_and_explicit_original_roster_agree(tmp_path): + from test_us_current_asec_income_routing import _member_row, _write_member + + path = tmp_path / "routing.csv" + _write_member(path, [_member_row()]) + default = owner.routing._read_capture(path, rows=1) + explicit = owner.routing._read_capture( + path, + rows=1, + columns=owner.routing.READ_COLUMNS, + patterns=owner.routing.amount_patterns(), + ) + pd.testing.assert_frame_equal(default, explicit) diff --git a/packages/microcosm-build/tests/test_us_current_asec_property_basis.py b/packages/microcosm-build/tests/test_us_current_asec_property_basis.py new file mode 100644 index 000000000..db4500db2 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_property_basis.py @@ -0,0 +1,552 @@ +"""Pure invented descriptions; no source files, qualification, models or engines.""" + +import copy + +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from microcosm.build.us_runtime.current_asec_property_basis import ( + build_asec_property_basis, +) +from microcosm.build.us_runtime.property_income_constants import ( + PROPERTY_COMPONENTS, + PROPERTY_REPORTED_TOTAL, +) + + +def code(frame, name, values, statuses=None): + frame[name + "_code"] = pd.array(values, dtype="Int16") + frame[name + "_literal_status"] = pd.array( + statuses or ["in_printed_range"] * len(values), dtype="string" + ) + + +def amount(frame, name, values, statuses=None): + if statuses is None: + statuses = ["known_nonreceipt" if v == 0 else "known_receipt" for v in values] + frame[name + "_published_amount"] = pd.array(values, dtype="Float64") + frame[name + "_literal_status"] = pd.array( + ["in_printed_range"] * len(values), dtype="string" + ) + frame[name + "_reporting_status"] = pd.array(statuses, dtype="string") + known = np.isin( + statuses, ["known_receipt", "known_nonreceipt", "observed_zero_component"] + ) + frame[name + "_amount_known"] = known + frame[name + "_amount"] = frame[name + "_published_amount"].where(known) + + +def inputs(n=3): + index = pd.Index(np.arange(n, dtype=np.int64) + 101, name="person_id") + base = pd.DataFrame( + { + "native_person_id": np.arange(n, dtype=np.int64) + 901, + "source_age": np.full(n, 40.0), + }, + index=index, + ) + interest, routing, dividend = (base.copy() for _ in range(3)) + amount(interest, "INT_VAL", [100.0] * n) + amount(interest, "TRDINT_VAL", [100.0] * n) + code(interest, "RINT_YN", [2] * n) + for k in (1, 2): + amount(interest, f"RINT_VAL{k}", [0.0] * n) + code(interest, f"RINT_SC{k}", [0] * n) + interest["TRINT_VAL1_literal"] = pd.array(["0"] * n, dtype="string") + interest["allocation_origin"] = pd.array( + ["publisher_allocated"] * n, dtype="string" + ) + routing["net_property_known_amount"] = np.full(n, -110.0) + routing["net_property_reporting_status"] = pd.array( + ["known_receipt"] * n, dtype="string" + ) + code(routing, "other_income_receipt", [2] * n) + code(routing, "other_income_category", [0] * n) + routing["other_income_reporting_status"] = pd.array( + ["known_nonreceipt"] * n, dtype="string" + ) + routing["other_income_routing_status"] = pd.array( + ["niu_category"] * n, dtype="string" + ) + amount(dividend, "DIV_VAL", [0.0] * n) + code(dividend, "DIV_YN", [2] * n) + code(dividend, "SUR_YN", [2] * n) + code(dividend, "SUR_SC1", [0] * n) + code(dividend, "SUR_SC2", [0] * n) + dividend["survivor_property_route_clear"] = pd.array([True] * n, dtype="boolean") + membership = pd.Series( + np.arange(n, dtype=np.int64) // 2 + 41, index=index, name="household_id" + ) + households = np.unique(membership) + weights = pd.Series( + np.arange(len(households), dtype=np.float64) * 3 + 2, + index=pd.Index(households, name="household_id"), + ) + return dict( + interest=interest, + income_routing=routing, + dividend=dividend, + original_household_membership=membership, + original_household_design_weights=weights, + ) + + +@pytest.mark.parametrize("rental,total", [(-110, -10), (-100, 0), (-40, 60), (50, 150)]) +def test_signed_anchors_preserve_positive_interest_and_losses(rental, total): + source = inputs() + source["income_routing"]["net_property_known_amount"] = float(rental) + before = copy.deepcopy(source) + result = build_asec_property_basis(**source) + assert result.person[PROPERTY_COMPONENTS[0]].eq(100).all() + assert result.person[PROPERTY_COMPONENTS[3]].eq(rental).all() + assert result.person[PROPERTY_REPORTED_TOTAL].eq(total).all() + assert result.person.joint_component_fit_eligible.all() + assert result.person.retirement_structural_zero_slot_count.eq(0).all() + assert ( + result.provenance["interest.allocation_origin"].eq("publisher_allocated").all() + ) + for name in ("interest", "income_routing", "dividend"): + assert_frame_equal(source[name], before[name]) + result.provenance.iloc[0, 0] = -9 + assert source["interest"].iloc[0, 0] == 901 + + +def test_named_unused_slot_derivation_keeps_published_niu_unknown(): + source = inputs(1) + i = source["interest"] + code(i, "RINT_YN", [1]) + code(i, "RINT_SC1", [4]) + amount(i, "RINT_VAL1", [20]) + amount(i, "RINT_VAL2", [0], ["unreported_account_slot"]) + amount(i, "INT_VAL", [120]) + result = build_asec_property_basis(**source) + row = result.person.iloc[0] + assert row.property_retirement_interest == 20 + assert row.joint_component_fit_eligible + assert row.retirement_structural_zero_slot_count == 1 + assert row.retirement_slot2_derived_unused_zero + assert ( + row.retirement_interest_derivation + == "declared_account_sum_with_unused_slot_zero" + ) + assert pd.isna(result.provenance.iloc[0]["interest.RINT_VAL2_amount"]) + assert result.provenance.iloc[0]["interest.RINT_VAL2_published_amount"] == 0 + + +@pytest.mark.parametrize( + "case", + [ + "zero_declared", + "missing_code", + "missing_amount", + "all_unused", + "niu_receipt", + "no_with_account", + ], +) +def test_unresolved_retirement_never_completes_unknown_slots(case): + source = inputs(1) + i = source["interest"] + code(i, "RINT_YN", [1]) + code(i, "RINT_SC1", [4]) + amount(i, "RINT_VAL1", [20]) + amount(i, "RINT_VAL2", [0], ["unreported_account_slot"]) + if case == "zero_declared": + code(i, "RINT_SC2", [4]) + amount(i, "RINT_VAL2", [0], ["ambiguous_recipient_zero"]) + elif case == "missing_code": + code(i, "RINT_SC2", [None], ["missing"]) + elif case == "missing_amount": + amount(i, "RINT_VAL2", [np.nan], ["missing_amount"]) + elif case == "all_unused": + code(i, "RINT_SC1", [0]) + amount(i, "RINT_VAL1", [0], ["unreported_account_slot"]) + elif case == "niu_receipt": + code(i, "RINT_YN", [0]) + else: + code(i, "RINT_YN", [2]) + row = build_asec_property_basis(**source).person.iloc[0] + assert np.isnan(row.property_retirement_interest) + assert row.reported_total_eligible + assert not row.joint_component_fit_eligible + assert row.retirement_structural_zero_slot_count == 0 + + +def test_two_active_accounts_sum_without_a_tax_label(): + source = inputs(1) + i = source["interest"] + code(i, "RINT_YN", [1]) + for n, value in ((1, 20), (2, 30)): + code(i, f"RINT_SC{n}", [n]) + amount(i, f"RINT_VAL{n}", [value]) + amount(i, "INT_VAL", [150]) + result = build_asec_property_basis(**source) + assert result.person.property_retirement_interest.iloc[0] == 50 + assert result.person.joint_component_fit_eligible.iloc[0] + assert ( + result.person.retirement_interest_derivation.iloc[0] == "declared_account_sum" + ) + + +def test_reported_aggregate_discrepancy_is_unprojected_and_excluded(): + source = inputs(1) + amount(source["interest"], "INT_VAL", [110]) + result = build_asec_property_basis(**source) + row = result.person.iloc[0] + assert row.property_reported_total == 0 + assert row.property_component_sum == -10 + assert row.interest_component_discrepancy == 10 + assert row.reported_minus_component_total == 10 + assert row.reported_total_eligible and not row.joint_component_fit_eligible + assert result.exclusions.interest_discrepancy_nonzero.iloc[0] + assert not any("cause" in column for column in result.person) + + +@pytest.mark.parametrize( + "receipt,category,status,route,clear,overlap", + [ + (1, 20, "known_receipt", "reported_category", True, False), + (1, 2, "ambiguous_recipient_zero", "reported_category", True, False), + (1, 5, "known_receipt", "reported_category", False, True), + (1, 8, "known_receipt", "reported_category", False, True), + (1, 19, "known_receipt", "reported_category", False, False), + (1, 0, "known_receipt", "receipt_without_category", False, False), + (2, 5, "known_nonreceipt", "category_without_receipt", False, False), + (0, 0, "niu", "niu_category", False, False), + ( + None, + None, + "missing_receipt_literal", + "missing_category_literal", + False, + False, + ), + ], +) +def test_other_income_routes_are_strict_and_never_added( + receipt, category, status, route, clear, overlap +): + source = inputs(1) + r = source["income_routing"] + code( + r, + "other_income_receipt", + [receipt], + ["missing" if receipt is None else "in_printed_range"], + ) + code( + r, + "other_income_category", + [category], + ["missing" if category is None else "in_printed_range"], + ) + r["other_income_reporting_status"] = status + r["other_income_routing_status"] = route + result = build_asec_property_basis(**source) + assert bool(result.person.reported_total_eligible.iloc[0]) == clear + assert bool(result.exclusions.other_income_possible_property.iloc[0]) == overlap + assert result.person.property_reported_total.iloc[0] == -10 + + +@pytest.mark.parametrize( + "receipt,first,second,declared", + [ + (1, 1, 0, True), + (1, 0, 9, True), + (1, 8, 0, False), + (1, 8, None, False), + (1, 10, 0, None), + (1, 1, None, None), + (1, 0, 0, None), + (2, 0, 0, True), + (2, 8, 0, None), + (0, 0, 0, None), + ], +) +def test_survivor_routes_keep_overlap_and_unknown_exclusions( + receipt, first, second, declared +): + source = inputs(1) + d = source["dividend"] + for name, value in (("SUR_YN", receipt), ("SUR_SC1", first), ("SUR_SC2", second)): + code(d, name, [value], ["missing" if value is None else "in_printed_range"]) + d["survivor_property_route_clear"] = pd.array([declared], dtype="boolean") + result = build_asec_property_basis(**source) + full_clear = receipt == 2 and declared is True + assert bool(result.person.reported_total_eligible.iloc[0]) is full_clear + assert bool(result.person.joint_component_fit_eligible.iloc[0]) is full_clear + assert bool(result.person.survivor_visible_routes_clear.iloc[0]) is ( + declared is True + ) + assert bool(result.exclusions.survivor_additional_sources_unresolved.iloc[0]) is ( + receipt == 1 + ) + assert bool(result.exclusions.survivor_possible_property.iloc[0]) is ( + declared is False + ) + + +def test_visible_survivor_clearance_does_not_clear_unobserved_extra_sources(): + source = inputs(3) + d = source["dividend"] + code(d, "SUR_YN", [1, 1, 2]) + code(d, "SUR_SC1", [1, 8, 0]) + d["survivor_property_route_clear"] = pd.array([True, False, True], dtype="boolean") + before = d.copy(deep=True) + result = build_asec_property_basis(**source) + assert result.person.survivor_visible_routes_clear.tolist() == [True, False, True] + assert result.person.survivor_full_scope_clear.tolist() == [False, False, True] + assert result.person.joint_component_fit_eligible.tolist() == [False, False, True] + assert result.person.property_reported_total.eq(-10).all() + reason = result.summary.loc["excluded:survivor_additional_sources_unresolved"] + assert reason.person_count == 2 and reason.household_count == 1 + assert reason.design_weighted_person_mass == 4 + assert reason.union_household_design_mass == 2 + assert result.exclusions.survivor_possible_property.tolist() == [False, True, False] + assert_frame_equal(source["dividend"], before) + assert_frame_equal( + result.provenance.filter(like="dividend."), before.add_prefix("dividend.") + ) + + +@pytest.mark.parametrize( + "axis", ["person", "native", "age", "duplicate_native", "duplicate_person"] +) +def test_both_identity_axes_and_age_must_match_exactly(axis): + source = inputs() + d = source["dividend"] + if axis == "person": + source["dividend"] = d.iloc[::-1] + elif axis == "native": + d["native_person_id"] = d.native_person_id.to_numpy()[::-1] + elif axis == "age": + d.iloc[0, d.columns.get_loc("source_age")] = 41 + elif axis == "duplicate_native": + d["native_person_id"] = 901 + else: + d.index = pd.Index([101, 101, 103], name="person_id") + with pytest.raises(ValueError): + build_asec_property_basis(**source) + + +def test_joint_permutation_and_household_weight_order_preserve_result(): + source = inputs() + original = build_asec_property_basis(**source) + for key in ( + "interest", + "income_routing", + "dividend", + "original_household_membership", + ): + source[key] = source[key].iloc[[2, 0, 1]] + source["original_household_design_weights"] = source[ + "original_household_design_weights" + ].iloc[::-1] + permuted = build_asec_property_basis(**source) + assert_frame_equal(original.person, permuted.person.loc[original.person.index]) + assert_frame_equal(original.summary, permuted.summary) + + +def test_weighted_exclusions_report_person_and_union_household_mass(): + source = inputs() + source["interest"].loc[101, "INT_VAL_amount"] = 110 + result = build_asec_property_basis(**source) + all_rows = result.summary.loc["all"] + excluded = result.summary.loc["excluded_joint_component_fit"] + assert all_rows.design_weighted_person_mass == 9 + assert all_rows.union_household_design_mass == 7 + assert excluded.person_count == 1 + assert excluded.design_weighted_person_mass == 2 + assert excluded.union_household_design_mass == 2 + + +@pytest.mark.parametrize( + "case", + [ + "extra_household", + "missing_household", + "duplicate_household", + "bad_person_order", + "negative_weight", + "infinite_weight", + ], +) +def test_only_exact_original_household_design_mapping_is_admitted(case): + source = inputs() + weights = source["original_household_design_weights"] + if case == "extra_household": + weights.loc[999] = 4 + elif case == "missing_household": + source["original_household_design_weights"] = weights.iloc[:1] + elif case == "duplicate_household": + weights.index = pd.Index([41, 41], name="household_id") + elif case == "bad_person_order": + source["original_household_membership"] = source[ + "original_household_membership" + ].iloc[::-1] + else: + weights.iloc[0] = -1 if case == "negative_weight" else np.inf + with pytest.raises(ValueError): + build_asec_property_basis(**source) + + +def test_empty_and_zero_weight_records_are_retained(): + empty = build_asec_property_basis(**inputs(0)) + assert len(empty.person) == 0 + assert empty.summary.person_count.eq(0).all() + source = inputs() + source["original_household_design_weights"].iloc[:] = 0 + result = build_asec_property_basis(**source) + assert len(result.person) == 3 and result.person.joint_component_fit_eligible.all() + assert result.summary.design_weighted_person_mass.eq(0).all() + + +def test_under15_unknowns_are_excluded_without_analytic_zero(): + source = inputs(1) + for key in ("interest", "income_routing", "dividend"): + source[key]["source_age"] = 14.0 + i = source["interest"] + for name in ("INT_VAL", "TRDINT_VAL", "RINT_VAL1", "RINT_VAL2"): + amount(i, name, [0], ["outside_reporting_universe"]) + code(i, "RINT_YN", [0]) + d = source["dividend"] + amount(d, "DIV_VAL", [0], ["outside_reporting_universe"]) + code(d, "DIV_YN", [0]) + code(d, "SUR_YN", [0]) + d["survivor_property_route_clear"] = pd.array([None], dtype="boolean") + r = source["income_routing"] + r["net_property_known_amount"] = np.nan + r["net_property_reporting_status"] = "outside_reporting_universe" + result = build_asec_property_basis(**source) + assert result.person[list(PROPERTY_COMPONENTS)].isna().all().all() + assert result.exclusions.under15.iloc[0] + assert not result.person.reported_total_eligible.iloc[0] + + +@pytest.mark.parametrize( + "case", + [ + "nonfinite", + "negative_interest", + "false_knownness", + "dividend_no_positive", + "rent_receipt_zero", + "survivor_false_clear", + "overflow", + ], +) +def test_invalid_descriptions_and_overflow_refuse(case): + source = inputs(1) + if case in ("nonfinite", "negative_interest"): + source["interest"].loc[101, "TRDINT_VAL_amount"] = ( + np.inf if case == "nonfinite" else -1 + ) + elif case == "false_knownness": + source["interest"]["INT_VAL_amount_known"] = False + elif case == "dividend_no_positive": + amount(source["dividend"], "DIV_VAL", [10]) + elif case == "rent_receipt_zero": + source["income_routing"]["net_property_known_amount"] = 0.0 + elif case == "survivor_false_clear": + source["dividend"]["survivor_property_route_clear"] = pd.array( + [False], dtype="boolean" + ) + else: + amount(source["interest"], "INT_VAL", [1.7e308]) + source["income_routing"]["net_property_known_amount"] = 1.7e308 + with pytest.raises((ValueError, FloatingPointError)): + build_asec_property_basis(**source) + + +def test_weight_summaries_are_stable_under_widely_different_weight_permutation(): + source = inputs() + source["original_household_membership"] = pd.Series( + [41, 42, 43], index=source["interest"].index, dtype="int64" + ) + source["original_household_design_weights"] = pd.Series( + [1e16, 1.0, 1.0], index=pd.Index([41, 42, 43], name="household_id") + ) + original = build_asec_property_basis(**source) + for key in ( + "interest", + "income_routing", + "dividend", + "original_household_membership", + ): + source[key] = source[key].iloc[::-1] + reordered = build_asec_property_basis(**source) + assert_frame_equal(original.summary, reordered.summary) + assert original.summary.loc["all", "design_weighted_person_mass"] == 1e16 + 2 + + +def test_fully_qualified_nonreceipt_zeros_remain_eligible(): + source = inputs(1) + for name in ("INT_VAL", "TRDINT_VAL"): + amount(source["interest"], name, [0]) + source["income_routing"]["net_property_known_amount"] = 0.0 + source["income_routing"]["net_property_reporting_status"] = "known_nonreceipt" + row = build_asec_property_basis(**source).person.iloc[0] + assert row[list(PROPERTY_COMPONENTS)].eq(0).all() + assert row.property_reported_total == 0 and row.joint_component_fit_eligible + + +@pytest.mark.parametrize("field", ["DIV_VAL", "RNT_VAL"]) +def test_receipt_yes_zero_stays_unknown_instead_of_eligible(field): + source = inputs(1) + if field == "DIV_VAL": + amount(source["dividend"], field, [0], ["ambiguous_recipient_zero"]) + code(source["dividend"], "DIV_YN", [1]) + else: + source["income_routing"]["net_property_known_amount"] = np.nan + source["income_routing"]["net_property_reporting_status"] = ( + "receipt_with_net_zero" + ) + result = build_asec_property_basis(**source) + assert np.isnan(result.person.property_reported_total.iloc[0]) + assert not result.person.reported_total_eligible.iloc[0] + assert not result.person.joint_component_fit_eligible.iloc[0] + + +def test_actual_pure_projector_column_composition_with_invented_literals(): + from test_us_current_asec_income_routing import _pure_rows, _set_amount + + from microcosm.build.us_runtime import current_asec_dividend_source as dividend + from microcosm.build.us_runtime import current_asec_income_routing_source as routing + from microcosm.build.us_runtime import current_asec_interest_source as interest + + source = inputs(1) + raw_interest = {name: "0" for name in interest.READ_COLUMNS} + raw_interest.update( + A_AGE="40", + INT_YN="1", + INT_VAL="120", + TRDINT_VAL="100", + RINT_YN="1", + RINT_SC1="4", + RINT_VAL1="20", + ) + raw_dividend = {name: "0" for name in dividend.READ_COLUMNS} + raw_dividend.update(A_AGE="40", DIV_YN="2", SUR_YN="2") + raw_routing, ages = _pure_rows(1, [40], RNT_YN=["1"], OI_YN=["2"]) + _set_amount(raw_routing, "RNT_VAL", [-120]) + projected = { + "interest": interest.project_interest_literals(pd.DataFrame([raw_interest])), + "income_routing": routing.project_income_routing(raw_routing, ages), + "dividend": dividend.project_dividend_literals(pd.DataFrame([raw_dividend])), + } + for name, frame in projected.items(): + frame.index = source[name].index + frame["native_person_id"] = source[name].native_person_id + source[name] = frame + result = build_asec_property_basis(**source) + row = result.person.iloc[0] + assert row.property_reported_total == 0 + assert row[list(PROPERTY_COMPONENTS)].tolist() == [100, 20, 0, -120] + assert row.joint_component_fit_eligible + assert row.retirement_slot2_derived_unused_zero + assert ( + result.provenance.iloc[0]["interest.RINT_VAL2_reporting_status"] + == "unreported_account_slot" + ) diff --git a/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py b/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py new file mode 100644 index 000000000..b98fef380 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py @@ -0,0 +1,693 @@ +"""Pure retirement candidate support over actual projectors and invented literals.""" + +import dataclasses +import importlib +import json + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_income_routing_source as routing +from microcosm.build.us_runtime import current_asec_retirement_detail_source as detail + + +@pytest.fixture +def owner(): + return importlib.import_module( + "microcosm.build.us_runtime.current_asec_retirement_basis" + ) + + +def options(owner, **changes): + return owner.RetirementCandidateAssumptions( + **{ + "pension_annuity_regularity": "unresolved", + "disability_pension_eligibility": "unresolved", + "survivor_annuity_overlap": "unresolved", + "withdrawal_regularity_netting": "unresolved", + "aggregate_accounting": "exact_visible_balance_only", + **changes, + } + ) + + +def inputs(owner, changes=({},), **assumptions): + rows = [] + for i, change in enumerate(changes): + row = { + **{n: "0" for n in detail.READ_COLUMNS}, + **{n: "0" for n in routing.RECEIPT_ENTRIES}, + **{n: "0" for n in routing.ACCOUNT_ENTRIES}, + **{n: "0" for n in routing.AMOUNT_FIELDS}, + "PERIDNUM": str(i + 1).zfill(22), + "PH_SEQ": str(i // 2 + 1), + "A_LINENO": str(i % 2 + 1), + "A_AGE": "40", + "OI_OFF": "0", + "PEN_YN": "2", + "DIS_YN": "2", + "SUR_YN": "2", + "ANN_YN": "2", + "DST_YN_YNG": "2", + "OI_YN": "2", + **change, + } + rows.append(row) + raw = pd.DataFrame(rows) + ages = np.array([int(r["A_AGE"]) for r in rows], dtype="int64") + values, statuses = {}, {} + for n in routing.AMOUNT_FIELDS: + data, labels = [], [] + for r in rows: + token = r[n] + value = np.nan if token == "" else float(token) + code = ( + routing.money.CodebookStatus.MISSING_NULL + if token == "" + else routing.money.CodebookStatus.DECLARED_NIU + if n == "ANN_VAL" and value == -1 + else routing.money.CodebookStatus.ZERO_NONE_OR_NIU + if value == 0 + else routing.money.CodebookStatus.AMOUNT_NONZERO + ) + data.append( + 0.0 if code == routing.money.CodebookStatus.DECLARED_NIU else value + ) + labels.append(int(code)) + values[n], statuses[n] = ( + np.array(data, dtype="float64"), + np.array(labels, dtype="u1"), + ) + arrays = { + "amounts": values, + "statuses": statuses, + "allocations": { + n: ([0] * len(rows), ["in_printed_range"] * len(rows)) + for n in routing.ALLOCATION_ENTRIES + }, + **{ + n: [r[n] for r in rows] + for n in (*routing.RECEIPT_ENTRIES, *routing.ACCOUNT_ENTRIES, "OI_OFF") + }, + } + projected = { + "retirement_detail": detail.project_retirement_detail_literals(raw), + "income_routing": routing.project_income_routing(arrays, ages), + } + index = pd.Index(np.arange(len(rows), dtype="int64") + 2**53 + 19, name="person_id") + for table in projected.values(): + table.index = index + table["native_person_id"] = np.arange(len(rows), dtype="int64") + 17 + membership = pd.Series(np.arange(len(rows), dtype="int64") // 2 + 101, index=index) + households = pd.Index(membership.unique(), dtype="int64", name="household_id") + return { + **projected, + "original_household_membership": membership, + "original_household_design_weights": pd.Series( + np.arange(len(households)) + 2.0, index=households + ), + "assumptions": options(owner, **assumptions), + } + + +def build(owner, changes=({},), **assumptions): + return owner.build_asec_retirement_basis(**inputs(owner, changes, **assumptions)) + + +def test_named_options_are_required_canonical_and_never_source_facts(owner): + value = options(owner) + assert json.loads(value.to_bytes())["pension_annuity_regularity"] == "unresolved" + assert ( + dataclasses.replace( + value, pension_annuity_regularity="assume_regular" + ).to_bytes() + != value.to_bytes() + ) + assert ( + dataclasses.replace( + value, disability_pension_eligibility="assume_qualifying" + ).to_bytes() + != value.to_bytes() + ) + with pytest.raises(TypeError): + owner.RetirementCandidateAssumptions() + for field, bad in ( + ("pension_annuity_regularity", True), + ("aggregate_accounting", "clip"), + ("withdrawal_regularity_netting", "all_regular"), + ): + with pytest.raises(ValueError): + options(owner, **{field: bad}) + + +@pytest.mark.parametrize( + "code,route", + [(1, "candidate"), (6, "candidate"), (7, "railroad"), (8, "unresolved")], +) +def test_balanced_pension_routes_and_named_regularity(owner, code, route): + row = {"PEN_YN": "1", "PEN_SC1": str(code), "PEN_VAL1": "100", "PNSN_VAL": "100"} + result = build(owner, [row]) + selected = result.slots.query("family == 'pension' and slot == 1").iloc[0] + assert selected.route == route and selected.source_known_amount == 100 + p = result.person.iloc[0] + assert p.pension_candidate_lower == 0 + assert p.pension_candidate_upper == (0 if code == 7 else 100) + assumed = build( + owner, [row], pension_annuity_regularity="assume_regular" + ).person.iloc[0] + assert assumed.pension_candidate_lower == (100 if code in (1, 6) else 0) + assert not p.point_identified and not p.fiscal_outputs_produced + assert ( + result.slots.query("family == 'pension' and slot == 2") + .iloc[0] + .structural_zero_comparison + ) + + +@pytest.mark.parametrize( + "total,status", + [(150, "additional_scope_unresolved"), (80, "contradictory_accounting")], +) +def test_unbalanced_pension_never_clips_or_allocates_residual(owner, total, status): + p = build( + owner, + [{"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": str(total)}], + ).person.iloc[0] + assert p.pension_accounting_difference == total - 100 + assert p.pension_status == status + assert np.isnan(p.pension_candidate_upper) + assert p.pension_observed_candidate_subtotal == 100 + assert not p.candidate_basis_eligible + + +@pytest.mark.parametrize("code", [1, 5, 8, 9, 10]) +def test_positive_survivor_never_claims_hidden_source_clearance(owner, code): + result = build( + owner, + [ + { + "SUR_YN": "1", + "SUR_SC1": str(code), + "SUR_VAL1": "100", + "SRVS_VAL": "100", + "ANN_YN": "1", + "ANN_VAL": "40", + } + ], + ) + p = result.person.iloc[0] + assert p.survivor_accounting_difference == 0 + assert p.survivor_status == "additional_survivor_scope_unresolved" + assert np.isnan(p.survivor_candidate_upper) and not p.candidate_basis_eligible + assert bool(result.exclusions.survivor_annuity_overlap.iloc[0]) == (code == 9) + assert not p.point_identified + + +@pytest.mark.parametrize( + "code,route", + [ + (2, "candidate"), + (5, "candidate"), + (6, "railroad"), + (9, "other_compensation"), + (10, "unresolved"), + ], +) +def test_disability_does_not_infer_eligibility_from_health_answers(owner, code, route): + row = { + "DIS_YN": "1", + "DIS_SC1": str(code), + "DIS_VAL1": "100", + "DSAB_VAL": "100", + "DIS_HP": "1", + "DIS_CS": "1", + } + result = build(owner, [row]) + p = result.person.iloc[0] + assert ( + result.slots.query("family == 'disability' and slot == 1").iloc[0].route + == route + ) + assert p.disability_candidate_lower == 0 + assert p.disability_candidate_upper == ( + 100 if route in ("candidate", "unresolved") else 0 + ) + changed = build(owner, [{**row, "DIS_HP": "2", "DIS_CS": "2"}]) + pd.testing.assert_frame_equal(result.person, changed.person, check_exact=True) + assumed = build( + owner, [row], disability_pension_eligibility="assume_qualifying" + ).person.iloc[0] + assert assumed.disability_candidate_lower == (100 if route == "candidate" else 0) + + +@pytest.mark.parametrize( + "amount,receipt,known", + [("-1", "0", False), ("0", "1", True), ("0", "2", True), ("40", "1", True)], +) +def test_annuity_niu_zero_and_regular_scenario_are_distinct( + owner, amount, receipt, known +): + source = inputs(owner, [{"ANN_YN": receipt, "ANN_VAL": amount}]) + p = owner.build_asec_retirement_basis(**source).person.iloc[0] + assert np.isfinite(p.annuity_candidate_upper) == known + if known: + assert p.annuity_candidate_lower == 0 and p.annuity_candidate_upper == float( + amount + ) + source["assumptions"] = options( + owner, pension_annuity_regularity="assume_regular" + ) + assert owner.build_asec_retirement_basis(**source).person.iloc[ + 0 + ].annuity_candidate_lower == float(amount) + + +@pytest.mark.parametrize("age", [57, 58]) +def test_distribution_account_is_not_frequency_or_taxability(owner, age): + young = age < 58 + suffix = "_YNG" if young else "" + row = { + "A_AGE": str(age), + "DST_YN_YNG": "1" if young else "0", + "DST_YN": "0" if young else "1", + "DST_SC1" + suffix: "4", + "DST_VAL1" + suffix: "100", + "DBTN_VAL": "0" if young else "100", + } + p = build(owner, [row]).person.iloc[0] + assert p.distribution_candidate_lower == 0 and p.distribution_candidate_upper == 100 + assert p.distribution_account_4_amount == 100 + assert not p.point_identified and not p.fiscal_outputs_produced + + +@pytest.mark.parametrize( + "change", + [ + {"DST_SC1_YNG": ""}, + {"DST_SC1_YNG": "9"}, + {"DST_VAL1": "1"}, + {"DST_YN": "2"}, + {"DST_SC2_YNG": "3"}, + ], +) +def test_distribution_unreadable_or_offroute_composition_stays_unknown(owner, change): + row = {"DST_YN_YNG": "1", "DST_SC1_YNG": "4", "DST_VAL1_YNG": "100", **change} + p = build(owner, [row]).person.iloc[0] + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_basis_eligible + + +@pytest.mark.parametrize( + "code,clear", + [(2, False), (13, False), (19, False), (1, True), (8, True), (20, True)], +) +def test_other_income_not_silently_added(owner, code, clear): + p = build( + owner, [{"OI_YN": "1", "OI_OFF": str(code), "OI_VAL": "100"}] + ).person.iloc[0] + assert bool(p.other_income_scope_clear) is clear + assert p.other_income_reported_amount == 100 + assert ( + (p.retirement_candidate_upper == 0) + if clear + else np.isnan(p.retirement_candidate_upper) + ) + + +@pytest.mark.parametrize("age", [0, 14]) +def test_under15_retains_evidence_without_analytic_zero(owner, age): + result = build( + owner, + [ + { + "A_AGE": str(age), + "PEN_YN": "0", + "ANN_YN": "0", + "DIS_YN": "0", + "SUR_YN": "0", + "DST_YN_YNG": "0", + "OI_YN": "0", + } + ], + ) + p = result.person.iloc[0] + assert all( + np.isnan(p[n]) + for n in p.index + if n.endswith(("_candidate_lower", "_candidate_upper")) + ) + assert result.exclusions.under15.iloc[0] and not p.candidate_basis_eligible + + +@pytest.mark.parametrize( + "change", [{"PEN_YN": ""}, {"PEN_SC1": ""}, {"PEN_VAL1": ""}, {"PEN_VAL1": "0"}] +) +def test_source_unknowns_and_ambiguous_zeros_remain_visible(owner, change): + row = { + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + **change, + } + p = build(owner, [row]).person.iloc[0] + assert np.isnan(p.pension_candidate_upper) and not p.candidate_basis_eligible + + +@pytest.mark.parametrize( + "defect", + [ + "person_order", + "native_id", + "age", + "shared_amount", + "known_mask", + "comparison", + "bad_weight", + "membership", + ], +) +def test_incompatible_descriptions_refuse(owner, defect): + source = inputs(owner, [{}, {}]) + d, r = source["retirement_detail"], source["income_routing"] + if defect == "person_order": + source["income_routing"] = r.iloc[::-1] + elif defect == "native_id": + r.iloc[0, r.columns.get_loc("native_person_id")] += 100 + elif defect == "age": + r.iloc[0, r.columns.get_loc("source_age")] += 1 + elif defect == "shared_amount": + r.iloc[0, r.columns.get_loc("pension_annuity_pension_source_total")] += 1 + elif defect == "known_mask": + d.iloc[0, d.columns.get_loc("PEN_VAL1_amount_known")] = False + elif defect == "comparison": + d.iloc[0, d.columns.get_loc("pension_total_minus_visible_slots")] = 1 + elif defect == "bad_weight": + source["original_household_design_weights"].iloc[0] = np.nan + else: + source["original_household_membership"] = source[ + "original_household_membership" + ].iloc[::-1] + with pytest.raises(ValueError): + owner.build_asec_retirement_basis(**source) + + +def test_exact_permutation_detachment_and_design_mass(owner): + source = inputs( + owner, + [{}, {"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": "80"}, {}], + ) + before = { + k: v.copy(deep=True) + for k, v in source.items() + if isinstance(v, (pd.DataFrame, pd.Series)) + } + result = owner.build_asec_retirement_basis(**source) + assert result.summary.loc["all", "design_weighted_person_mass"] == 7 + assert result.summary.loc["all", "union_household_design_mass"] == 5 + assert ( + result.summary.loc["candidate_basis_eligible", "design_weighted_person_mass"] + == 5 + ) + assert ( + result.summary.loc["candidate_basis_eligible", "union_household_design_mass"] + == 5 + ) + for name in ( + "retirement_detail", + "income_routing", + "original_household_membership", + ): + source[name] = source[name].iloc[::-1] + permuted = owner.build_asec_retirement_basis(**source) + pd.testing.assert_frame_equal( + permuted.person.loc[result.person.index], result.person, check_exact=True + ) + pd.testing.assert_frame_equal(permuted.summary, result.summary, check_exact=True) + result.provenance.iloc[0, 0] = "detached" + for name, original in before.items(): + current = source[name].loc[original.index] + if isinstance(original, pd.DataFrame): + pd.testing.assert_frame_equal(current, original, check_exact=True) + else: + pd.testing.assert_series_equal(current, original, check_exact=True) + + +def test_unknown_account_total_is_preserved_without_candidate_admission(owner): + source = inputs( + owner, [{"DST_YN_YNG": "1", "DST_SC1_YNG": "", "DST_VAL1_YNG": "100"}] + ) + assert source["income_routing"].retirement_distribution_known_amount.iloc[0] == 100 + p = owner.build_asec_retirement_basis(**source).person.iloc[0] + assert p.distribution_source_known_amount == 100 + assert np.isnan(p.distribution_candidate_upper) + assert not p.distribution_account_composition_known + + +def test_allocation_and_disclosure_flags_are_provenance_not_filters(owner): + row = {"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": "100"} + before = build(owner, [row]) + changed = build(owner, [{**row, "I_PENVAL1": "4", "TPEN_VAL1": "1"}]) + pd.testing.assert_frame_equal(before.person, changed.person, check_exact=True) + pd.testing.assert_frame_equal(before.summary, changed.summary, check_exact=True) + assert changed.provenance["retirement_detail.I_PENVAL1_code"].iloc[0] == 4 + + +def test_outside_universe_unreadable_and_contradictory_evidence_stay_distinct(owner): + unresolved = build( + owner, [{"A_AGE": "14", "PEN_YN": "0", "PEN_VAL1": ""}] + ).person.iloc[0] + contradictory = build( + owner, + [ + { + "A_AGE": "14", + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + } + ], + ).person.iloc[0] + assert unresolved.pension_status == "unresolved_outside_reporting_universe" + assert contradictory.pension_status == "contradictory_outside_reporting_universe" + assert np.isnan(unresolved.pension_candidate_upper) and np.isnan( + contradictory.pension_candidate_upper + ) + + +@pytest.mark.parametrize("total", [150, 50]) +def test_distribution_main_aggregate_difference_blocks_candidate_interval(owner, total): + result = build( + owner, + [ + { + "A_AGE": "60", + "DST_YN_YNG": "0", + "DST_YN": "1", + "DST_SC1": "4", + "DST_VAL1": "100", + "DBTN_VAL": str(total), + } + ], + ) + p = result.person.iloc[0] + assert p.distribution_accounting_difference == total - 100 + assert p.distribution_source_known_amount == 100 + assert p.distribution_account_4_amount == 100 + assert ( + result.provenance["retirement_detail.DBTN_VAL_published_amount"].iloc[0] + == total + ) + assert np.isnan(p.distribution_candidate_lower) + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_interval_available and not p.candidate_basis_eligible + assert p.distribution_status == ( + "additional_scope_unresolved" if total > 100 else "contradictory_accounting" + ) + + +@pytest.mark.parametrize("token", ["", "malformed"]) +@pytest.mark.parametrize( + "age,field,retained", + [ + (40, "DST_YN", "receipt_58"), + (40, "DST_SC1", "slot1_account"), + (40, "DST_SC2", "slot2_account"), + (60, "DST_YN_YNG", "receipt_young"), + (60, "DST_SC1_YNG", "slot1_young_account"), + (60, "DST_SC2_YNG", "slot2_young_account"), + ], +) +def test_unreadable_offroute_distribution_literals_prevent_admission( + owner, age, field, retained, token +): + young = age < 58 + suffix = "_YNG" if young else "" + result = build( + owner, + [ + { + "A_AGE": str(age), + "DST_YN_YNG": "1" if young else "0", + "DST_YN": "0" if young else "1", + "DST_SC1" + suffix: "4", + "DST_VAL1" + suffix: "100", + "DBTN_VAL": "0" if young else "100", + field: token, + } + ], + ) + p = result.person.iloc[0] + assert ( + result.provenance[ + "income_routing.retirement_distribution_" + retained + "_literal" + ].iloc[0] + == token + ) + assert not p.distribution_account_composition_known + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_basis_eligible + + +@pytest.mark.parametrize("age,field", [(40, "DST_VAL1"), (60, "DST_VAL1_YNG")]) +def test_unreadable_offroute_distribution_amount_is_not_assumed_zero(owner, age, field): + young = age < 58 + suffix = "_YNG" if young else "" + p = build( + owner, + [ + { + "A_AGE": str(age), + "DST_YN_YNG": "1" if young else "0", + "DST_YN": "0" if young else "1", + "DST_SC1" + suffix: "4", + "DST_VAL1" + suffix: "100", + "DBTN_VAL": "0" if young else "100", + field: "", + } + ], + ).person.iloc[0] + assert not p.distribution_account_composition_known + assert np.isnan(p.distribution_candidate_upper) + + +def test_account_subtotals_are_evidence_when_receipt_interval_is_unknown(owner): + p = build( + owner, [{"DST_YN_YNG": "", "DST_SC1_YNG": "4", "DST_VAL1_YNG": "100"}] + ).person.iloc[0] + assert p.distribution_account_composition_known + assert p.distribution_account_4_amount == 100 + assert np.isnan(p.distribution_source_known_amount) + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_interval_available + + +@pytest.mark.parametrize( + "family,prefix,total,code1,code2,lower,upper,status", + [ + ("pension", "PEN", "PNSN_VAL", 1, 8, 60, 100, "candidate_under_assumptions"), + ("pension", "PEN", "PNSN_VAL", 1, 7, 60, 60, "candidate_under_assumptions"), + ( + "disability", + "DIS", + "DSAB_VAL", + 2, + 10, + 60, + 100, + "candidate_under_assumptions", + ), + ("disability", "DIS", "DSAB_VAL", 2, 1, 60, 60, "candidate_under_assumptions"), + ("disability", "DIS", "DSAB_VAL", 6, 1, 0, 0, "route_outside_only"), + ], +) +def test_two_slot_family_routes_preserve_candidate_and_unresolved_bounds( + owner, family, prefix, total, code1, code2, lower, upper, status +): + p = build( + owner, + [ + { + prefix + "_YN": "1", + prefix + "_SC1": str(code1), + prefix + "_SC2": str(code2), + prefix + "_VAL1": "60", + prefix + "_VAL2": "40", + total: "100", + } + ], + pension_annuity_regularity="assume_regular", + disability_pension_eligibility="assume_qualifying", + ).person.iloc[0] + assert p[family + "_candidate_lower"] == lower + assert p[family + "_candidate_upper"] == upper + assert p[family + "_status"] == status + assert p[family + "_accounting_difference"] == 0 + assert ( + p[family + "_observed_candidate_subtotal"] + + p[family + "_observed_unresolved_subtotal"] + == upper + ) + + +@pytest.mark.parametrize( + "family,total", + [("pension", "PNSN_VAL"), ("disability", "DSAB_VAL"), ("survivor", "SRVS_VAL")], +) +def test_no_receipt_with_nonzero_total_and_zero_slots_remains_contradictory( + owner, family, total +): + p = build(owner, [{total: "100"}]).person.iloc[0] + assert p[family + "_status"] == "contradictory_accounting" + assert p[family + "_accounting_difference"] == 100 + assert p[family + "_observed_candidate_subtotal"] == 0 + assert np.isnan(p[family + "_candidate_upper"]) + assert not p.candidate_basis_eligible + + +def test_equal_bounds_under_explicit_scenarios_do_not_claim_identification(owner): + row = { + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + "DIS_YN": "1", + "DIS_SC1": "2", + "DIS_VAL1": "40", + "DSAB_VAL": "40", + "ANN_YN": "1", + "ANN_VAL": "20", + } + unresolved = build(owner, [row]).person.iloc[0] + p = build( + owner, + [row], + pension_annuity_regularity="assume_regular", + disability_pension_eligibility="assume_qualifying", + ).person.iloc[0] + assert ( + unresolved.retirement_candidate_lower == 0 + and unresolved.retirement_candidate_upper == 160 + ) + assert not unresolved.candidate_point_under_assumptions + assert p.retirement_candidate_lower == p.retirement_candidate_upper == 160 + assert p.candidate_point_under_assumptions and p.candidate_basis_eligible + assert not p.point_identified and not p.fiscal_outputs_produced + + +def test_diagnostic_household_mass_deduplicates_only_selected_households(owner): + row = {"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": "150"} + result = build(owner, [row, row, {}, row, {}]) + selected = result.summary.loc["diagnostic:pension_candidate_unresolved"] + assert selected.person_count == 3 and selected.household_count == 2 + assert selected.design_weighted_person_mass == 7 + assert selected.union_household_design_mass == 5 + + +def test_design_weights_refuse_an_unselected_household(owner): + source = inputs(owner, [{}]) + source["original_household_design_weights"].loc[999] = 4.0 + with pytest.raises(ValueError, match="DESIGN_MEMBERSHIP"): + owner.build_asec_retirement_basis(**source) diff --git a/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py b/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py new file mode 100644 index 000000000..21f6fe1b5 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py @@ -0,0 +1,459 @@ +"""Retirement detail source observations; no tax or regularity model.""" + +import copy +import hashlib +import importlib +import shutil +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + + +@pytest.fixture +def owner(): + return importlib.import_module( + "microcosm.build.us_runtime.current_asec_retirement_detail_source" + ) + + +def row(owner, **changes): + return { + **{name: "0" for name in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + "DIS_YN": "1", + "DIS_SC1": "6", + "DIS_VAL1": "200", + "DSAB_VAL": "200", + "SUR_YN": "1", + "SUR_SC1": "8", + "SUR_VAL1": "300", + "SRVS_VAL": "450", + **changes, + } + + +def project(owner, **changes): + return owner.project_retirement_detail_literals( + pd.DataFrame([row(owner, **changes)]) + ).iloc[0] + + +def test_source_components_and_extra_survivor_total_remain_distinct(owner): + value = project(owner) + assert value.PEN_VAL1_amount == 100 + assert value.DIS_VAL1_amount == 200 + assert value.SUR_VAL1_amount == 300 + assert value.SRVS_VAL_published_amount == 450 + assert value.survivor_total_minus_visible_slots == 150 + assert not value.survivor_visible_slots_exhaustive + assert not value.taxability_assigned + assert not value.regularity_assigned + assert not value.acs_retirement_component_assigned + + +@pytest.mark.parametrize( + "receipt,code,amount,status,known", + [ + ("1", "1", "100", "known_receipt", True), + ("1", "1", "0", "ambiguous_recipient_zero", False), + ("1", "0", "0", "unreported_source_slot", False), + ("1", "0", "100", "contradictory_source_slot", False), + ("2", "0", "0", "known_nonreceipt", True), + ("2", "1", "0", "contradictory_source_slot", False), + ("2", "0", "100", "contradictory_no_nonzero", False), + ("0", "0", "0", "niu", False), + ("", "1", "100", "missing_receipt_literal", False), + ("1", "", "100", "unresolved_source_slot", False), + ], +) +@pytest.mark.parametrize("family", ["PEN", "DIS", "SUR"]) +def test_slot_knownness_requires_receipt_and_source( + owner, family, receipt, code, amount, status, known +): + value = project( + owner, + **{family + "_YN": receipt, family + "_SC1": code, family + "_VAL1": amount}, + ) + assert value[family + "_VAL1_reporting_status"] == status + assert bool(value[family + "_VAL1_amount_known"]) is known + + +@pytest.mark.parametrize("age", ["0", "14"]) +def test_under15_has_no_analytical_zero(owner, age): + value = project(owner, A_AGE=age, PEN_YN="0", PEN_SC1="0", PEN_VAL1="0") + assert value.PEN_VAL1_reporting_status == "outside_reporting_universe" + assert not value.PEN_VAL1_amount_known + + +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing"), + ("-1", "malformed"), + ("1.2", "malformed"), + ("1000000", "malformed"), + ("NaN", "malformed"), + ], +) +def test_new_amount_literal_domains_are_strict(owner, token, status): + value = project(owner, SUR_VAL1=token) + assert value.SUR_VAL1_literal == token + assert value.SUR_VAL1_literal_status == status + assert not value.SUR_VAL1_amount_known + + +def test_public_amount_mapping_is_detached_and_values_immutable(owner): + entries = owner.amount_entries() + expected = entries["SUR_VAL1"] + entries["SUR_VAL1"] = entries["PEN_VAL1"] + assert owner.amount_entries()["SUR_VAL1"] == expected + + +def test_reference_ira_annuity_and_aggregate_do_not_acquire_new_meanings(owner): + value = project(owner, ANN_VAL="-1", DST_VAL1="100", DST_VAL2="50", DBTN_VAL="120") + assert value.ANN_VAL_published_amount == -1 + assert value.distribution_total_minus_main_slots == -30 + assert not value.regularity_assigned and not value.taxability_assigned + + +def test_full_physical_seal_covers_masked_backing_and_metadata(owner): + raw = pd.DataFrame([row(owner, SUR_VAL1="0")]) + result = owner.CurrentAsecRetirementDetailValues( + owner.project_retirement_detail_literals(raw), raw, {} + ) + seal = owner.retirement_detail_values_seal(result) + assert owner.retirement_detail_values_seal(copy.deepcopy(result)) == seal + for kind in ("masked_backing", "mask", "literal", "evidence"): + changed = copy.deepcopy(result) + if kind == "masked_backing": + changed.person.SUR_VAL1_amount.array._data[0] = 17.0 + elif kind == "mask": + changed.person.SUR_VAL1_amount.array._mask[0] = False + elif kind == "literal": + changed.asec_literals.loc[0, "SUR_VAL1"] = "1" + else: + changed.evidence["invented"] = True + assert owner.retirement_detail_values_seal(changed) != seal + + +@pytest.mark.parametrize("field", ["PEN_YN", "PEN_SC1", "PEN_VAL1"]) +def test_unreadable_under15_values_are_unresolved_not_contradictions(owner, field): + value = project(owner, **{"A_AGE": "14", field: ""}) + assert value.PEN_VAL1_reporting_status == "unresolved_outside_reporting_universe" + assert not value.PEN_VAL1_amount_known + + +def test_source_allocation_domains_do_not_inherit_full_header_range(owner): + value = project(owner, I_PENSC1="4", I_PENVAL1="4", TPEN_VAL1="1", I_SURVL2="") + assert value.I_PENSC1_literal_status == "outside_printed_range" + assert value.I_PENVAL1_literal_status == "in_printed_range" + assert value.I_SURVL2_literal_status == "missing" + assert value.PEN_VAL1_amount == 100 + assert value.TPEN_VAL1_code == 1 + assert ( + next(e for e in owner.ALLOCATION_ENTRIES if e[0] == "I_SURVL2")[4] + == "SURV_VAL2 > 0" + ) + + +@pytest.mark.parametrize("change", ["bound", "duplicate", "missing", "new_retained"]) +def test_live_money_roster_is_exact_for_existing_fields_only(owner, change): + entries = owner.amount_entries() + fields = [ + SimpleNamespace( + name=name, + entity="person", + grain="person", + column=name, + minimum=entries[name].encoded_minimum, + maximum=entries[name].encoded_maximum, + zero_semantics=entries[name].zero_semantics, + ) + for name in owner.RETAINED_MONEY_FIELDS + ] + owner._domain_agreement( + SimpleNamespace(bindings=SimpleNamespace(spec=SimpleNamespace(fields=fields))) + ) + if change == "bound": + fields[0].maximum += 1 + elif change == "duplicate": + fields.append(fields[0]) + elif change == "missing": + fields.pop() + else: + fields.append(SimpleNamespace(name="SRVS_VAL")) + with pytest.raises(ValueError): + owner._domain_agreement( + SimpleNamespace( + bindings=SimpleNamespace(spec=SimpleNamespace(fields=fields)) + ) + ) + + +def retirement_arguments(owner, tmp_path, monkeypatch, *, annuity_niu=False): + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + zeros = { + name: "0" + for name in ( + *owner.AMOUNT_FIELDS, + *owner.RECEIPT_ENTRIES, + *owner.SOURCE_ENTRIES, + ) + } + literals = { + 105: row(owner, I_PENVAL1="4", ANN_VAL="-1" if annuity_niu else "0"), + 106: row(owner, **zeros, A_AGE="14"), + 107: row( + owner, + **{**zeros, "A_AGE": "70", "PEN_YN": "2", "DIS_YN": "2", "SUR_YN": "2"}, + ), + 108: row(owner, PEN_VAL1="0", SUR_VAL1="0"), + } + changes = {} + for name in (*owner.RETAINED_MONEY_FIELDS, "A_AGE"): + values = people[name].to_numpy(copy=True) + for person_id, literal in literals.items(): + values[ids == person_id] = int(literal[name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in (*owner.RETAINED_MONEY_FIELDS, "A_AGE"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + for name in ( + set(owner.READ_COLUMNS) + - set(owner.routing.COORDINATE_COLUMNS) + - set(owner.RETAINED_MONEY_FIELDS) + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row(owner))[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "retirement-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(owner, tmp_path, monkeypatch): + return owner.routing.source.prepare_authenticated_survey_population( + **retirement_arguments(owner, tmp_path, monkeypatch) + ) + + +def test_actual_annuity_niu_preserves_literal_and_normalized_owner( + owner, tmp_path, monkeypatch +): + parent = owner.routing.source.prepare_authenticated_survey_population( + **retirement_arguments(owner, tmp_path, monkeypatch, annuity_niu=True) + ) + result = owner.qualify_current_asec_retirement_detail(parent) + values = result.person.set_index("native_person_id") + money = owner.routing.money + assert values.loc[105, "ANN_VAL_literal"] == "-1" + assert values.loc[105, "ANN_VAL_published_amount"] == -1 + assert values.loc[105, "ANN_VAL_parent_validity"] == 1 + assert ( + values.loc[105, "ANN_VAL_parent_statuses"] == money.CodebookStatus.DECLARED_NIU + ) + assert values.loc[105, "ANN_VAL_parent_zero_origin"] == money.ZeroOrigin.NOT_ZERO + assert values.loc[107, "ANN_VAL_literal"] == "0" + assert ( + values.loc[107, "ANN_VAL_parent_statuses"] != money.CodebookStatus.DECLARED_NIU + ) + literals = result.asec_literals + assert literals.loc[literals.ANN_VAL.eq("-1")].shape[0] == 1 + + capture = owner._capture_member + + def substitute_dollar_zero(*args): + raw = capture(*args) + raw.loc[raw.ANN_VAL.eq("-1"), "ANN_VAL"] = "0" + return raw + + monkeypatch.setattr(owner, "_capture_member", substitute_dollar_zero) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + + +@pytest.mark.parametrize("literal,niu_status", [("-1", False), ("0", True)]) +def test_annuity_literal_and_owner_niu_status_must_agree(owner, literal, niu_status): + money = owner.routing.money + status = ( + money.CodebookStatus.DECLARED_NIU + if niu_status + else money.CodebookStatus.ZERO_NONE_OR_NIU + ) + field = money.MoneyField( + "ANN_VAL", + np.array([0.0], dtype=np.float64).tobytes(), + np.array([status], dtype="u1").tobytes(), + np.array([1], dtype="u1").tobytes(), + np.array([0], dtype="u1").tobytes(), + ) + ready = SimpleNamespace(field=lambda name: field) + with pytest.raises(ValueError): + owner._compare_amount( + ready, np.array([0], dtype=np.int64), "ANN_VAL", [literal] + ) + + +def test_actual_owner_qualifies_new_literals_without_fabricating_money_fields( + owner, tmp_path, monkeypatch +): + parent = prepared(owner, tmp_path, monkeypatch) + compare, compared = owner._compare_amount, [] + + def seen(ready, positions, name, literals): + compared.append(name) + assert name not in owner.INDEPENDENT_LITERAL_FIELDS + return compare(ready, positions, name, literals) + + monkeypatch.setattr(owner, "_compare_amount", seen) + result = owner.qualify_current_asec_retirement_detail(parent) + assert tuple(compared) == owner.RETAINED_MONEY_FIELDS + values = result.person.set_index("native_person_id") + assert values.loc[105, "PEN_VAL1_amount"] == 100 + assert values.loc[105, "survivor_total_minus_visible_slots"] == 150 + assert values.loc[107, "SUR_VAL1_amount"] == 0 + assert not values.loc[106, "SUR_VAL1_amount_known"] + assert not values.loc[108, "SUR_VAL1_amount_known"] + assert not result.evidence["source_admission_issued"] + assert not result.evidence["taxability_assigned"] + assert owner.retirement_detail_values_seal( + owner.qualify_current_asec_retirement_detail(parent) + ) == owner.retirement_detail_values_seal(result) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(copy.copy(parent)) + + +@pytest.mark.parametrize("field", ["PERIDNUM", "A_AGE", "DIS_VAL1"]) +def test_actual_owner_requires_retained_identity_and_money( + owner, tmp_path, monkeypatch, field +): + parent = prepared(owner, tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if field == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(field)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + + +def test_actual_owner_pins_new_survivor_dollars_to_original_member( + owner, tmp_path, monkeypatch +): + parent = prepared(owner, tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(root, pin): + # Only the invented fixture is touched. Even an unretained money field + # must match the exact already admitted source-member bytes. + path = root / "asec" / pin[1] + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + raw["SRVS_VAL"] = "999" + raw.to_csv(path, index=False) + return capture(root, pin) + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "literal", "evidence"] +) +def test_actual_owner_final_seal_covers_returned_values( + owner, tmp_path, monkeypatch, mutation +): + parent = prepared(owner, tmp_path, monkeypatch) + seal, calls = owner.retirement_detail_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + array = value.person.SUR_VAL1_amount.array + if mutation == "amount_bit": + i = np.flatnonzero(~array._mask)[0] + array._data[i] = np.nextafter(array._data[i], np.inf) + elif mutation == "masked_backing": + array._data[np.flatnonzero(array._mask)[0]] = 17.0 + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("SUR_YN") + ] = "2" + else: + value.evidence["taxability_assigned"] = True + return before + + monkeypatch.setattr(owner, "retirement_detail_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_retirement_detail(parent) + + +def test_actual_owner_requalifies_parent_after_last_capture( + owner, tmp_path, monkeypatch +): + parent = prepared(owner, tmp_path, monkeypatch) + capture, calls = owner._capture_member, [] + + def changed(*args): + raw = capture(*args) + table = parent._checked()[2].frame.person + table.iloc[0, table.columns.get_loc("age")] += 1 + calls.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + assert calls diff --git a/packages/microcosm-build/tests/test_us_current_property_income_sources.py b/packages/microcosm-build/tests/test_us_current_property_income_sources.py new file mode 100644 index 000000000..351e0e8cb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_property_income_sources.py @@ -0,0 +1,439 @@ +"""Invented original-source composition; no PUF, QRF, native data or engine.""" + +import copy +import hashlib +import json +import shutil +from dataclasses import fields, is_dataclass, replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_property_income_sources as owner +from microcosm.build.us_runtime import graph_survey_population as graph +from microcosm.fit import model_input +from microcosm.frame import Frame, WeightKind, Weights + + +def _detached(value): + # Avoid Python 3.14 copy's __slotnames__ class-cache mutation on core + # slotted types. Real constructors preserve the same physical contents. + if isinstance(value, Frame): + return Frame( + {name: value.table(name).copy(deep=True) for name in value.entities}, + value.schema, + { + name: _detached(value.weights_for(name)) + for name in value.weighted_entities + }, + value.strata.copy(deep=True), + mass_log=value.mass_log, + metadata=value.metadata, + ) + if isinstance(value, Weights): + return Weights(value.values.copy(), value.kind) + if isinstance(value, pd.DataFrame): + return value.copy(deep=True) + if is_dataclass(value): + return type(value)( + **{ + field.name: _detached(getattr(value, field.name)) + for field in fields(value) + } + ) + if type(value) is tuple: + return tuple(_detached(item) for item in value) + if type(value) is list: + return [_detached(item) for item in value] + if type(value) is dict: + return {key: _detached(item) for key, item in value.items()} + return copy.deepcopy(value) + + +def source_arguments(tmp_path, monkeypatch): + import test_us_survey_population_preparation as fixture + from test_us_asec_coverage_authentication import _changed_parent + from test_us_current_asec_income_routing import routing_arguments + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + original = fixture._person + + def person(*args, **kwargs): + row = original(*args, **kwargs) + row.update(FINTP="0", FRETP="0", INTP="100", RETP="0") + if row["SERIALNO"] == "2024HU0000001" and int(row["SPORDER"]) == 1: + row.update(INTP="-800", ADJINC="1000133") + elif row["SERIALNO"] == "2024HU0000002": + row["INTP"] = "bad" + elif row["SERIALNO"] == "2024GQ0000001": + row.update(AGEP="14", INTP="", RETP="", WAGP="", SEMP="") + return row + + monkeypatch.setattr(fixture, "_person", person) + arguments = routing_arguments(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + literals = {} + for pid, age in zip(ids, people.A_AGE, strict=True): + literals[int(pid)] = { + **{ + name: "0" + for name in (*owner.interest.READ_COLUMNS, *owner.dividend.READ_COLUMNS) + }, + "A_AGE": str(int(age)), + "INT_YN": "2" if age >= 15 else "0", + "RINT_YN": "2" if age >= 15 else "0", + "DIV_YN": "2" if age >= 15 else "0", + "SUR_YN": "2" if age >= 15 else "0", + } + literals[105].update( + INT_VAL="140", + TRDINT_VAL="100", + INT_YN="1", + RINT_YN="1", + RINT_SC1="4", + RINT_VAL1="40", + DIV_VAL="140", + DIV_YN="1", + ) + literals[107].update( + INT_VAL="500", + TRDINT_VAL="500", + INT_YN="1", + DIV_VAL="500", + DIV_YN="1", + SUR_YN="1", + SUR_SC2="8", + ) + changes = {} + for name in ("INT_VAL", "DIV_VAL"): + values = people[name].to_numpy(copy=True) + for pid in (105, 106, 107, 108): + values[ids == pid] = int(literals[pid][name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in set( + (*owner.interest.READ_COLUMNS, *owner.dividend.READ_COLUMNS) + ) - set(owner.routing.COORDINATE_COLUMNS): + raw[name] = [ + literals[int(updated.loc[key, "person_id"])][name] + for key in raw.PERIDNUM + ] + # Historical parent values remain their own current-money observations. + for name in ("INT_VAL", "DIV_VAL"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "property-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return {**arguments, "store_root": tmp_path / "property-survey-store"} + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + patch = pytest.MonkeyPatch() + root = tmp_path_factory.mktemp("property-source-composition") + try: + arguments = source_arguments(root, patch) + live = graph.run_authenticated_survey_population( + **arguments, clones=True, return_values=True + ) + frame_seal = owner._frame_seal(live.preparation._checked()[2].frame) + result = owner.qualify_current_property_income_sources( + live.preparation, live.allocated_population, live.clone_population + ) + assert owner._frame_seal(live.preparation._checked()[2].frame) == frame_seal + (root / "composition-metadata.json").write_text( + json.dumps( + { + "scope": "invented source-composition fixture; no model or native data", + "survey_stage_order": list(live.compiled.order), + "source_people": result.source_frame.n("person"), + "source_households": result.source_frame.n("household"), + "allocated_people": live.allocated_population.frame.n("person"), + "clone_people": live.clone_population.frame.n("person"), + "eligible_donors": len(result.donor_columns), + "eligible_recipients": len(result.recipient_columns), + }, + indent=2, + ) + + "\n" + ) + yield live, result, arguments + finally: + patch.undo() + + +def test_actual_source_composition_preserves_design_and_signed_anchor(actual): + live, result, _ = actual + assert len(result.origins) == 9 + assert len(result.donor_columns) == 1 + assert len(result.recipient_columns) == 3 + assert result.donor_frame.weights_for("household").kind is WeightKind.DESIGN + assert ( + live.allocated_population.frame.weights_for("household").kind + is WeightKind.IMPORTANCE + ) + row = result.donor_columns.iloc[0] + assert row.property_ordinary_interest == 100 + assert row.property_retirement_interest == 40 + assert row.property_dividends == 140 + assert row.property_broad_receipts == -400 + assert row.property_reported_total == -120 + selected = result.recipient_columns.property_reported_total + assert (selected < 0).any() + amount = np.float64(-800) * (np.float64(1000133) / 1000000) + anchor = result.recipient_diagnostics + exact_id = anchor.index[ + anchor.SERIALNO.eq("2024HU0000001") & anchor.SPORDER.eq("1") + ][0] + assert selected.loc[exact_id].view("uint64") == amount.view("uint64") + pd.testing.assert_frame_equal( + model_input.decode_recipient_matrix(result.recipient_matrix).features, + result.recipient_columns, + ) + assert result.recipient_diagnostics.excluded_under15.sum() == 1 + assert result.recipient_diagnostics.excluded_unknown_anchor.sum() == 2 + assert result.evidence["source_admission_issued"] is False + assert not result.evidence["model_fitted"] + assert not result.evidence["clone_attachment_performed"] + assert json.loads(result.origin_document)["persons"]["rows"] + + +def _values(result): + return ( + result.shared_predictors, + result.acs_anchor_values, + result.asec_interest_values, + result.asec_routing_values, + result.asec_dividend_values, + ) + + +@pytest.mark.parametrize( + "target", + [ + "asec_person", + "asec_native", + "acs_person", + "acs_native", + "donor_features", + "recipient_matrix", + "origin_document", + ], +) +def test_source_composition_refuses_axis_mismatch(actual, target): + _, result, _ = actual + values = list(_detached(_values(result))) + origin = result.origin_document + if target == "asec_person": + values[2].person.index = values[2].person.index[::-1] + elif target == "asec_native": + values[3].person["native_person_id"] = values[ + 3 + ].person.native_person_id.to_numpy()[::-1] + elif target == "acs_person": + values[1].anchors.index = values[1].anchors.index[::-1] + elif target == "acs_native": + values[1].anchors["native_person_id"] = values[ + 1 + ].anchors.native_person_id.to_numpy()[::-1] + elif target == "donor_features": + values[0].donor_columns.index = values[0].donor_columns.index[::-1] + elif target == "recipient_matrix": + matrix = model_input.decode_recipient_matrix(values[0].matrix) + features = matrix.features.iloc[::-1] + values[0] = replace( + values[0], + matrix=model_input.encode_recipient_matrix( + features, entity="person", entity_ids=features.index.to_numpy() + ), + ) + else: + document = json.loads(origin) + document["persons"]["rows"].reverse() + origin = owner.shared.codec.encode_json(document) + with pytest.raises(ValueError, match="PROPERTY_INCOME_SOURCES"): + owner._compose(tuple(values), origin) + + +@pytest.mark.parametrize( + "target", + [ + "basis", + "nullable_storage", + "donor_columns", + "recipient_columns", + "origin_document", + "source_frame", + "projection", + ], +) +def test_complete_description_seal_detects_every_returned_surface(actual, target): + _, result, _ = actual + before = owner.property_income_sources_seal(result) + changed = _detached(result) + if target == "basis": + changed.donor_basis.person.iloc[ + 0, changed.donor_basis.person.columns.get_loc("property_reported_total") + ] += 1 + elif target == "nullable_storage": + array = changed.acs_anchor_values.anchors.property_income_amount.array + position = np.flatnonzero(array._mask)[0] + array._data[position] = 321.0 + elif target == "donor_columns": + changed.donor_columns.iloc[0, 0] += 1 + elif target == "recipient_columns": + changed.recipient_columns.iloc[0, 0] += 1 + elif target == "origin_document": + changed = replace(changed, origin_document=changed.origin_document + b" ") + elif target == "source_frame": + changed.source_frame.person.iloc[ + 0, changed.source_frame.person.columns.get_loc("age") + ] += 1 + else: + changed = replace(changed, projection=changed.projection + b" ") + assert owner.property_income_sources_seal(changed) != before + assert owner.property_income_sources_seal(result) == before + + +def test_no_eligible_recipient_is_explicit_without_empty_matrix_or_zero_fill(actual): + _, result, _ = actual + values = list(_detached(_values(result))) + anchors = values[1].anchors + anchors["property_income_known"] = False + anchors["property_income_amount"] = pd.array([None] * len(anchors), dtype="Float64") + anchors["property_income_status"] = "missing_source_amount" + rebuilt = owner._compose(tuple(values), result.origin_document) + assert rebuilt.recipient_frame is None and rebuilt.recipient_matrix is None + assert len(rebuilt.recipient_columns) == 0 + assert len(rebuilt.recipient_diagnostics) == 5 + + +@pytest.mark.parametrize("target", ["columns", "geography_config"]) +def test_final_io_cannot_change_already_composed_values(actual, monkeypatch, target): + live, result, _ = actual + # The first test already ran every real source owner. Here the same detached + # values isolate the final real preparation I/O fence; no new issuer or + # source admission is mocked. Any unchecked descriptive replay still has to + # survive the real original-owner check and final full-value seal. + values = list(_detached(_values(result))) + config = None + if target == "geography_config": + config = owner.shared.host.survey_budget.geography.AtomicSurveyReconstruction( + "/invented/support.json", + "a" * 64, + ( + ("district", "invented-district"), + ("population", "invented-population"), + ("puma", "invented-puma"), + ), + 1, + ) + # This is only the option-lifetime fault injection. No support file is + # opened or admitted, and no geography reconstruction is claimed. + values[0] = replace(values[0], geography_config_payload=config.to_bytes()) + for module, name, value in ( + (owner.shared, "qualify_current_survey_predictors", values[0]), + (owner.acs, "qualify_current_acs_income_anchors", values[1]), + (owner.interest, "qualify_current_asec_interest", values[2]), + (owner.routing, "qualify_current_asec_income_routing", values[3]), + (owner.dividend, "qualify_current_asec_dividend", values[4]), + ): + monkeypatch.setattr(module, name, lambda *a, _value=value, **kw: _value) + original = owner._compose + completed = [] + + def compose(*args): + value = original(*args) + completed.append(value) + return value + + monkeypatch.setattr(owner, "_compose", compose) + open_path = Path.open + mutated = [] + + def open_during_final_io(path, *args, **kwargs): + stream = open_path(path, *args, **kwargs) + if completed and not mutated: + if target == "columns": + completed[0].donor_columns.iloc[0, 0] += 1 + else: + object.__setattr__(config, "seed", 2) + mutated.append(str(path)) + return stream + + # Keep the issuer method itself untouched: replacing it is rightly refused + # by its own live-code seal before the intended last-I/O fence is reached. + monkeypatch.setattr(Path, "open", open_during_final_io) + reason = ( + "FINAL_DERIVED_VALUES_CHANGED" + if target == "columns" + else "FINAL_GEOGRAPHY_CONFIG_CHANGED" + ) + with pytest.raises(ValueError, match=reason): + owner.qualify_current_property_income_sources( + live.preparation, + live.allocated_population, + live.clone_population, + geography_config=config, + ) + + assert len(mutated) == 1 + + +def test_unissued_plain_object_refuses_without_source_io(): + with pytest.raises(ValueError, match="PREPARATION_TYPE"): + owner.qualify_current_property_income_sources(object(), None, None) + + +def test_constructor_weight_copy_preserves_source_live_identity(): + before = owner.shared.source._live() + copied = _detached(Weights(np.array([1.0]), WeightKind.DESIGN)) + assert copied.values.tolist() == [1.0] + assert owner.shared.source._live() == before + + +def test_unissued_preparation_copy_refuses_before_composition(actual): + live, _, _ = actual + before = owner.shared.source._live() + # Copy only the descriptive payload into an unissued identity. Avoid the + # unrelated stdlib copy class-cache side effect in this ownership control. + copied = object.__new__(type(live.preparation)) + object.__setattr__(copied, "payload", live.preparation.payload) + with pytest.raises(ValueError): + owner.qualify_current_property_income_sources( + copied, + live.allocated_population, + live.clone_population, + ) + assert owner.shared.source._live() == before diff --git a/packages/microcosm-build/tests/test_us_current_survey_amounts.py b/packages/microcosm-build/tests/test_us_current_survey_amounts.py new file mode 100644 index 000000000..89f92aecb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_amounts.py @@ -0,0 +1,316 @@ +"""Invented reporting bases and clone joins; no native data or country engine.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_unemployment_source as uc +from microcosm.build.us_runtime import current_survey_amounts as values +from microcosm.fit import model_input +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + +@pytest.mark.parametrize( + "amount,age,token,label,canonical", + [ + (100, 40, "1", "known_receipt", 100), + (0, 40, "2", "known_nonreceipt", 0), + (0, 40, "1", "ambiguous_recipient_zero", None), + (0, 40, "0", "niu", None), + (0, 14, "0", "outside_reporting_universe", None), + (0, 14, "2", "contradictory_outside_reporting_universe", None), + (100, 14, "1", "contradictory_outside_reporting_universe", None), + (100, 40, "2", "contradictory_no_positive", None), + (100, 40, "0", "contradictory_niu_positive", None), + (0, 40, "", "missing_receipt_literal", None), + (100, 40, "9", "unrecognized_receipt_literal", None), + (100, 40, "yes", "unrecognized_receipt_literal", None), + (100, 40, "01", "unrecognized_receipt_literal", None), + (np.nan, 40, "1", "missing_amount", None), + ], +) +def test_uc_raw_amount_and_reporting_knownness_are_separate( + amount, age, token, label, canonical +): + basis = uc.reporting_basis( + np.array([amount], dtype="float64"), np.array([age], dtype="float64"), [token] + ) + row = basis.iloc[0] + assert row.reporting_status == label + assert row.receipt_literal == token + assert bool(row.source_reporting_universe) == (age >= 15) + assert bool(row.canonical_amount_known) == (canonical is not None) + if canonical is None: + assert np.isnan(row.canonical_amount) + else: + assert row.canonical_amount == canonical + assert ( + np.isnan(row.source_amount) if np.isnan(amount) else row.source_amount == amount + ) + + +@pytest.mark.parametrize( + "amount,age", [(-1, 40), (100000, 40), (np.inf, 40), (0, -1), (0, 40.5)] +) +def test_uc_refuses_out_of_dictionary_numeric_domain(amount, age): + with pytest.raises(ValueError): + uc.reporting_basis( + np.array([amount], dtype="float64"), np.array([age], dtype="float64"), ["2"] + ) + + +def test_uc_bounded_literal_reader_and_duplicate_join_refusal(tmp_path): + path = tmp_path / "invented.csv" + header = ",".join(uc.READ_COLUMNS) + "\n" + row = "0000000000000000000001,1,1,40,0,2\n" + path.write_text(header + row) + result = uc._read_capture(path, rows=1) + assert result.iloc[0].UC_YN == "2" + path.write_text(header + row + row) + with pytest.raises(ValueError, match="DUPLICATE"): + uc._read_capture(path, rows=2) + path.write_text(header + row.replace(",0,2", ",0," + "9" * 65)) + with pytest.raises(ValueError, match="TOKEN_BOUND"): + uc._read_capture(path, rows=1) + + +def test_money_literal_join_retains_selected_and_unselected_missingness(): + field = uc.money.MoneyField( + "UC_VAL", + np.array([0.0, 0.0, 40.0], dtype=" 0 + assert type(receipt) is MappingProxyType + with pytest.raises(TypeError): + receipt["minimum_age"] = 0 + assert receipt["minimum_age"] == 15 + probe.person.loc[acs_index, "age"] = 15 + probe.person.loc[acs_index, "AGEP"] = 15 + with pytest.raises(ValueError, match="ACS_ELIGIBLE_EARNINGS_UNKNOWN"): + values._acs_earnings(probe) + matrix = graph.model_input.decode_recipient_matrix(qualified.matrix) + invented_draw = pd.DataFrame( + { + target: np.arange(1.0, 1.0 + len(matrix.entity_ids)) * (i + 1.0) + for i, target in enumerate(values.TARGETS) + }, + index=matrix.features.index, + ) + original_join = values.complete_predictor_columns( + qualified, clone.frame, invented_draw + ) + reversed_join = values.complete_predictor_columns( + replace( + qualified, + native_money=qualified.native_money.iloc[::-1], + origins=qualified.origins.iloc[::-1], + ), + clone.frame, + invented_draw, + ) + for name in original_join: + pd.testing.assert_series_equal(original_join[name], reversed_join[name]) + completed_interest = qualified.native_money["INT_VAL"].fillna( + invented_draw[values.TARGETS[0]] + ) + expected_interest = completed_interest.reindex( + clone.frame.person[values.provenance.support_source_id_column("person")] + ).to_numpy() + np.testing.assert_allclose( + original_join["person", "taxable_interest_income"] + + original_join["person", "tax_exempt_interest_income"], + expected_interest, + rtol=1e-15, + atol=0, + ) + assert allocated.frame.resolve_weights("person").kind is WeightKind.IMPORTANCE + if nonconstant: + origins = qualified.origins + ids = origins.index[origins.source.eq("asec")] + # Join the current-money projection by native identity, never row position. + ordered = origins.loc[ids].sort_values("native_person_id").index + assert origins.loc[ordered, "native_person_id"].tolist() == [105, 106, 107, 108] + for field, expected in observations.items(): + np.testing.assert_array_equal( + qualified.native_money.loc[ordered, field], expected + ) + for target in values.TARGETS: + assert qualified.donor_columns[target].nunique() == 3 + weights = qualified.donor_frame.resolve_weights("person") + assert weights.kind is WeightKind.DESIGN and (weights.values > 0).all() + np.testing.assert_array_equal(weights.values, [2552.12, 2552.12, 100.0, 100.0]) + else: + assert qualified.donor_columns.loc[:, list(values.TARGETS)].eq(0).all().all() + pins = {} + for edge in graph.host.current_survey_host_edges(): + receipt = live.manifest.node(edge.producer) + key = receipt.opaque_artifacts[edge.artifact] + pins[edge.name] = { + "producer_key": receipt.key, + "artifact_key": key, + "payload_sha256": graph.codec.sha(live.store.load_bytes(key)), + } + nodes = graph.current_survey_predictor_nodes( + qualified, clone.frame, host_pins=pins, n_estimators=2 + ) + assert nodes[1].base == survey.CREATE_NODE + assert nodes[0].population == graph.DONOR_NODE + assert tuple(edge.name for edge in nodes[1].artifact_inputs) == ("preparation",) + assert all("state" not in c and "female" not in c for c in values.FEATURES) + compiled = compile_graph( + replace(live.compiled.graph, nodes=(*live.compiled.graph.nodes, *nodes)) + ) + # Actual compiler dependencies retain allocation evidence without an + # ordinary node in CREATE depending on its own downstream allocation. + assert graph.PROJECTION_NODE not in compiled.predecessors[survey.ALLOCATION_NODE] + assert graph.DONOR_NODE in compiled.predecessors[graph.PROJECTION_NODE] + assert survey.ALLOCATION_NODE in compiled.predecessors[graph.PROJECTION_NODE] + assert graph.PROJECTION_NODE not in compiled.predecessors[graph.DONOR_NODE] + for cls in ( + graph.CurrentSurveyPredictorProjectionKernel, + graph.CurrentSurveyPredictorDonorFilterKernel, + graph.CurrentSurveyPredictorDonorColumnsKernel, + graph.CurrentSurveyPredictorAttachKernel, + ): + live.kernels.register( + cls(preparation, allocated, clone, host_pins=pins, n_estimators=2) + ) + live.kernels.register(LegacyQRFTrainKernel()) + live.kernels.register(LegacyQRFApplyMatrixKernel()) + artifacts = [ + ( + graph.PROJECTION_NODE, + "projection", + graph.PROJECTION_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ( + graph.PROJECTION_NODE, + "matrix", + graph.model_input.RECIPIENT_MATRIX_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ] + for i in range(3): + artifacts.extend( + [ + ( + f"{graph.FIT_PREFIX}.{i:03d}", + "model", + qrf_target.LEGACY_QRF_TARGET_TYPE, + LegacyQRFTrainKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "raw_draw", + graph.codec.RAW_TARGET_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "apply_state", + graph.MATRIX_APPLY_STATE_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ] + ) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume=resume, + _population_observer=lambda name, population, observed=observed: ( + observed.__setitem__(name, population) + ), + ) + donor_weights = observed[graph.DONOR_NODE].frame.resolve_weights("person") + assert donor_weights.kind is WeightKind.DESIGN + np.testing.assert_array_equal( + donor_weights.values, + qualified.donor_frame.resolve_weights("person").values, + ) + loaded = {} + for node_id, name, type_, cls in artifacts: + receipt = manifest.node(node_id) + key = opaque_artifact_key(receipt.key, name) + assert receipt.opaque_artifacts[name] == key + payload = live.store.load_bytes(key) + survey._final_artifact( + manifest, + live.store, + node_id=node_id, + name=name, + type_=type_, + payload=payload, + capabilities=cls.capabilities, + ) + loaded[node_id, name] = payload + for i, target in enumerate(values.TARGETS): + fit_id = f"{graph.FIT_PREFIX}.{i:03d}" + payload = loaded[fit_id, "model"] + # Actual typed producer/store verification above precedes pickle loading. + artifact = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=graph.codec.sha(payload) + ) + assert artifact.target == target + assert artifact._target_model.columns == ( + *values.FEATURES, + *values.TARGETS[:i], + ) + expected_regime = ( + qrf.Regime.ZERO_INFLATED_POSITIVE + if nonconstant + else qrf.Regime.DEGENERATE_ZERO + ) + assert artifact.regime == expected_regime + if nonconstant: + assert artifact._target_model.gate is not None + assert artifact._target_model.positive is not None + if resume == "require": + assert all(record.hit for record in manifest.nodes.values()) + raw = tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ) + states = tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ) + matrix_key = manifest.node(graph.PROJECTION_NODE).key + actual = observed[graph.ATTACH_NODE] + verification = dict( + population=actual, + projection=loaded[graph.PROJECTION_NODE, "projection"], + matrix=loaded[graph.PROJECTION_NODE, "matrix"], + matrix_producer_key=matrix_key, + raw_draws=raw, + apply_states=states, + host_pins=pins, + n_estimators=2, + ) + result = graph.verify_materialized_current_survey_predictors( + preparation, allocated, clone, **verification + ) + assert ( + result["all_output_cells_available"] is True + and result["release_eligible"] is False + ) + assert not actual.frame.person.loc[:, list(values.OUTPUTS)].isna().any().any() + actual_person = actual.frame.person + for _, rows in actual_person.groupby( + values.provenance.support_source_id_column("person") + ): + assert len(rows) == 2 + np.testing.assert_array_equal( + rows.loc[:, list(values.OUTPUTS)].iloc[0], + rows.loc[:, list(values.OUTPUTS)].iloc[1], + ) + for raw_name in ("WAGP", "SEMP", "ADJINC"): + pd.testing.assert_series_equal( + actual_person[raw_name], clone.frame.person[raw_name] + ) + puf_mask = support.puf_tax_detail_clone_mask( + actual.frame.table("tax_unit"), entity="tax_unit" + ) + features, _ = support._strict_recipient_predictor_surface( + actual.frame, + puf_mask, + support.PUF_TAX_DETAIL_DEFAULT_PREDICTORS, + person_outputs=(), + ) + assert features.shape[1] == 8 and np.isfinite(features.to_numpy()).all() + changed = _copy_population(actual) + changed.frame.person.loc[changed.frame.person.index[0], values.OUTPUTS[2]] += ( + 1.0 + ) + with pytest.raises(ValueError): + graph.verify_materialized_current_survey_predictors( + preparation, allocated, clone, **{**verification, "population": changed} + ) + with pytest.raises(ValueError, match="DRAW_MATRIX_BINDING"): + graph.read_current_survey_draws(qualified.matrix, "f" * 64, raw, states) + results.append(actual) + graph.replay.same_replayed_population(*results) diff --git a/packages/microcosm-build/tests/test_us_current_survey_property_graph.py b/packages/microcosm-build/tests/test_us_current_survey_property_graph.py new file mode 100644 index 000000000..3bba5a9bd --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_property_graph.py @@ -0,0 +1,566 @@ +"""Invented source owners and the actual property fragment; no native/engine.""" + +import json +from dataclasses import FrozenInstanceError, replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_property_income_sources import actual # noqa: F401 + +from microcosm.build.us_runtime import graph_current_survey_property as graph +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import WeightKind +from microcosm.graph import compile_graph, run_graph +from microcosm.graph import population as population_ops + +financial = graph.financial +shared = graph.shared +OPTIONS = dict(scales=(1.0, 1.0, 1.0, 1.0), atol=1e-10, rtol=1e-12, n_estimators=2) + + +def test_joint_fit_exclusion_preserves_independent_asec_components(actual): # noqa: F811 + """Actual invented source qualification and pure attachment, without fitting. + + Native fixture person107 reports interest/dividends but survivor source8 + excludes the person from the joint fit. Those independent observations + remain usable on both clones. ACS draw inputs below are invented numbers, + not fitted or authenticated model artifacts; they only exercise the + separate recipient assignment in the pure attachment function. + """ + from microcosm.build.us_runtime import current_property_income_sources as sources + from microcosm.build.us_runtime import graph_property_tax_leaves as tax + + live, qualified, _ = actual + basis = qualified.donor_basis + excluded = basis.person.index[basis.person.native_person_id.eq(107)] + assert len(excluded) == 1 + origin = excluded[0] + assert basis.exclusions.loc[origin, "survivor_possible_property"] + assert not basis.person.loc[origin, "joint_component_fit_eligible"] + assert origin not in qualified.donor_columns.index + assert origin not in qualified.recipient_columns.index + assert basis.provenance.loc[origin, "interest.TRDINT_VAL_amount_known"] + assert basis.provenance.loc[origin, "dividend.DIV_VAL_amount_known"] + assert basis.provenance.loc[origin, "dividend.SUR_SC2_code"] == 8 + source_seal = sources.property_income_sources_seal(qualified) + frame = live.clone_population.frame + frame_seal = sources._frame_seal(frame) + + draws = pd.DataFrame( + np.tile([11.0, 7.0, 13.0, -2.0], (len(qualified.recipient_columns), 1)), + index=qualified.recipient_columns.index, + columns=graph.PROPERTY_DRAW_COLUMNS, + ) + reconciled = graph._reconciliation_result( + qualified, graph.PropertyIncomeOptions(**OPTIONS), draws + ) + legacy = pd.DataFrame(0.0, index=draws.index, columns=shared.TARGETS) + columns = graph.complete_property_columns( + qualified, frame, draws, reconciled, legacy + ) + people = frame.person.copy(deep=True) + for (entity, name), values in columns.items(): + assert entity == "person" + np.testing.assert_array_equal(values.index, people.person_id) + people[name] = values.array.copy() + split = tax.split_property_tax_leaves(people) + source_ids = people[shared.provenance.support_source_id_column("person")] + paired = people.loc[source_ids.eq(origin)] + assert len(paired) == 2 + assert set(paired[shared.provenance.support_clone_index_column("person")]) == { + 0, + 1, + } + for component, known_column, primary, complement in ( + ( + graph.PROPERTY_COMPONENTS[0], + "interest.TRDINT_VAL_amount", + *tax.TAX_LEAF_COLUMNS[:2], + ), + ( + graph.PROPERTY_COMPONENTS[2], + "dividend.DIV_VAL_amount", + *tax.TAX_LEAF_COLUMNS[2:], + ), + ): + amount = np.float64(basis.provenance.loc[origin, known_column]) + assert amount == 500.0 + np.testing.assert_array_equal( + paired[component].to_numpy().view("uint64"), + np.repeat(amount, 2).view("uint64"), + ) + parts = split.loc[paired.person_id, [primary, complement]] + assert np.isfinite(parts.to_numpy()).all() + np.testing.assert_array_equal(parts.sum(axis=1), np.repeat(amount, 2)) + assert paired[list(graph.PROPERTY_DRAW_COLUMNS)].isna().all().all() + # Every original ASEC component, including unknowns, survives unfiltered. + for original_id, row in basis.person.iterrows(): + pair = people.loc[source_ids.eq(original_id)] + for component in graph.PROPERTY_COMPONENTS: + np.testing.assert_array_equal(pair[component], np.repeat(row[component], 2)) + assert sources.property_income_sources_seal(qualified) == source_seal + assert sources._frame_seal(frame) == frame_seal + + +def test_explicit_frozen_options_and_complete_owned_roster(): + options = graph.PropertyIncomeOptions(**OPTIONS) + assert graph.codec.decode_json(options.to_bytes()) == graph.codec.decode_json( + graph.codec.encode_json(OPTIONS) + ) + with pytest.raises(FrozenInstanceError): + options.atol = 1 + assert ( + len(graph.owned_columns()) + == len({o.column for o in graph.owned_columns()}) + == 24 + ) + assert not set(shared.OUTPUTS) & {o.column for o in graph.owned_columns()} + with pytest.raises(TypeError): + graph.PropertyIncomeOptions() + + +@pytest.mark.parametrize( + "key,value", + [ + ("scales", (1.0, 1.0, 1.0)), + ("scales", (1.0, 1.0, 1.0, 0.0)), + ("scales", (1.0, 1.0, 1.0, float("inf"))), + ("atol", -1.0), + ("rtol", float("nan")), + ("atol", 1), + ("n_estimators", 0), + ("n_estimators", True), + ], +) +def test_invalid_options_refuse(key, value): + with pytest.raises((ValueError, TypeError)): + graph.PropertyIncomeOptions(**{**OPTIONS, key: value}) + + +def _pins(live): + result = {} + for edge in graph.host.current_survey_host_edges(): + record = live.manifest.node(edge.producer) + key = record.opaque_artifacts[edge.artifact] + result[edge.name] = { + "producer_key": record.key, + "artifact_key": key, + "payload_sha256": graph.codec.sha(live.store.load_bytes(key)), + } + return result + + +@pytest.fixture(scope="module") +def complete(actual, tmp_path_factory): # noqa: F811 + live, qualified, _ = actual + pins = _pins(live) + options = graph.PropertyIncomeOptions(**OPTIONS) + legacy_nodes = financial.current_survey_predictor_nodes( + qualified.shared_predictors, + live.clone_population.frame, + host_pins=pins, + n_estimators=2, + ) + nodes = graph.current_survey_property_nodes( + qualified, live.clone_population.frame, host_pins=pins, options=options + ) + compiled = compile_graph( + replace( + live.compiled.graph, + nodes=(*live.compiled.graph.nodes, *legacy_nodes, *nodes), + ) + ) + for cls in ( + financial.CurrentSurveyPredictorProjectionKernel, + financial.CurrentSurveyPredictorDonorFilterKernel, + financial.CurrentSurveyPredictorDonorColumnsKernel, + financial.CurrentSurveyPredictorAttachKernel, + ): + live.kernels.register( + cls( + live.preparation, + live.allocated_population, + live.clone_population, + host_pins=pins, + n_estimators=2, + ) + ) + live.kernels.register(LegacyQRFTrainKernel()) + live.kernels.register(LegacyQRFApplyMatrixKernel()) + graph.register_property_kernels( + live.kernels, + live.preparation, + live.allocated_population, + live.clone_population, + host_pins=pins, + options=options, + ) + observed = {} + manifest = run_graph( + compiled, + sources=live.sources, + store=live.store, + kernels=live.kernels, + _population_observer=lambda node_id, p: observed.__setitem__(node_id, p), + ) + loaded = { + (node_id, name): live.store.load_bytes(key) + for node_id, record in manifest.nodes.items() + for name, key in record.opaque_artifacts.items() + } + results = graph.reconstruct_property_results( + qualified, + live.clone_population.frame, + host_pins=pins, + options=options, + artifacts=loaded, + legacy_matrix_producer_key=manifest.node(financial.PROJECTION_NODE).key, + ) + again_observed = {} + again = run_graph( + compiled, + sources=live.sources, + store=live.store, + kernels=live.kernels, + resume="require", + _population_observer=lambda node_id, p: again_observed.__setitem__(node_id, p), + ) + assert again.key == manifest.key and all(n.hit for n in again.nodes.values()) + metadata = { + "scope": "invented full source qualification plus actual legacy10/property16 graph; no native/country engine", + "nodes": len(compiled.order), + "property_nodes": len(nodes), + "cold_property_hits": {n.id: manifest.node(n.id).hit for n in nodes}, + "required_hits": len(again.nodes), + "source_people": len(qualified.origins), + "clone_people": len(live.clone_population.frame.person), + "donor_people": len(qualified.donor_columns), + "recipient_people": len(qualified.recipient_columns), + } + ( + tmp_path_factory.mktemp("property-fragment-metadata") / "acceptance.json" + ).write_text(json.dumps(metadata, indent=2)) + return ( + live, + qualified, + pins, + options, + compiled, + manifest, + observed, + loaded, + results, + again_observed, + ) + + +def test_sixteen_actual_nodes_replay_and_reconstructed_populations(complete): + ( + live, + qualified, + pins, + options, + compiled, + manifest, + observed, + loaded, + results, + again, + ) = complete + assert len(results) == 8 + assert sum(n.id.startswith("survey_property.") for n in compiled.graph.nodes) == 16 + assert financial.ATTACH_NODE in compiled.predecessors[graph.ATTACH_NODE] + assert set(shared.OUTPUTS).issubset( + next( + s.columns + for s in compiled.graph.node(graph.ATTACH_NODE).inputs + if s.entity == "person" + ) + ) + expected = {} + for node_id in compiled.order: + node = compiled.graph.node(node_id) + if node_id in results: + parent = ( + expected[node.base] + if node.structural is graph.StructuralDelta.FILTER + else expected.get(compiled.versions[node_id]) + ) + if parent is None: + # The first property operation in the pre-existing donor or + # clone version receives the actual independently checked prefix. + parent = ( + observed[financial.ATTACH_NODE] + if node_id == graph.ATTACH_NODE + else observed[financial.DONOR_NODE] + ) + result = results[node_id] + if node.structural is graph.StructuralDelta.FILTER: + mask = result.keep.reindex(parent.frame.person.person_id).to_numpy( + dtype=bool + ) + result = replace(result, frame=parent.frame.select(mask), keep=None) + current = population_ops.patch(parent, node, result) + graph.replay.same_replayed_population(current, observed[node_id]) + expected[compiled.versions[node_id]] = current + elif node_id.startswith(graph.PREFIX + "."): + assert node_id in { + f"{graph.PREFIX}.{kind}.{i:03d}" + for kind in ("fit", "apply") + for i in range(4) + } + else: + expected[compiled.versions[node_id]] = observed[node_id] + graph.replay.same_replayed_population(observed[node_id], again[node_id]) + final = observed[graph.ATTACH_NODE] + receipt = graph.verify_materialized_property_income( + live.preparation, + live.allocated_population, + live.clone_population, + legacy_population=observed[financial.ATTACH_NODE], + population=final, + host_pins=pins, + options=options, + artifacts=loaded, + legacy_matrix_producer_key=manifest.node(financial.PROJECTION_NODE).key, + ) + assert ( + receipt["tax_split_rebased"] is False + and receipt["capital_gains"] == graph.CAP_LIMITATION + ) + + +def test_exact_clone_pairs_unknowns_and_asec_qualified_values(complete): + live, qualified, _, _, _, _, observed, *_ = complete + final = observed[graph.ATTACH_NODE].frame + legacy = observed[financial.ATTACH_NODE].frame + for entity in legacy.entities: + pd.testing.assert_frame_equal( + final.table(entity)[list(legacy.table(entity))], legacy.table(entity) + ) + assert final.metadata == legacy.metadata and final.mass_log == legacy.mass_log + for entity in legacy.weighted_entities: + assert final.weights_for(entity).kind is legacy.weights_for(entity).kind + np.testing.assert_array_equal( + final.weights_for(entity).values, legacy.weights_for(entity).values + ) + ids = final.person[shared.provenance.support_source_id_column("person")].to_numpy() + columns = [o.column for o in graph.owned_columns()] + for origin in qualified.origins.index: + pair = final.person.loc[ids == origin, columns].reset_index(drop=True) + pd.testing.assert_series_equal(pair.iloc[0], pair.iloc[1], check_names=False) + basis = qualified.donor_basis.person + for origin in basis.index: + pair = final.person.loc[ids == origin] + for name in (*graph.PROPERTY_COMPONENTS, graph.PROPERTY_REPORTED_TOTAL): + np.testing.assert_array_equal( + pair[name].to_numpy(), np.repeat(basis.loc[origin, name], 2) + ) + assert pair[list(graph.PROPERTY_DRAW_COLUMNS)].isna().all().all() + excluded = qualified.recipient_diagnostics.index[ + ~qualified.recipient_diagnostics.eligible_recipient + ] + mask = np.isin(ids, excluded) + assert mask.sum() > 0 + assert ( + final.person.loc[ + mask, [*graph.PROPERTY_COMPONENTS, graph.PROPERTY_REPORTED_TOTAL] + ] + .isna() + .all() + .all() + ) + assert ( + not final.person.loc[ + mask, ["property_anchor_known", "property_components_known"] + ] + .to_numpy() + .any() + ) + assert ( + observed[graph.RECIPIENT_NODE].frame.person.person_id.to_numpy() + == qualified.recipient_columns.index.to_numpy() + ).all() + assert ( + observed[graph.DONOR_NODE].frame.resolve_weights("person").kind + is WeightKind.DESIGN + ) + assert ( + observed[graph.DONOR_NODE].frame.person.person_id.to_numpy() + == qualified.donor_columns.index.to_numpy() + ).all() + + +@pytest.mark.parametrize( + "which", ["raw", "state", "summary", "reconciliation", "projection"] +) +def test_tampered_graph_artifacts_refuse(complete, which): + live, q, pins, options, _, manifest, _, artifacts, *_ = complete + copied = dict(artifacts) + key = { + "raw": (graph.PREFIX + ".apply.001", "raw_draw"), + "state": (graph.PREFIX + ".apply.001", "apply_state"), + "summary": (graph.PREFIX + ".draws", "summary"), + "reconciliation": (graph.PREFIX + ".reconcile", "summary"), + "projection": (graph.PROJECTION_NODE, "projection"), + }[which] + if which == "raw": + raw = graph.codec.read_raw_target( + copied[key], + target=graph.PROPERTY_COMPONENTS[1], + index=q.recipient_frame.person.index, + ).copy() + raw[0] += 1 + copied[key] = graph.codec.encode_raw_target( + raw, + target=graph.PROPERTY_COMPONENTS[1], + index=q.recipient_frame.person.index, + ) + else: + data = graph.codec.decode_json(copied[key]) + data["unexpected"] = True + copied[key] = graph.codec.encode_json(data) + with pytest.raises((ValueError, KeyError)): + graph.reconstruct_property_results( + q, + live.clone_population.frame, + host_pins=pins, + options=options, + artifacts=copied, + legacy_matrix_producer_key=manifest.node(financial.PROJECTION_NODE).key, + ) + + +def test_scales_change_only_reconciliation_and_attachment_declarations(complete): + live, q, pins, options, *_ = complete + old = graph.current_survey_property_nodes( + q, live.clone_population.frame, host_pins=pins, options=options + ) + new = graph.current_survey_property_nodes( + q, + live.clone_population.frame, + host_pins=pins, + options=replace(options, scales=(2.0, 1.0, 1.0, 1.0)), + ) + assert [a.id for a, b in zip(old, new, strict=True) if a != b] == [ + graph.PREFIX + ".reconcile", + graph.ATTACH_NODE, + ] + + +def test_origin_identity_refusal_and_order_independence(complete): + live, q, pins, options, _, manifest, observed, artifacts, *_ = complete + ids, axis = graph._clone_lookup(q, live.clone_population.frame) + reordered = replace(q, origins=q.origins.iloc[::-1]) + assert np.array_equal( + graph._clone_lookup(reordered, live.clone_population.frame)[0], ids + ) + bad = q.origins.copy(deep=True) + bad.iloc[0, bad.columns.get_loc("native_person_id")] += 1 + with pytest.raises(ValueError, match="CLONE_ORIGIN_IDENTITY"): + graph._clone_lookup(replace(q, origins=bad), live.clone_population.frame) + assert axis.dtype == np.dtype("int64") + + +@pytest.mark.parametrize("target", ["donor_frame", "recipient_frame"]) +def test_empty_branch_refuses_without_model_defaults(complete, target): + live, q, pins, options, *_ = complete + with pytest.raises(ValueError, match="EMPTY_MODEL_BRANCH"): + graph.current_survey_property_nodes( + replace(q, **{target: None}), + live.clone_population.frame, + host_pins=pins, + options=options, + ) + + +def test_declaration_without_source_execution(monkeypatch): + from microcosm.frame import US_SCHEMA, Frame, Weights + + tables = { + e: pd.DataFrame( + { + US_SCHEMA.entity_id_column(e): np.array([1], dtype="int64"), + "fixture_" + e: np.array([1.0]), + } + ) + for e in US_SCHEMA.entities + } + for e in US_SCHEMA.group_entities: + tables["person"][US_SCHEMA.membership_column(e)] = np.array([1], dtype="int64") + tables["person"]["employment_income"] = 5.0 + frame = Frame( + tables, US_SCHEMA, {"household": Weights(np.array([1.0]), WeightKind.DESIGN)} + ) + features = (*shared.FEATURES, graph.PROPERTY_REPORTED_TOTAL) + q = SimpleNamespace( + source_frame=frame, + donor_frame=frame, + recipient_frame=frame, + donor_columns=pd.DataFrame(columns=(*features, *graph.PROPERTY_COMPONENTS)), + recipient_columns=pd.DataFrame(columns=features), + shared_predictors=SimpleNamespace( + donor_frame=frame, + demographic_conditioning=False, + geography_config_payload=None, + ), + ) + # Isolate declaration shape only. No fixture object claims source authority; + # the full graph fixture below uses the actual source qualifiers unchanged. + monkeypatch.setattr(graph, "_params", lambda *args: {"protocol": graph.PROTOCOL}) + nodes = graph.current_survey_property_nodes( + q, frame, host_pins={}, options=graph.PropertyIncomeOptions(**OPTIONS) + ) + assert len(nodes) == 16 + assert set(nodes[-1].params) == {"protocol", *OPTIONS} + + context_tables = {} + for entity in frame.entities: + structural = [US_SCHEMA.entity_id_column(entity)] + if entity == "person": + structural.extend( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + context_tables[entity] = frame.table(entity)[ + structural + [c for c in frame.table(entity) if c not in structural] + ].copy() + for name in shared.OUTPUTS: + context_tables["person"][name] = 0.0 + context = graph.KernelContext( + node=nodes[-1], + tables=context_tables, + weights={e: frame.resolve_weights(e) for e in frame.entities}, + strata=frame.strata, + params=nodes[-1].params, + rng=None, + ) + graph._clone_context(context, frame) + assert set(shared.OUTPUTS).issubset( + next(x.columns for x in nodes[-1].inputs if x.entity == "person") + ) + + +@pytest.mark.parametrize("field", ["property_reported_total", shared.OUTPUTS[0]]) +def test_final_verifier_refuses_changed_owned_or_retained_values(complete, field): + from test_us_current_survey_puf_transfer import _copy_population + + live, _, pins, options, _, manifest, observed, loaded, *_ = complete + changed = _copy_population(observed[graph.ATTACH_NODE]) + table = changed.frame.person + row = table.index[np.flatnonzero(np.isfinite(table[field].to_numpy()))[0]] + table.loc[row, field] += 1 + with pytest.raises((ValueError, AssertionError)): + graph.verify_materialized_property_income( + live.preparation, + live.allocated_population, + live.clone_population, + legacy_population=observed[financial.ATTACH_NODE], + population=changed, + host_pins=pins, + options=options, + artifacts=loaded, + legacy_matrix_producer_key=manifest.node(financial.PROJECTION_NODE).key, + ) diff --git a/packages/microcosm-build/tests/test_us_current_survey_puf_host.py b/packages/microcosm-build/tests/test_us_current_survey_puf_host.py new file mode 100644 index 000000000..50454a91d --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_puf_host.py @@ -0,0 +1,444 @@ +"""Invented host-only bridge; ordinary pytest, no donor/fit/engine/target access. + +The actual source test deliberately retains normal readiness and inventory gates. +Its proposal has not been executed. Fixture private file pins follow the existing +invented source helpers; no source/issuer function or money.ready is replaced. +""" + +import copy +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_survey_population import authenticated_arguments +from test_us_puf_detail_transfer import host as legacy_host +from test_us_survey_population_preparation import fixture as source_fixture + +from microcosm.build.us_runtime import asec_current_money as money +from microcosm.build.us_runtime import graph_combined_clone as clone +from microcosm.build.us_runtime import graph_puf_diagnostic_consumer as bridge +from microcosm.build.us_runtime import graph_survey_population as survey +from microcosm.build.us_runtime import puf_detail_transfer as detail +from microcosm.build.us_runtime import puf_diagnostic_consumer as consumer +from microcosm.build.us_runtime import support_provenance as provenance +from microcosm.build.us_runtime import survey_population_preparation as owner +from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.build.us_runtime.graph_sources import frame_column_declarations +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit.model_input import decode_recipient_matrix +from microcosm.frame import Frame, Weights +from microcosm.graph import NodeRejected, compile_graph, executor, run_graph +from microcosm.graph.keys import opaque_artifact_key + + +def _copy_frame(frame): + return Frame( + {e: frame.table(e).copy(deep=True) for e in frame.entities}, + frame.schema, + {e: frame.weights_for(e) for e in frame.weighted_entities}, + frame.strata.copy(deep=True), + metadata=frame.metadata, + mass_log=frame.mass_log, + ) + + +def _legacy_clone(): + return clone.clone_us_frame_for_puf_support(legacy_host()) + + +def test_existing_matrix_default_and_explicit_projection_have_identical_bytes(): + frame = _legacy_clone() + old, old_mask = detail.recipient_matrix(frame) + people = frame.person + values = [ + row["asec_reported_wage_income_2024_price"] + if row[provenance.support_channel_column("person")] == "asec" + else row["employment_income_before_lsr"] + for _, row in people.iterrows() + ] + wages = pd.Series( + values, index=pd.Index(people.person_id.to_numpy()), dtype="float64" + ) + new, new_mask = detail.recipient_matrix(frame, person_wages=wages) + assert new == old + np.testing.assert_array_equal(new_mask, old_mask) + # Independent hand expectations: actual JOINT spouse included, dependent + # wages 999/800 excluded, zero-weight single unit retained. + assert sorted( + decode_recipient_matrix(new).features.reported_wage_proxy.tolist() + ) == [10.0, 20.0, 30.0, 170.0] + with pytest.raises(ValueError, match="HOST_WAGE_ROW_AXIS"): + detail.recipient_matrix(frame, person_wages=wages.iloc[::-1]) + wrong = wages.copy() + role = people[provenance.support_clone_index_column("person")].to_numpy() + wrong.iloc[np.flatnonzero(role == 1)[0]] += 1 + with pytest.raises(ValueError, match="HOST_WAGE_PAIR_BYTES"): + detail.recipient_matrix(frame, person_wages=wrong) + missing = wages.copy() + first = people.iloc[0][provenance.support_source_id_column("person")] + pair = people[provenance.support_source_id_column("person")].eq(first).to_numpy() + missing.iloc[np.flatnonzero(pair)] = np.nan + with pytest.raises(ValueError, match="HOST_MISSING_FEATURE"): + detail.recipient_matrix(frame, person_wages=missing) + with pytest.raises(ValueError, match="HOST_WAGE_ROW_AXIS"): + detail.recipient_matrix(frame, person_wages=wages.astype("object")) + + +def test_current_route_is_explicit_and_legacy_declarations_are_separate(): + columns = frame_column_declarations(_legacy_clone()) + current = bridge.current_survey_host_nodes( + columns, + preparation_sha256="1" * 64, + allocation_sha256="2" * 64, + clone_sha256="3" * 64, + host_pins={ + e.name: { + "producer_key": "4" * 64, + "artifact_key": "5" * 64, + "payload_sha256": "6" * 64, + } + for e in bridge.current_survey_host_edges() + }, + ) + assert len(current) == 2 + assert all(not n.outputs and not n.sources for n in current) + assert [e.name for e in current[0].artifact_inputs] == [ + "preparation", + "allocation", + "frame_context", + ] + assert [e.name for e in current[1].artifact_inputs] == ["projection"] + assert current[0].params["host_route"] == consumer.CURRENT_SURVEY_ROUTE + assert len(bridge.host_edges(native_binding="native", clone_binding="clones")) == 11 + assert not any( + word in n.kernel + for n in current + for word in ("donor", "fit", "attach", "apply") + ) + + +def test_unissued_byte_equal_preparation_refuses_before_projection(): + fake = object.__new__(owner.AuthenticatedSurveyPopulationPreparation) + object.__setattr__(fake, "payload", b"{}") + with pytest.raises( + owner.SurveyPopulationPreparationError, match="UNISSUED_OR_CHANGED" + ): + bridge.qualify_current_survey_host(fake, None, None) + + +@pytest.mark.parametrize("year,field", [(2024, "ANN_VAL"), (2022, "WSAL_VAL")]) +def test_current_host_preserves_full_parent_readiness( + tmp_path, monkeypatch, year, field +): + arguments = source_fixture(tmp_path, monkeypatch, missing_asec_money=(year, field)) + live = survey.run_authenticated_survey_population( + **arguments, + store_root=tmp_path / "graph-store", + clones=True, + return_values=True, + ) + # Reaching this point establishes actual preparation and native issuance; + # an earlier SOURCE_CHANGED error is not the readiness result under test. + preparation_entry = owner._ISSUED[id(live.preparation)] + native = preparation_entry[2].native[1] + parent = owner.asec_native._ISSUED[id(native)][2].parent + decoded = money.classify_asec_money(parent.views(), parent.spec, parent.scope) + years = np.asarray(parent.scope.person_years) + absent = ~decoded.field(field).validity.astype(bool) + assert int(absent.sum()) == 1 + assert years[absent].tolist() == [year] + assert decoded.field("WSAL_VAL").validity[years == 2024].all() + with pytest.raises(money.MoneyRefusalError) as caught: + bridge.qualify_current_survey_host( + live.preparation, live.allocated_population, live.clone_population + ) + assert caught.value.reason == "MISSING_REQUIRED_AMOUNT" + assert caught.value.field == field + + +def test_actual_current_sources_cold_and_replayed_host(tmp_path, monkeypatch): + arguments = authenticated_arguments(tmp_path, monkeypatch) + live = survey.run_authenticated_survey_population( + **arguments, clones=True, return_values=True + ) + preparation, allocated, expanded = ( + live.preparation, + live.allocated_population, + live.clone_population, + ) + before = owner._frame_identity(expanded.frame) + projection, matrix, mask, evidence = bridge.qualify_current_survey_host( + preparation, allocated, expanded + ) + document = codec.decode_json(projection) + assert evidence["release_eligible"] is False + assert evidence["source_period_equivalence_claim"] is False + assert len(document["asec"]["rows"]) == 4 + assert [r[2] for r in document["asec"]["rows"]] == [2024] * 4 + assert [ + np.frombuffer(bytes.fromhex(r[3]), dtype=" 1 + for changed_slice in ( + replace(first, columns=first.columns[:-1]), + replace(first, columns=first.columns[::-1]), + replace(first, columns=(*first.columns, "__extra_input__")), + # A declared column lets Node construction succeed; this direct + # contract check must reject non-ALL rows before any mask evaluation. + replace(first, rows=first.columns[0]), + ): + bad_node = replace(nodes[0], inputs=(changed_slice, *nodes[0].inputs[1:])) + with pytest.raises(ValueError, match="^SURVEY_HOST_INPUTS$"): + bridge._current_context_frame( + replace(context, node=bad_node), expanded.frame + ) + for field, values, reason in ( + ("tables", context.tables, "SURVEY_HOST_TABLES"), + ("weights", context.weights, "SURVEY_HOST_WEIGHTS"), + ): + missing = dict(values) + del missing[next(iter(missing))] + extra = {**values, "__extra_entity__": next(iter(values.values()))} + for changed_values in (missing, extra): + with pytest.raises(ValueError, match="^" + reason + "$"): + bridge._current_context_frame( + replace(context, **{field: changed_values}), expanded.frame + ) + + person = context.tables[expanded.frame.schema.person_entity] + changed_tables = dict(context.tables) + changed_tables[expanded.frame.schema.person_entity] = person.iloc[:, ::-1] + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + changed_tables[expanded.frame.schema.person_entity] = person.rename_axis( + "__changed_columns__", axis="columns" + ) + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + changed_person = person.copy(deep=True) + changed_person.iloc[0, changed_person.columns.get_loc("age")] += 1 + changed_tables[expanded.frame.schema.person_entity] = changed_person + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_NATIVE_BITS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + + # Positive and negative zero are both valid weights; only their exact bits + # differ. This must reach the replay comparison rather than Frame refusal. + entity = next(e for e, w in context.weights.items() if np.any(w.values == 0)) + value = context.weights[entity] + changed_values = value.values.copy() + zero = int(np.flatnonzero(changed_values == 0)[0]) + changed_values[zero] = 0.0 if np.signbit(changed_values[zero]) else -0.0 + changed_weights = {**context.weights, entity: Weights(changed_values, value.kind)} + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_WEIGHT_BYTES$"): + bridge._current_context_frame( + replace(context, weights=changed_weights), expanded.frame + ) + with pytest.raises(ValueError, match="^SURVEY_HOST_STRATA_NAME$"): + bridge._current_context_frame( + replace(context, strata=context.strata.rename("__changed_name__")), + expanded.frame, + ) + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, strata=context.strata.rename_axis("__changed_index__")), + expanded.frame, + ) + assert owner._frame_identity(expanded.frame) == before + + graph = replace(live.compiled.graph, nodes=(*live.compiled.graph.nodes, *nodes)) + compiled = compile_graph(graph) + assert len(compiled.graph.nodes) == 6 + live.kernels.register( + bridge.CurrentSurveyHostProjectionKernel(preparation, allocated, expanded) + ) + live.kernels.register(bridge.CurrentSurveyMatrixKernel()) + outputs = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume=resume, + _population_observer=lambda name, population, observed=observed: ( + observed.__setitem__(name, population) + ), + ) + actual = [] + for node_id, name, type_, expected, kernel in ( + ( + bridge.CURRENT_PROJECTION_NODE, + "projection", + bridge.CURRENT_PROJECTION_TYPE, + projection, + bridge.CurrentSurveyHostProjectionKernel, + ), + ( + bridge.CURRENT_MATRIX_NODE, + "matrix", + bridge.model_input.RECIPIENT_MATRIX_TYPE, + matrix, + bridge.CurrentSurveyMatrixKernel, + ), + ): + survey._final_artifact( + manifest, + live.store, + node_id=node_id, + name=name, + type_=type_, + payload=expected, + capabilities=kernel.capabilities, + ) + receipt = manifest.node(node_id) + assert receipt.opaque_artifacts[name] == opaque_artifact_key( + receipt.key, name + ) + actual.append(live.store.load_bytes(receipt.opaque_artifacts[name])) + replay.same_replayed_population(expanded, observed[node_id]) + checked = bridge.verify_materialized_current_survey_host( + preparation, + allocated, + expanded, + population=observed[bridge.CURRENT_MATRIX_NODE], + projection=actual[0], + matrix=actual[1], + ) + assert checked == evidence + outputs.append(tuple(actual)) + assert outputs[0] == outputs[1] == (projection, matrix) + assert owner._frame_identity(expanded.frame) == before + wrong_pins = copy.deepcopy(pins) + wrong_pins["preparation"]["producer_key"] = "f" * 64 + bad_projection_node = replace( + nodes[0], + params={ + **dict(nodes[0].params), + "host_edges": codec.encode_json(wrong_pins).decode(), + }, + ) + bad_graph = replace( + graph, nodes=(*live.compiled.graph.nodes, bad_projection_node, nodes[1]) + ) + with pytest.raises(NodeRejected, match="SURVEY_HOST_EDGE_PIN"): + run_graph( + compile_graph(bad_graph), + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume="auto", + ) + + # A well-formed amount mutation can pass the pure value parser; the live + # materialized verifier must reject it against the retained current owner. + changed = copy.deepcopy(document) + changed["asec"]["rows"][0][3] = np.asarray([60001.0], dtype=" None: + # Real Chronicle CBO rows declare source_projection. Missing assertions in + # this fixture previously disguised the compiler's observation-only default. + fact = _cbo_income_source_projection_fact(2024, income_source, value=125_000) + registry = compile_us_fiscal_target_registry( + [*packaged_reference_facts(), fact], target_period=2024, age_targets=True + ) + spec = _aged_spec_by_source_record_id(registry, fact["lineage"]["source_record_id"]) + + assert spec.value == 125_000 + assert spec.period == 2024 + assert spec.metadata["ledger_assertion"] == "source_projection" + assert spec.metadata["ledger_resolved_assertion"] == "source_projection" + assert spec.metadata["ledger_assertion_policy"] == "allow_source_projection" + assert fact["assertion"] == "source_projection" + + +def test_unapproved_non_cbo_projection_is_still_refused() -> None: + fact = _dynamic_ledger_fact( + source_record_id="usda_snap.fy2024.total_benefits", + source_name="usda_snap", + measure_id="total_benefits", + value=125_000, + ) + fact["assertion"] = "source_projection" + + with pytest.raises(ValueError, match="assertion_policy='observed_only'"): + compile_us_fiscal_target_registry([*packaged_reference_facts(), fact]) + + +@pytest.mark.parametrize( + ("income_source", "measure_id"), + [ + ("unmapped_income", "projected_amount"), + ("adjusted_gross_income", "other_amount"), + ], +) +def test_unmapped_cbo_projection_does_not_become_a_target( + income_source, measure_id +) -> None: + fact = _cbo_income_source_projection_fact(2024, income_source, value=125_000) + fact["layout"]["measure_id"] = measure_id + fact["observed_measure"]["source_measure_id"] = measure_id + registry = compile_us_fiscal_target_registry([*packaged_reference_facts(), fact]) + + assert fact["lineage"]["source_record_id"] not in { + spec.metadata["ledger_source_record_id"] for spec in registry.specs + } + + def test_age_targets_defaults_off_leaves_surface_unchanged() -> None: facts = [ *packaged_reference_facts(), diff --git a/packages/microcosm-build/tests/test_us_full_puf_enrichment.py b/packages/microcosm-build/tests/test_us_full_puf_enrichment.py new file mode 100644 index 000000000..f173636ba --- /dev/null +++ b/packages/microcosm-build/tests/test_us_full_puf_enrichment.py @@ -0,0 +1,477 @@ +"""Ordinary canonical-column controls; no genuine donor/source authority mocks.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import full_puf_enrichment as full +from microcosm.build.us_runtime import puf_support as support +from microcosm.build.us_runtime.acs_income_universe import ( + apply_acs_pums_earnings_universe_zeros, +) +from microcosm.build.us_runtime.spine_assembly import assemble_spines +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactOutput, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.codecs import load_frame_store +from microcosm.graph.keys import opaque_artifact_key + + +def _donor_columns(): + links = np.repeat(np.arange(1001, 1005, dtype=np.int64), 2) + person = pd.DataFrame( + { + "person_id": np.arange(101, 109, dtype=np.int64), + "person_tax_unit_id": links, + } + ) + for index, target in enumerate(full.PERSON_OUTPUTS): + person[target] = ( + np.arange(8) % 2 == 1 + if target in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + else (np.arange(8, dtype=np.float64) + 1) * (index + 1) + ) + person["short_term_capital_gains"] = [-10.0, 0.0, 5.0, 10.0, -3.0, 0.0, 12.0, 20.0] + tax_unit = pd.DataFrame( + { + "tax_unit_id": np.arange(1001, 1005, dtype=np.int64), + "weight": [1.0, 0.0, 2.0, 3.0], + "filing_status_code": [1.0, 2.0, 3.0, 4.0], + } + ) + for target in full.TAX_UNIT_OUTPUTS: + tax_unit[target] = ( + [0.0, 2010.0, 2020.0, 2024.0] + if target in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS + else [1.0, 2.0, 3.0, 4.0] + ) + pk = pd.DataFrame(True, index=person.index, columns=full.PERSON_OUTPUTS) + tk = pd.DataFrame( + True, + index=tax_unit.index, + columns=("weight", "filing_status_code", *full.TAX_UNIT_OUTPUTS), + ) + return person, tax_unit, pk, tk + + +def _donor(): + p, t, pk, tk = _donor_columns() + return full.canonical_full_puf_donor(p, t, person_known=pk, tax_unit_known=tk) + + +def _recipient(): + ids = np.arange(1, 5, dtype=np.int64) + links = np.repeat(ids, 2) + person = pd.DataFrame({"person_id": np.arange(1, 9, dtype=np.int64)}) + for entity in US_SCHEMA.group_entities: + person[US_SCHEMA.membership_column(entity)] = links + person["age"] = [14.0, 45.0] * 4 + for name, amount in ( + ("employment_income_before_lsr", 100.0), + ("self_employment_income_before_lsr", 20.0), + ("taxable_interest_income", 5.0), + ("qualified_dividend_income", 2.0), + ("non_qualified_dividend_income", 3.0), + ("short_term_capital_gains", 10.0), + ("long_term_capital_gains_before_response", 12.0), + ): + person[name] = [0.0, amount] * 4 + for name in support.PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS: + person[name] = [0.0, 10.0] * 4 + person["is_full_time_college_student"] = [False, True] * 4 + tables = { + entity: pd.DataFrame({US_SCHEMA.entity_id_column(entity): ids}) + for entity in US_SCHEMA.group_entities + } + tables["tax_unit"]["filing_status_input"] = [ + "SINGLE", + "JOINT", + "SEPARATE", + "HEAD_OF_HOUSEHOLD", + ] + tables["person"] = person + native = Frame( + tables, + US_SCHEMA, + {"household": Weights(np.array([1.0, 0.0, 2.0, 3.0]), WeightKind.DESIGN)}, + metadata={"scope": "invented_full_puf_contract", "source_admission": False}, + ) + asec = native.select(person.person_household_id.le(2)) + acs = native.select(person.person_household_id.gt(2)) + acs_person = acs.table("person") + child = acs_person.age.lt(15) + for mapped, raw in ( + ("employment_income_before_lsr", "WAGP"), + ("self_employment_income_before_lsr", "SEMP"), + ): + acs_person[raw] = acs_person[mapped] + acs_person.loc[child, [mapped, raw]] = np.nan + assembled = assemble_spines( + {"asec": asec, "acs": acs}, + household_mass_shares={"asec": 0.5, "acs": 0.5}, + ) + cloned = support.clone_us_frame_for_puf_support(assembled) + application = apply_acs_pums_earnings_universe_zeros( + cloned, boundary="invented full65 recipient ACS materialization" + ) + # This is the maintained source-universe operator, with its actual receipt; + # the mapped child zeros are produced here and the raw blanks survive. + assert application.receipt["structurally_absent_person_rows"] == 4 + people = application.frame.table("person") + acs_child = people[support.support_channel_column("person")].eq( + "acs" + ) & people.age.lt(15) + assert people.loc[acs_child, ["WAGP", "SEMP"]].isna().all().all() + return application.frame + + +def _known(frame): + table = frame.table("tax_unit") + mask = support.puf_tax_detail_clone_mask(table, entity="tax_unit") + return pd.DataFrame( + True, + columns=full.PREDICTORS, + index=pd.Index(table.loc[mask, "tax_unit_id"].to_numpy(), name="tax_unit_id"), + ) + + +def test_complete_canonical_reduction_retains_zero_weights_signed_values_and_mortgage(): + p, t, pk, tk = _donor_columns() + before = p.copy(deep=True) + donor = full.canonical_full_puf_donor(p, t, person_known=pk, tax_unit_known=tk) + assert len(full.TARGETS) == 65 and len(full.PERSON_OUTPUTS) == 56 + assert "prior_year_wages" not in full.TARGETS + assert tuple(donor) == (*full.PREDICTORS, *full.TARGETS, "weight") + assert donor.weight.tolist() == [1.0, 0.0, 2.0, 3.0] + assert donor.short_term_capital_gains.tolist() == [-10.0, 15.0, -3.0, 32.0] + assert donor.business_is_sstb.tolist() == [1.0] * 4 + # Canonical mortgage has already been interpreted upstream: no E19200 split. + np.testing.assert_array_equal( + donor.home_mortgage_interest, + p.home_mortgage_interest.groupby(p.person_tax_unit_id).sum(), + ) + pd.testing.assert_frame_equal(p, before) + + +@pytest.mark.parametrize( + "value", + [None, np.nan, np.inf, "0", True, 1 + 2j, pd.Timestamp("2024-01-01"), 2**53 + 1], +) +def test_rejects_bad_member_before_groupby_can_turn_it_into_a_valid_total(value): + p, t, pk, tk = _donor_columns() + p["taxable_interest_income"] = p.taxable_interest_income.astype(object) + p.loc[0, "taxable_interest_income"] = value + with pytest.raises( + ValueError, match="PUF_(UNKNOWN|NONFINITE|PHYSICAL_TYPE|FLOAT64_INTEGER_RANGE)" + ): + full.canonical_full_puf_donor(p, t, person_known=pk, tax_unit_known=tk) + + +def test_unknown_zero_and_numeric_boolean_are_not_complete_canonical_values(): + p, t, pk, tk = _donor_columns() + p.loc[0, "taxable_interest_income"] = 0.0 + pk.loc[0, "taxable_interest_income"] = False + with pytest.raises(ValueError, match="PUF_UNKNOWN:person.taxable_interest_income"): + full.canonical_full_puf_donor(p, t, person_known=pk, tax_unit_known=tk) + pk.loc[0, "taxable_interest_income"] = True + p["business_is_sstb"] = p.business_is_sstb.astype(np.int64) + with pytest.raises(ValueError, match="PUF_PHYSICAL_TYPE:person.business_is_sstb"): + full.canonical_full_puf_donor(p, t, person_known=pk, tax_unit_known=tk) + + +def test_explicit_return_grain_for_person_destination_and_collision_refusal(): + p, t, pk, tk = _donor_columns() + name = "self_employed_pension_contributions_desired" + t[name] = [10.0, 20.0, 30.0, 40.0] + tk[name] = True + tk = tk.loc[:, ["weight", "filing_status_code", name, *full.TAX_UNIT_OUTPUTS]] + kwargs = dict( + person_known=pk.drop(columns=name), + tax_unit_known=tk, + person_targets_at_tax_unit=(name,), + ) + with pytest.raises(ValueError, match="PUF_DONOR_GRAIN_COLLISION"): + full.canonical_full_puf_donor(p, t, **kwargs) + donor = full.canonical_full_puf_donor(p.drop(columns=name), t, **kwargs) + assert donor[name].tolist() == [10.0, 20.0, 30.0, 40.0] + + +def test_full_recipient_surface_keeps_zero_weight_rows_and_rejects_unknowns(): + frame, donor = _recipient(), _donor() + known = _known(frame) + inputs = full.prepare_full_puf_inputs(frame, donor, predictor_known=known) + matrix = model_input.decode_recipient_matrix(inputs.matrix) + assert len(matrix.features) == 4 and tuple(matrix.features) == full.PREDICTORS + assert matrix.entity_ids.tolist() == known.index.tolist() + assert inputs.recipient_universe["structurally_absent_person_rows"] == 2 + assert matrix.features[full.PREDICTORS[2]].eq(100.0).all() + assert matrix.features[full.PREDICTORS[3]].eq(20.0).all() + assert (frame.weights_for("household").values == 0).sum() == 2 + assert ( + inputs.recipient_universe["person_output_allocation"][ + "out_of_universe_person_rows" + ] + == 4 + ) + known.iloc[0, 2] = False + with pytest.raises(ValueError, match="PUF_UNKNOWN:recipient_predictor"): + full.prepare_full_puf_inputs(frame, donor, predictor_known=known) + known.iloc[0, 2] = True + person = frame.table("person") + person.loc[person.index[-1], "taxable_interest_income"] = np.nan + with pytest.raises(ValueError, match="missing values before coercion"): + full.prepare_full_puf_inputs(frame, donor, predictor_known=known) + + +def test_return_only_donor_does_not_invent_persons(): + expected = _donor() + _, returns, _, _ = _donor_columns() + columns = ( + "weight", + "filing_status_code", + "tax_unit_person_count", + *full.PERSON_OUTPUTS, + *full.TAX_UNIT_OUTPUTS, + ) + returns["tax_unit_person_count"] = expected[full.PREDICTORS[1]] + for name in full.PERSON_OUTPUTS: + returns[name] = expected[name] + known = pd.DataFrame(True, index=returns.index, columns=columns) + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PERSON_OUTPUTS, + ) + pd.testing.assert_frame_equal(actual, expected) + + +class InventedSource(KernelBase): + ref = "test.full_puf.frame_source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + return KernelResult( + frame=load_frame_store(context.sources[context.node.sources[0]]) + ) + + +class InventedMatrix(KernelBase): + ref = "test.full_puf.matrix@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.PLATFORM_BITWISE + ) + + def run(self, context): + table = context.tables["tax_unit"] + ids = table.tax_unit_id.to_numpy() + features = table.loc[:, list(full.PREDICTORS)].copy() + features.index = pd.Index(ids, name="tax_unit_id") + return KernelResult( + artifacts={ + "matrix": model_input.encode_recipient_matrix( + features, entity="tax_unit", entity_ids=ids + ) + } + ) + + +def test_real_full_65_target_graph_draw_chain_finalization_and_replay(tmp_path): + frame, donor = _recipient(), _donor() + known = _known(frame) + prepared = full.prepare_full_puf_inputs(frame, donor, predictor_known=known) + matrix = model_input.decode_recipient_matrix(prepared.matrix) + # An explicit minimal recipient model Frame for the ordinary graph control; + # finalization below consumes the actual retained US clone Frame. + table = matrix.features.copy() + table.insert(0, "tax_unit_id", matrix.entity_ids) + from microcosm.frame import EntitySchema + + recipient = Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + { + "person_id": matrix.entity_ids, + "person_tax_unit_id": matrix.entity_ids, + } + ), + }, + EntitySchema(group_entities=("tax_unit",)), + { + "tax_unit": Weights( + frame.resolve_weights("tax_unit").values[ + support.puf_tax_detail_clone_mask( + frame.table("tax_unit"), entity="tax_unit" + ) + ], + WeightKind.DESIGN, + ) + }, + ) + store = ContentStore(tmp_path / "store", codecs={"frame-store": load_frame_store}) + sources, nodes = {}, [] + for name, value in (("donor", prepared.donor_frame), ("recipient", recipient)): + key = full.codec.sha(("invented-full-puf:" + name).encode()) + sources[name] = store.put_frame(key, value) + nodes.append( + Node( + name, + InventedSource.ref, + sources=(name,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned("tax_unit", col, "float64") + for col in value.table("tax_unit") + if col != "tax_unit_id" + ), + ) + ) + fits, applies = full.full_puf_train_apply_nodes( + donor_population="donor", + recipient_population="recipient", + matrix_producer="matrix", + seed=578, + n_estimators=2, + zero_atol=0, + ) + assert len(fits) == len(applies) == 65 + assert [ + e.name for e in applies[-1].artifact_inputs if e.name.startswith("prior_") + ] == [f"prior_{i:03d}" for i in range(64)] + nodes.append( + Node( + "matrix", + InventedMatrix.ref, + population="recipient", + inputs=(Slice("tax_unit", full.PREDICTORS),), + artifact_outputs=( + ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE), + ), + ) + ) + compiled = compile_graph( + Graph( + "us", + tuple(SourceRef(name, "frame-store") for name in sources), + (*nodes, *fits, *applies), + ) + ) + registry = KernelRegistry() + for kernel in ( + InventedSource(), + InventedMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + registry.register(kernel) + actual = run_graph(compiled, sources=sources, store=store, kernels=registry) + + def payload(manifest, node_id, artifact): + record = manifest.node(node_id) + key = opaque_artifact_key(record.key, artifact) + assert key == record.opaque_artifacts[artifact] + return store.load_bytes(key) + + raw = { + target: payload(actual, node.id, "raw_draw") + for target, node in zip(full.TARGETS, applies, strict=True) + } + kwargs = dict( + matrix=payload(actual, "matrix", "matrix"), + matrix_producer_key=actual.node("matrix").key, + raw_draws=raw, + apply_state=payload(actual, applies[-1].id, "apply_state"), + training_state=payload(actual, fits[-1].id, "training_state"), + seed=578, + ) + assert kwargs["matrix"] == prepared.matrix + before = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + last_model = payload(actual, fits[-1].id, "model") + result, receipt = full.finalize_full_puf( + frame, donor, predictor_known=known, last_model=last_model, **kwargs + ) + assert receipt["release_eligible"] is False and receipt["tail_bounds"] + assert receipt["target_order"] == list(full.TARGETS) + for entity in frame.entities: + pd.testing.assert_frame_equal(frame.table(entity), before[entity]) + np.testing.assert_array_equal( + result.weights_for("household").values, frame.weights_for("household").values + ) + for entity, outputs in ( + ("person", full.PERSON_OUTPUTS), + ("tax_unit", full.TAX_UNIT_OUTPUTS), + ): + mask = support.puf_tax_detail_clone_mask(result.table(entity), entity=entity) + for output in outputs: + assert result.table(entity).loc[mask, output].notna().all() + if output not in before[entity]: + assert result.table(entity).loc[~mask, output].isna().all() + else: + np.testing.assert_array_equal( + result.table(entity).loc[~mask, output], + before[entity].loc[~mask, output], + ) + people = result.table("person") + child = support.puf_tax_detail_clone_mask(people, entity="person") & people.age.lt( + 15 + ) + assert people.loc[child, "employment_income_before_lsr"].eq(0).all() + changed_raw = dict(raw) + values = full.codec.read_raw_target( + raw[full.TARGETS[0]], target=full.TARGETS[0], index=matrix.features.index + ).copy() + values[0] += 1.0 + changed_raw[full.TARGETS[0]] = full.codec.encode_raw_target( + values, target=full.TARGETS[0], index=matrix.features.index + ) + with pytest.raises(ValueError, match="PUF_RAW_HISTORY"): + full.decode_full_puf_draws(**{**kwargs, "raw_draws": changed_raw}) + with pytest.raises(ValueError, match="PUF_FULL_MATRIX_BINDING"): + full.decode_full_puf_draws(**{**kwargs, "matrix_producer_key": "f" * 64}) + changed_donor = donor.copy(deep=True) + changed_donor.loc[0, "casualty_loss"] += 1.0 + with pytest.raises(ValueError, match="PUF_DONOR_CONSUMED_BYTES"): + full.finalize_full_puf( + frame, changed_donor, predictor_known=known, last_model=last_model, **kwargs + ) + incomplete = dict(raw) + incomplete.pop(full.TARGETS[-1]) + with pytest.raises(ValueError, match="PUF_RAW_ROSTER"): + full.decode_full_puf_draws(**{**kwargs, "raw_draws": incomplete}) + changed_training = full.codec.decode_json(kwargs["training_state"]) + changed_training["models"][0]["sha256"] = "e" * 64 + with pytest.raises(ValueError, match="PUF_FULL_CHAIN_IDENTITY"): + full.decode_full_puf_draws( + **{**kwargs, "training_state": full.codec.encode_json(changed_training)} + ) + warm = run_graph( + compiled, sources=sources, store=store, kernels=registry, resume="require" + ) + assert warm.key == actual.key and all(record.hit for record in warm.nodes.values()) + for node in applies: + assert payload(warm, node.id, "raw_draw") == payload( + actual, node.id, "raw_draw" + ) diff --git a/packages/microcosm-build/tests/test_us_full_puf_output_profiles.py b/packages/microcosm-build/tests/test_us_full_puf_output_profiles.py new file mode 100644 index 000000000..53fa59024 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_full_puf_output_profiles.py @@ -0,0 +1,1118 @@ +"""Ordinary profile controls with invented donors and real graph/QRF execution. + +The helper fixtures retain their explicit lack of source admission. These tests +run ordinary maintained fit/draw/finalizer and Population/store/replay paths; +they do not replace source qualification or an independent production gate. +""" + +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_full_puf_enrichment import ( + InventedMatrix, + InventedSource, + _donor_columns, + _known, +) +from test_us_graph_full_puf_enrichment import ( + InventedCompleteBoundary, + _complete_frame, + _copy_population, + _load_artifact, +) +from test_us_graph_full_puf_enrichment import ( + attached_full65 as original_attached_full65, +) + +from microcosm.build.us_runtime import full_puf_enrichment as full +from microcosm.build.us_runtime import graph_full_puf_enrichment as placement +from microcosm.build.us_runtime import puf_support as support +from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.build.us_runtime.graph_sources import frame_column_declarations +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ContentStore, + Graph, + KernelRegistry, + KernelResult, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph import population as population_ops +from microcosm.graph.codecs import load_frame_store +from microcosm.graph.population import dtype_for_token + +SCF_MORTGAGE_OUTPUTS = ( + "first_home_mortgage_balance", + "second_home_mortgage_balance", + "first_home_mortgage_interest", + "second_home_mortgage_interest", + "first_home_mortgage_origination_year", + "second_home_mortgage_origination_year", +) +PUF59_TAX_UNIT_OUTPUTS = ( + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "health_savings_account_ald", +) + +# Register the existing ordinary fixture unchanged, without invoking its body. +attached_full65 = original_attached_full65 + + +def _puf59_columns(): + person, tax_unit, person_known, tax_unit_known = _donor_columns() + # Invented upstream leaves, not a source measurement implementation or receipt. + tax_unit[full.PUF59.predictors[0]] = [1.0, 2.0, 3.0, 4.0] + tax_unit[full.PUF59.predictors[1]] = [3.0, 5.0, 2.0, 4.0] + for name in full.PUF59.predictors[:2]: + tax_unit_known[name] = True + return ( + person, + tax_unit.drop(columns=list(SCF_MORTGAGE_OUTPUTS)), + person_known, + tax_unit_known.drop(columns=list(SCF_MORTGAGE_OUTPUTS)), + ) + + +def _puf59_donor(): + person, tax_unit, person_known, tax_unit_known = _puf59_columns() + return full.canonical_full_puf_donor( + person, + tax_unit, + person_known=person_known, + tax_unit_known=tax_unit_known, + profile=full.PUF59, + ) + + +def _puf59_return_only_columns(): + """Invented return incidence bridge; no physical people or source admission.""" + reduced = _puf59_donor() + _, returns, _, _ = _puf59_columns() + for name in full.PUF59.person_outputs: + returns[name] = ( + np.array([0.0, 1.0, 1.0, 0.0]) + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + else reduced[name] + ) + returns["puf_person_incidence_capacity"] = 1.0 + known = pd.DataFrame( + True, + index=returns.index, + columns=( + "weight", + "filing_status_code", + "puf_person_incidence_capacity", + *full.PUF59.person_outputs, + *full.PUF59.tax_unit_outputs, + *full.PUF59.predictors[:2], + ), + ) + return returns, known + + +def _puf59_return_only_donor(): + returns, known = _puf59_return_only_columns() + return full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + + +def _puf59_frame(): + frame = _complete_frame() + tax_unit = frame.table("tax_unit") + tax_unit[full.PUF59.predictors[0]] = np.resize( + np.array([2.0, 4.0, 1.0, 3.0]), len(tax_unit) + ) + tax_unit[full.PUF59.predictors[1]] = np.resize( + np.array([5.0, 4.0, 3.0, 2.0]), len(tax_unit) + ) + return frame + + +def _puf59_known(frame): + return _known(frame).set_axis(full.PUF59.predictors, axis="columns") + + +class InventedProfileMatrix(InventedMatrix): + """Real ordinary projection of the declared selected predictor columns.""" + + ref = "test.puf_profile.matrix@1" + + def run(self, context): + table = context.tables["tax_unit"] + ids = table.tax_unit_id.to_numpy() + features = table.loc[:, list(context.node.inputs[0].columns)].copy() + features.index = pd.Index(ids, name="tax_unit_id") + return KernelResult( + artifacts={ + "matrix": model_input.encode_recipient_matrix( + features, entity="tax_unit", entity_ids=ids + ) + } + ) + + +def _finalizer_arguments(case, result): + artifacts = result.artifacts + return dict( + predictor_known=case.binding.predictor_known, + matrix=artifacts["matrix"].payload, + matrix_producer_key=artifacts["matrix"].producer_key, + raw_draws={ + target: artifacts[f"raw_{index:03d}"].payload + for index, target in enumerate(case.binding.profile.targets) + }, + apply_state=artifacts["apply_state"].payload, + training_state=artifacts["training_state"].payload, + last_model=artifacts["last_model"].payload, + seed=578, + ) + + +def _independent_puf59_population(binding, nodes, artifacts): + """Actual finalizer and graph patch, independent of attachment result code.""" + case = SimpleNamespace(binding=binding) + arguments = _finalizer_arguments(case, SimpleNamespace(artifacts=artifacts)) + candidate, receipt = full.finalize_full_puf( + binding.expected_population.frame, + binding.donor, + **arguments, + profile=full.PUF59, + ) + expected = binding.expected_population + masks = {} + for entity in ("person", "tax_unit"): + table = expected.frame.table(entity) + id_column = expected.frame.schema.entity_id_column(entity) + masks[entity] = pd.Series( + support.puf_tax_detail_clone_mask(table, entity=entity), + index=pd.Index(table[id_column], name=id_column), + dtype=bool, + ) + expected = population_ops.patch( + expected, + nodes[0], + KernelResult( + columns={ + (entity, placement.MASKS[entity]): mask + for entity, mask in masks.items() + } + ), + ) + columns = {} + for owned in nodes[1].outputs: + table = candidate.table(owned.entity) + mask = masks[owned.entity].to_numpy() + id_column = candidate.schema.entity_id_column(owned.entity) + columns[(owned.entity, owned.column)] = pd.Series( + table.loc[mask, owned.column].array, + index=pd.Index(table.loc[mask, id_column], name=id_column), + dtype=dtype_for_token(owned.dtype), + ) + return population_ops.patch( + expected, nodes[1], KernelResult(columns=columns) + ), receipt + + +@pytest.fixture(scope="module", params=("absent", "incumbent")) +def attached_puf59(tmp_path_factory, request): + root = tmp_path_factory.mktemp("puf59_" + request.param) + frame = _puf59_frame() + donor = _puf59_return_only_donor() if request.param == "absent" else _puf59_donor() + if request.param == "incumbent": + # The actual Frame contract requires global column-name uniqueness. + # These six details belong to tax units; the separate observed person + # home_mortgage_interest destination remains part of the PUF59 profile. + table = frame.table("tax_unit") + for index, name in enumerate(SCF_MORTGAGE_OUTPUTS): + # Independent incumbent bytes, including nulls and signed zero, + # survive on native AND clone rows without entering the PUF model. + table[name] = np.resize( + np.array([-0.0, np.nan, -12.5, 1800.5 + index]), len(table) + ) + assert name not in frame.person + frame.revalidate() + known = _puf59_known(frame) + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + decoded = model_input.decode_recipient_matrix(prepared.matrix) + table = decoded.features.copy() + table.insert(0, "tax_unit_id", decoded.entity_ids) + matrix_frame = Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + { + "person_id": decoded.entity_ids, + "person_tax_unit_id": decoded.entity_ids, + } + ), + }, + EntitySchema(group_entities=("tax_unit",)), + {"tax_unit": Weights(np.ones(len(table), dtype="float64"), WeightKind.DESIGN)}, + ) + retained_store = ContentStore( + root / "retained", codecs={"frame-store": load_frame_store} + ) + sources, nodes = {}, [] + for name, value in ( + ("survey", frame), + ("donor", prepared.donor_frame), + ("matrix_input", matrix_frame), + ): + source = "invented." + name + sources[source] = retained_store.put_frame( + full.codec.sha(("ordinary-puf59:" + request.param + ":" + name).encode()), + value, + ) + outputs = ( + frame_column_declarations(value) + if name == "survey" + else tuple( + Owned("tax_unit", name, "float64") + for name in value.table("tax_unit") + if name != "tax_unit_id" + ) + ) + nodes.append( + Node( + name, + InventedSource.ref, + sources=(source,), + structural=StructuralDelta.CREATE, + outputs=outputs, + ) + ) + boundary = Node( + "complete_upstream", + InventedCompleteBoundary.ref, + base="survey", + structural=StructuralDelta.FILTER, + inputs=(Slice("person", ("age",)),), + ) + matrix_node = Node( + "matrix", + InventedProfileMatrix.ref, + population="matrix_input", + inputs=(Slice("tax_unit", full.PUF59.predictors),), + artifact_outputs=(ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE),), + ) + base_graph = Graph( + "us", + tuple(SourceRef(name, "frame-store") for name in sources), + (*nodes, boundary, matrix_node), + ) + kernels = KernelRegistry() + for kernel in ( + InventedSource(), + InventedCompleteBoundary(), + InventedProfileMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + kernels.register(kernel) + observed = {} + retained_manifest = run_graph( + compile_graph(base_graph), + sources=sources, + store=retained_store, + kernels=kernels, + _population_observer=lambda name, value: observed.__setitem__(name, value), + ) + upstream = observed[boundary.id] + matrix = _load_artifact( + retained_manifest, + retained_store, + ArtifactInput( + "matrix", matrix_node.id, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ), + ) + assert matrix.payload == prepared.matrix + fits, applies = full.full_puf_train_apply_nodes( + donor_population="donor", + recipient_population=boundary.id, + matrix_producer=matrix_node.id, + seed=578, + n_estimators=2, + zero_atol=0, + profile=full.PUF59, + ) + retention_arguments = dict( + population=upstream, + expected_population=_copy_population(upstream), + population_node=boundary, + input_owners=dict(upstream.owners), + donor=donor, + predictor_known=known, + matrix=matrix, + fit_nodes=fits, + apply_nodes=applies, + profile=full.PUF59, + ) + binding = placement.retain_full_puf_attachment(**retention_arguments) + attachment_nodes = placement.full_puf_attachment_nodes(binding) + kernels.register(placement.FullPufMaskKernel(binding)) + kernels.register(placement.FullPufAttachKernel(binding)) + compiled = compile_graph( + replace( + base_graph, nodes=(*base_graph.nodes, *fits, *applies, *attachment_nodes) + ) + ) + store = ContentStore(root / "application", codecs={"frame-store": load_frame_store}) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=lambda name, value, target=observed: ( + target.__setitem__(name, value) + ), + ) + assert all( + record.hit is (resume == "require") for record in manifest.nodes.values() + ) + artifacts, producer_keys = placement.load_full_puf_attachment_artifacts( + binding, compiled=compiled, manifest=manifest, store=store + ) + actual = observed[attachment_nodes[-1].id] + evidence = placement.verify_materialized_full_puf_attachment( + binding, + upstream_population=observed[boundary.id], + population=actual, + artifacts=artifacts, + producer_keys=producer_keys, + ) + expected, receipt = _independent_puf59_population( + binding, attachment_nodes, artifacts + ) + replay.same_replayed_population(expected, actual) + frame_key = full.codec.sha( + ( + "puf59-materialized:" + manifest.node(attachment_nodes[-1].id).key + ).encode() + ) + store.put_frame(frame_key, actual.frame) + replay.same_replayed_frame(expected.frame, store.load_frame(frame_key)) + results.append( + SimpleNamespace( + manifest=manifest, + population=_copy_population(actual), + upstream=_copy_population(observed[boundary.id]), + artifacts=artifacts, + producer_keys=producer_keys, + evidence=evidence, + finalizer_receipt=receipt, + ) + ) + replay.same_replayed_population(results[0].population, results[1].population) + assert results[0].artifacts == results[1].artifacts + assert results[0].manifest.key == results[1].manifest.key + return SimpleNamespace( + binding=binding, + retention_arguments=retention_arguments, + nodes=attachment_nodes, + fits=fits, + applies=applies, + results=results, + compiled=compiled, + sources=sources, + store=store, + kernels=kernels, + mortgage_mode=request.param, + ) + + +def test_closed_profiles_keep_observed_mortgage_and_original_conditioning_order(): + assert ( + full.FULL65.person_outputs == full.PUF59.person_outputs == full.PERSON_OUTPUTS + ) + assert full.FULL65.tax_unit_outputs == full.TAX_UNIT_OUTPUTS + assert full.FULL65.targets == full.TARGETS + assert full.PUF59.tax_unit_outputs == PUF59_TAX_UNIT_OUTPUTS + assert len(full.PUF59.person_outputs) == 56 and len(full.PUF59.targets) == 59 + assert full.PUF59.targets == tuple( + t for t in full.TARGETS if t not in SCF_MORTGAGE_OUTPUTS + ) + assert "home_mortgage_interest" in full.PUF59.person_outputs + assert not {"prior_year_wages", "employment_income_last_year"} & set( + full.PUF59.targets + ) + assert full.FULL65.predictors == full.PREDICTORS + assert full.FULL65.donor_auxiliary_columns == () + assert full.PUF59.donor_auxiliary_columns == ("puf_person_incidence_capacity",) + assert full.PUF59.predictors == ( + "puf_2015_filing_status_code", + "puf_2015_capped_return_size", + *full.PREDICTORS[2:], + ) + + +@pytest.mark.parametrize("invalid", (None, "puf59", "full65", 59, True, ("puf59",))) +def test_profiles_require_explicit_enum_members(invalid): + with pytest.raises(ValueError, match="PUF_OUTPUT_PROFILE"): + full.require_puf_output_profile(invalid) + + +def test_puf59_canonical_donor_accepts_truly_absent_scf_columns_and_keeps_observed_interest(): + person, tax_unit, pk, tk = _puf59_columns() + # Actual canonical arithmetic after an invented QBI allocation: entirely + # SSTB positive, ordinary positive, entirely SSTB loss, ordinary positive. + person["self_employment_income_before_lsr"] = [ + 0.0, + 0.0, + 50.0, + 25.0, + 0.0, + 0.0, + 700.0, + 300.0, + ] + person["sstb_self_employment_income_before_lsr"] = [ + 900.0, + 100.0, + 0.0, + 0.0, + -250.0, + -50.0, + 0.0, + 0.0, + ] + before = tuple(value.copy(deep=True) for value in (person, tax_unit, pk, tk)) + donor = full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + assert tuple(donor) == ( + *full.PUF59.predictors, + *full.PUF59.targets, + "weight", + "puf_person_incidence_capacity", + ) + assert not set(SCF_MORTGAGE_OUTPUTS) & set(donor) + np.testing.assert_array_equal( + donor.home_mortgage_interest, + person.home_mortgage_interest.groupby(person.person_tax_unit_id).sum(), + ) + assert donor.weight.tolist() == [1.0, 0.0, 2.0, 3.0] + np.testing.assert_array_equal( + donor[full.PUF59.predictors[3]], [1000.0, 75.0, -300.0, 1000.0] + ) + np.testing.assert_array_equal( + donor.self_employment_income_before_lsr, [0.0, 75.0, 0.0, 1000.0] + ) + np.testing.assert_array_equal( + donor.sstb_self_employment_income_before_lsr, [1000.0, 0.0, -300.0, 0.0] + ) + for actual, expected in zip((person, tax_unit, pk, tk), before, strict=True): + pd.testing.assert_frame_equal(actual, expected) + with pytest.raises(ValueError, match="PUF_DONOR_COLUMNS:tax_unit"): + full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk + ) + + +def test_puf59_donor_does_not_validate_excluded_years_or_consume_scf_payloads(): + person, tax_unit, pk, tk = _puf59_columns() + expected = _puf59_donor() + for name in SCF_MORTGAGE_OUTPUTS: + tax_unit[name] = ["independent SCF", None, False, 1800.5] + actual = full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + pd.testing.assert_frame_equal(actual, expected) + person["business_is_sstb"] = person.business_is_sstb.astype("float64") + with pytest.raises(ValueError, match="PUF_PHYSICAL_TYPE:person.business_is_sstb"): + full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + + +def test_puf59_return_only_donor_uses_explicit_leaves_without_inventing_persons(): + expected = _puf59_donor() + returns, known = _puf59_return_only_columns() + before = returns.copy(deep=True), known.copy(deep=True) + assert "tax_unit_person_count" not in returns + assert "tax_unit_person_count" not in known + for name in full.PUF59.person_outputs: + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + assert returns[name].dtype == np.dtype("float64") + assert set(returns[name]) == {0.0, 1.0} + expected[name] = returns[name] + expected["puf_person_incidence_capacity"] = 1.0 + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + pd.testing.assert_frame_equal(actual, expected) + assert bool((returns.sstb_self_employment_income_before_lsr != 0).any()) + np.testing.assert_array_equal( + actual[full.PUF59.predictors[3]], + returns.self_employment_income_before_lsr + + returns.sstb_self_employment_income_before_lsr, + ) + pd.testing.assert_frame_equal(returns, before[0]) + pd.testing.assert_frame_equal(known, before[1]) + + +@pytest.mark.parametrize( + ("failure", "error"), + ( + ("absent", "PUF_DONOR_COLUMNS:tax_unit"), + ("physical_count_alias", "PUF_DONOR_COLUMNS:tax_unit"), + ("unknown", "PUF_UNKNOWN:tax_unit.puf_person_incidence_capacity"), + ("missing_knownness", "PUF_KNOWNNESS_AXIS:tax_unit"), + ), +) +def test_puf59_return_only_requires_exact_known_incidence_capacity(failure, error): + returns, known = _puf59_return_only_columns() + name = "puf_person_incidence_capacity" + if failure == "absent": + returns.drop(columns=name, inplace=True) + elif failure == "physical_count_alias": + returns.rename(columns={name: "tax_unit_person_count"}, inplace=True) + known.rename(columns={name: "tax_unit_person_count"}, inplace=True) + elif failure == "unknown": + known.loc[0, name] = False + else: + known.drop(columns=name, inplace=True) + with pytest.raises(ValueError, match=error): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + + +@pytest.mark.parametrize( + ("value", "error"), + ( + (0.0, "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN"), + (-1.0, "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN"), + (0.5, "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN"), + ("1", "PUF_PHYSICAL_TYPE:puf_person_incidence_capacity"), + (True, "PUF_PHYSICAL_TYPE:puf_person_incidence_capacity"), + (None, "PUF_UNKNOWN:puf_person_incidence_capacity"), + (np.inf, "PUF_NONFINITE:puf_person_incidence_capacity"), + ), +) +def test_puf59_return_only_incidence_capacity_requires_physical_positive_integer( + value, error +): + returns, known = _puf59_return_only_columns() + name = "puf_person_incidence_capacity" + returns[name] = returns[name].astype(object) + returns.loc[0, name] = value + with pytest.raises(ValueError, match=error): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + + +def test_puf59_return_only_rejects_boolean_incidence_above_source_capacity(): + returns, known = _puf59_return_only_columns() + returns.loc[0, "business_is_sstb"] = 2.0 + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + donor = _puf59_return_only_donor() + donor.loc[0, "business_is_sstb"] = 2.0 + frame = _puf59_frame() + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +@pytest.mark.parametrize("predictor_index", range(8)) +def test_puf59_requires_all_eight_known_predictors(predictor_index): + frame, donor = _puf59_frame(), _puf59_donor() + known = _puf59_known(frame) + known.iloc[0, predictor_index] = False + with pytest.raises(ValueError, match="PUF_UNKNOWN:recipient_predictor"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + known.iloc[0, predictor_index] = True + with pytest.raises(ValueError, match="PUF_KNOWNNESS_AXIS:recipient_predictor"): + full.prepare_full_puf_inputs( + frame, + donor, + predictor_known=known.drop(columns=known.columns[predictor_index]), + profile=full.PUF59, + ) + + +@pytest.mark.parametrize("predictor_index", (0, 1)) +def test_puf59_requires_new_leaves_and_never_falls_back_to_generic_measurements( + predictor_index, +): + frame, donor = _puf59_frame(), _puf59_donor() + known = _puf59_known(frame) + before = frame.table("tax_unit").filing_status_input.copy(deep=True) + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + assert prepared.profile is full.PUF59 + decoded = model_input.decode_recipient_matrix(prepared.matrix) + assert tuple(decoded.features) == full.PUF59.predictors + tax_unit = frame.table("tax_unit") + mask = support.puf_tax_detail_clone_mask(tax_unit, entity="tax_unit") + for name in full.PUF59.predictors[:2]: + np.testing.assert_array_equal(decoded.features[name], tax_unit.loc[mask, name]) + pd.testing.assert_series_equal(tax_unit.filing_status_input, before) + name = full.PUF59.predictors[predictor_index] + tax_unit.drop(columns=name, inplace=True) + with pytest.raises(ValueError, match="PUF_PROFILE_PREDICTOR_SOURCE:" + name): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + # A same-named person column is also insufficient for the explicit return-grain leaf. + frame.table("person")[name] = 1.0 + with pytest.raises(ValueError, match="PUF_PROFILE_PREDICTOR_SOURCE:" + name): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + person, returns, pk, tk = _puf59_columns() + with pytest.raises(ValueError, match="PUF_DONOR_COLUMNS:tax_unit"): + full.canonical_full_puf_donor( + person, + returns.drop(columns=name), + person_known=pk, + tax_unit_known=tk.drop(columns=name), + profile=full.PUF59, + ) + + +@pytest.mark.parametrize("value", (0.5, 6.0)) +def test_puf59_prepare_still_rejects_invalid_selected_boolean_counts(value): + frame, donor = _puf59_frame(), _puf59_donor() + donor.loc[0, "business_is_sstb"] = value + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +@pytest.mark.parametrize( + ("predictor_index", "value", "error"), + ( + (0, "1", "PUF_PHYSICAL_TYPE"), + (1, "2", "PUF_PHYSICAL_TYPE"), + (0, True, "PUF_PHYSICAL_TYPE"), + (1, False, "PUF_PHYSICAL_TYPE"), + (0, None, "missing values before coercion"), + (1, np.inf, "PUF_NONFINITE"), + (0, 5.0, "PUF_PROFILE_FILING_STATUS_DOMAIN"), + (1, 1.5, "PUF_PROFILE_RETURN_SIZE_DOMAIN"), + ), +) +def test_puf59_measured_predictor_leaves_require_physical_values_and_domains( + predictor_index, value, error +): + frame, donor = _puf59_frame(), _puf59_donor() + tax_unit = frame.table("tax_unit") + selected = support.puf_tax_detail_clone_mask(tax_unit, entity="tax_unit") + name = full.PUF59.predictors[predictor_index] + tax_unit[name] = tax_unit[name].astype(object) + tax_unit.loc[tax_unit.index[selected][0], name] = value + with pytest.raises(ValueError, match=error): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +def test_puf59_person_donor_derives_incidence_capacity_from_actual_membership(): + person, returns, pk, tk = _puf59_columns() + person.loc[:1, "business_is_sstb"] = True + returns.loc[0, full.PUF59.predictors[1]] = 1.0 + # The bound is derived from membership here, not read as a return source fact. + returns["puf_person_incidence_capacity"] = "not a source fact" + returns["tax_unit_person_count"] = "not a source fact" + donor = full.canonical_full_puf_donor( + person, returns, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + assert ( + donor.loc[0, "business_is_sstb"] + == donor.loc[0, "puf_person_incidence_capacity"] + == 2 + ) + assert donor.loc[0, full.PUF59.predictors[1]] == 1 + frame = _puf59_frame() + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + for auxiliary in ("puf_person_incidence_capacity", "tax_unit_person_count"): + assert auxiliary not in prepared.donor + assert auxiliary not in prepared.donor_frame.table("tax_unit") + donor.loc[0, "puf_person_incidence_capacity"] = 1.0 + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +@pytest.mark.parametrize( + "year", + ("first_home_mortgage_origination_year", "second_home_mortgage_origination_year"), +) +def test_default_full65_retains_selected_discrete_year_checks(year): + person, tax_unit, pk, tk = _donor_columns() + tax_unit.loc[0, year] = 2010.5 + with pytest.raises(ValueError, match="PUF_YEAR_DOMAIN:" + year): + full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk + ) + + +def test_default_full65_return_only_retains_legacy_person_count_contract(): + person, returns, person_known, tax_unit_known = _donor_columns() + for name in full.FULL65.person_outputs: + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + person.loc[:1, name] = True + expected = full.canonical_full_puf_donor( + person, + returns, + person_known=person_known, + tax_unit_known=tax_unit_known, + ) + returns["tax_unit_person_count"] = 2.0 + for name in full.FULL65.person_outputs: + returns[name] = expected[name] + known = pd.DataFrame( + True, + index=returns.index, + columns=( + "weight", + "filing_status_code", + "tax_unit_person_count", + *full.FULL65.person_outputs, + *full.FULL65.tax_unit_outputs, + ), + ) + assert "puf_person_incidence_capacity" not in returns + for arguments in ({}, {"profile": full.FULL65}): + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.FULL65.person_outputs, + **arguments, + ) + pd.testing.assert_frame_equal(actual, expected) + assert actual.loc[0, "business_is_sstb"] == 2.0 + # FULL65 compatibility is explicit: its historical alias excludes the + # nonzero SSTB component rather than silently changing old evidence. + assert bool((returns.sstb_self_employment_income_before_lsr != 0).any()) + np.testing.assert_array_equal( + actual[full.FULL65.predictors[3]], returns.self_employment_income_before_lsr + ) + returns["puf_person_incidence_capacity"] = 1.0 + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.FULL65.person_outputs, + ) + pd.testing.assert_frame_equal(actual, expected) + returns.drop(columns="tax_unit_person_count", inplace=True) + with pytest.raises(ValueError, match="PUF_DONOR_COLUMNS:tax_unit"): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.FULL65.person_outputs, + ) + + +def test_real_puf59_chain_finalization_attachment_and_required_replay(attached_puf59): + case = attached_puf59 + assert case.binding.profile is full.PUF59 + assert len(case.fits) == len(case.applies) == 59 + assert [ + edge.name + for edge in case.applies[-1].artifact_inputs + if edge.name.startswith("prior_") + ] == [f"prior_{i:03d}" for i in range(58)] + assert tuple(owned.column for owned in case.nodes[-1].outputs) == full.PUF59.targets + for node in (*case.fits, *case.applies): + assert node.params["phase"] == full.PUF59.phase + assert not { + *SCF_MORTGAGE_OUTPUTS, + "puf_person_incidence_capacity", + "tax_unit_person_count", + } & {column for item in node.inputs for column in item.columns} + assert all(node.params["predictors"] == full.PUF59.predictors for node in case.fits) + assert all(node.params["targets"] == full.PUF59.targets for node in case.fits) + assert "raw_058" in case.results[0].artifacts + assert "raw_059" not in case.results[0].artifacts + if case.mortgage_mode == "absent": + assert case.binding.donor.puf_person_incidence_capacity.eq(1.0).all() + for name in full.PUF59.person_outputs: + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + assert set(case.binding.donor[name]) == {0.0, 1.0} + for result in case.results: + assert result.finalizer_receipt["output_profile"] == "puf59" + assert result.finalizer_receipt["person_target_count"] == 56 + assert result.finalizer_receipt["tax_unit_target_count"] == 3 + assert result.finalizer_receipt["target_count"] == 59 + assert result.finalizer_receipt["predictor_order"] == list( + full.PUF59.predictors + ) + assert result.finalizer_receipt["donor_auxiliary_columns"] == [ + "puf_person_incidence_capacity" + ] + assert result.finalizer_receipt["target_order"] == list(full.PUF59.targets) + assert result.finalizer_receipt["person_target_count"] == 56 + assert result.finalizer_receipt["tax_unit_target_count"] == 3 + assert result.finalizer_receipt["target_count"] == 59 + assert result.evidence["output_profile"] == "puf59" + assert result.evidence["source_admission_issued"] is False + assert result.evidence["release_eligible"] is False + for entity in ("person", "tax_unit"): + before, after = ( + result.upstream.frame.table(entity), + result.population.frame.table(entity), + ) + for name in SCF_MORTGAGE_OUTPUTS: + if case.mortgage_mode == "absent" or entity != "tax_unit": + assert name not in before and name not in after + else: + assert population_ops.storage_equal( + before[name], after[name], np.ones(len(before), dtype=bool) + ) + assert ( + result.population.owners[(entity, name)] + == result.upstream.owners[(entity, name)] + ) + for owned in case.nodes[-1].outputs: + before = result.upstream.frame.table(owned.entity) + after = result.population.frame.table(owned.entity) + mask = support.puf_tax_detail_clone_mask(before, entity=owned.entity) + assert after.loc[mask, owned.column].notna().all() + if owned.column in before: + assert population_ops.storage_equal( + before[owned.column], after[owned.column], ~mask + ) + else: + assert after.loc[~mask, owned.column].isna().all() + + +def test_puf59_incidence_capacity_remains_bound_to_attachment(attached_puf59): + case = attached_puf59 + donor = case.binding.donor.copy(deep=True) + donor["puf_person_incidence_capacity"] += 1.0 + stale = replace(case.binding, donor=donor) + with pytest.raises(ValueError, match="FULL_PUF_DONOR_CHANGED"): + placement.full_puf_attachment_nodes(stale) + rebound = placement.retain_full_puf_attachment( + **{**case.retention_arguments, "donor": donor} + ) + assert rebound.fit_nodes == case.binding.fit_nodes + assert rebound.apply_nodes == case.binding.apply_nodes + assert rebound.matrix == case.binding.matrix + nodes = placement.full_puf_attachment_nodes(rebound) + for original, changed in zip(case.nodes, nodes, strict=True): + assert original.params["donor_values"] != changed.params["donor_values"] + + +def test_default_full65_matches_explicit_full65_and_runs_actual_complete_chain( + attached_full65, +): + case = attached_full65 + kwargs = dict( + donor_population="donor", + recipient_population="complete_upstream", + matrix_producer="matrix", + seed=578, + n_estimators=2, + zero_atol=0, + ) + default = full.full_puf_train_apply_nodes(**kwargs) + explicit = full.full_puf_train_apply_nodes(**kwargs, profile=full.FULL65) + assert default == explicit == (case.fits, case.applies) + assert len(case.fits) == len(case.applies) == 65 + assert tuple(owned.column for owned in case.nodes[-1].outputs) == full.TARGETS + assert case.binding.profile is full.FULL65 + assert set(SCF_MORTGAGE_OUTPUTS) <= set(full.FULL65.tax_unit_outputs) + for result in case.results: + assert "raw_064" in result.artifacts + assert result.finalizer_receipt["output_profile"] == "full65" + assert result.finalizer_receipt["target_order"] == list(full.TARGETS) + + +def test_actual_wrong_profile_chains_and_manifests_are_refused( + attached_puf59, attached_full65 +): + for correct, wrong in ( + (attached_puf59, attached_full65), + (attached_full65, attached_puf59), + ): + with pytest.raises(ValueError, match="FULL_PUF_CHAIN_ROSTER"): + placement.retain_full_puf_attachment( + **{**correct.retention_arguments, "profile": wrong.binding.profile} + ) + result = correct.results[-1] + arguments = _finalizer_arguments(correct, result) + arguments.pop("predictor_known") + arguments.pop("last_model") + with pytest.raises(ValueError, match="PUF_"): + full.decode_full_puf_draws(**arguments, profile=wrong.binding.profile) + # Feed the real other-profile compiled graph, manifest and ContentStore; + # no fabricated manifest, source issuer or artifact readiness is involved. + with pytest.raises(ValueError, match="FULL_PUF_"): + placement.load_full_puf_attachment_artifacts( + correct.binding, + compiled=wrong.compiled, + manifest=wrong.results[-1].manifest, + store=wrong.store, + ) + + +def test_puf59_wrong_profile_model_and_trimmed_full65_history_are_refused( + attached_puf59, attached_full65 +): + case, result = attached_puf59, attached_puf59.results[-1] + arguments = _finalizer_arguments(case, result) + arguments["last_model"] = ( + attached_full65.results[-1].artifacts["last_model"].payload + ) + with pytest.raises(ValueError, match="Legacy QRF artifact content digest mismatch"): + full.finalize_full_puf( + case.binding.expected_population.frame, + case.binding.donor, + **arguments, + profile=full.PUF59, + ) + full_arguments = _finalizer_arguments(attached_full65, attached_full65.results[-1]) + full_arguments.pop("predictor_known") + full_arguments.pop("last_model") + full_arguments["raw_draws"] = { + target: full_arguments["raw_draws"][target] for target in full.PUF59.targets + } + with pytest.raises(ValueError, match="PUF_"): + full.decode_full_puf_draws(**full_arguments, profile=full.PUF59) + + +def test_puf59_rebuilt_binding_survives_nullable_frame_store_normalization( + attached_puf59, tmp_path +): + case = attached_puf59 + retained = _copy_population(case.binding.expected_population) + nullable = retained.frame.table("person").unrelated_nullable.array + missing = nullable._mask.copy() + assert missing.any() + nullable._data[missing] = True + arguments = {**case.retention_arguments, "expected_population": retained} + physical_binding = placement.retain_full_puf_attachment(**arguments) + assert placement.full_puf_attachment_nodes(physical_binding) == case.nodes + + directory = tmp_path / "puf59-new-session" + key = full.codec.sha(b"invented-puf59-retained-upstream") + ContentStore(directory).put_frame(key, retained.frame) + # Fresh ContentStore decoding performs the actual v2 masked-byte normalization. + restored_frame = ContentStore(directory).load_frame(key) + assert ( + not restored_frame.table("person").unrelated_nullable.array._data[missing].any() + ) + replay.same_replayed_frame(retained.frame, restored_frame) + restored = replace(_copy_population(retained), frame=restored_frame) + upstream = _copy_population(case.results[-1].upstream) + rebuilt = placement.retain_full_puf_attachment( + **{ + **arguments, + "population": upstream, + "expected_population": restored, + "input_owners": dict(upstream.owners), + } + ) + assert physical_binding.expected_stamp != rebuilt.expected_stamp + nodes = placement.full_puf_attachment_nodes(rebuilt) + assert nodes == case.nodes + replacements = {node.id: node for node in nodes} + compiled = compile_graph( + replace( + case.compiled.graph, + nodes=tuple( + replacements.get(node.id, node) for node in case.compiled.graph.nodes + ), + ) + ) + registry = KernelRegistry() + for kernel in ( + InventedSource(), + InventedCompleteBoundary(), + InventedProfileMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + placement.FullPufMaskKernel(rebuilt), + placement.FullPufAttachKernel(rebuilt), + ): + registry.register(kernel) + observed = {} + manifest = run_graph( + compiled, + sources=case.sources, + store=case.store, + kernels=registry, + resume="require", + _population_observer=lambda name, value: observed.__setitem__(name, value), + ) + assert all(record.hit for record in manifest.nodes.values()) + assert manifest.key == case.results[-1].manifest.key + artifacts, producer_keys = placement.load_full_puf_attachment_artifacts( + rebuilt, compiled=compiled, manifest=manifest, store=case.store + ) + evidence = placement.verify_materialized_full_puf_attachment( + rebuilt, + upstream_population=observed[rebuilt.population_node.id], + population=observed[nodes[-1].id], + artifacts=artifacts, + producer_keys=producer_keys, + ) + assert evidence["output_profile"] == "puf59" + replay.same_replayed_population(case.results[-1].population, observed[nodes[-1].id]) diff --git a/packages/microcosm-build/tests/test_us_geography_integration.py b/packages/microcosm-build/tests/test_us_geography_integration.py new file mode 100644 index 000000000..7e2521de9 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_geography_integration.py @@ -0,0 +1,276 @@ +"""Ordinary invented geography controls for the combined graph contract. + +All NPZ/CSV bytes are created in the test. The national-sized fixture satisfies +the shape validator with invented equal cell masses and district allocations; +it is not a Census crosswalk or evidence of genuine source admission. +""" + +import hashlib +import json +from collections import Counter +from dataclasses import fields +from io import BytesIO + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_geography as geography +from microcosm.build.us_runtime.congressional_district_vintage import ( + CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, + SOURCE_CONGRESSIONAL_DISTRICT_PREFIX, +) +from microcosm.build.us_runtime.puma_ladder import ( + assign_us_puma_ladder, + decode_us_puma_ladder, + load_us_puma_ladder, +) +from microcosm.frame import WeightKind, Weights +from microcosm.graph import ArtifactValue, KernelContext, Owned +from microcosm.graph.kernel import NumericScope + + +def _ordinary_lookup_bytes(**overrides): + # The production validator requires 436 districts across 51 jurisdictions. + # Allocate invented counts: 7 at-large + 33*10 + 11*9 = 436. + states = [i for i in range(1, 57) if i not in {3, 7, 14, 43, 52}] + at_large = {2, 10, 11, 38, 46, 50, 56} + multi = [i for i in states if i not in at_large] + rows, crosswalk = [], [] + for state in states: + districts = ( + [0] + if state in at_large + else range(1, 11 if multi.index(state) < 33 else 10) + ) + for district in districts: + county = max(1, district) + if state == 36: + county = 61 if district == 1 else (1 if district == 2 else district) + puma = state * 100_000 + 100 + tract = (state * 1000 + county) * 1_000_000 + 100 + cd = state * 100 + district + rows.append((puma, tract, cd)) + crosswalk.append( + { + "source_geography_id": f"{SOURCE_CONGRESSIONAL_DISTRICT_PREFIX}{cd:04d}", + "target_geography_id": f"{CURRENT_CONGRESSIONAL_DISTRICT_PREFIX}{cd:04d}", + "weight": 1.0, + "pair_population": 1, + } + ) + rows.sort() + pumas = Counter(row[0] for row in rows) + arrays = { + "puma": np.asarray(sorted(pumas), dtype=np.int64), + "puma_population": np.asarray([pumas[p] for p in sorted(pumas)]), + "joint_overlap_puma": np.asarray([r[0] for r in rows]), + "joint_overlap_tract": np.asarray([r[1] for r in rows]), + "joint_overlap_cd": np.asarray([r[2] for r in rows]), + "joint_overlap_population": np.ones(len(rows), dtype=np.int64), + "metadata_json": np.asarray( + json.dumps( + { + "schema_version": 2, + "kind": "us_puma_ladder", + "puma_vintage": "2020_puma", + "sampling_basis": "population", + "layers": { + name: { + "vintage": vintage, + "source": "invented ordinary fixture", + } + for name, vintage in ( + ("congressional_district", "119th_congress"), + ("county", "2020_census"), + ("tract", "2020_census"), + ) + }, + } + ) + ), + } + for layer, coordinate in (("cd", 2), ("county", 1), ("tract", 1)): + counts = Counter( + ( + row[0], + row[coordinate] // 1_000_000 if layer == "county" else row[coordinate], + ) + for row in rows + ) + keys = sorted(counts) + arrays[f"{layer}_overlap_puma"] = np.asarray([key[0] for key in keys]) + arrays[f"{layer}_overlap_{layer}"] = np.asarray([key[1] for key in keys]) + arrays[f"{layer}_overlap_population"] = np.asarray( + [counts[key] for key in keys] + ) + arrays.update(overrides) + stream = BytesIO() + np.savez_compressed(stream, **arrays) + return stream.getvalue(), pd.DataFrame(crosswalk).to_csv(index=False).encode() + + +def _nodes(): + return geography.us_geography_nodes( + (Owned("person", "age", "int64"), Owned("household", "state_fips", "int64")), + base="ordinary_base", + context_producer="ordinary_context", + ) + + +def _context(household, weights, ladder_bytes, crosswalk_bytes): + artifacts = {} + for alias, payload, type_ in ( + ("ladder", ladder_bytes, geography.US_PUMA_LOOKUP_TYPE), + ("crosswalk", crosswalk_bytes, geography.US_CD_CROSSWALK_TYPE), + ): + artifacts[alias] = ArtifactValue( + payload, + type_, + hashlib.sha256(payload).hexdigest(), + "a" * 64, + NumericScope(), + ) + node = _nodes()[-1] + return KernelContext( + node=node, + tables={"household": household}, + weights={"household": Weights(np.asarray(weights), WeightKind.DESIGN)}, + strata=pd.Series(dtype="int64"), + params=node.params, + rng=np.random.default_rng(0), + artifacts=artifacts, + ) + + +def _households(): + # Same ordinary mass control as test_us_puma_ladder._gated_household: + # NYC = 2.6% nationally and 2.6/6.2 of New York's mass. + return pd.DataFrame( + { + "household_id": [1, 2, 3, 4, 5], + "state_fips": [36, 36, 6, 48, 1], + "puma": ["3600100", "3600100", "0600100", "4800100", "0100100"], + "county_fips": ["36061", "36001", "06001", "48001", "01001"], + "congressional_district_geoid": [3601, 3602, 601, 4801, 101], + } + ) + + +def test_byte_and_path_joint_decode_and_assignments_match(tmp_path): + payload, _ = _ordinary_lookup_bytes() + path = tmp_path / "invented-ladder.npz" + path.write_bytes(payload) + path_ladder = load_us_puma_ladder(path) + byte_ladder = decode_us_puma_ladder(payload) + for field in fields(path_ladder): + left, right = getattr(path_ladder, field.name), getattr(byte_ladder, field.name) + if isinstance(left, np.ndarray): + np.testing.assert_array_equal(left, right) + assert left.dtype == right.dtype + else: + assert left == right + for assign_tract in (False, True): + pd.testing.assert_frame_equal( + assign_us_puma_ladder( + _households(), path_ladder, seed=17, assign_tract=assign_tract + ), + assign_us_puma_ladder( + _households(), byte_ladder, seed=17, assign_tract=assign_tract + ), + ) + + +@pytest.mark.parametrize("defect", ["schema_v1", "joint_population"]) +def test_byte_and_path_reject_the_same_joint_defect(tmp_path, defect): + payload, _ = _ordinary_lookup_bytes() + with np.load(BytesIO(payload), allow_pickle=False) as archive: + arrays = {key: archive[key] for key in archive.files} + if defect == "schema_v1": + metadata = json.loads(str(arrays["metadata_json"].item())) + metadata["schema_version"] = 1 + arrays["metadata_json"] = np.asarray(json.dumps(metadata)) + else: + arrays["joint_overlap_population"][0] += 1 + path = tmp_path / "invalid-invented-ladder.npz" + np.savez_compressed(path, **arrays) + with pytest.raises(ValueError) as path_error: + load_us_puma_ladder(path) + with pytest.raises(ValueError) as byte_error: + decode_us_puma_ladder(path.read_bytes()) + assert str(path_error.value) == str(byte_error.value) + + +@pytest.mark.parametrize("wrapper", [bytearray, memoryview]) +def test_byte_decoder_requires_immutable_bytes(wrapper): + with pytest.raises(TypeError, match="immutable bytes"): + decode_us_puma_ladder(wrapper(b"not an archive")) + + +@pytest.mark.parametrize("unsupported_weight", [0.0, 1.0]) +def test_graph_gate_rejects_real_unsupported_pair_even_at_zero_weight( + unsupported_weight, +): + ladder_bytes, crosswalk_bytes = _ordinary_lookup_bytes() + household = _households() + weights = [2.6, 3.6, 73.8, 20.0, unsupported_weight] + kernel = geography.USGeographyGateKernel() + assert ( + kernel.run(_context(household, weights, ladder_bytes, crosswalk_bytes)).receipt[ + "outcome" + ] + == "pass" + ) + # County 01001 and CD 102 are individually present in PUMA 0100100, + # but the only supported pairs are county 01001/CD 101, county 01002/CD 102. + household.loc[4, "congressional_district_geoid"] = 102 + result = kernel.run(_context(household, weights, ladder_bytes, crosswalk_bytes)) + assert result.receipt["outcome"] == "fail" + expected_mass = geography.us_puma_ladder_gate(household, np.asarray(weights)) + ladder = decode_us_puma_ladder(ladder_bytes) + assert (100100, 1001) in set( + zip(ladder.county_overlap_puma, ladder.county_overlap_county, strict=True) + ) + assert (100100, 102) in set( + zip(ladder.cd_overlap_puma, ladder.cd_overlap_cd, strict=True) + ) + assert (100100, 1001, 102) not in set( + zip( + ladder.joint_overlap_puma, + ladder.joint_overlap_tract // 1_000_000, + ladder.joint_overlap_cd, + strict=True, + ) + ) + expected_joint = geography.us_puma_ladder_joint_support_gate(household, ladder) + assert expected_mass.passed + assert not expected_joint.passed + assert expected_joint.details["unsupported_rows"] == 1 + assert ( + result.receipt["evidence"] + == geography.GateReport((expected_mass, expected_joint)).to_manifest() + ) + + +def test_graph_gate_retains_independent_legacy_mass_failure(): + ladder_bytes, crosswalk_bytes = _ordinary_lookup_bytes() + household = _households() + weights = np.asarray([0.0, 6.2, 73.8, 20.0, 0.0]) + ladder = decode_us_puma_ladder(ladder_bytes) + assert geography.us_puma_ladder_joint_support_gate(household, ladder).passed + assert not geography.us_puma_ladder_gate(household, weights).passed + result = geography.USGeographyGateKernel().run( + _context(household, weights, ladder_bytes, crosswalk_bytes) + ) + assert result.receipt["outcome"] == "fail" + + +def test_graph_gate_declares_both_lookup_producer_dependencies(): + lookup, _, _, gate = _nodes() + assert { + (item.name, item.producer, item.artifact, item.type) + for item in gate.artifact_inputs + } == { + ("ladder", lookup.id, "ladder", geography.US_PUMA_LOOKUP_TYPE), + ("crosswalk", lookup.id, "crosswalk", geography.US_CD_CROSSWALK_TYPE), + } diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py b/packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py new file mode 100644 index 000000000..f26803e21 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py @@ -0,0 +1,318 @@ +"""Issued 19/35-node financial graphs over invented source files and blocks.""" + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments +from test_us_current_property_income_sources import source_arguments +from test_us_graph_atomic_survey_population import _support_payload +from test_us_survey_social_security import _social_security_arguments + +from microcosm.build.us_runtime import graph_atomic_survey_financial as runner +from microcosm.build.us_runtime import graph_current_survey_property as property_graph +from microcosm.build.us_runtime import graph_puf55_survey_recipients as puf_recipient +from microcosm.graph import compile_graph + + +@pytest.fixture(scope="module") +def property_host_source(tmp_path_factory): + import test_us_current_asec_demographics as demographic_fixture_module + import test_us_current_asec_income_routing as routing_fixture + import test_us_survey_population_preparation as survey_fixture_module + + from microcosm.build.us_runtime import current_asec_demographics as demographics + + root = tmp_path_factory.mktemp("atomic-property-financial") + with pytest.MonkeyPatch.context() as patch: + # Compose complete invented member bytes before the real source issuer + # runs. Only fixture constructors and their registry pins are replaced. + # The nesting order is original source -> Social Security -> demographics + # -> routing -> property. Each extension preserves prior columns and + # rebuilds the retained attachment; preparation is issued only afterward. + def social_security_fixture(path, monkeypatch, *, zero): + assert zero is False + return _social_security_arguments(path, monkeypatch, ambiguous=False) + + def demographic_fixture(path, monkeypatch): + return _demographic_arguments(path, monkeypatch, unknown=False, zero=False) + + patch.setattr(demographic_fixture_module, "fixture", social_security_fixture) + patch.setattr(routing_fixture, "fixture", demographic_fixture) + original_person = survey_fixture_module._person + + def person(*args, **kwargs): + row = original_person(*args, **kwargs) + # The property fixture makes this person age 14. Its other income + # questions must also be blank before the source archive is built. + if row["SERIALNO"] == "2024GQ0000001": + row.update(SSP="", SSIP="") + return row + + patch.setattr(survey_fixture_module, "_person", person) + arguments = source_arguments(root, patch) + patch.setattr( + demographics.demographic, + "_MEMBER_PINS", + property_graph.sources.routing.coverage._MEMBER_PINS, + ) + payload, source_ids = _support_payload() + support = root / "invented-block-support.npz" + support.write_bytes(payload) + config = runner.reconstruction.AtomicSurveyReconstruction( + support_path=str(support), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + options = property_graph.PropertyIncomeOptions( + scales=(1.0, 1.0, 1.0, 1.0), atol=1e-10, rtol=1e-12, n_estimators=2 + ) + call = { + **arguments, + "geography_config": config, + "demographic_conditioning": True, + "n_estimators": 2, + "return_values": True, + } + yield SimpleNamespace(root=root, call=call, options=options) + + +@pytest.fixture(scope="module") +def property_host_base(property_host_source): + case = property_host_source + run = runner.run_atomic_survey_financial( + **case.call, property_income=None, resume="auto" + ) + yield run + runner.values.source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture(scope="module") +def property_host_cold(property_host_source): + case = property_host_source + run = runner.run_atomic_survey_financial( + **case.call, property_income=case.options, resume="auto" + ) + yield run + runner.values.source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture(scope="module") +def property_host_warm(property_host_source, property_host_cold): + case = property_host_source + run = runner.run_atomic_survey_financial( + **case.call, property_income=case.options, resume="require" + ) + yield run + runner.values.source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture(scope="module") +def actual_property_host(request, property_host_source): + # Explicit ordering demonstrates reuse of the unchanged 19-node prefix. + # A targeted PUF test requests only the cold host and need not rerun these + # separately accepted baseline/replay checks. + base = request.getfixturevalue("property_host_base") + cold = request.getfixturevalue("property_host_cold") + warm = request.getfixturevalue("property_host_warm") + (property_host_source.root / "host-acceptance.json").write_text( + json.dumps( + { + "scope": "invented source-issued financial host; no native or engine", + "base_nodes": len(base.compiled.order), + "extended_nodes": len(cold.compiled.order), + "cold_hits": sum(n.hit for n in cold.manifest.nodes.values()), + "required_hits": sum(n.hit for n in warm.manifest.nodes.values()), + "persons": cold.financial_population.frame.n("person"), + "households": cold.financial_population.frame.n("household"), + "owned_property_columns": [ + o.column for o in property_graph.owned_columns() + ], + "tax_split_rebased": False, + }, + indent=2, + ) + + "\n" + ) + return SimpleNamespace( + base=base, cold=cold, warm=warm, options=property_host_source.options + ) + + +def test_default_graph_and_payload_remain_nineteen_nodes(actual_property_host): + run = actual_property_host.base + document = runner.codec.decode_json(run.checked_view().payload) + assert len(run.compiled.order) == 19 + assert runner.financial_output_node(run) == runner.financial.ATTACH_NODE + assert document["owned_columns"] == list(runner.values.OUTPUTS) + assert ( + not {"property_income", "property_node_count", "tax_split_rebased"} + & document.keys() + ) + assert not any( + n.startswith(property_graph.PREFIX + ".") for n in run.compiled.order + ) + + +def test_actual_extended_host_cold_required_and_checked_issuance(actual_property_host): + case = actual_property_host + assert len(case.cold.compiled.order) == len(case.warm.compiled.order) == 35 + assert case.cold.manifest.key == case.warm.manifest.key + assert sum(n.hit for n in case.cold.manifest.nodes.values()) == 19 + assert all(n.hit for n in case.warm.manifest.nodes.values()) + runner.atomic.same_replayed_population( + case.cold.financial_population, case.warm.financial_population + ) + checked = [run.checked_view() for run in (case.cold, case.warm)] + assert checked[0].payload == checked[1].payload + document = runner.codec.decode_json(checked[0].payload) + assert document["property_income"] == runner.codec.decode_json( + case.options.to_bytes() + ) + assert document["property_node_count"] == 16 + assert document["tax_split_rebased"] is False + assert document["capital_gains_conditioning"] == property_graph.CAP_LIMITATION + assert document["release_eligible"] is False + assert runner.financial_output_node(case.cold) == property_graph.ATTACH_NODE + + +def test_extended_host_retains_complete_legacy_population(actual_property_host): + base, extended = actual_property_host.base, actual_property_host.cold + state = runner._run_entry(extended)[2] + runner.atomic.same_replayed_population( + base.financial_population, state.legacy_financial_population + ) + before, after = base.financial_population, extended.financial_population + for entity in before.frame.entities: + pd.testing.assert_frame_equal( + after.frame.table(entity)[before.frame.table(entity).columns], + before.frame.table(entity), + check_exact=True, + ) + assert after.version == before.version + assert after.frame.schema == before.frame.schema + assert after.frame.links == before.frame.links + assert after.frame.metadata == before.frame.metadata + assert after.mass_ledger == before.mass_ledger + assert after.frame.mass_log == before.frame.mass_log + pd.testing.assert_series_equal(after.frame.strata, before.frame.strata) + for entity in before.design_weights: + np.testing.assert_array_equal( + after.design_weights[entity], before.design_weights[entity] + ) + np.testing.assert_array_equal( + after.frame.weights_for(entity).values, + before.frame.weights_for(entity).values, + ) + assert dict(after.owners) == { + **before.owners, + **{ + ("person", o.column): property_graph.ATTACH_NODE + for o in property_graph.owned_columns() + }, + } + + +def test_property_values_and_unknowns_follow_original_clone_identity( + actual_property_host, +): + run = actual_property_host.cold + people = run.financial_population.frame.person + source_id = runner.values.provenance.support_source_id_column("person") + columns = [o.column for o in property_graph.owned_columns()] + for _, pair in people.groupby(source_id, sort=False): + assert len(pair) == 2 + pd.testing.assert_series_equal( + pair.iloc[0][columns], pair.iloc[1][columns], check_names=False + ) + assert (~people.property_anchor_known).any() + assert ( + people.loc[ + ~people.property_anchor_known, property_graph.PROPERTY_REPORTED_TOTAL + ] + .isna() + .all() + ) + known = people.property_components_known + assert known.any() and (~known).any() + channel = runner.values.provenance.support_channel_column("person") + modeled = known & people[channel].astype(str).eq("acs") + assert modeled.any() + values = people.loc[modeled, list(property_graph.PROPERTY_COMPONENTS)] + np.testing.assert_allclose( + values.sum(axis=1), + people.loc[modeled, property_graph.PROPERTY_REPORTED_TOTAL], + atol=1e-10, + rtol=1e-12, + ) + assert (values.iloc[:, :3] >= 0).all().all() + assert (people.property_reported_total < 0).any() + + +@pytest.mark.parametrize("defect", ("options", "legacy", "final")) +def test_retained_host_lifetime_rejects_mutation(actual_property_host, defect): + run = actual_property_host.cold + state = runner._run_entry(run)[2] + if defect == "options": + original = state.property_income.atol + object.__setattr__(state.property_income, "atol", original + 1.0) + try: + with pytest.raises( + ValueError, match="PROPERTY_OPTIONS_OR_LEGACY_POPULATION_CHANGED" + ): + run.checked_view() + finally: + object.__setattr__(state.property_income, "atol", original) + else: + population = ( + state.legacy_financial_population + if defect == "legacy" + else run.financial_population + ) + table = population.frame.person + column = ( + runner.values.OUTPUTS[0] + if defect == "legacy" + else property_graph.PROPERTY_REPORTED_TOTAL + ) + index = table.index[table[column].notna()][0] + original = table.loc[index, column] + table.loc[index, column] = original + 1.0 + try: + with pytest.raises( + ValueError, + match=( + "PROPERTY_OPTIONS_OR_LEGACY_POPULATION_CHANGED" + "|FINANCIAL_RUN_POPULATION_CHANGED" + "|FINANCIAL_RUN_ATTACHED_POPULATION_CHANGED" + ), + ): + run.checked_view() + finally: + table.loc[index, column] = original + runner._pure_run(run, runner._run_entry(run)) + + +def test_detached_run_copy_has_no_issuer_authority(actual_property_host): + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + replace(actual_property_host.cold).checked_view() + + +def test_puf_recipient_reads_complete_final_property_writer(property_host_cold): + run = property_host_cold + qualified = puf_recipient.values.qualify_puf55_survey_recipients(run) + nodes = puf_recipient.puf55_survey_recipient_nodes(qualified) + compiled = compile_graph( + replace(run.compiled.graph, nodes=(*run.compiled.graph.nodes, *nodes)) + ) + assert len(compiled.order) == 37 + for node in nodes: + assert property_graph.ATTACH_NODE in compiled.predecessors[node.id] + inputs = next(s.columns for s in node.inputs if s.entity == "person") + assert {o.column for o in property_graph.owned_columns()}.issubset(inputs) + assert qualified.financial_run is run diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py b/packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py new file mode 100644 index 000000000..1f9d7330b --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py @@ -0,0 +1,254 @@ +"""Optional 38-node financial host over actual invented source owners.""" + +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_atomic_property_financial import property_host_source # noqa: F401 + +from microcosm.build.us_runtime import graph_atomic_survey_financial as runner +from microcosm.build.us_runtime import graph_current_survey_property as property_graph +from microcosm.build.us_runtime import graph_property_tax_leaves as tax +from microcosm.build.us_runtime import graph_puf55_survey_recipients as recipients +from microcosm.graph import describe, explain_html +from microcosm.graph.store import StoreCorrupt + + +@pytest.fixture(scope="module") +def tax_host_cold(request): + case = request.getfixturevalue("property_host_source") + return runner.run_atomic_survey_financial( + **case.call, + property_income=replace(case.options, completion_routing=True), + rebase_property_taxes=True, + resume="auto", + ) + + +@pytest.fixture(scope="module") +def tax_host_warm(request, tax_host_cold): + case = request.getfixturevalue("property_host_source") + return runner.run_atomic_survey_financial( + **case.call, + property_income=replace(case.options, completion_routing=True), + rebase_property_taxes=True, + resume="require", + ) + + +def test_opt_in_cold_required_issued_output(tax_host_cold, tax_host_warm): + cold, warm = tax_host_cold, tax_host_warm + assert len(cold.compiled.order) == len(warm.compiled.order) == 38 + assert cold.manifest.key == warm.manifest.key + assert all(record.hit for record in warm.manifest.nodes.values()) + runner.atomic.same_replayed_population( + cold.financial_population, warm.financial_population + ) + checked = cold.checked_view() + assert checked.payload == warm.checked_view().payload + document = runner.codec.decode_json(checked.payload) + assert document["property_income"]["completion_routing"] is True + assert document["tax_split_rebased"] is True + assert document["tax_rebase_node_count"] == 3 + assert document["tax_leaf_complete"] is False + assert document["release_eligible"] is False + assert runner.financial_output_node(cold) == tax.GATE_NODE + assert cold.financial_population.version == tax.RECEIVING_NODE + + +def test_opt_in_preserves_full_prior_populations_and_unknowns(tax_host_cold): + run = tax_host_cold + state = runner._run_entry(run)[2] + legacy, before, after = ( + state.legacy_financial_population, + state.property_population, + run.financial_population, + ) + assert before.version == legacy.version != after.version + for entity in before.frame.entities: + kept = [ + name + for name in before.frame.table(entity) + if name not in tax.TAX_LEAF_COLUMNS + ] + pd.testing.assert_frame_equal( + before.frame.table(entity)[kept], + after.frame.table(entity)[kept], + check_exact=True, + ) + assert before.frame.schema == after.frame.schema + assert before.frame.metadata == after.frame.metadata + assert before.frame.mass_log == after.frame.mass_log + assert after.mass_ledger[:-1] == before.mass_ledger + assert after.mass_ledger[-1].node_id == tax.RECEIVING_NODE + pd.testing.assert_series_equal(before.frame.strata, after.frame.strata) + for entity in before.design_weights: + np.testing.assert_array_equal( + before.design_weights[entity], after.design_weights[entity] + ) + expected = tax.split_property_tax_leaves(before.frame.person) + for name in tax.TAX_LEAF_COLUMNS: + np.testing.assert_array_equal( + after.frame.person[name].to_numpy().view("uint64"), + expected[name].to_numpy().view("uint64"), + ) + assert after.frame.person[list(tax.TAX_LEAF_COLUMNS)].isna().any().any() + assert legacy.frame.person[list(tax.TAX_LEAF_COLUMNS)].notna().all().all() + + +def test_incomplete_development_run_refuses_puf_before_source_qualification( + tax_host_cold, +): + run = tax_host_cold + assert ( + runner.codec.decode_json(runner._run_entry(run)[2].tax_verification)["complete"] + is False + ) + with pytest.raises(ValueError, match="PROPERTY_TAX_INPUTS_INCOMPLETE"): + recipients.values.qualify_puf55_survey_recipients(run) + edges = recipients._edges(run) + edge = next(e for e in edges if e.name == "property_tax_verification") + assert edge.producer == tax.GATE_NODE and edge.type == tax.VERIFICATION_TYPE + assert recipients._pins(run)[edge.name]["payload_sha256"] == runner.codec.sha( + runner._run_entry(run)[2].tax_verification + ) + + +@pytest.mark.parametrize("target", ["property", "leaf", "unknown"]) +def test_retained_tax_lifetime_refuses_changes(tax_host_cold, target): + run = tax_host_cold + state = runner._run_entry(run)[2] + population = ( + state.property_population if target == "property" else run.financial_population + ) + table = population.frame.person + column = ( + tax.PROPERTY_COMPONENTS[0] if target == "property" else tax.TAX_LEAF_COLUMNS[0] + ) + row = table.index[ + table[column].isna() if target == "unknown" else table[column].notna() + ][0] + old = table.loc[row, column] + table.loc[row, column] = 17.0 + try: + with pytest.raises(ValueError, match="POPULATION_CHANGED"): + run.checked_view() + finally: + table.loc[row, column] = old + runner._pure_run(run, runner._run_entry(run)) + + +def test_rebased_dataclass_copy_remains_unissued(tax_host_cold): + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + replace(tax_host_cold).checked_view() + + +@pytest.mark.parametrize("flag", [None, 1, "yes", True]) +def test_invalid_or_unconfigured_rebase_refuses_before_source_io(flag): + with pytest.raises( + ValueError, match="PROPERTY_TAX_FLAG|PROPERTY_TAX_REQUIRES_PROPERTY" + ): + runner.run_atomic_survey_financial( + None, + snapshot_root=None, + store_root=None, + fraction=1.0, + seed=17, + geography_config=None, + rebase_property_taxes=flag, + ) + + +def test_completion_private_artifact_and_aggregate_receipt_replay( + tax_host_cold, tax_host_warm +): + """Actual enabled host/required path, with no additional source qualification.""" + private_payloads, receipts = [], [] + for run in (tax_host_cold, tax_host_warm): + state = runner._run_entry(run) + record = run.manifest.node(property_graph.PROJECTION_NODE) + assert set(record.opaque_artifacts) == { + "projection", + "donor_matrix", + "recipient_matrix", + "completion_routing", + } + payload = run.store.load_bytes(record.opaque_artifacts["completion_routing"]) + runner.survey._final_artifact( + run.manifest, + run.store, + node_id=property_graph.PROJECTION_NODE, + name="completion_routing", + type_=property_graph.completion.PROPERTY_COMPLETION_TYPE, + payload=payload, + capabilities=run.kernels.get( + property_graph.CurrentSurveyPropertyProjectionKernel.ref + ).capabilities, + ) + assert ( + property_graph.PROJECTION_NODE, + "completion_routing", + runner.codec.sha(payload), + ) in state[2].artifact_hashes + private = json.loads(payload) + assert private["stage"] == "before_property_models" + assert set(private["tables"]) == { + "person", + "components", + "reasons", + "clones", + "summary", + } + assert len( + private["tables"]["clones"]["rows"] + ) == run.financial_population.frame.n("person") + receipt = record.receipt["completion_routing"] + assert set(receipt) == { + "summary", + "private_artifact", + "private_artifact_sha256", + } + assert receipt["private_artifact"] == "completion_routing" + assert receipt["private_artifact_sha256"] == runner.codec.sha(payload) + summary = receipt["summary"] + assert ( + summary["source_authority"] is False + and summary["amounts_assigned"] is False + ) + assert summary["model_executed"] is False + assert summary["routes_mutually_exclusive"] is True + assert summary["reasons_overlap"] is True + assert all( + set(row) == {"selection", *property_graph.completion._PUBLIC_MEASURES} + for row in summary["summary"] + ) + public = explain_html(run.compiled, run.manifest) + describe( + run.compiled, property_graph.PROJECTION_NODE, run.manifest + ) + assert "completion_routing" in public and "carry_known_components" in public + assert payload.decode() not in public + private_payloads.append(payload) + receipts.append(receipt) + runner._pure_run(run, state) + assert private_payloads[0] == private_payloads[1] + assert receipts[0] == receipts[1] + + +def test_completion_artifact_retained_lifetime_refuses_store_tampering(tax_host_cold): + """Keep this destructive artifact control last; restore only test-owned bytes.""" + run = tax_host_cold + record = run.manifest.node(property_graph.PROJECTION_NODE) + path = ( + run.store.object_path(record.opaque_artifacts["completion_routing"]) + / "payload.bin" + ) + original = path.read_bytes() + try: + path.write_bytes(original + b" ") + with pytest.raises(StoreCorrupt): + run.checked_view() + finally: + path.write_bytes(original) + runner._pure_run(run, runner._run_entry(run)) diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py b/packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py new file mode 100644 index 000000000..0ae0c3af0 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py @@ -0,0 +1,294 @@ +"""Nineteen actual graph nodes over invented, source-issued survey fixtures.""" + +import hashlib +import sys +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments +from test_us_graph_atomic_survey_population import _support_payload + +from microcosm.build.us_runtime import graph_atomic_survey_financial as runner +from microcosm.graph import ArtifactType, ArtifactValue, NumericScope +from microcosm.graph.keys import opaque_artifact_key + +financial = runner.financial +values = financial.values + + +@pytest.fixture(scope="module") +def known_financial_run(tmp_path_factory): + root = tmp_path_factory.mktemp("atomic-survey-financial") + with pytest.MonkeyPatch.context() as patch: + arguments = _demographic_arguments(root, patch, unknown=False, zero=False) + payload, source_ids = _support_payload() + support_path = root / "invented-block-support.npz" + support_path.write_bytes(payload) + config = runner.reconstruction.AtomicSurveyReconstruction( + support_path=str(support_path), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + call = { + **arguments, + "store_root": root / "store", + "geography_config": config, + "demographic_conditioning": True, + "n_estimators": 2, + "return_values": True, + } + cold = runner.run_atomic_survey_financial(**call, resume="auto") + warm = runner.run_atomic_survey_financial(**call, resume="require") + yield SimpleNamespace(call=call, cold=cold, warm=warm) + for run in (cold, warm): + values.source.verify_survey_population_preparation(run.prefix.preparation) + + +def test_nineteen_node_financial_cold_and_required_replay(known_financial_run): + case = known_financial_run + assert case.cold.manifest.key == case.warm.manifest.key + assert all(n.hit for n in case.warm.manifest.nodes.values()) + for run in (case.cold, case.warm): + assert len(run.compiled.order) == 19 + edge = financial._geography_edge() + assert edge.producer in run.compiled.predecessors[financial.DONOR_NODE] + gate = run.manifest.node(edge.producer) + assert gate.opaque_artifacts[edge.artifact] == opaque_artifact_key( + gate.key, edge.artifact + ) + assert run.store.load_bytes( + gate.opaque_artifacts[edge.artifact] + ) == financial.codec.encode_json( + financial.codec.decode_json(run.projection)["atomic_geography"][ + "validation_receipt" + ] + ) + assert run.prefix.clone_population is run.prefix.geography_population + assert "census_block_geoid" not in run.prefix.expanded_population.frame.table( + "household" + ) + assert set(run.compiled.order) == { + *run.prefix.compiled.order, + financial.PROJECTION_NODE, + financial.DONOR_NODE, + financial.DONOR_COLUMNS_NODE, + *(f"{financial.FIT_PREFIX}.{i:03d}" for i in range(3)), + *(f"{financial.APPLY_PREFIX}.{i:03d}" for i in range(3)), + financial.ATTACH_NODE, + } + before, after = run.prefix.clone_population, run.financial_population + assert before is not after + assert "census_block_geoid" not in run.prefix.allocated_population.frame.table( + "household" + ) + assert "census_block_geoid" in before.frame.table("household") + assert ( + tuple(financial.model_input.decode_recipient_matrix(run.matrix).features) + == values.DEMOGRAPHIC_FEATURES + ) + evidence = financial.codec.decode_json(run.projection) + assert ( + evidence["model_judgments"]["demographic_conditioning"][ + "asec_sex_and_state_observation_year" + ] + == 2025 + ) + assert evidence["atomic_geography"]["config_sha256"] == financial.codec.sha( + run.prefix.geography_config.to_bytes() + ) + assert evidence["release_eligible"] is False + assert not any("prior" in c for c in values.DEMOGRAPHIC_FEATURES) + assert len(values.OUTPUTS) == 8 + for entity in before.frame.entities: + retained = [ + c + for c in before.frame.table(entity) + if entity != "person" or c not in values.OUTPUTS + ] + pd.testing.assert_frame_equal( + after.frame.table(entity)[retained], + before.frame.table(entity)[retained], + check_exact=True, + ) + assert after.version == before.version + assert after.weight_kind == before.weight_kind + assert after.mass_ledger == before.mass_ledger + assert after.frame.metadata == before.frame.metadata + assert after.frame.mass_log == before.frame.mass_log + assert after.frame.schema == before.frame.schema + assert after.frame.links == before.frame.links + pd.testing.assert_series_equal( + after.frame.strata, before.frame.strata, check_exact=True + ) + assert dict(after.owners) == { + **before.owners, + **{("person", c): financial.ATTACH_NODE for c in values.OUTPUTS}, + } + for entity in before.design_weights: + np.testing.assert_array_equal( + after.design_weights[entity], before.design_weights[entity] + ) + np.testing.assert_array_equal( + after.frame.weights_for(entity).values, + before.frame.weights_for(entity).values, + ) + # Source-origin pairing must carry each modeled ACS draw to both arms. + person = after.frame.person + source_id = values.provenance.support_source_id_column("person") + for _, rows in person.groupby(source_id, sort=False): + assert len(rows) == 2 + for column in values.OUTPUTS: + assert rows[column].notna().all() + assert rows[column].nunique() == 1 + runner.survey._same_frame(after.frame, run.manifest.population(after.version)) + runner.atomic.same_replayed_population( + case.cold.financial_population, case.warm.financial_population + ) + + +def test_geography_option_preserves_default_three_features(known_financial_run): + run = known_financial_run.cold + prefix = run.prefix + default = values.qualify_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + geography_config=prefix.geography_config, + ) + opted = values.qualify_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + geography_config=prefix.geography_config, + demographic_conditioning=True, + ) + assert default.demographic_conditioning is False + assert ( + tuple(financial.model_input.decode_recipient_matrix(default.matrix).features) + == values.FEATURES + ) + assert len(values.FEATURES) == 3 + assert len(values.DEMOGRAPHIC_FEATURES) == 5 + pd.testing.assert_frame_equal( + default.native_money, opted.native_money, check_exact=True + ) + assert ( + default.geography_config_payload + == opted.geography_config_payload + == prefix.geography_config.to_bytes() + ) + # A geography-enriched clone cannot be admitted through the historical arm. + with pytest.raises(ValueError, match="CLONE_FRAME_VALUES"): + values.qualify_current_survey_predictors( + prefix.preparation, prefix.allocated_population, prefix.clone_population + ) + + +def test_final_support_return_cannot_mutate_detached_money(known_financial_run): + case, fired = known_financial_run, [] + + def profile(frame, event, arg): + caller = frame.f_back + if ( + event == "return" + and frame.f_code is runner.reconstruction._read_support.__code__ + and caller is not None + and caller.f_code is values.qualify_current_survey_predictors.__code__ + and "result" in caller.f_locals + and not fired + ): + fired.append(True) + result = caller.f_locals["result"] + index = result.origins.index[result.origins.source.eq("asec")][0] + result.native_money.loc[index, "WSAL_VAL"] += 1 + + previous = sys.getprofile() + sys.setprofile(profile) + try: + prefix = case.cold.prefix + with pytest.raises(ValueError, match="FINAL_DERIVED_VALUES"): + values.qualify_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + geography_config=prefix.geography_config, + demographic_conditioning=True, + ) + finally: + sys.setprofile(previous) + assert fired == [True] + + +def test_final_owner_return_cannot_mutate_materialized_geography(known_financial_run): + case, fired = known_financial_run, [] + + def wrapper_profile(frame, event, arg): + caller = frame.f_back + if ( + event == "return" + and frame.f_code + is financial.verify_materialized_current_survey_predictors.__code__ + and caller is not None + and caller.f_code is runner.run_atomic_survey_financial.__code__ + and "result" in caller.f_locals + and not fired + ): + fired.append(True) + table = caller.f_locals["result"].financial_population.frame.table( + "household" + ) + table.loc[table.index[0], "survey_observed_state"] = "99" + + previous = sys.getprofile() + sys.setprofile(wrapper_profile) + try: + with pytest.raises(ValueError, match="ATOMIC_FINAL_POPULATION_MUTATION"): + runner.run_atomic_survey_financial(**case.call, resume="require") + finally: + sys.setprofile(previous) + assert fired == [True] + + +@pytest.mark.parametrize( + "defect,reason", + ( + ("payload", "GEOGRAPHY_VALIDATION_BINDING"), + ("producer", "GEOGRAPHY_VALIDATION_BINDING"), + ("type", "DETAIL_ARTIFACT_IDENTITY"), + ), +) +def test_financial_donor_requires_actual_geography_gate( + known_financial_run, defect, reason +): + run = known_financial_run.cold + node = run.compiled.graph.node(financial.DONOR_NODE) + artifacts = {} + for edge in node.artifact_inputs: + producer = run.manifest.node(edge.producer) + key = producer.opaque_artifacts[edge.artifact] + artifacts[edge.name] = ArtifactValue( + run.store.load_bytes(key), edge.type, key, producer.key, NumericScope() + ) + # A detached consumer-context negative, using actual source-owned run keys. + # This does not pretend that constructing ArtifactValue issues an authority. + edge = financial._geography_edge() + original = artifacts[edge.name] + if defect == "payload": + artifacts[edge.name] = replace(original, payload=original.payload + b" ") + elif defect == "producer": + other = "0" * 64 if original.producer_key != "0" * 64 else "1" * 64 + artifacts[edge.name] = replace( + original, producer_key=other, key=opaque_artifact_key(other, edge.artifact) + ) + else: + artifacts[edge.name] = replace( + original, type=ArtifactType("invented.wrong_gate", 1) + ) + context = SimpleNamespace(node=node, sources={}, artifacts=artifacts) + kernel = run.kernels.get(financial.CurrentSurveyPredictorDonorFilterKernel.ref) + with pytest.raises(ValueError, match=reason): + kernel._qualified(context) diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py b/packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py new file mode 100644 index 000000000..43dc09b50 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py @@ -0,0 +1,396 @@ +"""Actual survey/geography/clone execution over pinned invented originals. + +Normalized invented support proves no publisher provenance or native acceptance. +Invented registry pins bind source fixtures; real issuers and kernels execute. +""" + +import hashlib +import json +import sys +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments + +from microcosm.build import atomic_geography as atomic +from microcosm.build.us_runtime import atomic_block_support as blocks +from microcosm.build.us_runtime import current_survey_geography as observed +from microcosm.build.us_runtime import graph_atomic_survey_population as runner +from microcosm.build.us_runtime import graph_combined_clone as clone +from microcosm.build.us_runtime import graph_current_survey_geography as projection +from microcosm.build.us_runtime import graph_survey_population as survey +from microcosm.build.us_runtime import puf_support +from microcosm.build.us_runtime import survey_atomic_geography as reconstruction +from microcosm.build.us_runtime import survey_population_preparation as source +from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.frame import WeightKind + + +def _support_payload(*, first_population=2): + areas = tuple( + int(value) + for value in ( + "060010201001000", + "060010201001001", + "060010202001000", + "360010001001000", + ) + ) + source_ids = { + name: "invented-" + name for name in ("district", "population", "puma") + } + payload = blocks.assemble_atomic_block_support( + block_population=dict(zip(areas, (first_population, 6, 3, 5), strict=True)), + cd_by_block=dict(zip(areas, (601, 602, 603, 3601), strict=True)), + puma_by_tract={ + areas[0] // 10000: 612345, + areas[2] // 10000: 699999, + areas[3] // 10000: 3600001, + }, + source_ids=source_ids, + ) + return payload, source_ids + + +@pytest.fixture(scope="module") +def known_atomic_run(tmp_path_factory): + root = tmp_path_factory.mktemp("atomic-survey-population") + with pytest.MonkeyPatch.context() as patch: + arguments = _demographic_arguments(root, patch, unknown=False) + payload, source_ids = _support_payload() + support_path = root / "invented-block-support.npz" + support_path.write_bytes(payload) + config = reconstruction.AtomicSurveyReconstruction( + support_path=str(support_path), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + store_root = root / "store" + runs = tuple( + runner.run_atomic_survey_population( + **arguments, + store_root=store_root, + geography_config=config, + resume=resume, + return_values=True, + ) + for resume in ("auto", "require") + ) + yield SimpleNamespace( + arguments=arguments, + payload=payload, + config=config, + store_root=store_root, + cold=runs[0], + warm=runs[1], + ) + for run in runs: + source.verify_survey_population_preparation(run.preparation) + + +def _assert_complete_clone(before, after): + expected = puf_support.clone_us_frame_for_puf_support( + before.frame, clone_attachment_fraction=1.0, clone_attachment_seed=0 + ) + replay.same_replayed_frame(expected, after.frame) + # Independently pair both roles by assembly source ID, so inheritance is + # checked independently of the clone operator's row ordering/remapping. + for entity in before.frame.entities: + source_id = puf_support.support_source_id_column(entity) + clone_index = puf_support.support_clone_index_column(entity) + original = before.frame.table(entity).set_index( + source_id, verify_integrity=True + ) + expanded = after.frame.table(entity) + assert len(expanded) == 2 * len(original) + assert set(expanded[clone_index]) == {0, 1} + remapped = {before.frame.schema.entity_id_column(entity), clone_index} + if entity == before.frame.schema.person_entity: + remapped.update( + before.frame.schema.membership_column(group) + for group in before.frame.schema.group_entities + ) + inherited = [column for column in original if column not in remapped] + for role in (0, 1): + arm = expanded.loc[expanded[clone_index].eq(role)].set_index( + source_id, verify_integrity=True + ) + pd.testing.assert_index_equal(arm.index, original.index, exact=True) + pd.testing.assert_frame_equal( + arm.loc[original.index, inherited], + original.loc[:, inherited], + check_exact=True, + ) + assert after.version == clone.COMBINED_CLONE_NODE + assert after.mass_ledger[:-1] == before.mass_ledger + assert ( + after.weight_kind == before.weight_kind == {"household": WeightKind.IMPORTANCE} + ) + np.testing.assert_array_equal( + after.frame.weights_for("household").values, + np.tile(before.frame.weights_for("household").values / 2, 2), + ) + assert set(after.design_weights) == set(before.design_weights) == {"household"} + np.testing.assert_array_equal( + after.design_weights["household"], + np.tile(before.design_weights["household"], 2), + ) + assert dict(after.owners) == { + (entity, column): ( + clone.COMBINED_CLONE_CLAIM_NODE + if column == puf_support.support_clone_index_column(entity) + else clone.COMBINED_CLONE_NODE + ) + for entity in after.frame.entities + for column in after.frame.table(entity) + } + + +def test_cold_and_required_replay_assign_after_complete_clone(known_atomic_run): + case = known_atomic_run + assert case.cold.manifest.key == case.warm.manifest.key + assert all(record.hit for record in case.warm.manifest.nodes.values()) + assert all( + not record.hit + for name, record in case.cold.manifest.nodes.items() + if name not in {survey.CREATE_NODE, survey.ALLOCATION_NODE} + ) + for field in ( + "allocated_population", + "observed_population", + "expanded_population", + "geography_population", + "clone_population", + ): + replay.same_replayed_population( + getattr(case.cold, field), getattr(case.warm, field) + ) + support = atomic.decode_atomic_support(case.payload) + for run in (case.cold, case.warm): + raw, observed_population = run.allocated_population, run.observed_population + expanded, geography = run.expanded_population, run.geography_population + assert run.clone_population is geography + assert raw.version == observed_population.version == survey.ALLOCATION_NODE + assert expanded.version == geography.version == clone.COMBINED_CLONE_NODE + assert len(run.compiled.order) == 9 + stages = ( + survey.ALLOCATION_NODE, + projection.NODE, + clone.COMBINED_CLONE_NODE, + clone.COMBINED_CLONE_CLAIM_NODE, + "geography.assign", + "geography.derive", + "geography.gate", + ) + positions = tuple(run.compiled.order.index(name) for name in stages) + assert positions == tuple(sorted(positions)) + assert ( + clone.COMBINED_CLONE_CLAIM_NODE + in run.compiled.predecessors["geography.assign"] + ) + rebuilt = reconstruction.reconstruct_atomic_survey_geography( + run.preparation, raw, case.config + ) + for expected, actual in ( + (rebuilt.observed_population, observed_population), + (rebuilt.expanded_population, expanded), + (rebuilt.population, geography), + ): + replay.same_replayed_population(expected, actual) + receipt, definition = ( + json.loads(rebuilt.receipt), + json.loads(rebuilt.definition), + ) + assert receipt["protocol"] == "microcosm.us.atomic-survey-reconstruction.v2" + assert receipt["assignment_identity"] == list( + reconstruction.composition.ASSIGNMENT_IDENTITY + ) + assert rebuilt.support_payload == case.payload + assert receipt["support_sha256"] == case.config.support_sha256 + for name in ( + "publisher_provenance_established", + "source_admission_issued", + "population_admission_issued", + "release_eligible", + ): + assert receipt[name] is False + assigned = tuple(definition["outputs"].values()) + derived = tuple(layer["output"] for layer in definition["systems"][0]["layers"]) + assert not set((*observed.COLUMNS, *assigned, *derived)) & set( + raw.frame.table("household") + ) + assert not set((*assigned, *derived)) & set(expanded.frame.table("household")) + _assert_complete_clone(observed_population, expanded) + # Both ordinary projection steps preserve every incumbent cell, entity, + # membership, axis, weight, design anchor and context field exactly. + for before, after, added in ( + (raw, observed_population, observed.COLUMNS), + (expanded, geography, (*assigned, *derived)), + ): + assert set(after.frame.table("household")) - set( + before.frame.table("household") + ) == set(added) + for entity in before.frame.entities: + pd.testing.assert_frame_equal( + after.frame.table(entity).loc[ + :, before.frame.table(entity).columns + ], + before.frame.table(entity), + check_exact=True, + ) + assert after.frame.schema == before.frame.schema + assert after.frame.entities == before.frame.entities + assert after.frame.links == before.frame.links == () + assert after.frame.metadata == before.frame.metadata + assert after.frame.mass_log == before.frame.mass_log + assert after.mass_ledger == before.mass_ledger + assert after.weight_kind == before.weight_kind + assert after.frame.weighted_entities == before.frame.weighted_entities + pd.testing.assert_series_equal( + after.frame.strata, before.frame.strata, check_exact=True + ) + for entity in before.design_weights: + np.testing.assert_array_equal( + after.design_weights[entity], before.design_weights[entity] + ) + np.testing.assert_array_equal( + after.frame.weights_for(entity).values, + before.frame.weights_for(entity).values, + ) + assert dict(observed_population.owners) == { + **raw.owners, + **{("household", name): projection.NODE for name in observed.COLUMNS}, + } + assert dict(geography.owners) == { + **expanded.owners, + **{("household", name): "geography.assign" for name in assigned}, + **{("household", name): "geography.derive" for name in derived}, + } + replay.same_replayed_frame( + observed_population.frame, run.manifest.population(survey.ALLOCATION_NODE) + ) + replay.same_replayed_frame( + geography.frame, run.manifest.population(clone.COMBINED_CLONE_NODE) + ) + households = geography.frame.table("household") + assert len(households) == 12 + assert not households.duplicated( + list(reconstruction.composition.ASSIGNMENT_IDENTITY) + ).any() + keys = households[observed.COLUMNS[0]].map(json.loads) + acs = keys.map(lambda key: key[0] == "acs") + assert acs.sum() == 8 + assert households.loc[acs, "survey_observed_puma"].eq("0612345").all() + assert households.loc[acs, "assigned_puma_geoid"].eq("0612345").all() + assert households.loc[~acs, "survey_observed_puma"].isna().all() + for native, state in (("00007", "06"), ("00008", "36")): + selected = keys.map( + lambda key, native=native: key[0] == "asec" and key[-1] == native + ) + assert selected.sum() == 2 + assert households.loc[selected, "survey_observed_state"].tolist() == [ + state, + state, + ] + assert households.loc[selected, "assigned_state_fips"].tolist() == [ + state, + state, + ] + assert households.loc[acs, "assigned_state_fips"].eq("06").all() + assert set(households.census_block_geoid) <= set(support.arrays["area"]) + atomic.validate_geography(households, definition, {blocks.SYSTEM: support}) + gate = run.manifest.node("geography.gate") + assert gate.receipt["outcome"] == "pass" + assert ( + run.store.load_bytes(gate.opaque_artifacts["validation"]) + == rebuilt.stages[-1].receipt + ) + + +def test_changed_normalized_support_is_refused_by_exact_digest( + known_atomic_run, tmp_path +): + case = known_atomic_run + path = tmp_path / "changed-invented-support.npz" + changed, _ = _support_payload(first_population=3) + assert changed != case.payload + atomic.decode_atomic_support(changed) + path.write_bytes(changed) + config = replace(case.config, support_path=str(path)) + with pytest.raises( + ValueError, match="ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_CHANGED" + ): + reconstruction.reconstruct_atomic_survey_geography( + case.cold.preparation, case.cold.allocated_population, config + ) + + +def test_enriched_population_cannot_substitute_for_raw_allocation(known_atomic_run): + run = known_atomic_run.cold + with pytest.raises(ValueError, match="MATERIALIZED_FRAME_VALUES"): + reconstruction.reconstruct_atomic_survey_geography( + run.preparation, run.geography_population, known_atomic_run.config + ) + + +@pytest.mark.parametrize( + "change,reason", + ( + ("observed_cell", "SURVEY_POPULATION_REPLAY_"), + ("observed_snapshot", "SURVEY_POPULATION_REPLAY_"), + ("expanded_snapshot", "SURVEY_POPULATION_REPLAY_"), + ("source_container", "ATOMIC_FINAL_RECONSTRUCTION"), + ), +) +def test_late_returned_population_or_sources_mutation_refuses( + known_atomic_run, change, reason +): + case, fired = known_atomic_run, [] + + def trace(frame, event, arg): + caller = frame.f_back + if ( + event == "return" + and frame.f_code + is reconstruction.reconstruct_atomic_survey_geography.__code__ + and caller is not None + and caller.f_code is runner.run_atomic_survey_population.__code__ + and "result" in caller.f_locals + and not fired + ): + fired.append(True) + result = caller.f_locals["result"] + if change == "observed_cell": + households = result.geography_population.frame.table("household") + households.loc[households.index[0], "survey_observed_state"] = "99" + elif change in {"observed_snapshot", "expanded_snapshot"}: + value = ( + result.observed_population + if change == "observed_snapshot" + else result.expanded_population + ) + households = value.frame.table("household") + households.loc[households.index[0], "survey_observed_state"] = "99" + else: + result.sources[blocks.SOURCE] = ( + str(result.sources[blocks.SOURCE]) + ".changed" + ) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError, match=reason): + runner.run_atomic_survey_population( + **case.arguments, + store_root=case.store_root, + geography_config=case.config, + resume="require", + return_values=True, + ) + finally: + sys.setprofile(previous) + assert fired == [True] diff --git a/packages/microcosm-build/tests/test_us_graph_current_survey_geography.py b/packages/microcosm-build/tests/test_us_graph_current_survey_geography.py new file mode 100644 index 000000000..eadbedd74 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_current_survey_geography.py @@ -0,0 +1,353 @@ +"""Observed-geography graph contracts over actual issuers and invented originals.""" + +import json +import sys +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments + +from microcosm.build.us_runtime import current_survey_geography as qualifier +from microcosm.build.us_runtime import graph_current_survey_geography as graph +from microcosm.build.us_runtime import graph_survey_population as population +from microcosm.build.us_runtime import survey_population_preparation as source +from microcosm.graph import ArtifactType, StructuralDelta, compile_graph, run_graph + + +def _executed(tmp_path, monkeypatch, *, unknown=False): + """Construct privately pinned invented originals for a complete graph run.""" + arguments = _demographic_arguments(tmp_path, monkeypatch, unknown=unknown) + return _execute(arguments, store_root=tmp_path / "store") + + +def _execute(arguments, *, store_root): + """Keep real preparation/artifact owners and capture a real kernel context.""" + prefix = population.run_authenticated_survey_population( + **arguments, + store_root=store_root, + clones=False, + return_values=True, + ) + qualified = qualifier.qualify_current_survey_geography(prefix.preparation) + node = graph.current_survey_geography_node( + preparation_sha256=population._sha(prefix.preparation.payload), + projection_receipt_sha256=population._sha(qualified.receipt), + population=population.ALLOCATION_NODE, + ) + compiled = compile_graph( + replace(prefix.compiled.graph, nodes=(*prefix.compiled.graph.nodes, node)) + ) + kernel = graph.CurrentSurveyGeographyKernel(prefix.preparation) + prefix.kernels.register(kernel) + contexts = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is kernel.run.__code__: + contexts.append(frame.f_locals["context"]) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + manifest = run_graph( + compiled, + sources=prefix.sources, + store=prefix.store, + kernels=prefix.kernels, + ) + finally: + sys.setprofile(previous) + assert len(contexts) == 1 + return SimpleNamespace( + arguments=arguments, + prefix=prefix, + qualified=qualified, + node=node, + compiled=compiled, + kernel=kernel, + context=contexts[0], + manifest=manifest, + ) + + +@pytest.fixture(scope="module") +def known_run(tmp_path_factory): + """Share one real execution while its invented registry pins remain active.""" + with pytest.MonkeyPatch.context() as patch: + run = _executed(tmp_path_factory.mktemp("survey-geography-known"), patch) + yield run + # All consumers of this fixture must leave its issued source untouched. + source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture +def fresh_source_run(known_run, tmp_path): + """Reissue independent mutable Frames over the shared immutable originals.""" + run = _execute(known_run.arguments, store_root=tmp_path / "fresh-source-store") + assert run.prefix.preparation is not known_run.prefix.preparation + assert run.prefix.preparation.frame is not known_run.prefix.preparation.frame + yield run + source.verify_survey_population_preparation(known_run.prefix.preparation) + + +def _detached_context(context): + """Each negative gets independent tables, weights and container mappings.""" + return replace( + context, + tables={name: table.copy(deep=True) for name, table in context.tables.items()}, + weights={ + name: replace(value, values=value.values.copy()) + for name, value in context.weights.items() + }, + strata=context.strata.copy(deep=True), + params=dict(context.params), + sources=dict(context.sources), + artifacts=dict(context.artifacts), + tolerances=dict(context.tolerances), + numerics=dict(context.numerics), + ) + + +@pytest.mark.parametrize("unknown", (False, True)) +def test_real_projection_graph_execution_and_replay_preserve_inputs( + tmp_path, monkeypatch, unknown +): + run = _executed(tmp_path, monkeypatch, unknown=unknown) + assert run.compiled.order == ( + population.CREATE_NODE, + population.ALLOCATION_NODE, + graph.NODE, + ) + assert run.node.structural is StructuralDelta.NONE + assert run.node.sources == run.node.artifact_outputs == () + assert run.node.weights is None + assert tuple(o.column for o in run.node.outputs) == qualifier.COLUMNS + assert all( + o.entity == "household" and o.dtype == "string" and not o.rewrite + for o in run.node.outputs + ) + assert run.node.artifact_inputs[0].producer == population.CREATE_NODE + assert run.node.artifact_inputs[0].type == population.PREPARATION_TYPE + assert not run.manifest.node(graph.NODE).hit + original = run.prefix.allocated_population.frame + version = run.compiled.versions[graph.NODE] + projected = run.manifest.population(version) + for entity in original.entities: + pd.testing.assert_frame_equal( + projected.table(entity).loc[:, original.table(entity).columns], + original.table(entity), + check_exact=True, + ) + assert ( + projected.weights_for("household").kind + is original.weights_for("household").kind + ) + assert ( + projected.weights_for("household").values.tobytes() + == original.weights_for("household").values.tobytes() + ) + assert ( + run.manifest.mass_ledger(version) == run.prefix.allocated_population.mass_ledger + ) + table = projected.table("household").set_index("household_id") + actual = table.loc[:, list(graph.OUTPUT_COLUMNS)] + pd.testing.assert_frame_equal(actual, run.qualified.household, check_exact=True) + assert len(actual) == 6 + assert actual.survey_observed_state.isna().sum() == int(unknown) + assert actual.survey_observed_puma.notna().sum() == 4 + unknown_asec = actual.survey_geography_origin_key.eq('["asec",2024,2025,"00008"]') + assert unknown_asec.sum() == 1 + if unknown: + assert actual.loc[unknown_asec, "survey_observed_state"].isna().all() + else: + assert actual.loc[unknown_asec, "survey_observed_state"].tolist() == ["36"] + receipt = run.manifest.node(graph.NODE).receipt + assert receipt["projection_receipt_sha256"] == population._sha( + run.qualified.receipt + ) + assert receipt["source_projection"]["households"] == 6 + assert ( + not receipt["population_admission_issued"] and not receipt["release_eligible"] + ) + + warm_calls = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is run.kernel.run.__code__: + warm_calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + warm = run_graph( + run.compiled, + sources=run.prefix.sources, + store=run.prefix.store, + kernels=run.prefix.kernels, + resume="require", + ) + finally: + sys.setprofile(previous) + assert warm_calls == [] + assert all(node.hit for node in warm.nodes.values()) + assert warm.key == run.manifest.key + population._same_frame(projected, warm.population(version)) + # Cached graph output is not source admission. Requalify the retained + # preparation independently, as the country runner must on every replay. + current = qualifier.qualify_current_survey_geography(run.prefix.preparation) + pd.testing.assert_frame_equal(current.household, actual, check_exact=True) + assert current.receipt == run.qualified.receipt + + +@pytest.mark.parametrize( + "change,reason", + ( + ("preparation_digest", "DECLARATION"), + ("projection_digest", "PROJECTION_RECEIPT"), + ("artifact_payload", "ARTIFACT_BINDING"), + ("artifact_type", "ARTIFACT_BINDING"), + ("rewrite_declaration", "DECLARATION"), + ("source_roster", "CONTEXT_ROSTER"), + ), +) +def test_mismatched_declaration_or_typed_evidence_refuses( + known_run, tmp_path, change, reason +): + run = known_run + context = _detached_context(run.context) + if change in {"preparation_digest", "projection_digest"}: + field = ( + "preparation_sha256" + if change == "preparation_digest" + else "projection_receipt_sha256" + ) + node = replace(context.node, params={**context.node.params, field: "f" * 64}) + context = replace(context, node=node, params=node.params) + elif change in {"artifact_payload", "artifact_type"}: + artifact = context.artifacts["preparation"] + artifact = ( + replace(artifact, payload=artifact.payload + b" ") + if change == "artifact_payload" + else replace( + artifact, type=ArtifactType(population.PREPARATION_TYPE.name, 99) + ) + ) + context = replace(context, artifacts={"preparation": artifact}) + elif change == "rewrite_declaration": + first, *rest = context.node.outputs + context = replace( + context, + node=replace(context.node, outputs=(replace(first, rewrite=True), *rest)), + ) + else: + context = replace(context, sources={"undeclared": tmp_path}) + with pytest.raises(ValueError, match=reason): + run.kernel.run(context) + + +def test_same_household_ids_with_wrong_native_origin_refuse(known_run): + run = known_run + context = _detached_context(run.context) + table = context.tables["household"] + column = population.spine_source_id_column("household") + table.loc[table.index[0], column] += 1 + with pytest.raises(ValueError, match="HOUSEHOLD_ORIGIN"): + run.kernel.run(context) + + +def test_returned_series_are_detached_and_do_not_authorize_reuse(known_run): + run = known_run + before = source._frame_identity(run.prefix.preparation.frame) + result = run.kernel.run(_detached_context(run.context)) + column = ("household", "survey_observed_state") + result.columns[column].iloc[0] = "99" + result.receipt["source_projection"]["households"] = 999 + fresh = run.kernel.run(_detached_context(run.context)) + assert fresh.receipt["source_projection"]["households"] == 6 + for name in graph.OUTPUT_COLUMNS: + pd.testing.assert_series_equal( + fresh.columns[("household", name)], + run.qualified.household[name], + check_exact=True, + ) + assert source._frame_identity(run.prefix.preparation.frame) == before + assert ( + fresh.frame + is fresh.keep + is fresh.expand + is fresh.weights + is fresh.strata + is None + ) + assert not fresh.artifacts + + +def _assert_final_mutation_refuses(run, change): + context = _detached_context(run.context) + state = source._ISSUED[id(run.prefix.preparation)][2] + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code + is source.AuthenticatedSurveyPopulationPreparation._checked.__code__ + and frame.f_back.f_code is run.kernel.run.__code__ + and "result" in frame.f_back.f_locals + and not fired + ): + fired.append(True) + result = frame.f_back.f_locals["result"] + if change == "source_frame": + state.frame.table("household")["state_fips"] = 99 + elif change == "returned_column": + result.columns[("household", "survey_observed_state")].iloc[0] = "99" + else: + result.receipt["phase"] = "changed" + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError): + run.kernel.run(context) + finally: + sys.setprofile(previous) + assert fired == [True] + + +@pytest.mark.parametrize("change", ("returned_column", "receipt")) +def test_mutation_at_final_owner_return_refuses(known_run, change): + _assert_final_mutation_refuses(known_run, change) + + +def test_source_frame_mutation_at_final_owner_return_refuses(fresh_source_run): + _assert_final_mutation_refuses(fresh_source_run, "source_frame") + + +def test_changed_qualifier_return_cannot_hide_behind_unchanged_receipt( + known_run, +): + run = known_run + context = _detached_context(run.context) + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code is qualifier.qualify_current_survey_geography.__code__ + and frame.f_back.f_code is run.kernel.run.__code__ + and not fired + ): + fired.append(True) + assert isinstance(json.loads(arg.receipt)["projection_sha256"], str) + arg.household.loc[arg.household.index[0], "survey_observed_state"] = "99" + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError, match="QUALIFIED_PROJECTION"): + run.kernel.run(context) + finally: + sys.setprofile(previous) + assert fired == [True] diff --git a/packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py b/packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py new file mode 100644 index 000000000..7f69dec3f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py @@ -0,0 +1,473 @@ +"""Invented common-frame fiscal solve through existing grouped Adam and graph.""" + +import hashlib +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_fiscal_measurement import InventedSource, context, declaration, frame + +from microcosm.build.us_runtime import graph_fiscal_dense_calibration as stage +from microcosm.build.us_runtime import graph_fiscal_measurement as measurement +from microcosm.calibrate import TargetRegistry, group_bounds +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactOutput, + ArtifactValue, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + NodeRejectedError, + Numeric, + NumericScope, + Owned, + Slice, + SourceRef, + StoreMissError, + StructuralDelta, + WeightTransition, + compile_graph, + run_graph, +) +from microcosm.graph.keys import opaque_artifact_key + + +def setup(*, incoming=None, row_upper=None, ids=None): + value = frame() + if ids is not None: + old = value.table("household").household_id.to_numpy().copy() + mapping = dict(zip(old.tolist(), ids.tolist(), strict=True)) + value.table("household")["household_id"] = ids + value.table("person")["person_household_id"] = np.array( + [mapping[v] for v in value.person.person_household_id], dtype=ids.dtype + ) + incoming = np.array([1.0, 2.0, 3.0] if incoming is None else incoming) + value = Frame( + {e: value.table(e) for e in value.entities}, + value.schema, + {"household": Weights(incoming, WeightKind.IMPORTANCE)}, + ) + args = declaration() + # Exact common-frame targets correspond to desired weights [2, 2, 4]. + args["registry"] = TargetRegistry( + [ + replace(s, value=v) + for s, v in zip(args["registry"], [10, 6, 4, 4, 2, 4], strict=True) + ], + country="us", + ) + if incoming[1] == 0: + args["registry"] = TargetRegistry( + [ + replace(s, value=v) + for s, v in zip(args["registry"], [8, 4, 4, 4, 0, 4], strict=True) + ], + country="us", + ) + mc = context(args, value) + measured = measurement.FiscalMeasurementKernel().run(mc).artifacts["measurement"] + hh_ids = value.table("household").household_id.to_numpy() + groups = group_bounds.GroupedUpperBounds(hh_ids, np.array([0, 0, 1]), [6.0, 5.0]) + bounds = stage.encode_fiscal_calibration_bounds( + population=mc.node.population, + household_ids=hh_ids, + original_design_weights=Weights(np.array([8.0, 9.0, 10.0]), WeightKind.DESIGN), + incoming_weights=value.weights_for("household"), + grouped_upper_bounds=groups, + row_upper=np.array([6.0, 6.0, 5.0] if row_upper is None else row_upper), + budget_sha256=hashlib.sha256(b"invented budget; not an issuer").hexdigest(), + ) + node = stage.fiscal_dense_calibration_node( + measurement_node=mc.node, + bounds_node="invented.bounds", + epochs=120, + learning_rate=0.03, + ) + artifacts = {} + for name, payload, type_, output in ( + ("measurement", measured, measurement.MEASUREMENT_TYPE, "measurement"), + ("bounds", bounds, stage.BOUNDS_TYPE, "numeric_bounds"), + ): + key = hashlib.sha256(name.encode()).hexdigest() + artifacts[name] = ArtifactValue( + payload, + type_, + opaque_artifact_key(key, output), + key, + NumericScope(Numeric.BITWISE), + ) + return value, mc, replace(mc, node=node, params=node.params, artifacts=artifacts) + + +def test_real_grouped_solver_rebuilds_exact_matrix_and_residuals(monkeypatch): + value, mc, c = setup() + captured = [] + original_calibrate = stage.calibrate + + def record(*args, **kwargs): + result = original_calibrate(*args, **kwargs) + captured.append(result) + return result + + monkeypatch.setattr(stage, "calibrate", record) + result = stage.FiscalDenseCalibrationKernel().run(c) + solved = captured[0] + expected = measurement.decode_fiscal_measurement(c.artifacts["measurement"].payload) + np.testing.assert_array_equal( + solved.problem.matrix.toarray(), expected.matrix.toarray() + ) + assert solved.problem.names == tuple( + s.to_target().row_name for s in expected.registry + ) + assert solved.problem.target_vector.tobytes() == expected.target_values.tobytes() + assert solved.final_loss < solved.initial_loss + assert result.weights.kind is WeightKind.CALIBRATED + assert result.frame is None and not result.columns + assert result.receipt["release_eligible"] is False + assert result.receipt["source_admission"] == "required_from_complete_parent_owner" + diagnostic = json.loads(result.artifacts["diagnostics"]) + assert diagnostic["schema_version"] == 8 + origin = json.loads(result.artifacts["origin_diagnostics"]) + achieved = expected.matrix @ result.weights.values + for row, actual in zip(diagnostic["targets"], achieved, strict=True): + assert row["final_estimate"] == actual + assert origin["accepted_weights"] == measurement._array(result.weights.values) + assert origin["group_totals"] == measurement._array( + np.array([result.weights.values[:2].sum(), result.weights.values[2]]) + ) + assert origin["fixed_zero_rows"] == 0 + assert diagnostic["build"]["matrix_matches_measurement"] is True + # The direct kernel receives observations only, never a writable parent. + for entity in value.entities: + if entity in mc.tables: + pd.testing.assert_frame_equal(c.tables[entity], mc.tables[entity]) + assert ( + value.weights_for("household").values.tobytes() + == np.array([1.0, 2.0, 3.0]).tobytes() + ) + + +def test_uint64_order_original_design_and_strict_zero_support(): + ids = np.array([2**63, 2**63 + 1, 2**64 - 1], dtype=np.uint64) + _, _, c = setup(incoming=[1.0, -0.0, 3.0], ids=ids) + bounds = stage.verify_fiscal_calibration_bounds( + c.artifacts["bounds"].payload, + household_ids=ids, + incoming_weights=c.weights["household"], + original_design_weights=Weights(np.array([8.0, 9.0, 10.0]), WeightKind.DESIGN), + ) + assert bounds.household_ids.dtype == ids.dtype + assert ( + bounds.original_design.values.tobytes() == np.array([8.0, 9.0, 10.0]).tobytes() + ) + output = stage.FiscalDenseCalibrationKernel().run(c) + assert output.weights.values[1:2].tobytes() == np.array([-0.0]).tobytes() + assert np.all(output.weights.values[[0, 2]] > 0) + stage.check_fiscal_calibration_weights(bounds, output.weights.values) + assert json.loads(output.artifacts["origin_diagnostics"])["fixed_zero_rows"] == 1 + with pytest.raises(ValueError, match="DESIGN_ANCHOR"): + stage.verify_fiscal_calibration_bounds( + c.artifacts["bounds"].payload, + household_ids=ids, + incoming_weights=c.weights["household"], + original_design_weights=Weights( + np.array([8.0, 8.0, 10.0]), WeightKind.DESIGN + ), + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("epochs", 0), + ("epochs", True), + ("epochs", 1.5), + ("learning_rate", float("nan")), + ("learning_rate", float("inf")), + ("learning_rate", 0.0), + ("learning_rate", True), + ], +) +def test_solver_settings_refuse_nonfinite_or_undeclared_modes(field, value): + _, mc, _ = setup() + with pytest.raises(ValueError): + stage.fiscal_dense_calibration_node( + measurement_node=mc.node, bounds_node="bounds", **{field: value} + ) + + +def test_misaligned_measurement_bounds_and_calibrated_parent_refuse_before_solve( + monkeypatch, +): + _, _, c = setup() + + def forbidden(*a, **kw): + pytest.fail("solver ran before admission") + + monkeypatch.setattr(stage, "calibrate", forbidden) + with pytest.raises(ValueError, match="IMPORTANCE"): + stage.FiscalDenseCalibrationKernel().run( + replace( + c, + weights={ + "household": Weights( + np.array([1.0, 2.0, 3.0]), WeightKind.CALIBRATED + ) + }, + ) + ) + changed = dict(c.tables) + changed["person"] = c.tables["person"].copy() + changed["person"]["amount"] += 1 + with pytest.raises(ValueError, match="PROJECTION_BINDING"): + stage.FiscalDenseCalibrationKernel().run(replace(c, tables=changed)) + raw = json.loads(c.artifacts["bounds"].payload) + raw["household_ids"] = measurement._array(np.array([30, 20, 10], dtype=np.int64)) + artifacts = dict(c.artifacts) + artifacts["bounds"] = replace( + artifacts["bounds"], payload=measurement.canonical_json(raw) + ) + with pytest.raises(ValueError, match="HOUSEHOLD_ALIGNMENT"): + stage.FiscalDenseCalibrationKernel().run(replace(c, artifacts=artifacts)) + + +def test_group_and_row_caps_refuse_without_clipping(): + _, _, c = setup(row_upper=[1.0, 6.0, 5.0]) + with pytest.raises(ValueError, match="ROW_REFERENCE_CAP"): + stage.FiscalDenseCalibrationKernel().run(c) + bounds = stage.decode_fiscal_calibration_bounds(c.artifacts["bounds"].payload) + with pytest.raises(ValueError, match="frozen absolute bounds"): + stage.check_fiscal_calibration_weights(bounds, np.array([5.0, 2.0, 3.0])) + + +@pytest.mark.parametrize( + "change,reason", + [ + ("names", "TARGET_ALIGNMENT"), + ("matrix", "MATRIX_IDENTITY"), + ("diagnostic", "DIAGNOSTIC_ESTIMATES"), + ], +) +def test_solver_return_with_wrong_matrix_or_target_order_is_refused( + monkeypatch, change, reason +): + _, _, c = setup() + actual = stage.calibrate + + def changed(*a, **kw): + r = actual(*a, **kw) + if change == "names": + return replace( + r, problem=replace(r.problem, names=tuple(reversed(r.problem.names))) + ) + if change == "matrix": + matrix = r.problem.matrix.copy() + matrix.data[0] += 1 + return replace(r, problem=replace(r.problem, matrix=matrix)) + return replace( + r, + diagnostics=( + replace(r.diagnostics[0], final_estimate=999.0), + *r.diagnostics[1:], + ), + ) + + monkeypatch.setattr(stage, "calibrate", changed) + with pytest.raises(ValueError, match=reason): + stage.FiscalDenseCalibrationKernel().run(c) + + +class InventedImportance(KernelBase): + ref = "test.fiscal_dense_importance@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def run(self, context): + return KernelResult( + weights=Weights(np.array(context.params["incoming"]), WeightKind.IMPORTANCE) + ) + + +class InventedBounds(KernelBase): + ref = "test.fiscal_dense_bounds@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): + payload = context.sources["bounds"].read_bytes() + stage.decode_fiscal_calibration_bounds(payload) + return KernelResult(artifacts={"numeric_bounds": payload}) + + +def graph_fixture(tmp_path): + value, mc, c = setup(incoming=[1.0, 0.0, 3.0]) + value = Frame( + {e: value.table(e) for e in value.entities}, + value.schema, + {"household": Weights(np.array([8.0, 9.0, 10.0]), WeightKind.DESIGN)}, + ) + create = Node( + "invented.design", + InventedSource.ref, + sources=("fixture",), + structural=StructuralDelta.CREATE, + outputs=( + Owned("person", "amount", "float64"), + Owned("person", "keep", "bool"), + Owned("tax_unit", "tax_unit_amount", "float64"), + Owned("household", "state_fips", "int64"), + Owned("household", "congressional_district_geoid", "int64"), + ), + ) + importance = Node( + "invented", + InventedImportance.ref, + base=create.id, + structural=StructuralDelta.REWEIGHT, + inputs=(Slice("household", ("state_fips",)),), + weights=WeightTransition("household", "importance", mass="free"), + mass="free", + params={"incoming": (1.0, 0.0, 3.0)}, + ) + bounds = Node( + "invented.bounds", + InventedBounds.ref, + population=importance.id, + sources=("bounds",), + artifact_outputs=(ArtifactOutput("numeric_bounds", stage.BOUNDS_TYPE),), + ) + nodes = (create, importance, bounds, mc.node, c.node) + sources = (SourceRef("fixture", "frame-store"), SourceRef("bounds", "raw-bytes-v1")) + kernels = KernelRegistry() + for kernel in ( + InventedSource(), + InventedImportance(), + InventedBounds(), + measurement.FiscalMeasurementKernel(), + stage.FiscalDenseCalibrationKernel(), + ): + kernels.register(kernel) + store = ContentStore(tmp_path / "store") + path = store.put_frame( + hashlib.sha256(b"invented fiscal dense fixture").hexdigest(), value + ) + bound_path = tmp_path / "bounds.json" + bound_path.write_bytes(c.artifacts["bounds"].payload) + arguments = { + "store": store, + "kernels": kernels, + "sources": {"fixture": path, "bounds": bound_path}, + } + return value, sources, nodes, arguments + + +def test_real_graph_cold_required_replay_preserves_parent_and_original_anchors( + tmp_path, +): + original, sources, nodes, arguments = graph_fixture(tmp_path) + compiled = compile_graph(Graph("us", sources, nodes)) + anchors = {} + + def observe(node_id, population): + anchors[node_id] = population.design_weights["household"].tobytes() + + cold = run_graph(compiled, **arguments, _population_observer=observe) + cold_anchors = dict(anchors) + warm = run_graph( + compiled, **arguments, resume="require", _population_observer=observe + ) + assert cold.key == warm.key and all(n.store_hit for n in warm.nodes.values()) + before = warm.population("invented") + after = warm.population(nodes[-1].id) + # The graph performs precisely DESIGN -> IMPORTANCE -> CALIBRATED. + assert after.weights_for("household").kind is WeightKind.CALIBRATED + assert before.weights_for("household").kind is WeightKind.IMPORTANCE + for entity in original.entities: + pd.testing.assert_frame_equal(before.table(entity), original.table(entity)) + pd.testing.assert_frame_equal(after.table(entity), original.table(entity)) + assert after.weights_for("household").values[1] == 0 + assert ( + cold.population(nodes[-1].id).weights_for("household").values.tobytes() + == after.weights_for("household").values.tobytes() + ) + # Detached observer snapshots expose original anchors without owner authority. + assert anchors == cold_anchors + assert anchors[nodes[-1].id] == np.array([8.0, 9.0, 10.0]).tobytes() + payload = arguments["store"].load_bytes( + warm.node(nodes[-1].id).opaque_artifacts["origin_diagnostics"] + ) + assert json.loads(payload)["fixed_zero_rows"] == 1 + # A repeated CALIBRATED -> CALIBRATED stage is refused before another solve. + new_measure = replace(nodes[-2], id="repeat.measurement", population=nodes[-1].id) + repeated = stage.fiscal_dense_calibration_node( + measurement_node=new_measure, + bounds_node=nodes[2].id, + node_id="repeat.calibration", + ) + compiled_repeat = compile_graph( + Graph("us", sources, (*nodes, new_measure, repeated)) + ) + with pytest.raises( + NodeRejectedError, match="(?i)(importance|calibrated|transition|forward)" + ): + run_graph(compiled_repeat, **arguments) + + +def test_real_graph_replay_refuses_changed_bounds_and_target_declaration(tmp_path): + _, sources, nodes, arguments = graph_fixture(tmp_path) + compiled = compile_graph(Graph("us", sources, nodes)) + run_graph(compiled, **arguments) + changed_measure = measurement.fiscal_measurement_node(**declaration()) + changed_solve = stage.fiscal_dense_calibration_node( + measurement_node=changed_measure, bounds_node=nodes[2].id + ) + changed = compile_graph( + Graph("us", sources, (*nodes[:3], changed_measure, changed_solve)) + ) + with pytest.raises( + (NodeRejectedError, StoreMissError), match="(?i)(require|cache|missing)" + ): + run_graph(changed, **arguments, resume="require") + path = arguments["sources"]["bounds"] + raw = json.loads(path.read_bytes()) + raw["group_upper"] = measurement._array(np.array([5.5, 5.0])) + path.write_bytes(measurement.canonical_json(raw)) + with pytest.raises( + (NodeRejectedError, StoreMissError), match="(?i)(require|cache|missing)" + ): + run_graph(compiled, **arguments, resume="require") + + +def test_bound_codec_refuses_noncanonical_values_and_wrong_types(): + _, _, c = setup() + payload = c.artifacts["bounds"].payload + for field, value in ( + ("incoming", np.array([1.0, float("nan"), 3.0])), + ("group_indices", np.array([0, 0, 9], dtype=np.int64)), + ("original_design", np.array([8.0, 9.0])), + ("household_ids", np.array([10, 10, 30], dtype=np.int64)), + ): + raw = json.loads(payload) + raw[field] = measurement._array(value) + with pytest.raises(ValueError): + stage.decode_fiscal_calibration_bounds(measurement.canonical_json(raw)) + with pytest.raises(ValueError, match="CANONICAL"): + stage.decode_fiscal_calibration_bounds(payload + b"\n") + + +def test_implementation_identity_binds_existing_solver_and_target_math( + monkeypatch, tmp_path +): + # Point source identity at changed synthetic bytes without editing package source. + baseline = stage.FiscalDenseCalibrationKernel().implementation_hash() + changed = tmp_path / "changed_target.py" + changed.write_text("# invented changed target arithmetic\n") + monkeypatch.setattr(stage.target_module, "__file__", str(changed)) + assert stage.FiscalDenseCalibrationKernel().implementation_hash() != baseline diff --git a/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py b/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py new file mode 100644 index 000000000..ad80a0f09 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py @@ -0,0 +1,552 @@ +"""Invented fiscal measures through the actual interpreter, CSR and graph store.""" + +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_fiscal_measurement as stage +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate import target as target_math +from microcosm.calibrate.hierarchy import ( + CalibrationHierarchy, + HierarchyCategory, + HierarchyGeography, + HierarchyNode, +) +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.frame import materialize as frame_materialize +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + NodeRejectedError, + Owned, + SourceRef, + StoreMissError, + StructuralDelta, + compile_graph, + load_source, + run_graph, + source_hash, +) +from microcosm.graph.canonical import canonical_json + + +def frame(): + tables = { + "person": pd.DataFrame( + { + "person_id": np.array([400, 100, 300, 200], dtype=np.int64), + **{ + f"person_{e}_id": np.array([30, 10, 20, 10], dtype=np.int64) + for e in US_SCHEMA.group_entities + }, + "amount": [40.0, 10.0, 30.0, 20.0], + "keep": [True, True, False, True], + } + ), + **{ + e: pd.DataFrame({f"{e}_id": np.array([10, 20, 30], dtype=np.int64)}) + for e in US_SCHEMA.group_entities + }, + } + tables["household"]["state_fips"] = np.array([6, 6, 36], dtype=np.int64) + tables["household"]["congressional_district_geoid"] = np.array( + [601, 602, 3601], dtype=np.int64 + ) + tables["tax_unit"]["tax_unit_amount"] = [100.0, 200.0, 300.0] + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.array([1.0, 2.0, 3.0]), WeightKind.DESIGN)}, + ) + + +def declaration(*, value_variable="person_count", entity="person", model_outputs=()): + definitions = ( + ("national", "0100000US", None, None), + ("state", "0400000US06", "state_fips", 6), + ("state", "0400000US36", "state_fips", 36), + ( + "congressional_district", + "5001900US0601", + "congressional_district_geoid", + 601, + ), + ( + "congressional_district", + "5001900US0602", + "congressional_district_geoid", + 602, + ), + ( + "congressional_district", + "5001900US3601", + "congressional_district_geoid", + 3601, + ), + ) + scopes = tuple( + stage.FiscalGeographyScope(HierarchyGeography(id_, id_, level), column, value) + for level, id_, column, value in definitions + ) + specs, bindings = [], {} + for i, scope in enumerate(scopes): + name = f"invented_{i}" + specs.append( + TargetSpec( + name, + entity, + 1.0, + f"measure_{i}", + period=2024, + source="Invented test; not an administrative observation", + family="invented", + metadata={"contract_target_id": name}, + hierarchy=CalibrationHierarchy( + HierarchyNode("test", "Test"), + HierarchyCategory("count", "Count", "test"), + scope.geography, + (), + HierarchyNode(name, name), + ), + ) + ) + bindings[name] = { + "bindings": {"policyengine": {"value_variable": value_variable}} + } + return { + "registry": TargetRegistry(specs, country="us"), + "population": "invented", + "input_columns": { + "person": ("amount", "keep"), + "tax_unit": ("tax_unit_amount",), + "household": ("state_fips", "congressional_district_geoid"), + }, + "contract_targets": bindings, + "advertised_scopes": scopes, + "geography_vintage": "invented CD119 mapping", + "period": 2024, + "model_outputs": model_outputs, + } + + +def context(arguments=None, value=None): + value = frame() if value is None else value + arguments = declaration() if arguments is None else arguments + node = stage.fiscal_measurement_node(**arguments) + tables = {} + for slice_ in node.inputs: + entity = slice_.entity + structural = [f"{entity}_id"] + if entity == "person": + structural += [f"person_{e}_id" for e in US_SCHEMA.group_entities] + tables[entity] = ( + value.table(entity) + .loc[:, list(dict.fromkeys([*structural, *slice_.columns]))] + .copy() + ) + return KernelContext( + node, + tables, + {"household": value.weights_for("household")}, + value.strata, + node.params, + np.random.default_rng(0), + ) + + +def test_actual_measurement_maps_person_counts_and_tax_units_to_households(): + original = frame() + c = context(value=original) + result = stage.FiscalMeasurementKernel().run(c) + payload = result.artifacts["measurement"] + measured = stage.verify_fiscal_measurement( + payload, context=c, expected_sha256=hashlib.sha256(payload).hexdigest() + ) + np.testing.assert_array_equal(measured.household_ids, [10, 20, 30]) + np.testing.assert_array_equal( + measured.matrix.toarray(), + [[2, 1, 1], [2, 1, 0], [0, 0, 1], [2, 0, 0], [0, 1, 0], [0, 0, 1]], + ) + np.testing.assert_array_equal( + measured.matrix.toarray()[3:].sum(axis=0), measured.matrix.toarray()[0] + ) + assert result.frame is None and result.weights is None and not result.columns + assert result.receipt["fit_performed"] is False + assert result.receipt["release_eligible"] is False + for entity in original.entities: + pd.testing.assert_frame_equal(original.table(entity), frame().table(entity)) + c = context(declaration(value_variable="tax_unit_amount", entity="tax_unit")) + values = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel().run(c).artifacts["measurement"] + ) + np.testing.assert_array_equal(values.matrix.toarray()[0], [100, 200, 300]) + + +def test_actual_shared_interpreter_sum_and_filter(): + arguments = declaration(value_variable="amount") + for target in arguments["contract_targets"].values(): + target["bindings"]["policyengine"] = { + "value_expression": "amount + amount", + "filters": [{"variable": "amount", "operator": ">", "value": 5}], + } + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel().run(context(arguments)).artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [60, 60, 80]) + + +def test_boolean_indicator_expression_is_an_arithmetic_sum(): + arguments = declaration() + for target in arguments["contract_targets"].values(): + target["bindings"]["policyengine"] = {"value_expression": "keep + keep"} + value = frame() + value.person["keep"] = [True, False, True, True] + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel() + .run(context(arguments, value)) + .artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [2, 2, 2]) + + +@pytest.mark.parametrize("kind", ["float", "nullable_boolean"]) +@pytest.mark.parametrize("location", ["binding", "target_filter"]) +def test_missing_consumed_predicate_is_refused(kind, location): + arguments, value = declaration(), frame() + variable = "amount" if kind == "float" else "keep" + if kind == "float": + value.person.loc[1, variable] = np.nan + predicate = {"variable": variable, "operator": ">", "value": 5} + else: + value.person[variable] = pd.Series([True, pd.NA, True, True], dtype="boolean") + predicate = {"variable": variable, "equals": True} + if location == "binding": + for binding in arguments["contract_targets"].values(): + binding["bindings"]["policyengine"]["filters"] = [predicate] + else: + arguments["registry"] = TargetRegistry( + [replace(s, filter=variable) for s in arguments["registry"]], country="us" + ) + with pytest.raises( + ValueError, match="MISSING_PREDICATE_INPUT:person\\." + variable + ): + stage.FiscalMeasurementKernel().run(context(arguments, value)) + + +@pytest.mark.parametrize("location", ["binding", "target_filter"]) +def test_known_false_predicates_and_unconsumed_missing_inputs_are_preserved(location): + arguments, value = declaration(), frame() + value.person["keep"] = [True, False, True, True] + value.person.loc[0, "amount"] = np.nan # This measure does not read amount. + if location == "binding": + for binding in arguments["contract_targets"].values(): + binding["bindings"]["policyengine"]["filters"] = [ + {"variable": "keep", "equals": True} + ] + else: + arguments["registry"] = TargetRegistry( + [replace(s, filter="keep") for s in arguments["registry"]], country="us" + ) + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel() + .run(context(arguments, value)) + .artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [1, 1, 1]) + + +def test_implementation_identity_includes_target_arithmetic_source( + tmp_path, monkeypatch +): + before = stage.FiscalMeasurementKernel().implementation_hash() + changed = tmp_path / "changed_target.py" + source = Path(target_math.__file__).read_text() + original = " return values * filter_mask\n" + assert source.count(original) == 1 + changed.write_text( + source.replace(original, " return 2.0 * values * filter_mask\n") + ) + monkeypatch.setattr(target_math, "__file__", str(changed)) + assert stage.FiscalMeasurementKernel().implementation_hash() != before + + +def test_implementation_identity_includes_engine_table_transform(tmp_path, monkeypatch): + assert stage.policyengine_us.engine_tables is frame_materialize.engine_tables + before = stage.FiscalMeasurementKernel().implementation_hash() + changed = tmp_path / "changed_engine_tables.py" + source = Path(frame_materialize.__file__).read_text() + original = " ).values\n" + assert source.count(original) == 1 + changed.write_text(source.replace(original, " ).values * 2.0\n")) + monkeypatch.setattr(frame_materialize, "__file__", str(changed)) + assert stage.FiscalMeasurementKernel().implementation_hash() != before + + +@pytest.mark.parametrize( + "change,reason", + [ + ("binding", "UNSUPPORTED_BINDING"), + ("coverage", "ADVERTISED_LEVELS"), + ("period", "TARGET_PERIOD_HIERARCHY"), + ("collision", "INPUT_MEASURE_COLLISION"), + ("unknown", "UNMATERIALIZED_TARGETS"), + ("empty_area", "EMPTY_ADVERTISED_GEOGRAPHY"), + ("zero_support", "UNSUPPORTED_TARGET"), + ("zero_weight", "UNSUPPORTED_TARGET"), + ], +) +def test_declared_surface_and_advertised_support_fail_closed(change, reason): + arguments, value = declaration(), frame() + if change == "binding": + arguments["contract_targets"]["invented_0"]["bindings"]["policyengine"][ + "kind" + ] = "input_substitution_counterfactual" + elif change == "coverage": + arguments["advertised_scopes"] = arguments["advertised_scopes"][:3] + elif change == "period": + arguments["period"] = 2025 + elif change == "collision": + arguments["input_columns"]["person"] += ("measure_0",) + elif change == "unknown": + arguments["contract_targets"]["invented_0"]["bindings"]["policyengine"][ + "value_variable" + ] = "not_produced" + elif change == "empty_area": + value.table("household").loc[2, "congressional_district_geoid"] = 3699 + elif change == "zero_support": + arguments["contract_targets"]["invented_5"]["bindings"]["policyengine"][ + "filters" + ] = [{"variable": "amount", "operator": ">", "value": 999}] + elif change == "zero_weight": + value = Frame( + {e: value.table(e) for e in value.entities}, + US_SCHEMA, + {"household": Weights(np.array([1.0, 2.0, 0.0]), WeightKind.DESIGN)}, + ) + with pytest.raises(ValueError, match=reason): + stage.FiscalMeasurementKernel().run(context(arguments, value)) + + +def test_artifact_digest_projection_and_order_tamper_refusals(): + c = context() + payload = stage.FiscalMeasurementKernel().run(c).artifacts["measurement"] + digest = hashlib.sha256(payload).hexdigest() + changed = json.loads(payload) + changed["data"]["hex"] = np.full(9, 99.0, dtype="