From b7af94fa2a1abe4ff62e61be65e78b1ebd0a76bc Mon Sep 17 00:00:00 2001 From: CS Date: Sat, 14 Nov 2026 12:00:07 -0800 Subject: [PATCH 1/2] feat(gateway): add operator receipt resolutions Signed-off-by: CS --- CHANGELOG.md | 10 + gateway/TROUBLESHOOTING.md | 2 +- gateway/app/db/models.py | 2 + gateway/app/main.py | 1 + gateway/app/routers/receipts.py | 46 ++ gateway/app/schemas/resolutions.py | 35 +- gateway/app/services/mcp_dispatch_attempts.py | 12 +- gateway/app/services/operator_resolution.py | 278 ++++++++++ gateway/app/services/permits.py | 18 +- gateway/app/services/receipt_resolutions.py | 97 +++- gateway/docs/failure-semantics.md | 11 +- gateway/docs/openapi.json | 185 ++++++- gateway/docs/operator-resolution-runbook.md | 85 +++ gateway/docs/partner-first-tool-runbook.md | 3 +- gateway/docs/schema-058-rollout.md | 28 + .../versions/058_resolution_evidence.py | 39 ++ gateway/scripts/amw_resolutions.py | 59 ++ gateway/tests/test_amw_resolutions_cli.py | 53 ++ gateway/tests/test_operator_resolution.py | 520 ++++++++++++++++++ .../test_repair_api_lineage_migration.py | 2 +- .../test_resolution_evidence_migration.py | 62 +++ sdk/python/src/b2a_sdk/resolution_verifier.py | 19 +- sdk/python/tests/test_resolution_verifier.py | 25 + 23 files changed, 1568 insertions(+), 24 deletions(-) create mode 100644 gateway/app/services/operator_resolution.py create mode 100644 gateway/docs/operator-resolution-runbook.md create mode 100644 gateway/docs/schema-058-rollout.md create mode 100644 gateway/migrations/versions/058_resolution_evidence.py create mode 100644 gateway/scripts/amw_resolutions.py create mode 100644 gateway/tests/test_amw_resolutions_cli.py create mode 100644 gateway/tests/test_operator_resolution.py create mode 100644 gateway/tests/test_resolution_evidence_migration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4abaa6279..58924c365 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ This repository starts with a fresh history. Releases are listed here once tagge ## [Unreleased] +### Operator receipt resolution (SEL-351 follow-up) + +- Add a bootstrap-admin-only resolution endpoint and operator CLI for signed + delivery_uncertain receipts, with bounded evidence and explicit refunds. +- Add schema 058 evidence columns and resolution-v0.2 evidence-hash signatures; + preserve offline verification of resolution-v0.1 and original receipt bytes. +- Keep original replays unchanged; repeat requests finish interrupted refunds + through separate ledger entries. Optional fixture lookups remain unregistered + by default and refuse unfenced negative answers. + - Receipts: additive signed fields `permit_digest`, `approval_digest`, `counterparty`, `sequence` (per-permit, from 0 when a caller opts in; left unset otherwise), and `prev_receipt_digest` (omitted at sequence 0). Digests diff --git a/gateway/TROUBLESHOOTING.md b/gateway/TROUBLESHOOTING.md index ba262b7fa..71aa9496c 100644 --- a/gateway/TROUBLESHOOTING.md +++ b/gateway/TROUBLESHOOTING.md @@ -107,7 +107,7 @@ If the upstream exposes a status endpoint, a deployment can register an optional - The provider says the call never ran: you still get `delivery_uncertain` and no refund. A paused worker that holds the claim could still send, so this answer is logged as `mcp_dispatch_status_lookup_not_delivered_unfenced` for your manual reconciliation. - The lookup is not registered, raises, times out, returns nothing, returns a malformed answer or a non-boolean `delivered`, or returns a result that fails the checks: you still get `delivery_uncertain`. -The gateway never re-sends the call on this path. Your lookup is your own code, so keep it read-only. The lookup must be an `async def` function, an object with an `async def __call__`, or a `functools.partial` of either. A plain function raises `TypeError` at registration, because the timeout cannot interrupt synchronous code. A non-error (`isError` false) result whose `structuredContent` is missing or does not match the tool's `outputSchema` stays `delivery_uncertain`, as on the live path. `isError` results skip the `outputSchema` check. A lookup that returns something other than an awaitable, such as a sync function marked with `inspect.markcoroutinefunction`, also stays `delivery_uncertain`. The gateway never touches an attempt that already has a signed `delivery_uncertain` receipt, so those still need manual reconciliation. For action-permit calls, the query's `forwarded_idempotency_key` is `None` because the native key sent upstream is not recorded, so match on `invocation_id`. The cleanup log field `dispatch_status_resolved` counts the attempts the lookup resolved. +The gateway never re-sends the call on this path. Your lookup is your own code, so keep it read-only. The lookup must be an `async def` function, an object with an `async def __call__`, or a `functools.partial` of either. A plain function raises `TypeError` at registration, because the timeout cannot interrupt synchronous code. A non-error (`isError` false) result whose `structuredContent` is missing or does not match the tool's `outputSchema` stays `delivery_uncertain`, as on the live path. `isError` results skip the `outputSchema` check. A lookup that returns something other than an awaitable, such as a sync function marked with `inspect.markcoroutinefunction`, also stays `delivery_uncertain`. The automatic reconciler never revisits an attempt that already has a signed `delivery_uncertain` receipt. The bootstrap-admin-only operator endpoint can append an evidence-bound resolution; it never changes that receipt or redispatches. The charge stands unless the operator records `confirmed_not_delivered` with evidence and explicitly requests a separate refund. See the [operator resolution runbook](docs/operator-resolution-runbook.md). For action-permit calls, the query's `forwarded_idempotency_key` is `None` because the native key sent upstream is not recorded, so match on `invocation_id`. The cleanup log field `dispatch_status_resolved` counts the attempts the lookup resolved. --- diff --git a/gateway/app/db/models.py b/gateway/app/db/models.py index 7218c5e89..78fd85ed9 100644 --- a/gateway/app/db/models.py +++ b/gateway/app/db/models.py @@ -1013,6 +1013,8 @@ class ReceiptResolutionModel(SQLModel, table=True): prev_hash: Optional[str] = Field(default=None, max_length=64) receipt_hash: str = Field(max_length=64, index=True) outcome: str = Field(max_length=32) + evidence: Optional[str] = Field(default=None, sa_column=Column(Text, nullable=True)) + evidence_hash: Optional[str] = Field(default=None, max_length=64) created_at: datetime = Field(sa_type=NaiveUTCDateTime, default_factory=utc_now) payload_hash: str = Field(max_length=64) resolution_hash: str = Field(max_length=64, unique=True, index=True) diff --git a/gateway/app/main.py b/gateway/app/main.py index 8abe0b551..abd2062f6 100644 --- a/gateway/app/main.py +++ b/gateway/app/main.py @@ -838,6 +838,7 @@ def mount_dormant_trust_surfaces(application: FastAPI) -> None: ), ) app.include_router(api_keys.admin_router) +app.include_router(receipts.admin_router) # Self-serve dev keys: the handler is triple-gated at runtime (its own # ENABLE_DEV_KEY_SELF_PROVISION flag answers 404, production-like environments diff --git a/gateway/app/routers/receipts.py b/gateway/app/routers/receipts.py index 5b74b2cf9..87b014c27 100644 --- a/gateway/app/routers/receipts.py +++ b/gateway/app/routers/receipts.py @@ -10,6 +10,7 @@ from app.core.wallet_resolution import resolve_wallet_id from app.schemas.resolutions import ( ResolutionListResponse, + ResolutionRequest, ResolutionResponse, resolution_to_response, ) @@ -382,3 +383,48 @@ async def verify_receipt( else: auth.require_bootstrap_admin() return ReceiptVerifyResponse(valid=valid, reason=reason, receipt=receipt) + + +admin_router = APIRouter(prefix="/v1/admin/receipts", tags=["Operator Receipts"]) + + +@admin_router.post("/{receipt_id}/resolution", response_model=ResolutionResponse) +async def resolve_uncertain_receipt( + receipt_id: str, + request: ResolutionRequest, + auth: AuthContext = Depends(get_auth_context), +) -> ResolutionResponse: + from app.services.operator_resolution import resolve_receipt + from app.trust import sha256_hex + + error = None + try: + auth.require_bootstrap_admin() + model = await resolve_receipt(receipt_id, request) + return resolution_to_response(model) + except ReceiptResolutionError as exc: + error = exc.reason + raise HTTPException( + status_code=404 if error == "receipt_not_found" else 409, detail=error + ) from exc + except HTTPException: + error = "admin_access_denied" + raise + except Exception: + error = "resolution_failed" + raise HTTPException(status_code=503, detail=error) from None + finally: + await record_audit_event( + event="receipt.resolution.attempt", + endpoint=f"/v1/admin/receipts/{receipt_id}/resolution", + auth_source=auth.source, + key_id=auth.key_id, + ok=error is None, + error=error, + metadata={ + "receipt_id": receipt_id, + "outcome": request.outcome, + "operator_id": auth.key_id or sha256_hex(auth.raw_key.encode("utf-8")), + "refund_credits": request.refund_credits, + }, + ) diff --git a/gateway/app/schemas/resolutions.py b/gateway/app/schemas/resolutions.py index 102da9370..9717a8d19 100644 --- a/gateway/app/schemas/resolutions.py +++ b/gateway/app/schemas/resolutions.py @@ -3,9 +3,10 @@ from __future__ import annotations import json -from typing import Literal, cast +from typing import Any, Literal, cast -from pydantic import BaseModel +from datetime import datetime +from pydantic import BaseModel, ConfigDict, Field, field_validator from app.core.config import public_api_origin from app.db.models import ReceiptResolutionModel @@ -13,7 +14,9 @@ class ResolutionResponse(BaseModel): - version: Literal["resolution-v0.1"] + version: Literal["resolution-v0.1", "resolution-v0.2"] + evidence: dict[str, Any] | None = None + evidence_hash: str | None = None issuer: str resolution_id: str receipt_id: str @@ -47,6 +50,8 @@ def resolution_to_response(model: ReceiptResolutionModel) -> ResolutionResponse: return ResolutionResponse( version=signed["version"], issuer=public_api_origin(), + evidence=json.loads(model.evidence) if model.evidence else None, + evidence_hash=model.evidence_hash, resolution_id=model.resolution_id, receipt_id=model.receipt_id, wallet_id=model.wallet_id, @@ -63,3 +68,27 @@ def resolution_to_response(model: ReceiptResolutionModel) -> ResolutionResponse: signature=model.signature, kid=model.signature_key_id, ) + + +class ResolutionEvidence(BaseModel): + model_config = ConfigDict(extra="forbid") + source: str = Field(min_length=1, max_length=256) + upstream_reference_ids: list[str] = Field(default_factory=list, max_length=64) + observed_at: datetime + note: str = Field(min_length=1, max_length=4096) + raw_lookup_answer: dict[str, Any] | None = None + + @field_validator("observed_at") + @classmethod + def require_timezone(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("observed_at requires a timezone") + return value + + +class ResolutionRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + outcome: Literal["confirmed_success", "confirmed_not_delivered"] + evidence: ResolutionEvidence + use_lookup: bool = False + refund_credits: bool = False diff --git a/gateway/app/services/mcp_dispatch_attempts.py b/gateway/app/services/mcp_dispatch_attempts.py index fd3780b49..b8fb26a27 100644 --- a/gateway/app/services/mcp_dispatch_attempts.py +++ b/gateway/app/services/mcp_dispatch_attempts.py @@ -1883,7 +1883,17 @@ async def mark_debit_refunded( "dispatch_refund_ledger_conflict" ) if attempt.state != "returned_error": - raise DispatchAttemptConflictError("dispatch_refund_state_invalid") + from app.services.receipt_resolutions import ( + operator_refund_authorized, + ) + + if ( + attempt.state != "delivery_uncertain" + or not await operator_refund_authorized(session, attempt_id) + ): + raise DispatchAttemptConflictError( + "dispatch_refund_state_invalid" + ) refund = ( await session.execute( select(LedgerEntryModel).where( diff --git a/gateway/app/services/operator_resolution.py b/gateway/app/services/operator_resolution.py new file mode 100644 index 000000000..484216c39 --- /dev/null +++ b/gateway/app/services/operator_resolution.py @@ -0,0 +1,278 @@ +"""Operator adjudication of immutable uncertain receipts, without dispatching.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Awaitable, Callable +from typing import Any, cast + +import httpx +from sqlalchemy import select +from sqlalchemy.sql.elements import ColumnElement + +from app.core.resilience import run_with_write_conflict_retry +from app.db.database import get_session_factory +from app.db.models import ( + LedgerEntryModel, + McpDispatchAttemptModel, + ReceiptModel, + ReceiptResolutionModel, +) +from app.schemas.resolutions import ResolutionRequest +from app.services.agent_money import get_agent_money +from app.services.idempotency import ACTION_MCP_IDEMPOTENCY_ENDPOINT +from app.services.mcp_dispatch_attempts import ( + get_mcp_dispatch_attempt_service, +) +from app.services.mcp_dispatch_reconciliation import ( + STATUS_LOOKUP_TIMEOUT_SECONDS, + UpstreamDispatchStatusQuery, + _require_async_lookup, +) +from app.services.ownership import CallerIdentity +from app.services.permits import get_permit_service +from app.services.receipt_resolutions import ( + ReceiptResolutionError, + append_resolution, + canonical_evidence, + get_resolution, +) + +ReceiptStatusLookup = Callable[[UpstreamDispatchStatusQuery], Awaitable[str | None]] +_LOOKUPS: dict[str, ReceiptStatusLookup] = {} + + +def register_receipt_status_lookup(origin: str, lookup: ReceiptStatusLookup) -> None: + """Explicit startup opt-in only. No adapter is registered by default.""" + _require_async_lookup(lookup) + _LOOKUPS[origin] = lookup + + +class RefundFixtureStatusLookup: + """Read the refund fixture status contract; never retain its raw answer. + + Deployment code supplies an origin and token. Redirects are forbidden and + only the status enum is exposed, so reflected credentials cannot enter + evidence, responses, or logs. A transport may be injected for offline tests. + """ + + def __init__( + self, + origin: str, + token: str, + *, + transport: httpx.AsyncBaseTransport | None = None, + ) -> None: + url = httpx.URL(origin) + if ( + url.scheme not in {"http", "https"} + or not url.host + or url.userinfo + or url.query + or url.fragment + or url.path not in {"", "/"} + ): + raise ValueError("fixture origin must be an origin without credentials") + self._origin = origin.rstrip("/") + self._token = token + self._transport = transport + + async def __call__(self, query: UpstreamDispatchStatusQuery) -> str | None: + if query.upstream_origin.rstrip("/") != self._origin: + return None + async with httpx.AsyncClient( + transport=self._transport, + timeout=STATUS_LOOKUP_TIMEOUT_SECONDS, + follow_redirects=False, + trust_env=False, + ) as client: + async with client.stream( + "POST", + f"{self._origin}/status", + headers={"Authorization": f"Bearer {self._token}"}, + json={ + "tool": query.upstream_tool_name, + "invocation_id": query.invocation_id, + "idempotency_key": query.forwarded_idempotency_key, + }, + ) as response: + response.raise_for_status() + data = bytearray() + async for chunk in response.aiter_bytes(): + data.extend(chunk) + if len(data) > 16384: + return None + answer = json.loads(data) + status = answer.get("status") if isinstance(answer, dict) else None + return ( + status + if status in {"executed", "not_executed", "pending", "unknown"} + else None + ) + + +def _redact(value: Any) -> Any: + if isinstance(value, dict): + return { + key: "[REDACTED]" + if any( + word in key.lower() + for word in ( + "token", + "secret", + "authorization", + "password", + "api_key", + "credential", + ) + ) + else _redact(item) + for key, item in value.items() + } + if isinstance(value, list): + return [_redact(item) for item in value] + return value + + +async def resolve_receipt( + receipt_id: str, request: ResolutionRequest +) -> ReceiptResolutionModel: + evidence = _redact(request.evidence.model_dump(mode="json", exclude_none=True)) + if request.refund_credits and request.outcome != "confirmed_not_delivered": + raise ReceiptResolutionError("resolution_refund_outcome_invalid") + async with get_session_factory()() as session: + receipt = await session.get(ReceiptModel, receipt_id) + if receipt is None: + raise ReceiptResolutionError("receipt_not_found") + attempt = ( + await session.get(McpDispatchAttemptModel, receipt.dispatch_attempt_id) + if receipt.dispatch_attempt_id + else None + ) + if attempt is None and receipt.idempotency_record_id: + attempt = ( + await session.execute( + select(McpDispatchAttemptModel).where( + cast( + ColumnElement[bool], + McpDispatchAttemptModel.idempotency_record_id + == receipt.idempotency_record_id, + ) + ) + ) + ).scalar_one_or_none() + if attempt is not None and ( + attempt.wallet_id != receipt.wallet_id + or attempt.ledger_entry_id != receipt.ledger_entry_id + ): + raise ReceiptResolutionError("resolution_dispatch_linkage_invalid") + if request.refund_credits: + debit = ( + await session.get(LedgerEntryModel, receipt.ledger_entry_id) + if receipt.ledger_entry_id + else None + ) + if ( + debit is None + or debit.wallet_id != receipt.wallet_id + or debit.action != "debit" + or abs(debit.amount) != receipt.credits_charged + ): + raise ReceiptResolutionError("resolution_debit_invalid") + # A product-derived ledger identity, signed before the refund. Its + # presence is durable intent; retry finishes an interrupted refund. + evidence["refund_entry_id"] = f"refund-{debit.entry_id}" + evidence["use_lookup"] = request.use_lookup + if request.use_lookup: + if attempt is None: + raise ReceiptResolutionError("resolution_inconclusive") + evidence["lookup_origin"] = attempt.upstream_origin + evidence["lookup_status"] = ( + "executed" if request.outcome == "confirmed_success" else "not_executed" + ) + canonical_evidence(evidence) + existing = await get_resolution(receipt_id) + if request.use_lookup and existing is None: + assert attempt is not None + context = ( + await get_mcp_dispatch_attempt_service().get_context(attempt.attempt_id) + if attempt + else None + ) + lookup = _LOOKUPS.get(attempt.upstream_origin) if attempt else None + if context is None or lookup is None: + raise ReceiptResolutionError("resolution_inconclusive") + query = UpstreamDispatchStatusQuery( + attempt_id=attempt.attempt_id, + invocation_id=attempt.idempotency_record_id, + idempotency_key=context.idempotency_key, + forwarded_idempotency_key=None + if context.endpoint == ACTION_MCP_IDEMPOTENCY_ENDPOINT + else context.idempotency_key, + upstream_origin=attempt.upstream_origin, + upstream_tool_name=attempt.upstream_tool_name, + request_hash=attempt.request_hash, + dispatched_at=attempt.dispatched_at, + ) + try: + status = await asyncio.wait_for( + lookup(query), timeout=STATUS_LOOKUP_TIMEOUT_SECONDS + ) + except Exception: + raise ReceiptResolutionError("resolution_inconclusive") from None + # Terminalizing a claim cannot fence a paused sender. Only attempts + # with no historical claim or send can prove non-delivery locally. + negative_safe = ( + attempt.state == "returned_error" + and attempt.completed_at is not None + and attempt.dispatch_claim_hash is None + and attempt.dispatched_at is None + ) + expected = ( + "confirmed_success" + if status == "executed" + else "confirmed_not_delivered" + if status == "not_executed" and negative_safe + else None + ) + if expected is None or expected != request.outcome: + raise ReceiptResolutionError("resolution_inconclusive") + model = await append_resolution(receipt_id, request.outcome, evidence) + if request.refund_credits: + assert receipt.ledger_entry_id is not None + charge_entry_id = receipt.ledger_entry_id + + async def compensate() -> None: + await get_agent_money().refund_charge( + wallet_id=receipt.wallet_id, + charge_entry_id=charge_entry_id, + description=f"Operator resolution {model.resolution_id}", + caller=CallerIdentity.for_wallet(receipt.wallet_id), + ) + + await run_with_write_conflict_retry( + compensate, + on_exhausted=lambda exc: ReceiptResolutionError( + "resolution_refund_contended" + ), + ) + if attempt: + + async def finish_compensation() -> None: + await get_mcp_dispatch_attempt_service().mark_debit_refunded( + attempt_id=attempt.attempt_id, + ledger_entry_id=charge_entry_id, + ) + await get_permit_service().release_dispatch_budget_once( + attempt.attempt_id, + caller=CallerIdentity.for_wallet(receipt.wallet_id), + ) + + await run_with_write_conflict_retry( + finish_compensation, + on_exhausted=lambda exc: ReceiptResolutionError( + "resolution_refund_contended" + ), + ) + return model diff --git a/gateway/app/services/permits.py b/gateway/app/services/permits.py index 01f5fde74..066af0cdf 100644 --- a/gateway/app/services/permits.py +++ b/gateway/app/services/permits.py @@ -1694,7 +1694,23 @@ async def _once() -> bool: # reservation that no longer exists. This pre-check is the # contract; the guarded claim below is the once-only gate. if attempt.state != "returned_error": - raise PermitError("dispatch_budget_release_state_invalid") + from app.db.models import LedgerEntryModel + from app.services.receipt_resolutions import ( + operator_refund_authorized, + ) + + refund = await session.get( + LedgerEntryModel, f"refund-{attempt.ledger_entry_id}" + ) + if ( + attempt.state != "delivery_uncertain" + or not await operator_refund_authorized(session, attempt_id) + or refund is None + or refund.wallet_id != attempt.wallet_id + or refund.action != "refund" + or refund.correlation_id != attempt.ledger_entry_id + ): + raise PermitError("dispatch_budget_release_state_invalid") if attempt.budget_released_at is not None: return False now = utc_now() diff --git a/gateway/app/services/receipt_resolutions.py b/gateway/app/services/receipt_resolutions.py index 997c11db0..36f72cb0c 100644 --- a/gateway/app/services/receipt_resolutions.py +++ b/gateway/app/services/receipt_resolutions.py @@ -4,6 +4,7 @@ import asyncio import random +import json import uuid from typing import Any, NoReturn, cast @@ -18,6 +19,7 @@ from app.db.database import get_session_factory from app.db.models import ( ReceiptModel, + McpDispatchAttemptModel, ReceiptResolutionModel, ResolutionChainHeadModel, ) @@ -28,7 +30,8 @@ sha256_hex, ) -RESOLUTION_VERSION = "resolution-v0.1" +RESOLUTION_VERSION = "resolution-v0.2" +MAX_EVIDENCE_BYTES = 16384 RESOLUTION_OUTCOMES = frozenset({"confirmed_success", "confirmed_not_delivered"}) @@ -43,8 +46,8 @@ class _HeadConflict(Exception): def _signed_fields(model: ReceiptResolutionModel) -> dict[str, Any]: - return { - "version": RESOLUTION_VERSION, + fields = { + "version": RESOLUTION_VERSION if model.evidence_hash else "resolution-v0.1", "resolution_id": model.resolution_id, "receipt_id": model.receipt_id, "wallet_id": model.wallet_id, @@ -55,6 +58,24 @@ def _signed_fields(model: ReceiptResolutionModel) -> dict[str, Any]: "created_at": model.created_at, } + if model.evidence_hash is not None: + fields["evidence_hash"] = model.evidence_hash + return fields + + +def canonical_evidence(evidence: dict[str, Any]) -> tuple[str, str]: + try: + encoded = canonical_json(evidence) + if ( + not isinstance(evidence, dict) + or not evidence + or len(encoded.encode("utf-8")) > MAX_EVIDENCE_BYTES + ): + raise ValueError("evidence size") + except (ValueError, TypeError, RecursionError) as exc: + raise ReceiptResolutionError("resolution_evidence_invalid") from exc + return encoded, sha256_hex(encoded.encode("utf-8")) + def resolution_signing_input( model: ReceiptResolutionModel, key_id: str @@ -88,6 +109,15 @@ async def _assert_valid_resolution( *, expected_outcome: str | None = None, ) -> None: + if model.evidence_hash is not None: + try: + encoded, digest = canonical_evidence(json.loads(model.evidence or "null")) + except (ValueError, TypeError) as exc: + raise ReceiptResolutionError("resolution_integrity_invalid") from exc + if encoded != model.evidence or digest != model.evidence_hash: + raise ReceiptResolutionError("resolution_integrity_invalid") + elif model.evidence is not None: + raise ReceiptResolutionError("resolution_integrity_invalid") receipt, receipt_input = await _receipt_input(session, model.receipt_id) if expected_outcome is not None and model.outcome != expected_outcome: raise ReceiptResolutionError("resolution_outcome_conflict") @@ -272,14 +302,17 @@ async def reject_or_retry() -> NoReturn: await reject_or_retry() -async def append_resolution(receipt_id: str, outcome: str) -> ReceiptResolutionModel: - """Sign and append one final outcome; exact webhook replays return its row.""" - # TODO(SEL-351, PR #187, claude/sel-351): After #187 merges, hook - # append_resolution into gateway/app/services/mcp_dispatch_reconciliation.py. - # _lookup_status decides the provider outcome; _reconcile_active records it - # once via its guarded self._dispatch.complete() call. Append the matching - # resolution there after completion; not-delivered uses - # STATUS_NOT_DELIVERED_ERROR_CODE. Migration 044 may need renumbering. +async def append_resolution( + receipt_id: str, outcome: str, evidence: dict[str, Any] | None = None +) -> ReceiptResolutionModel: + """Append immutable evidence; replay only an identical outcome and digest. + + Evidence-free calls retain the historical v0.1 format for compatibility. + The operator boundary always supplies evidence. + """ + encoded, digest = ( + canonical_evidence(evidence) if evidence is not None else (None, None) + ) if outcome not in RESOLUTION_OUTCOMES: raise ReceiptResolutionError("resolution_outcome_invalid") @@ -287,6 +320,8 @@ async def append_resolution(receipt_id: str, outcome: str) -> ReceiptResolutionM if existing is not None: if existing.outcome != outcome: raise ReceiptResolutionError("resolution_outcome_conflict") + if existing.evidence_hash != digest: + raise ReceiptResolutionError("resolution_evidence_conflict") return existing # Refuse invalid source receipts before provisioning a new signing key. @@ -316,6 +351,8 @@ async def append_resolution(receipt_id: str, outcome: str) -> ReceiptResolutionM await _assert_valid_resolution( session, existing, expected_outcome=outcome ) + if existing.evidence_hash != digest: + raise ReceiptResolutionError("resolution_evidence_conflict") return existing await signing_keys.validate_prepared_signing_key( @@ -333,6 +370,8 @@ async def append_resolution(receipt_id: str, outcome: str) -> ReceiptResolutionM prev_hash=prev_hash, receipt_hash=sha256_hex(receipt_input.encode("utf-8")), outcome=outcome, + evidence=encoded, + evidence_hash=digest, created_at=utc_now(), payload_hash="", resolution_hash="", @@ -403,3 +442,39 @@ async def append_resolution(receipt_id: str, outcome: str) -> ReceiptResolutionM finally: await session.close() raise AssertionError("resolution_retry_budget_invalid") + + +async def operator_refund_authorized(session: AsyncSession, attempt_id: str) -> bool: + """Validate signed refund intent before releasing a reservation/checkpoint.""" + attempt = await session.get(McpDispatchAttemptModel, attempt_id) + if attempt is None: + return False + result = ( + await session.execute( + select(ReceiptResolutionModel, ReceiptModel) + .join( + ReceiptModel, + cast( + ColumnElement[bool], + ReceiptModel.receipt_id == ReceiptResolutionModel.receipt_id, + ), + ) + .where( + cast( + ColumnElement[bool], + ReceiptModel.idempotency_record_id == attempt.idempotency_record_id, + ) + ) + ) + ).one_or_none() + if result is None: + return False + resolution, receipt = result + await _assert_valid_resolution(session, resolution) + evidence = json.loads(resolution.evidence or "{}") + return ( + resolution.outcome == "confirmed_not_delivered" + and bool(resolution.evidence_hash) + and receipt.ledger_entry_id is not None + and evidence.get("refund_entry_id") == f"refund-{receipt.ledger_entry_id}" + ) diff --git a/gateway/docs/failure-semantics.md b/gateway/docs/failure-semantics.md index 4b94f9221..811371022 100644 --- a/gateway/docs/failure-semantics.md +++ b/gateway/docs/failure-semantics.md @@ -137,7 +137,7 @@ Proven by `tests/test_audit_chain_contention_surface.py` and `tests/test_receipt_write_contention_surface.py`, which also pin that an ordinary bug in the same window still returns `-32603`. -## Why `delivery_uncertain` cannot be "fixed" +## Resolving `delivery_uncertain` without changing the receipt The durable dispatch claim is written immediately before the network send (`before_dispatch` in `app/services/upstream_mcp.py`). Every failure @@ -155,6 +155,15 @@ honestly* rather than guess: - resolution belongs to the caller and the upstream provider, using the forwarded idempotency metadata (below) and the signed evidence. +An operator can append a signed, evidence-bound resolution through the +bootstrap-admin-only endpoint. The original receipt and replay remain +`delivery_uncertain`; the gateway never redispatches. The charge stands unless +an operator records `confirmed_not_delivered` with evidence and explicitly +requests a refund. That refund is a separate ledger entry. Exact retries finish +interrupted compensation; conflicting evidence or outcomes are refused. +See the [operator resolution runbook](operator-resolution-runbook.md) for +lookup limitations, sender fencing, refunds, and offline verification. + Retaining the charge is a deliberate incentive choice: an ambiguous call that auto-refunded would pay callers to induce timeouts against providers whose side effects completed. diff --git a/gateway/docs/openapi.json b/gateway/docs/openapi.json index ace858d7f..803e5a50f 100644 --- a/gateway/docs/openapi.json +++ b/gateway/docs/openapi.json @@ -5781,6 +5781,82 @@ } } }, + "/v1/admin/receipts/{receipt_id}/resolution": { + "post": { + "tags": [ + "Operator Receipts" + ], + "summary": "Resolve Uncertain Receipt", + "operationId": "resolve_uncertain_receipt_v1_admin_receipts__receipt_id__resolution_post", + "security": [ + { + "APIKeyHeader": [] + }, + { + "HTTPBearer": [] + } + ], + "parameters": [ + { + "name": "receipt_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Receipt Id" + } + }, + { + "name": "authorization", + "in": "header", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Authorization" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResolutionRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResolutionResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v1/demo/keys": { "get": { "tags": [ @@ -10569,6 +10645,55 @@ ], "title": "RefundReconciliationRetryResponse" }, + "ResolutionEvidence": { + "properties": { + "source": { + "type": "string", + "maxLength": 256, + "minLength": 1, + "title": "Source" + }, + "upstream_reference_ids": { + "items": { + "type": "string" + }, + "type": "array", + "maxItems": 64, + "title": "Upstream Reference Ids" + }, + "observed_at": { + "type": "string", + "format": "date-time", + "title": "Observed At" + }, + "note": { + "type": "string", + "maxLength": 4096, + "minLength": 1, + "title": "Note" + }, + "raw_lookup_answer": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Raw Lookup Answer" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "source", + "observed_at", + "note" + ], + "title": "ResolutionEvidence" + }, "ResolutionListResponse": { "properties": { "records": { @@ -10617,13 +10742,71 @@ ], "title": "ResolutionListResponse" }, + "ResolutionRequest": { + "properties": { + "outcome": { + "type": "string", + "enum": [ + "confirmed_success", + "confirmed_not_delivered" + ], + "title": "Outcome" + }, + "evidence": { + "$ref": "#/components/schemas/ResolutionEvidence" + }, + "use_lookup": { + "type": "boolean", + "title": "Use Lookup", + "default": false + }, + "refund_credits": { + "type": "boolean", + "title": "Refund Credits", + "default": false + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "outcome", + "evidence" + ], + "title": "ResolutionRequest" + }, "ResolutionResponse": { "properties": { "version": { "type": "string", - "const": "resolution-v0.1", + "enum": [ + "resolution-v0.1", + "resolution-v0.2" + ], "title": "Version" }, + "evidence": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Evidence" + }, + "evidence_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Evidence Hash" + }, "issuer": { "type": "string", "title": "Issuer" diff --git a/gateway/docs/operator-resolution-runbook.md b/gateway/docs/operator-resolution-runbook.md new file mode 100644 index 000000000..c06abb8b5 --- /dev/null +++ b/gateway/docs/operator-resolution-runbook.md @@ -0,0 +1,85 @@ +# Operator receipt resolution + +Use this path only for an existing, signed `delivery_uncertain` receipt after +checking authoritative upstream state. It never dispatches a tool, changes the +original receipt, or changes the original idempotency replay. The charge stands +unless the operator records `confirmed_not_delivered` with evidence and +explicitly requests a refund. Only the bootstrap admin credential is accepted; +agent keys, wallet keys, demo keys, and permits cannot resolve receipts. This +route is absent from MCP tools and agent discovery. + +## Collect and review evidence + +Export the original portable receipt and retain the public trust keys. Find the +upstream invocation from its dispatch attempt and idempotency record. Check the +provider's own reference ids and observation time. Prepare a local JSON request +file with `outcome` (`confirmed_success` or `confirmed_not_delivered`), an +`evidence` object containing `source`, `upstream_reference_ids` (a list), +`observed_at` (an ISO timestamp with timezone), and `note`. Optionally include +`raw_lookup_answer`, with secrets removed. Evidence is readable by the wallet +and any authorized receipt counterparty, so include only information they may +see. Common credential-named fields are redacted automatically, but review +free text and provider-specific fields yourself. Canonical evidence is limited +to 16 KiB. Use actual provider references, never invented evidence. + +Set `refund_credits` explicitly to true only to refund a confirmed non-delivery. +It defaults to false. Success cannot refund. Before manually asserting +non-delivery, stop every process that could hold the dispatch claim and ensure +no worker can resume sending. Document how this was established in the note. +A terminal database row alone does not fence a paused sender. + +Run the thin CLI on the operator host with the bootstrap key already configured +in `VALID_API_KEYS`: + +```bash +python gateway/scripts/amw_resolutions.py "$RECEIPT_ID" "$REQUEST_FILE" \ + --base-url "$AMW_GATEWAY_URL" +``` + +This calls `POST /v1/admin/receipts/{receipt_id}/resolution`. Keep the request +file: an exact replay returns the same signed resolution and finishes any +interrupted refund. Changing the evidence, outcome, lookup flag, or refund +intent conflicts with the existing record (HTTP 409). Never delete it to retry. +A signed `evidence.refund_entry_id` records refund intent; confirm the separate +ledger entry exists before treating the refund as complete. An interruption +may leave the signed intent before the money transaction finishes. + +## Optional refund fixture status lookup + +A deployment can explicitly register `RefundFixtureStatusLookup(origin, token)` +with `register_receipt_status_lookup` in `app.services.operator_resolution` +during startup. Nothing is registered by default. Supply the bearer token from +the deployment's configuration without logging it. Set `use_lookup` to true +in the reviewed request file. The adapter sends `POST {origin}/status` with +`tool`, `invocation_id`, and the forwarded `idempotency_key`. For action-permit +calls that key is unavailable and null; the fixture must match invocation id. +Redirects are disabled, response bytes are bounded, and the lookup has a +five-second bound. Only the status enum is retained as evidence, so credentials +reflected in the raw provider response are discarded. + +`executed` proves `confirmed_success`. `not_executed` can prove non-delivery +only for a terminal pre-dispatch failure with no historical send or dispatch claim. +For the usual uncertain attempt with a historical claim it is inconclusive: +a paused holder could still send after the observation. `pending`, `unknown`, +timeout, malformed answers, errors, an unregistered origin, and an answer that +disagrees with the requested outcome return HTTP 409 `resolution_inconclusive`. +No resolution or refund is written. The audit records the attempt. + +If the provider cannot answer, retain the uncertain receipt and the charge. +Do not redispatch automatically. Obtain authoritative evidence and establish +that all senders have stopped before an operator asserts a final outcome. + +## Verify offline and inspect the chain + +Read `GET /v1/receipts/{receipt_id}/resolution` and the wallet chain at +`GET /v1/receipts/resolutions/wallet/{wallet_id}` with an authorized credential. +Export the original `GET /v1/receipts/{receipt_id}/portable` bundle and public +`/.well-known/trust-keys.json` document. On an offline host, load these actual +files and call the SDK's `key_set_from_document`, then +`verify_resolution(resolution, receipt_bundle, keys)` from +`b2a_sdk.resolution_verifier` (also exported through `amw`). Require `.ok`. +Use `verify_resolution_chain` with the complete ordered chain for chain checks. +Verification covers original receipt bytes, resolution signature and hash, +and the v0.2 evidence hash. v0.1 records remain verifiable without evidence. +The verifier does not prove the truth of an operator's assertion or that the +refund ledger transaction completed; check the ledger separately. diff --git a/gateway/docs/partner-first-tool-runbook.md b/gateway/docs/partner-first-tool-runbook.md index 2f751ec3b..cd467efa6 100644 --- a/gateway/docs/partner-first-tool-runbook.md +++ b/gateway/docs/partner-first-tool-runbook.md @@ -143,7 +143,8 @@ exactly-once side effects. 7. Call the out-of-scope tool under the same permit → deny (`permit_tool_not_allowed`). 8. Call `YOUR_TOOL_ID` with no permit → deny (`permit_required`). 9. Force a post-dispatch timeout → signed `delivery_uncertain`, charge retained, - and no automatic retry. + and no automatic retry. Use the [operator resolution runbook](operator-resolution-runbook.md) + if authoritative upstream evidence later becomes available. 10. **Partner verifies the receipt offline, in their own environment.** Export the portable bundle and the public key set, then verify with no gateway access and no credentials: diff --git a/gateway/docs/schema-058-rollout.md b/gateway/docs/schema-058-rollout.md new file mode 100644 index 000000000..c5c5849c5 --- /dev/null +++ b/gateway/docs/schema-058-rollout.md @@ -0,0 +1,28 @@ +# Schema 058: evidence for operator receipt resolutions + +Migration `058_resolution_evidence` revises `057_receipt_permit_seq_unique`. +It adds nullable `evidence` (canonical JSON text) and `evidence_hash` (SHA-256) +columns to `receipt_resolutions`. Existing rows remain unchanged and retain +resolution-v0.1 signatures. Operator writes use resolution-v0.2 and sign the +evidence hash. Application writes cap canonical evidence at 16 KiB. + +## Compatibility release + +Boot requires the packaged migration head. Qualify the 058 image and upgrade +from 057 against disposable PostgreSQL before scheduling the production +maintenance window. Apply migration 058 before starting the 058 image. No +staging or production migration is part of this change. + +## Rollback boundary + +Downgrading drops both evidence columns only while all rows are v0.1. The +migration refuses downgrade if evidence is present, preserving signed v0.2 +evidence. +Export resolutions, original portable receipts, and trusted public keys first. +An older application cannot validate v0.2 signatures. Restore a database backup +from before the first v0.2 append if an older image is required. Downgrade and +re-upgrade are safe only while all rows are v0.1 (or the table is empty). + +Refund ledger entries survive downgrade. A schema rollback does not reverse +credits or recover the signed refund intent. Never delete a resolution to +retry a refund; repeat the original operator request on the 058 application. diff --git a/gateway/migrations/versions/058_resolution_evidence.py b/gateway/migrations/versions/058_resolution_evidence.py new file mode 100644 index 000000000..1a4e09bd5 --- /dev/null +++ b/gateway/migrations/versions/058_resolution_evidence.py @@ -0,0 +1,39 @@ +"""Evidence bound to append-only receipt resolutions. + +Revision ID: 058_resolution_evidence +Revises: 057_receipt_permit_seq_unique +""" + +from alembic import op +import sqlalchemy as sa + +revision = "058_resolution_evidence" +down_revision = "057_receipt_permit_seq_unique" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "receipt_resolutions", sa.Column("evidence", sa.Text(), nullable=True) + ) + op.add_column( + "receipt_resolutions", sa.Column("evidence_hash", sa.String(64), nullable=True) + ) + + +def downgrade() -> None: + if op.get_bind().dialect.name == "postgresql": + op.execute( + sa.text("LOCK TABLE receipt_resolutions IN SHARE ROW EXCLUSIVE MODE") + ) + if op.get_bind().scalar( + sa.text( + "SELECT COUNT(*) FROM receipt_resolutions WHERE evidence_hash IS NOT NULL OR evidence IS NOT NULL" + ) + ): + raise RuntimeError( + "cannot drop signed resolution evidence; restore a pre-v0.2 backup instead" + ) + op.drop_column("receipt_resolutions", "evidence_hash") + op.drop_column("receipt_resolutions", "evidence") diff --git a/gateway/scripts/amw_resolutions.py b/gateway/scripts/amw_resolutions.py new file mode 100644 index 000000000..d8ab30d4e --- /dev/null +++ b/gateway/scripts/amw_resolutions.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Submit an operator resolution from a reviewed JSON request file.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import sys +from urllib.parse import quote + +import httpx + +from amw_approvals import ( + BASE_URL_ENV, + DEFAULT_BASE_URL, + _client, + _read_api_key, + _refusal_reason, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("receipt_id") + parser.add_argument("request_file", type=Path) + parser.add_argument( + "--base-url", default=os.environ.get(BASE_URL_ENV, DEFAULT_BASE_URL) + ) + args = parser.parse_args(argv) + if _refusal_reason(args.base_url): + print("error: refusing to send the admin key to an unsafe URL", file=sys.stderr) + return 1 + try: + key = _read_api_key() + if not key: + raise ValueError("missing admin key") + payload = json.loads(args.request_file.read_text()) + with _client(args.base_url, key) as client: + response = client.post( + f"/v1/admin/receipts/{quote(args.receipt_id, safe='')}/resolution", + json=payload, + ) + if response.is_error: + print(f"resolution failed: HTTP {response.status_code}", file=sys.stderr) + return 1 + print(json.dumps(response.json(), indent=2)) + return 0 + except (OSError, ValueError, httpx.HTTPError): + print( + "resolution request failed; check configuration and retry the same file", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/gateway/tests/test_amw_resolutions_cli.py b/gateway/tests/test_amw_resolutions_cli.py new file mode 100644 index 000000000..d86d5bbf6 --- /dev/null +++ b/gateway/tests/test_amw_resolutions_cli.py @@ -0,0 +1,53 @@ +"""Operator CLI delegates reviewed JSON without exposing credentials.""" + +import importlib +import json +from pathlib import Path + +import httpx + + +def test_cli_posts_request_file(monkeypatch, tmp_path, capsys): + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "scripts")) + cli = importlib.import_module("amw_resolutions") + from tests.test_operator_resolution import body + + payload = body() + path = tmp_path / "request.json" + path.write_text(json.dumps(payload)) + monkeypatch.setattr(cli, "_read_api_key", lambda: "test-admin") + calls = [] + + def handler(request): + assert request.headers["x-api-key"] == "test-admin" + assert request.url.path == "/v1/admin/receipts/receipt-argument/resolution" + calls.append(json.loads(request.content)) + return httpx.Response(200, json={"accepted": True}) + + monkeypatch.setattr( + cli, + "_client", + lambda origin, key: httpx.Client( + base_url=origin, + headers={"X-API-Key": key}, + transport=httpx.MockTransport(handler), + ), + ) + assert cli.main(["receipt-argument", str(path)]) == 0 + assert calls == [payload] + assert "test-admin" not in capsys.readouterr().out + + +def test_cli_refuses_unsafe_destination_before_reading_key(monkeypatch, capsys): + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "scripts")) + cli = importlib.import_module("amw_resolutions") + + def forbidden(): + raise AssertionError("key must not be read") + + monkeypatch.setattr(cli, "_read_api_key", forbidden) + assert ( + cli.main(["receipt-argument", "request.json", "--base-url", "http://192.0.2.1"]) + == 1 + ) + assert "refusing" in capsys.readouterr().err diff --git a/gateway/tests/test_operator_resolution.py b/gateway/tests/test_operator_resolution.py new file mode 100644 index 000000000..01d878539 --- /dev/null +++ b/gateway/tests/test_operator_resolution.py @@ -0,0 +1,520 @@ +"""Operator boundary, immutable evidence and interrupted compensation proofs.""" + +from __future__ import annotations + +import asyncio +from copy import deepcopy +import json + +import httpx +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import func, select + +from tests.support import db_gate +from app.main import app +from app.db.database import get_session_factory +from app.db.models import ControlPlaneAuditEventModel, ReceiptResolutionModel +from app.services import operator_resolution as service +from app.services.agent_money import get_agent_money +from app.services.mcp_dispatch_reconciliation import ( + get_mcp_dispatch_reconciliation_service, +) +from app.services.receipts import get_receipt_service +from b2a_sdk.receipt_verifier import key_set_from_document +from b2a_sdk.resolution_verifier import verify_resolution +from tests.test_trust_helpers import BOOTSTRAP_HEADERS +from tests.test_mcp_dispatch_reconciliation import ( + _seed_attempt, + _ledger_counts, + _replay, + _attempt, +) + + +@pytest.fixture +async def client(): + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test" + ) as client: + yield client + + +@pytest.fixture(autouse=True) +def lookup_registry(monkeypatch): + monkeypatch.setattr(service, "_LOOKUPS", {}) + + +def body(outcome="confirmed_success", *, refund=False, lookup=False): + return { + "outcome": outcome, + "refund_credits": refund, + "use_lookup": lookup, + "evidence": { + "source": "operator", + "upstream_reference_ids": [], + "observed_at": "2026-11-14T12:00:00Z", + "note": "Reviewed authoritative upstream state; sender stopped.", + }, + } + + +async def uncertain(client, suffix): + seed = await _seed_attempt(client, suffix=suffix, state="dispatched") + result = await get_mcp_dispatch_reconciliation_service().reconcile(idle_seconds=300) + assert result.dispatched_uncertain == 1 + attempt = await _attempt(seed.attempt_id) + receipt = await get_receipt_service().get_receipt_by_idempotency_record_id( + attempt.idempotency_record_id + ) + assert receipt.outcome == "delivery_uncertain" + return seed, receipt + + +@pytest.mark.anyio +@pytest.mark.parametrize( + ("outcome", "refund"), + [ + ("confirmed_success", False), + ("confirmed_not_delivered", False), + ("confirmed_not_delivered", True), + ], +) +async def test_operator_resolution_replay_and_original_unchanged( + client, clean_database, outcome, refund +): + seed, receipt = await uncertain(client, f"operator-{outcome}-{refund}") + path = f"/v1/admin/receipts/{receipt.receipt_id}/resolution" + original = await get_receipt_service().signing_input(receipt.receipt_id) + replay_before = await _replay(seed) + request = body(outcome, refund=refund) + first = await client.post(path, headers=BOOTSTRAP_HEADERS, json=request) + assert first.status_code == 200, first.text + second = await client.post(path, headers=BOOTSTRAP_HEADERS, json=request) + assert second.status_code == 200, second.text + assert second.json() == first.json() + assert first.json()["version"] == "resolution-v0.2" + assert await _ledger_counts(seed.wallet_id) == (1, int(refund)) + assert await get_receipt_service().signing_input(receipt.receipt_id) == original + assert ( + await get_receipt_service().get_receipt(receipt.receipt_id) + ).signature == receipt.signature + assert await _replay(seed) == replay_before + if refund: + attempt = await _attempt(seed.attempt_id) + assert attempt.budget_released_at is not None + assert attempt.debit_refunded_at is not None + assert ( + first.json()["evidence"]["refund_entry_id"] + == f"refund-{seed.ledger_entry_id}" + ) + portable = ( + await client.get( + f"/v1/receipts/{receipt.receipt_id}/portable", headers=BOOTSTRAP_HEADERS + ) + ).json() + keys = key_set_from_document( + (await client.get("/.well-known/trust-keys.json")).json() + ) + assert verify_resolution(first.json(), portable, keys).ok + tampered = deepcopy(first.json()) + tampered["evidence"]["note"] += "changed" + assert not verify_resolution(tampered, portable, keys).ok + changed = deepcopy(request) + changed["evidence"]["note"] += " changed" + assert ( + await client.post(path, headers=BOOTSTRAP_HEADERS, json=changed) + ).status_code == 409 + changed = deepcopy(request) + changed["outcome"] = ( + "confirmed_not_delivered" + if outcome == "confirmed_success" + else "confirmed_success" + ) + assert ( + await client.post(path, headers=BOOTSTRAP_HEADERS, json=changed) + ).status_code == 409 + async with get_session_factory()() as session: + assert ( + await session.scalar( + select(func.count()).select_from(ReceiptResolutionModel) + ) + == 1 + ) + audits = ( + ( + await session.execute( + select(ControlPlaneAuditEventModel).where( + ControlPlaneAuditEventModel.event + == "receipt.resolution.attempt" + ) + ) + ) + .scalars() + .all() + ) + assert len(audits) == 4 + assert all(json.loads(row.metadata_json)["operator_id"] for row in audits) + + +@pytest.mark.anyio +@pytest.mark.parametrize("source", ["api_key", "demo", "permit", "wallet"]) +async def test_non_admin_refused(client, clean_database, monkeypatch, source): + from app.core.auth import AuthContext, get_auth_context + + app.dependency_overrides[get_auth_context] = lambda: AuthContext( + source=source, raw_key="test", wallet_id="test-wallet" + ) + try: + response = await client.post( + "/v1/admin/receipts/missing/resolution", json=body() + ) + assert response.status_code == 403 + finally: + app.dependency_overrides.pop(get_auth_context) + + +@pytest.mark.anyio +async def test_interrupt_after_append_retry_finishes_refund( + client, clean_database, monkeypatch +): + seed, receipt = await uncertain(client, "operator-interrupt") + money = get_agent_money() + original = money.refund_charge + + async def fail(**kwargs): + raise RuntimeError("injected interruption") + + monkeypatch.setattr(money, "refund_charge", fail) + path = f"/v1/admin/receipts/{receipt.receipt_id}/resolution" + request = body("confirmed_not_delivered", refund=True) + assert ( + await client.post(path, headers=BOOTSTRAP_HEADERS, json=request) + ).status_code == 503 + assert await _ledger_counts(seed.wallet_id) == (1, 0) + monkeypatch.setattr(money, "refund_charge", original) + for _ in range(2): + response = await client.post(path, headers=BOOTSTRAP_HEADERS, json=request) + assert response.status_code == 200, response.text + assert await _ledger_counts(seed.wallet_id) == (1, 1) + async with get_session_factory()() as session: + assert ( + await session.scalar( + select(func.count()).select_from(ReceiptResolutionModel) + ) + == 1 + ) + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "status", + [ + "pending", + "unknown", + "not_executed", + "malformed", + "error", + "timeout", + "unregistered", + ], +) +async def test_inconclusive_writes_no_resolution( + client, clean_database, monkeypatch, status +): + seed, receipt = await uncertain(client, f"operator-{status}") + + async def lookup(query): + if status == "timeout": + await asyncio.sleep(1) + if status == "error": + raise RuntimeError("injected") + return status + + if status != "unregistered": + service.register_receipt_status_lookup("https://partner.example", lookup) + monkeypatch.setattr(service, "STATUS_LOOKUP_TIMEOUT_SECONDS", 0.001) + response = await client.post( + f"/v1/admin/receipts/{receipt.receipt_id}/resolution", + headers=BOOTSTRAP_HEADERS, + json=body("confirmed_not_delivered", refund=True, lookup=True), + ) + assert response.status_code == 409, response.text + assert response.json()["detail"] == "resolution_inconclusive" + assert await _ledger_counts(seed.wallet_id) == (1, 0) + async with get_session_factory()() as session: + assert ( + await session.scalar( + select(func.count()).select_from(ReceiptResolutionModel) + ) + == 0 + ) + + +@pytest.mark.anyio +async def test_fixture_lookup_success_and_stable_retry(client, clean_database): + seed, receipt = await uncertain(client, "operator-lookup") + calls = [] + + def handler(request): + assert request.url.path == "/status" + assert request.headers["authorization"] == "Bearer test-token" + calls.append(json.loads(request.content)) + return httpx.Response(200, json={"status": "executed", "token": "test-token"}) + + service.register_receipt_status_lookup( + "https://partner.example", + service.RefundFixtureStatusLookup( + "https://partner.example", + "test-token", + transport=httpx.MockTransport(handler), + ), + ) + path = f"/v1/admin/receipts/{receipt.receipt_id}/resolution" + for _ in range(2): + response = await client.post( + path, headers=BOOTSTRAP_HEADERS, json=body(lookup=True) + ) + assert response.status_code == 200, response.text + assert "test-token" not in response.text + assert response.json()["evidence"]["lookup_status"] == "executed" + assert len(calls) == 1 + assert calls[0]["idempotency_key"] == seed.idempotency_key + assert ( + calls[0]["invocation_id"] + == (await _attempt(seed.attempt_id)).idempotency_record_id + ) + + +@pytest.mark.anyio +async def test_lost_response_restart_retry_and_offline_operator_resolution( + client, clean_database, monkeypatch +): + from fastapi import FastAPI + from app.routers import mcp, receipts + from app.services import ( + idempotency, + mcp_dispatch_attempts, + receipts as receipt_module, + agent_money, + ) + from app.services.service_registry import get_service_registry + from tests.test_mcp_upstream_governed import ( + FakeUpstreamExecutor, + _register_upstream, + _call_body, + ) + from tests.test_trust_helpers import provision_agent_wallet, create_tool_permit + + provisioned = await provision_agent_wallet(client) + tool = "operator-restart-tool" + executor = FakeUpstreamExecutor("delivery_uncertain") + _register_upstream(tool, executor) + try: + permit = await create_tool_permit( + client, + wallet_id=provisioned["agent_wallet_id"], + key_id=provisioned["key_id"], + tool_name=tool, + idem_key="operator-restart-permit", + ) + request = _call_body( + tool_name=tool, + wallet_id=provisioned["agent_wallet_id"], + permit_id=permit["permit_id"], + idempotency_key="operator-restart-invoke", + ) + first = await client.post( + "/mcp/messages", headers=provisioned["agent_headers"], json=request + ) + error = first.json()["error"] + receipt_id = error["data"]["receipt"]["receipt_id"] + portable = ( + await client.get( + f"/v1/receipts/{receipt_id}/portable", headers=BOOTSTRAP_HEADERS + ) + ).json() + # Fresh application and fresh service objects, same durable database. + monkeypatch.setattr(idempotency, "_service", None) + monkeypatch.setattr(mcp_dispatch_attempts, "_service", None) + monkeypatch.setattr(receipt_module, "_service", None) + monkeypatch.setattr(agent_money, "_agent_money", None) + fresh = FastAPI() + fresh.include_router(mcp.router) + fresh.include_router(receipts.router) + fresh.include_router(receipts.admin_router) + service.register_receipt_status_lookup( + "https://partner.example", + service.RefundFixtureStatusLookup( + "https://partner.example", + "test", + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json={"status": "executed"}) + ), + ), + ) + async with AsyncClient( + transport=ASGITransport(app=fresh), base_url="http://test" + ) as restarted: + replay = await restarted.post( + "/mcp/messages", headers=provisioned["agent_headers"], json=request + ) + assert replay.json()["error"]["code"] == error["code"] + assert replay.json()["error"]["data"]["receipt"] == error["data"]["receipt"] + assert ( + replay.json()["error"]["data"]["dispatch"] == error["data"]["dispatch"] + ) + resolved = await restarted.post( + f"/v1/admin/receipts/{receipt_id}/resolution", + headers=BOOTSTRAP_HEADERS, + json=body(lookup=True), + ) + assert resolved.status_code == 200, resolved.text + after = await restarted.get( + f"/v1/receipts/{receipt_id}/portable", headers=BOOTSTRAP_HEADERS + ) + assert after.json() == portable + replay_after = await restarted.post( + "/mcp/messages", headers=provisioned["agent_headers"], json=request + ) + assert ( + replay_after.json()["error"]["data"]["receipt"] + == error["data"]["receipt"] + ) + keys = key_set_from_document( + (await client.get("/.well-known/trust-keys.json")).json() + ) + assert verify_resolution(resolved.json(), portable, keys).ok + assert executor.dispatch_count == 1 + assert await _ledger_counts(provisioned["agent_wallet_id"]) == (1, 0) + finally: + get_service_registry().unregister_local(tool) + + +@pytest.mark.asyncio(loop_scope="session") +@db_gate.collection_mark( + flag="RUN_POSTGRES_CONCURRENCY_TESTS", reason="requires disposable PostgreSQL" +) +async def test_two_operators_racing_converge(monkeypatch): + from httpx import AsyncClient, ASGITransport + from app.db import database + from app.main import app + from tests.test_operator_resolution import uncertain, body + from tests.test_trust_helpers import BOOTSTRAP_HEADERS + from tests.test_mcp_dispatch_reconciliation import _ledger_counts + + with monkeypatch.context() as patch: + patch.setattr(database, "_engine", None) + patch.setattr(database, "_session_factory", None) + engine = database.get_engine() + try: + db_gate.require_live_postgres( + flag="RUN_POSTGRES_CONCURRENCY_TESTS", + reason="requires disposable PostgreSQL", + ) + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test" + ) as client: + seed, receipt = await uncertain(client, "operator-postgres-race") + path = f"/v1/admin/receipts/{receipt.receipt_id}/resolution" + responses = await asyncio.gather( + *( + client.post( + path, + headers=BOOTSTRAP_HEADERS, + json=body("confirmed_not_delivered", refund=True), + ) + for _ in range(2) + ) + ) + assert [response.status_code for response in responses] == [200, 200], [ + r.text for r in responses + ] + assert responses[0].json() == responses[1].json() + assert await _ledger_counts(seed.wallet_id) == (1, 1) + finally: + await engine.dispose() + + +@pytest.mark.anyio +async def test_real_agent_key_refused_and_evidence_bounded(client, clean_database): + from tests.test_trust_helpers import provision_agent_wallet + + provisioned = await provision_agent_wallet(client) + response = await client.post( + "/v1/admin/receipts/missing/resolution", + headers=provisioned["agent_headers"], + json=body(), + ) + assert response.status_code == 403 + seed, receipt = await uncertain(client, "operator-large-evidence") + request = body() + request["evidence"]["raw_lookup_answer"] = {"data": "x" * 16384} + response = await client.post( + f"/v1/admin/receipts/{receipt.receipt_id}/resolution", + headers=BOOTSTRAP_HEADERS, + json=request, + ) + assert response.status_code == 409 + assert response.json()["detail"] == "resolution_evidence_invalid" + assert await _ledger_counts(seed.wallet_id) == (1, 0) + + +@pytest.mark.anyio +async def test_interrupt_after_refund_before_budget_release( + client, clean_database, monkeypatch +): + from app.services.permits import get_permit_service + + seed, receipt = await uncertain(client, "operator-after-refund") + permits = get_permit_service() + release = permits.release_dispatch_budget_once + + async def fail(*args, **kwargs): + raise RuntimeError("injected interruption after refund") + + monkeypatch.setattr(permits, "release_dispatch_budget_once", fail) + path = f"/v1/admin/receipts/{receipt.receipt_id}/resolution" + request = body("confirmed_not_delivered", refund=True) + assert ( + await client.post(path, headers=BOOTSTRAP_HEADERS, json=request) + ).status_code == 503 + assert await _ledger_counts(seed.wallet_id) == (1, 1) + monkeypatch.setattr(permits, "release_dispatch_budget_once", release) + response = await client.post(path, headers=BOOTSTRAP_HEADERS, json=request) + assert response.status_code == 200, response.text + assert await _ledger_counts(seed.wallet_id) == (1, 1) + assert (await _attempt(seed.attempt_id)).budget_released_at is not None + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "answer", + [ + {}, + {"status": []}, + {"status": "unknown"}, + {"status": "pending"}, + {"status": "not_executed"}, + ], +) +async def test_fixture_adapter_inconclusive(client, clean_database, answer): + seed, receipt = await uncertain(client, "operator-fixture-inconclusive") + service.register_receipt_status_lookup( + "https://partner.example", + service.RefundFixtureStatusLookup( + "https://partner.example", + "test", + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json=answer) + ), + ), + ) + response = await client.post( + f"/v1/admin/receipts/{receipt.receipt_id}/resolution", + headers=BOOTSTRAP_HEADERS, + json=body("confirmed_not_delivered", lookup=True, refund=True), + ) + assert response.status_code == 409 + assert response.json()["detail"] == "resolution_inconclusive" + assert await _ledger_counts(seed.wallet_id) == (1, 0) diff --git a/gateway/tests/test_repair_api_lineage_migration.py b/gateway/tests/test_repair_api_lineage_migration.py index edcd74b6f..c87f40f99 100644 --- a/gateway/tests/test_repair_api_lineage_migration.py +++ b/gateway/tests/test_repair_api_lineage_migration.py @@ -84,7 +84,7 @@ def test_missing_objects_are_recreated_with_main_shape(tmp_path, monkeypatch): conn.execute(text(f"DROP TABLE {name}")) engine.dispose() - _run(command.upgrade, config, "head") + _run(command.upgrade, config, "053_repair_api_lineage") shape, views = _shape(sync_url) assert shape == expected[0] assert VIEW in views diff --git a/gateway/tests/test_resolution_evidence_migration.py b/gateway/tests/test_resolution_evidence_migration.py new file mode 100644 index 000000000..874c977e4 --- /dev/null +++ b/gateway/tests/test_resolution_evidence_migration.py @@ -0,0 +1,62 @@ +"""Disposable PostgreSQL migration and concurrent operator proofs.""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path + +from alembic import command +from alembic.config import Config +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from tests.support import db_gate +from tests.support.action_database_guard import ( + require_action_database_url, + require_action_test_environment, +) + + +@db_gate.collection_mark( + url_env="RESOLUTION_MIGRATION_DATABASE_URL", + reason="requires fresh disposable migration PostgreSQL", +) +def test_resolution_evidence_migration_roundtrip(monkeypatch): + target = os.environ["RESOLUTION_MIGRATION_DATABASE_URL"] + require_action_database_url(target) + require_action_test_environment(os.environ.get("ENVIRONMENT")) + monkeypatch.setenv("DATABASE_URL", target) + root = Path(__file__).resolve().parents[1] + config = Config(str(root / "alembic.ini")) + config.set_main_option("script_location", str(root / "migrations")) + + async def query(statement): + engine = create_async_engine(target) + try: + async with engine.begin() as connection: + result = await connection.execute(text(statement)) + return result.all() + finally: + await engine.dispose() + + def sql(statement): + return asyncio.run(query(statement)) + + assert sql("SELECT tablename FROM pg_tables WHERE schemaname='public'") == [], ( + "refuse reused database" + ) + command.upgrade(config, "057_receipt_permit_seq_unique") + for _ in range(2): + command.upgrade(config, "058_resolution_evidence") + assert sql( + "SELECT column_name, is_nullable FROM information_schema.columns WHERE table_name='receipt_resolutions' AND column_name IN ('evidence','evidence_hash') ORDER BY column_name" + ) == [("evidence", "YES"), ("evidence_hash", "YES")] + command.downgrade(config, "057_receipt_permit_seq_unique") + assert ( + sql( + "SELECT column_name FROM information_schema.columns WHERE table_name='receipt_resolutions' AND column_name IN ('evidence','evidence_hash')" + ) + == [] + ) + command.upgrade(config, "head") diff --git a/sdk/python/src/b2a_sdk/resolution_verifier.py b/sdk/python/src/b2a_sdk/resolution_verifier.py index 969a1723c..50ab5d6fd 100644 --- a/sdk/python/src/b2a_sdk/resolution_verifier.py +++ b/sdk/python/src/b2a_sdk/resolution_verifier.py @@ -139,9 +139,22 @@ def _verify_record( if not verify_ed25519(key, signature, signing_bytes): return _fail(VerificationStatus.INVALID, "resolution signature does not verify") - if claims.get("alg") != "Ed25519" or claims.get("version") != _VERSION: + if claims.get("alg") != "Ed25519" or claims.get("version") not in {_VERSION, "resolution-v0.2"}: return _fail(VerificationStatus.UNSUPPORTED, "unsupported signed resolution format") - if set(claims) != _SIGNED_FIELDS: + signed_fields = _SIGNED_FIELDS | ( + {"evidence_hash"} if claims["version"] == "resolution-v0.2" else set() + ) + if "evidence_hash" in signed_fields: + evidence = envelope.get("evidence") + if not isinstance(evidence, dict) or not _is_hash(claims.get("evidence_hash")): + return _fail(VerificationStatus.MALFORMED, "resolution evidence missing") + try: + digest = hashlib.sha256(canonical_json(evidence).encode("utf-8")).hexdigest() + except (ValueError, TypeError, RecursionError): + return _fail(VerificationStatus.MALFORMED, "resolution evidence invalid") + if digest != claims["evidence_hash"]: + return _fail(VerificationStatus.MISMATCH, "resolution evidence hash disagrees") + if set(claims) != signed_fields: return _fail(VerificationStatus.MALFORMED, "signed resolution fields are incomplete") for name in ("resolution_id", "receipt_id", "wallet_id", "kid"): if not isinstance(claims[name], str) or not claims[name]: @@ -178,7 +191,7 @@ def _verify_record( return _fail(VerificationStatus.MISMATCH, "resolution canonicalization disagrees") if envelope.get("resolution_hash") != resolution_hash: return _fail(VerificationStatus.MISMATCH, "resolution_hash disagrees with signing_input") - for name in _SIGNED_FIELDS: + for name in signed_fields: if name not in envelope: return _fail(VerificationStatus.MALFORMED, f"resolution envelope has no {name}") if type(envelope[name]) is not type(claims[name]) or envelope[name] != claims[name]: diff --git a/sdk/python/tests/test_resolution_verifier.py b/sdk/python/tests/test_resolution_verifier.py index af2c16cbb..2f87eee0b 100644 --- a/sdk/python/tests/test_resolution_verifier.py +++ b/sdk/python/tests/test_resolution_verifier.py @@ -323,3 +323,28 @@ def test_unknown_key_is_not_reported_as_tampering(signer): assert result.status is VerificationStatus.UNKNOWN_KEY assert not result.is_rejected + + +@pytest.mark.parametrize("tamper", [False, True]) +def test_v02_evidence_hash(signer, tamper): + import json + + receipt = _receipt(signer) + resolution = _resolution(signer, receipt) + claims = json.loads(resolution["signing_input"]) + claims.pop("payload_hash") + claims["version"] = "resolution-v0.2" + evidence = {"source": "operator", "note": "Reviewed upstream state"} + claims["evidence_hash"] = _hash(canonical_json(evidence)) + claims["payload_hash"] = _hash(canonical_json(claims)) + signing_input = canonical_json(claims) + resolution.update(claims) + resolution.update( + signing_input=signing_input, + signature=signer[2](signing_input), + resolution_hash=_hash(signing_input), + evidence=evidence, + ) + if tamper: + resolution["evidence"]["note"] += " changed" + assert verify_resolution(resolution, receipt, signer[1]).ok is not tamper From 2cfb9ea0a0bb77f661a9748985fa2d7e499eea3f Mon Sep 17 00:00:00 2001 From: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:20:31 -0800 Subject: [PATCH 2/2] fix(gateway): address operator resolution review findings Written by the Codex CLI engine. Reject refunds for receipts without a dispatch attempt before signing intent; exclude durably refunded operator non-deliveries from budget reconciliation; normalize lookup origins at register and lookup; return audited 409 resolution_inconclusive for dispatch-context conflicts. Signed-off-by: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> --- gateway/app/services/operator_resolution.py | 48 ++++---- gateway/app/services/permits.py | 32 +++++- gateway/tests/test_operator_resolution.py | 120 +++++++++++++++++++- 3 files changed, 175 insertions(+), 25 deletions(-) diff --git a/gateway/app/services/operator_resolution.py b/gateway/app/services/operator_resolution.py index 484216c39..d08e6d7fb 100644 --- a/gateway/app/services/operator_resolution.py +++ b/gateway/app/services/operator_resolution.py @@ -23,6 +23,7 @@ from app.services.agent_money import get_agent_money from app.services.idempotency import ACTION_MCP_IDEMPOTENCY_ENDPOINT from app.services.mcp_dispatch_attempts import ( + DispatchAttemptConflictError, get_mcp_dispatch_attempt_service, ) from app.services.mcp_dispatch_reconciliation import ( @@ -46,7 +47,23 @@ def register_receipt_status_lookup(origin: str, lookup: ReceiptStatusLookup) -> None: """Explicit startup opt-in only. No adapter is registered by default.""" _require_async_lookup(lookup) - _LOOKUPS[origin] = lookup + _LOOKUPS[_normalize_origin(origin)] = lookup + + +def _normalize_origin(origin: str) -> str: + url = httpx.URL(origin) + if ( + url.scheme not in {"http", "https"} + or not url.host + or url.userinfo + or url.query + or url.fragment + or url.path not in {"", "/"} + ): + raise ValueError("fixture origin must be an origin without credentials") + if (url.scheme, url.port) in {("http", 80), ("https", 443)}: + url = url.copy_with(port=None) + return str(url).rstrip("/") class RefundFixtureStatusLookup: @@ -64,22 +81,12 @@ def __init__( *, transport: httpx.AsyncBaseTransport | None = None, ) -> None: - url = httpx.URL(origin) - if ( - url.scheme not in {"http", "https"} - or not url.host - or url.userinfo - or url.query - or url.fragment - or url.path not in {"", "/"} - ): - raise ValueError("fixture origin must be an origin without credentials") - self._origin = origin.rstrip("/") + self._origin = _normalize_origin(origin) self._token = token self._transport = transport async def __call__(self, query: UpstreamDispatchStatusQuery) -> str | None: - if query.upstream_origin.rstrip("/") != self._origin: + if _normalize_origin(query.upstream_origin) != self._origin: return None async with httpx.AsyncClient( transport=self._transport, @@ -168,6 +175,8 @@ async def resolve_receipt( ): raise ReceiptResolutionError("resolution_dispatch_linkage_invalid") if request.refund_credits: + if attempt is None: + raise ReceiptResolutionError("resolution_refund_unsupported") debit = ( await session.get(LedgerEntryModel, receipt.ledger_entry_id) if receipt.ledger_entry_id @@ -195,12 +204,13 @@ async def resolve_receipt( existing = await get_resolution(receipt_id) if request.use_lookup and existing is None: assert attempt is not None - context = ( - await get_mcp_dispatch_attempt_service().get_context(attempt.attempt_id) - if attempt - else None - ) - lookup = _LOOKUPS.get(attempt.upstream_origin) if attempt else None + try: + context = await get_mcp_dispatch_attempt_service().get_context( + attempt.attempt_id + ) + except DispatchAttemptConflictError: + raise ReceiptResolutionError("resolution_inconclusive") from None + lookup = _LOOKUPS.get(_normalize_origin(attempt.upstream_origin)) if context is None or lookup is None: raise ReceiptResolutionError("resolution_inconclusive") query = UpstreamDispatchStatusQuery( diff --git a/gateway/app/services/permits.py b/gateway/app/services/permits.py index 066af0cdf..87fb03e87 100644 --- a/gateway/app/services/permits.py +++ b/gateway/app/services/permits.py @@ -1864,6 +1864,7 @@ async def _consumed_credits(self, session: Any, permit_id: str) -> Decimal: IdempotencyRecordModel, LedgerEntryModel, ReceiptModel, + ReceiptResolutionModel, ) receipt_rows = ( @@ -1902,7 +1903,8 @@ async def _consumed_credits(self, session: Any, permit_id: str) -> Decimal: ledger_entry_id, _wallet_id, ) in receipt_rows - if outcome == "failed_unrefunded" and ledger_entry_id is not None + if outcome in {"failed_unrefunded", "delivery_uncertain"} + and ledger_entry_id is not None ] exact_refunded_ledger_ids: set[str] = set() if pending_ledger_ids: @@ -1937,7 +1939,8 @@ async def _consumed_credits(self, session: Any, permit_id: str) -> Decimal: ledger_entry_id, wallet_id, ) in receipt_rows - if outcome == "failed_unrefunded" and ledger_entry_id is not None + if outcome in {"failed_unrefunded", "delivery_uncertain"} + and ledger_entry_id is not None } exact_refunded_ledger_ids = { correlation_id @@ -1997,6 +2000,26 @@ async def _consumed_credits(self, session: Any, permit_id: str) -> Decimal: and state.get("ledger_entry_id") == failed_receipt_by_id[receipt_id] ): resolved_receipt_ids.add(receipt_id) + # Signed intent alone is insufficient: require the exact durable refund. + for receipt_id, outcome, _credits, ledger_id, _wallet in receipt_rows: + if ( + outcome != "delivery_uncertain" + or ledger_id not in exact_refunded_ledger_ids + ): + continue + resolution = await session.scalar( + select(ReceiptResolutionModel).where( + ReceiptResolutionModel.receipt_id == receipt_id, + cast( + ColumnElement[bool], + ReceiptResolutionModel.outcome == "confirmed_not_delivered", + ), + ) + ) + if resolution is not None: + evidence = json.loads(resolution.evidence or "{}") + if evidence.get("refund_entry_id") == f"refund-{ledger_id}": + resolved_receipt_ids.add(receipt_id) consumed_decimal = sum( ( Decimal(str(credits)) @@ -2007,7 +2030,10 @@ async def _consumed_credits(self, session: Any, permit_id: str) -> Decimal: _ledger_entry_id, _wallet_id, ) in receipt_rows - if outcome in {"success", "delivery_uncertain", "response_rejected"} + if ( + outcome in {"success", "delivery_uncertain", "response_rejected"} + and receipt_id not in resolved_receipt_ids + ) or ( outcome == "failed_unrefunded" and receipt_id not in resolved_receipt_ids diff --git a/gateway/tests/test_operator_resolution.py b/gateway/tests/test_operator_resolution.py index 01d878539..633a551f9 100644 --- a/gateway/tests/test_operator_resolution.py +++ b/gateway/tests/test_operator_resolution.py @@ -252,7 +252,10 @@ async def lookup(query): @pytest.mark.anyio -async def test_fixture_lookup_success_and_stable_retry(client, clean_database): +@pytest.mark.parametrize( + "origin", ["https://partner.example", "HTTPS://PARTNER.EXAMPLE:443/"] +) +async def test_fixture_lookup_success_and_stable_retry(client, clean_database, origin): seed, receipt = await uncertain(client, "operator-lookup") calls = [] @@ -263,9 +266,9 @@ def handler(request): return httpx.Response(200, json={"status": "executed", "token": "test-token"}) service.register_receipt_status_lookup( - "https://partner.example", + origin, service.RefundFixtureStatusLookup( - "https://partner.example", + origin, "test-token", transport=httpx.MockTransport(handler), ), @@ -518,3 +521,114 @@ async def test_fixture_adapter_inconclusive(client, clean_database, answer): assert response.status_code == 409 assert response.json()["detail"] == "resolution_inconclusive" assert await _ledger_counts(seed.wallet_id) == (1, 0) + + +@pytest.mark.anyio +async def test_no_attempt_refund_rejected_before_intent(client, clean_database): + from app.db.models import ReceiptModel + + seed, receipt = await uncertain(client, "operator-no-attempt") + async with get_session_factory()() as session: + model = await session.get(ReceiptModel, receipt.receipt_id) + model.dispatch_attempt_id = None + model.idempotency_record_id = None + session.add(model) + await session.commit() + response = await client.post( + f"/v1/admin/receipts/{receipt.receipt_id}/resolution", + headers=BOOTSTRAP_HEADERS, + json=body("confirmed_not_delivered", refund=True), + ) + assert response.status_code == 409 + assert response.json()["detail"] == "resolution_refund_unsupported" + assert await _ledger_counts(seed.wallet_id) == (1, 0) + async with get_session_factory()() as session: + assert ( + await session.scalar( + select(func.count()).select_from(ReceiptResolutionModel) + ) + == 0 + ) + + +@pytest.mark.anyio +@pytest.mark.parametrize("complete", [False, True]) +async def test_budget_reconcile_requires_completed_operator_refund( + client, clean_database, monkeypatch, complete +): + from datetime import timedelta + from decimal import Decimal + from app.core.time import utc_now + from app.db.models import PermitModel + from app.services.permits import get_permit_service + + seed, receipt = await uncertain(client, f"operator-budget-{complete}") + if not complete: + + async def fail(**kwargs): + raise RuntimeError("interrupted refund") + + monkeypatch.setattr(get_agent_money(), "refund_charge", fail) + response = await client.post( + f"/v1/admin/receipts/{receipt.receipt_id}/resolution", + headers=BOOTSTRAP_HEADERS, + json=body("confirmed_not_delivered", refund=True), + ) + assert response.status_code == (200 if complete else 503) + async with get_session_factory()() as session: + permit = await session.get(PermitModel, seed.permit_id) + expected = Decimal("0") if complete else receipt.credits_charged + assert permit.spent_credits == expected + permit.expires_at = utc_now() - timedelta(hours=1) + permit.updated_at = utc_now() - timedelta(hours=1) + session.add(permit) + await session.commit() + await get_permit_service().reconcile_budgets(idle_seconds=1) + async with get_session_factory()() as session: + permit = await session.get(PermitModel, seed.permit_id) + assert permit.spent_credits == expected + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "reason", ["dispatch_permit_missing", "dispatch_approval_binding_mismatch"] +) +async def test_lookup_context_conflict_is_audited_inconclusive( + client, clean_database, monkeypatch, reason +): + from app.services.mcp_dispatch_attempts import ( + DispatchAttemptConflictError, + get_mcp_dispatch_attempt_service, + ) + + seed, receipt = await uncertain(client, f"operator-context-{reason}") + + async def fail(*args): + raise DispatchAttemptConflictError(reason) + + monkeypatch.setattr(get_mcp_dispatch_attempt_service(), "get_context", fail) + response = await client.post( + f"/v1/admin/receipts/{receipt.receipt_id}/resolution", + headers=BOOTSTRAP_HEADERS, + json=body(lookup=True), + ) + assert response.status_code == 409 + assert response.json()["detail"] == "resolution_inconclusive" + assert await _ledger_counts(seed.wallet_id) == (1, 0) + async with get_session_factory()() as session: + assert ( + await session.scalar( + select(func.count()).select_from(ReceiptResolutionModel) + ) + == 0 + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ControlPlaneAuditEventModel) + .where( + ControlPlaneAuditEventModel.event == "receipt.resolution.attempt" + ) + ) + == 1 + )