From e5082eb62bbe9d1431a4ec1800039d7f07c9f7fb Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 12:52:03 -0800 Subject: [PATCH 01/19] feat(website): sync the clock of, snapshot, and configure a multi env's websites as its MCP servers (FD-4683) A website env, and a multi env's websites, get the harness treatment MCP servers get: sync_env_clock syncs them (and tells them main's timezone), snapshot_env exports each through its data plane and fails on a site whose card can't be read, and an apply_server_config broadcast reaches them; a site without the clock, a data plane or the config extension is skipped. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/agent_env/env/legacy_protocol.py | 53 +++++- .../task_steps/apply_server_config.py | 75 ++++---- .../task_step/task_steps/snapshot_env.py | 90 ++++++--- .../task_step/task_steps/sync_env_clock.py | 57 ++++-- tst/unit/env/test_legacy_protocol.py | 19 ++ .../env_gateway_provider_test.py | 12 ++ tst/unit/task_step/sync_env_clock_test.py | 153 ++++++++++++++- .../task_step/test_apply_server_config.py | 91 +++++++++ tst/unit/task_step/test_snapshot_env.py | 174 ++++++++++++++++++ 9 files changed, 635 insertions(+), 89 deletions(-) diff --git a/src/agent_env/env/legacy_protocol.py b/src/agent_env/env/legacy_protocol.py index 16899025b..8790157ac 100644 --- a/src/agent_env/env/legacy_protocol.py +++ b/src/agent_env/env/legacy_protocol.py @@ -9,7 +9,8 @@ import asyncio import logging -from typing import TYPE_CHECKING, Optional +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Optional import httpx from agentenv_protocol import RPC_PATH, DataPart, client as protocol_v1 @@ -33,20 +34,62 @@ def environment_base_url(gateway_url: Optional[str], environment_name: str, mcp: return f"{gateway_url}/svc/{prefix}{environment_name}" +@dataclass(frozen=True) +class BackingServer: + """A server behind an env's gateway, by its environment_name: an MCP server, at ``/svc/mcp-{name}``, or a website, + at ``/svc/{name}``.""" + + environment_name: str + mcp: bool = True + + +def backing_servers(env: Any) -> list[BackingServer]: + """The servers behind env's gateway: a MultiEnv's MCP servers then its websites, a WebsiteEnv's website, or else + the env's one MCP server; empty for an env with none.""" + from agent_env.env.envs.website import WebsiteEnv + + if isinstance(env, WebsiteEnv): + return [BackingServer(env.environment_name, mcp=False)] + servers = [BackingServer(e.environment_name) for e in getattr(env, "mcp_server_envs", None) or []] + servers += [BackingServer(e.environment_name, mcp=False) for e in getattr(env, "website_envs", None) or []] + if not servers and getattr(env, "environment_name", None): + servers = [BackingServer(env.environment_name)] + return servers + + +def shared_names(servers: list[BackingServer]) -> set[str]: + """The environment_names an MCP server and a website in ``servers`` share, which a stored env card can't tell apart.""" + return {s.environment_name for s in servers if s.mcp} & {s.environment_name for s in servers if not s.mcp} + + +def mcp_wins(servers: list[BackingServer]) -> list[BackingServer]: + """``servers`` without each website that shares its environment_name with an MCP server, as a MultiEnv routes an + artifact of that name to the MCP server.""" + shared = shared_names(servers) + return [s for s in servers if s.mcp or s.environment_name not in shared] + + async def v1_base_url(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str, mcp: bool = True) -> Optional[str]: - """A child env's v1 data-plane base from the stored env card, or None for legacy; without a stored card, the live probe decides.""" + """A child env's v1 data-plane base from the stored env card, or None for legacy; without a stored card, the live probe + decides. A website's probe is a card read that is None only on a 404 and raises on any other failure, so a website whose + card could not be read is never taken for one serving no data plane.""" if deployed is None or not deployed.environment_card: base_url = environment_base_url(gateway_url, environment_name, mcp=mcp) + if not mcp: + _, card = await child_env_card(None, gateway_url, environment_name, mcp=False) + return base_url if card is not None else None return base_url if await protocol_v1.supports_v1(base_url) else None child = deployed.get_child_env_card(environment_name) return deployed.environment_url + child.get("url", RPC_PATH).removesuffix(RPC_PATH) if child else None -async def child_env_card(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str, timeout: int = 10) -> tuple[str, Optional[dict]]: - """An MCP child env's card and the base its endpoints resolve against: from the stored env card, or without one a live read (None on 404).""" +async def child_env_card(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str, timeout: int = 10, + mcp: bool = True) -> tuple[str, Optional[dict]]: + """A child env's card and the base its endpoints resolve against: from the stored env card, or without one a live read + at the MCP server's gateway path, or with ``mcp=False`` the website's (None on 404).""" if deployed is None or not deployed.environment_card: - base_url = environment_base_url((gateway_url or "").rstrip("/"), environment_name) + base_url = environment_base_url((gateway_url or "").rstrip("/"), environment_name, mcp=mcp) try: return base_url, await protocol_v1.get_card(base_url, timeout=timeout) except httpx.HTTPStatusError as e: diff --git a/src/agent_env/task_step/task_steps/apply_server_config.py b/src/agent_env/task_step/task_steps/apply_server_config.py index 41f82ffd0..c8c8e1f08 100644 --- a/src/agent_env/task_step/task_steps/apply_server_config.py +++ b/src/agent_env/task_step/task_steps/apply_server_config.py @@ -1,15 +1,15 @@ """Task step that applies a task's behavior config to deployed servers. At setup (after the env is deployed + universe loaded, before the agent runs) -this step invokes each backing MCP server's ``set_*`` config extensions over the -gateway. Extensions are advertised on the server's ``EnvironmentCard`` (served at +this step invokes each backing MCP server's or website's ``set_*`` config extensions +over the gateway. Extensions are advertised on the server's ``EnvironmentCard`` (served at ``/.well-known/agent-env.json``) under ``capabilities.extensions[]`` and invoked at their advertised REST endpoint (e.g. ``/agentenv/ext/set_errors``). Config lives on the backing server, not on the gateway itself. Each server's card comes from the env card stored on the deployed record, whose extension endpoints the -gateway has already rewritten to its ``/svc/mcp-{service}`` proxy; a record without a -stored card reads it live there. The agent never sees the card or these endpoints; +gateway has already rewritten to its ``/svc/mcp-{service}`` (or a website's ``/svc/{service}``) +proxy; a record without a stored card reads it live there. The agent never sees the card or these endpoints; only this harness step invokes them. """ @@ -76,7 +76,7 @@ class ApplyServerConfigStep(TaskStep): entity_refs = (EntityRef.env("env_id"),) # A directive whose ``service`` is this sentinel is a broadcast: at execution - # it expands to one directive per MCP server in the env (same uri + args). + # it expands to one directive per MCP server and website in the env (same uri + args). # Lets a caller apply an env-wide config (e.g. set_acting_user across every # server) without enumerating services itself. Pairs naturally with # ``tolerate_unadvertised`` so servers that don't opt in are simply skipped. @@ -131,44 +131,51 @@ def from_dict(cls, data: dict) -> "ApplyServerConfigStep": tolerate_unadvertised=data.get("tolerate_unadvertised", False), ) - def _expand_directives(self, deployed) -> list[ConfigDirective]: + def _expand_directives(self, deployed) -> tuple[list[ConfigDirective], set[str], set[int]]: """Expand any broadcast directive (service == ``*``) into one directive - per MCP server in the env, resolved from the env definition. The env's - child service names aren't otherwise enumerable here (the deployed card - list carries no roster), so we read them from ``Env.get`` — matching how - deploy_env / load_artifact resolve the env. Non-broadcast directives - pass through untouched.""" + per MCP server and website in the env, resolved from the env definition, + and return them with the names that are websites, whose cards a live read + finds at ``/svc/{name}``, and the positions of the directives a broadcast + made for a website. Those are skipped, as under ``tolerate_unadvertised``, + when the website has no card or doesn't advertise the URI, so a website + the task didn't name never fails it; a directive naming a website, and + every MCP server, stays strict. The env's child service names aren't otherwise + enumerable here (the deployed card list carries no roster), so we read + them from ``Env.get`` — matching how deploy_env / load_artifact resolve + the env. A website sharing its name with an MCP server is left to the MCP + server, as the stored env card can't tell the two apart. Non-broadcast + directives pass through untouched.""" if not any(d.service == self._BROADCAST for d in self.directives): - return self.directives + return self.directives, set(), set() + from agent_env.env import legacy_protocol from agent_env.env.env import Env env = Env.get(deployed.env_id, deployed.env_version) - environment_names = [ - e.environment_name for e in (getattr(env, "mcp_server_envs", None) or []) - ] # A single-server env (a bare MCPServerEnv) exposes `environment_name` and no - # `mcp_server_envs`; mirror snapshot_env / exporter enumeration and fall - # back to it, else a broadcast silently matches nothing on single-server envs. - if not environment_names and getattr(env, "environment_name", None): - environment_names = [env.environment_name] + # `mcp_server_envs`; backing_servers falls back to it, as snapshot_env does, + # else a broadcast silently matches nothing on single-server envs. + servers = legacy_protocol.mcp_wins(legacy_protocol.backing_servers(env)) # A broadcast that resolves to zero services would apply nothing while the # step reports success: the exact silent misconfig this step prevents. Fail # loud (tolerate_unadvertised governs advertisement, not missing servers). - if not environment_names: + if not servers: raise ApplyServerConfigError( - f"apply_server_config: broadcast directive matched no MCP servers " - f"in env {self.env_id!r} (env exposes neither mcp_server_envs nor a " + f"apply_server_config: broadcast directive matched no MCP servers or websites " + f"in env {self.env_id!r} (env exposes neither mcp_server_envs, website_envs nor a " f"environment_name); nothing to apply, likely a task authoring error" ) expanded: list[ConfigDirective] = [] + lenient: set[int] = set() for d in self.directives: if d.service != self._BROADCAST: expanded.append(d) continue - for svc in environment_names: - expanded.append(ConfigDirective(service=svc, uri=d.uri, args=d.args)) - return expanded + for server in servers: + if not server.mcp: + lenient.add(len(expanded)) + expanded.append(ConfigDirective(service=server.environment_name, uri=d.uri, args=d.args)) + return expanded, {server.environment_name for server in servers if not server.mcp}, lenient async def execute(self, context: TaskStepContext) -> TaskStepContext: deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) @@ -180,7 +187,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # actually armed with — a partial apply must not look like a no-op. changes = context.metadata.setdefault("server_config_changes", []) # Directives skipped because the target server doesn't advertise the - # extension (only when tolerate_unadvertised). Kept separate from + # extension (when tolerate_unadvertised, or for a website a broadcast reached). Kept separate from # `changes` so the audit trail never shows a skip as an applied change. skipped = context.metadata.setdefault("server_config_skipped", []) # Expand broadcast (service="*") directives to one-per-env-service. @@ -188,7 +195,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # network), so wrap it to keep the broadcast path's failure contract the # same ApplyServerConfigError as the per-directive HTTP calls below. try: - directives = self._expand_directives(deployed) + directives, websites, lenient = self._expand_directives(deployed) except ApplyServerConfigError: raise except Exception as e: @@ -204,15 +211,17 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # card: a server predating env-card support, one whose card name differs, or one # whose card the deploy couldn't read. cards: dict[str, tuple[str, Optional[dict]]] = {} - for directive in directives: + for position, directive in enumerate(directives): + tolerate = self.tolerate_unadvertised or position in lenient try: if directive.service not in cards: cards[directive.service] = await legacy_protocol.child_env_card( - deployed, gateway_url_of(deployed), directive.service, timeout=self.timeout_seconds + deployed, gateway_url_of(deployed), directive.service, timeout=self.timeout_seconds, + mcp=directive.service not in websites, ) base_url, card = cards[directive.service] if card is None: - if not self.tolerate_unadvertised: + if not tolerate: raise ApplyServerConfigError( f"Service {directive.service!r} has no env card (env={self.env_id}), so {directive.uri!r} " f"can't be applied: it serves none, its card isn't named {directive.service!r}, or the deploy couldn't read it" @@ -220,7 +229,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning( f"apply_server_config: service {directive.service!r} serves no " f"environment card (env={self.env_id}); skipping {directive.uri!r} " - f"(tolerate_unadvertised)" + f"({'tolerate_unadvertised' if self.tolerate_unadvertised else 'broadcast to a website'})" ) skipped.append({ "step_id": self.id, @@ -233,11 +242,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: continue ext = protocol_v1.find_extension(card, directive.uri) if ext is None: - if self.tolerate_unadvertised: + if tolerate: logger.warning( f"apply_server_config: {directive.uri!r} not advertised on " f"service {directive.service!r} (env={self.env_id}); skipping " - f"(tolerate_unadvertised)" + f"({'tolerate_unadvertised' if self.tolerate_unadvertised else 'broadcast to a website'})" ) skipped.append({ "step_id": self.id, diff --git a/src/agent_env/task_step/task_steps/snapshot_env.py b/src/agent_env/task_step/task_steps/snapshot_env.py index 48fc9d6dc..9415dac98 100644 --- a/src/agent_env/task_step/task_steps/snapshot_env.py +++ b/src/agent_env/task_step/task_steps/snapshot_env.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import PurePosixPath -from typing import ClassVar, Optional +from typing import TYPE_CHECKING, ClassVar, Optional from urllib.parse import urlparse import httpx @@ -37,6 +37,9 @@ from agent_env.entity.scopes import FieldScope from agent_env.task_step.task_step import TaskStep, TaskStepDependency +if TYPE_CHECKING: + from agent_env.env.legacy_protocol import BackingServer + logger = logging.getLogger(__name__) DEFAULT_EXPORT_TIMEOUT_SECONDS = 600 @@ -167,6 +170,9 @@ class SnapshotEnvTaskStep(TaskStep): Output: ``context.metadata["env_snapshotted_universes"][self.id]`` = ``{"id", "version"}``. Fails the step if any service fails to export. + Websites are snapshotted through their v1 data plane; a website serving + none is left out, and one sharing its name with an MCP server is left to + the MCP server, as a MultiEnv loads that name's artifact into it. ``include_env_changes`` also stores who changed what (urn:agentenv:changes/v1) under ``metadata["env_changes"]``. @@ -469,28 +475,36 @@ def _resolve_loaded_universe_ref(deployed) -> Optional[tuple[str, Optional[int]] return None return (su["id"], su.get("version")) if su and su.get("id") else None + @classmethod + def _enumerate_environments(cls, env) -> list[str]: + """environment names for a multi env, a website env or a single MCP server env.""" + return [server.environment_name for server in cls._enumerate_servers(env)] + @staticmethod - def _enumerate_environments(env) -> list[str]: - """environment names for a multi env or a single MCP server env.""" - if getattr(env, "mcp_server_envs", None): - return [e.environment_name for e in env.mcp_server_envs] - if getattr(env, "environment_name", None): - return [env.environment_name] - raise RuntimeError( - f"Env '{env.id}' v{env.version} has no MCP services to snapshot " - "(expected mcp_server_envs or environment_name)" - ) + def _enumerate_servers(env) -> list[BackingServer]: + """The servers to snapshot: the env's MCP servers and websites, a website sharing its name with an MCP + server left to it.""" + from agent_env.env import legacy_protocol + + servers = legacy_protocol.mcp_wins(legacy_protocol.backing_servers(env)) + if not servers: + raise RuntimeError( + f"Env '{env.id}' v{env.version} has no MCP services or websites to snapshot " + "(expected mcp_server_envs, website_envs or environment_name)" + ) + return servers @staticmethod async def _export_environment_to_file( gateway_url: Optional[str], environment_name: str, tmp_path: str, timeout_seconds: float, deployed=None, - data_get_params: Optional[dict] = None, - ) -> str | tuple[GetDataResponse, Optional[_SnapshotUpload]]: - """Export one service; return the suffix written (".json"/".zip") to - ``tmp_path``, or the object url a FilePart names when the service uploaded the - bundle itself (nothing written to ``tmp_path`` then — export_one registers it), or, for - an environment whose card declares a file tree, its data/get answer and the upload it - answered under, which export_one keeps file by file. ``data_get_params`` go to its data/get. + data_get_params: Optional[dict] = None, mcp: bool = True, + ) -> Optional[str | tuple[GetDataResponse, Optional[_SnapshotUpload]]]: + """Export one service, an MCP server or with ``mcp=False`` a website; return the + suffix written (".json"/".zip") to ``tmp_path``, or the object url a FilePart names + when the service uploaded the bundle itself (nothing written to ``tmp_path`` then — + export_one registers it), or, for an environment whose card declares a file tree, its + data/get answer and the upload it answered under, which export_one keeps file by file, + or None for a website serving no v1 data plane. ``data_get_params`` go to its data/get. v1 ``get_data``, handed a namespace grant when the service takes the data-objects form: an upload through it → its object url; ``DataPart`` → json; ``FilePart`` → @@ -499,10 +513,12 @@ async def _export_environment_to_file( guard. Errors are caught by the caller (fails just this service).""" from agent_env.env import legacy_protocol - base_url = await legacy_protocol.v1_base_url(deployed, gateway_url, environment_name, mcp=True) + base_url = await legacy_protocol.v1_base_url(deployed, gateway_url, environment_name, mcp=mcp) + if base_url is None and not mcp: + return None if base_url is not None: headers = deployed.control_headers_for(base_url) if deployed is not None else None - card_base, card = await legacy_protocol.child_env_card(deployed, gateway_url, environment_name) + card_base, card = await legacy_protocol.child_env_card(deployed, gateway_url, environment_name, mcp=mcp) if returns_file_tree(card or {}): upload = await asyncio.to_thread(_snapshot_upload, card or {}, timeout_seconds, getattr(deployed, "sandbox_type", None), FILE_TREE_LIMITS) @@ -611,7 +627,10 @@ async def snapshot_env_state( from agent_env.artifact import FileArtifact, EnvironmentArtifact, EnvironmentUniverseArtifact from agent_env.env.env import gateway_url_of - services = cls._enumerate_environments(env) if gateway_url else _card_environments(deployed) + from agent_env.env.legacy_protocol import BackingServer + + servers = cls._enumerate_servers(env) if gateway_url else [BackingServer(name) for name in _card_environments(deployed)] + services = [server.environment_name for server in servers] gateway = gateway_url.rstrip("/") if gateway_url else None # `gateway` identifies which deployment was exported: env= alone doesn't when one env_id has several. logger.info( @@ -643,9 +662,12 @@ async def snapshot_env_state( # The stored card describes the deployment's own gateway; an override pointing elsewhere keeps the live probe. card_record = deployed if deployed is not None and (gateway is None or (gateway_url_of(deployed) or "").rstrip("/") == gateway) else None sem = asyncio.Semaphore(concurrency) + no_data_plane: set[str] = set() - async def export_one(environment_name: str): - """Returns EnvironmentArtifact on success, (public_err, log_err) on failure.""" + async def export_one(server: BackingServer): + """Returns EnvironmentArtifact on success, (public_err, log_err) on failure, None for a website serving no data plane.""" + environment_name = server.environment_name + website = {} if server.mcp else {"mcp": False} async with sem: # Suffix (.json/.zip) is reported by _export_environment_to_file. fd, tmp_path = tempfile.mkstemp(prefix=f"{environment_name}-") @@ -656,7 +678,7 @@ async def export_one(environment_name: str): try: result = await cls._export_environment_to_file( gateway, environment_name, tmp_path, export_timeout_seconds, - deployed=card_record, data_get_params=(data_get_params or {}).get(environment_name), + deployed=card_record, data_get_params=(data_get_params or {}).get(environment_name), **website, ) except httpx.HTTPStatusError as e: body_snippet = "" @@ -671,6 +693,9 @@ async def export_one(environment_name: str): return public, f"{public}: {e}" except ValueError as e: return str(e), str(e) + if result is None: + no_data_plane.add(environment_name) + return None if isinstance(result, tuple): try: @@ -744,7 +769,7 @@ def _register_uploaded_bundle(): except OSError: pass - results = await asyncio.gather(*(export_one(name) for name in services)) + results = await asyncio.gather(*(export_one(server) for server in servers)) environment_artifacts = [] skipped: dict[str, tuple[str, str]] = {} @@ -767,9 +792,18 @@ def _register_uploaded_bundle(): f"{len(skipped)}/{len(services)} services were not exported " f"(missing={sorted(skipped)}): {public_summary}" ) - if empty := [name for name, result in zip(services, results) if result is None]: - if len(empty) == len(services): - raise RuntimeError(f"{', '.join(map(repr, empty))} returned no files to snapshot") + without_data_plane = [name for name in services if name in no_data_plane] + if without_data_plane: + logger.warning( + f"snapshot_env: websites {without_data_plane} of {env.id} serve no v1 data plane; " + "their state is not in the snapshot" + ) + empty = [name for name, result in zip(services, results) if result is None and name not in no_data_plane] + if not environment_artifacts: + reasons = [f"its websites {without_data_plane} serve no v1 data plane"] if without_data_plane else [] + reasons += [f"{', '.join(map(repr, empty))} returned no files to snapshot"] if empty else [] + raise RuntimeError(f"Env '{env.id}' v{env.version} exported no state: {'; '.join(reasons)}") + if empty: logger.info(f"snapshot_env: {empty} returned no files to keep, so {snapshot_id} leaves them out") universe = await asyncio.to_thread( diff --git a/src/agent_env/task_step/task_steps/sync_env_clock.py b/src/agent_env/task_step/task_steps/sync_env_clock.py index 76ff196e8..c80cef8a8 100644 --- a/src/agent_env/task_step/task_steps/sync_env_clock.py +++ b/src/agent_env/task_step/task_steps/sync_env_clock.py @@ -1,16 +1,16 @@ """Arm a deployed env gateway's virtual clock, then sync its backing servers to it. Arms the clock (set_time) and reads the gateway's server-reachable env_get_time_url (state), both as the -stored env card declares clock/v1, then invokes sync_time on each backing server whose child env card -advertises it. Non-advertisers are skipped when tolerate_missing_sync_time (default True). Place right -before prompt_agent so the virtual timeline starts at agent-start. +stored env card declares clock/v1, then invokes sync_time on each backing server, MCP server or website, +whose child env card advertises it. Non-advertisers are skipped when tolerate_missing_sync_time (default +True), and websites always. Place right before prompt_agent so the virtual timeline starts at agent-start. """ from __future__ import annotations import logging from datetime import datetime, timezone as _utc_tz -from typing import ClassVar, Optional +from typing import TYPE_CHECKING, ClassVar, Optional from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx @@ -22,6 +22,9 @@ from agent_env.entity.refs import EntityRef from agent_env.task_step.task_step import TaskStep, TaskStepDependency +if TYPE_CHECKING: + from agent_env.env.legacy_protocol import BackingServer + logger = logging.getLogger(__name__) @@ -107,16 +110,15 @@ def from_dict(cls, data: dict) -> SyncEnvClockTaskStep: timezone=data.get("timezone"), ) - def _service_names(self, deployed) -> list[str]: - """Backing MCP server names for the env (mirrors apply_server_config's enumeration).""" + def _backing_servers(self, deployed) -> list[BackingServer]: + """The env's backing MCP servers and websites (``legacy_protocol.backing_servers``). A website sharing its + name with an MCP server has its card read live, since the stored env card can't tell the two apart.""" + from agent_env.env import legacy_protocol from agent_env.env.env import Env - env = Env.get(deployed.env_id, deployed.env_version) - names = [e.environment_name for e in (getattr(env, "mcp_server_envs", None) or [])] - if not names and getattr(env, "environment_name", None): - names = [env.environment_name] - return names + return legacy_protocol.backing_servers(Env.get(deployed.env_id, deployed.env_version)) async def execute(self, context: TaskStepContext) -> TaskStepContext: + from agent_env.env import legacy_protocol from agent_env.env.env import gateway_url_of deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) @@ -131,10 +133,17 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: state = await deployed.invoke(EXT_CLOCK_URI, "state", timeout=self.timeout_seconds) env_get_time_url = state.get("env_get_time_url") + servers = self._backing_servers(deployed) + shared = legacy_protocol.shared_names(servers) synced, skipped = [], [] - for service in self._service_names(deployed): - outcome = await self._sync_one(deployed, gateway_url_of(deployed), service, env_get_time_url) - (synced if outcome.get("synced") else skipped).append({"service": service, "environment": service, **outcome}) + for server in servers: + service = server.environment_name + record = None if service in shared and not server.mcp else deployed + outcome = await self._sync_one(record, gateway_url_of(deployed), service, env_get_time_url, mcp=server.mcp, + tolerate=self.tolerate_missing_sync_time or not server.mcp, + headers=deployed.control_headers) + entry = {"service": service, "environment": service, **({} if server.mcp else {"website": True}), **outcome} + (synced if outcome.get("synced") else skipped).append(entry) configuration = { "step_id": self.id, "env_id": self.env_id, @@ -151,17 +160,24 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: f"({[(s['service'], s.get('reason')) for s in skipped]}) — every tool still answers wall time") return context - async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time_url: Optional[str]) -> dict: - """Invoke sync_time on one server if its card advertises clock/v1; skip (or raise) per the flag.""" + async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time_url: Optional[str], mcp: bool = True, + tolerate: Optional[bool] = None, headers: Optional[dict] = None) -> dict: + """Invoke sync_time on one server, an MCP server or with ``mcp=False`` a website, if its card advertises + clock/v1; skip a server with no card or no clock/v1 when ``tolerate`` (tolerate_missing_sync_time by default), + else raise. ``execute`` tolerates every website, so one that doesn't follow the clock never fails the step; one + that follows it but can't take the task's timezone does, as an MCP server does. ``headers`` are the env's control + headers, ``deployed``'s by default; ``execute`` passes them because it hands a website that shares its name with + an MCP server no record, so its card is read live.""" + tolerate = self.tolerate_missing_sync_time if tolerate is None else tolerate from agent_env.env import legacy_protocol - base_url, card = await legacy_protocol.child_env_card(deployed, gateway_url, service, timeout=self.timeout_seconds) + base_url, card = await legacy_protocol.child_env_card(deployed, gateway_url, service, timeout=self.timeout_seconds, mcp=mcp) if card is None: - if self.tolerate_missing_sync_time: + if tolerate: return {"synced": False, "reason": "no_env_card"} raise RuntimeError(f"{service} has no env card: it serves none, its card isn't named {service!r}, or the deploy couldn't read it") if protocol_v1.find_extension(card, EXT_CLOCK_URI) is None: - if self.tolerate_missing_sync_time: + if tolerate: logger.info(f"sync_env_clock: {service} does not advertise {EXT_CLOCK_URI}; skipping") return {"synced": False, "reason": "not_advertised"} raise RuntimeError(f"{service} does not advertise {EXT_CLOCK_URI}") @@ -176,5 +192,6 @@ async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time params["timezone"] = self.timezone result = await protocol_v1.invoke_extension(base_url, card, EXT_CLOCK_URI, params=params, - timeout=self.timeout_seconds, headers=deployed.control_headers) + timeout=self.timeout_seconds, + headers=deployed.control_headers if headers is None else headers) return {"synced": True, "result": result} diff --git a/tst/unit/env/test_legacy_protocol.py b/tst/unit/env/test_legacy_protocol.py index 9508e8d2e..7d107ad34 100644 --- a/tst/unit/env/test_legacy_protocol.py +++ b/tst/unit/env/test_legacy_protocol.py @@ -177,3 +177,22 @@ async def test_a_v1_services_state_is_the_data_it_answers_with_else_its_export_s assert await legacy_protocol.service_state(None, "http://gw", "slack") == state assert ("GET /svc/mcp-slack/export-state" in asked) is reads_export_state + + +def test_backing_servers_are_an_envs_mcp_servers_then_its_websites(): + from types import SimpleNamespace + + from agent_env.env.envs.website import WebsiteEnv + from agent_env.env.legacy_protocol import BackingServer + + named = lambda *names: [SimpleNamespace(environment_name=n) for n in names] + multi = SimpleNamespace(mcp_server_envs=named("slack", "email"), website_envs=named("shop", "slack")) + + assert legacy_protocol.backing_servers(multi) == [BackingServer("slack"), BackingServer("email"), + BackingServer("shop", mcp=False), BackingServer("slack", mcp=False)] + assert legacy_protocol.shared_names(legacy_protocol.backing_servers(multi)) == {"slack"} + assert legacy_protocol.mcp_wins(legacy_protocol.backing_servers(multi)) == [ + BackingServer("slack"), BackingServer("email"), BackingServer("shop", mcp=False)] + assert legacy_protocol.backing_servers(WebsiteEnv(id="w", version=1, environment_name="shop")) == [BackingServer("shop", mcp=False)] + assert legacy_protocol.backing_servers(SimpleNamespace(environment_name="notes")) == [BackingServer("notes")] + assert legacy_protocol.backing_servers(SimpleNamespace()) == [] diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index a2f9958bc..f604e42cf 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -2466,6 +2466,18 @@ def test_a_one_image_website_env_is_fronted_with_only_its_server_image(): assert topology.website_images == [server] +def test_a_websites_server_reaches_the_gateways_clock_at_the_url_the_gateway_hands_it(): + """sync_env_clock hands each server http://gateway:/clock/time: the gateway's compose service is that host, + listening on that port, on the one network the website's server is on.""" + from agent_env.env.gateway.constants import GATEWAY_COMPOSE_HOST + + services = yaml.safe_load(_render([], [WebsiteConfig(backend_image="shop-be", frontend_image=None, environment_name="shop")]))["services"] + gateway, site = services[GATEWAY_COMPOSE_HOST], services["shop-website-backend"] + + assert "MCP_PORT=18765" in gateway["environment"] + assert gateway["networks"] == site["networks"] == ["env-network"] + + def _sites(handler): """Route the deploy's site checks to ``handler(request) -> httpx.Response``.""" real = httpx.AsyncClient diff --git a/tst/unit/task_step/sync_env_clock_test.py b/tst/unit/task_step/sync_env_clock_test.py index 5ab7bee8e..5b83c51d7 100644 --- a/tst/unit/task_step/sync_env_clock_test.py +++ b/tst/unit/task_step/sync_env_clock_test.py @@ -5,6 +5,7 @@ from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, EnvCapabilityUnsupported from agent_env.env.gateway.constants import EXT_CLOCK_URI +from agent_env.env.legacy_protocol import BackingServer from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps import sync_env_clock as mod from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep, resolve_virtual_time @@ -94,7 +95,7 @@ async def test_sync_one_skips_or_raises_for_a_child_env_missing_from_the_stored_ @pytest.mark.asyncio async def test_execute_syncs_each_child_env_through_the_stored_card(monkeypatch): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_service_names", lambda self, deployed: ["email"]) + monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))]) await _step().execute(context) @@ -124,6 +125,96 @@ async def test_execute_keeps_the_arm_failure_message(monkeypatch): await _step().execute(context) +@pytest.mark.asyncio +async def test_execute_syncs_a_multi_envs_websites_alongside_its_mcp_servers(monkeypatch): + from types import SimpleNamespace + from unittest.mock import patch + + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], + website_envs=[SimpleNamespace(environment_name="shop"), SimpleNamespace(environment_name="blog")]) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT], "shop": [_CLOCK_EXT], "blog": []}))]) + + with patch("agent_env.env.env.Env.get", return_value=env): + await _step().execute(context) + + assert [(r.method, str(r.url)) for r in sent][2:] == [ + ("POST", f"{_CARD}/svc/mcp-email/agentenv/ext/sync_time"), ("POST", f"{_CARD}/svc/shop/agentenv/ext/sync_time"), + ] + record = context.metadata["clock_configurations"][0] + assert [(s["service"], s.get("website")) for s in record["synced"]] == [("email", None), ("shop", True)] + assert [(s["service"], s["website"], s["reason"]) for s in record["skipped"]] == [("blog", True, "not_advertised")] + + +@pytest.mark.asyncio +async def test_execute_syncs_a_single_website_env(monkeypatch): + from unittest.mock import patch + + from agent_env.env.envs.website import WebsiteEnv + + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"shop": [_CLOCK_EXT]}))]) + + with patch("agent_env.env.env.Env.get", return_value=WebsiteEnv(id="e1", version=1, environment_name="shop")): + await _step().execute(context) + + assert [(r.method, str(r.url)) for r in sent][2:] == [("POST", f"{_CARD}/svc/shop/agentenv/ext/sync_time")] + assert json.loads(sent[2].content) == {"env_get_time_url": CLOCK_URL} + + +@pytest.mark.asyncio +async def test_sync_one_reads_a_websites_card_live_at_its_own_gateway_path(monkeypatch): + read = [] + + async def fake_card(base_url, **kw): + read.append(base_url) + return {"capabilities": {"extensions": []}} + + monkeypatch.setattr(mod.protocol_v1, "get_card", fake_card) + assert await _step()._sync_one(None, "http://gw", "shop", CLOCK_URL, mcp=False) == {"synced": False, "reason": "not_advertised"} + assert read == ["http://gw/svc/shop"] + + +@pytest.mark.asyncio +async def test_execute_reads_a_website_sharing_an_mcp_servers_name_live(monkeypatch): + from types import SimpleNamespace + from unittest.mock import patch + + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], + website_envs=[SimpleNamespace(environment_name="email")]) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))]) + + with patch("agent_env.env.env.Env.get", return_value=env): + await _step().execute(context) + + assert [(r.method, str(r.url)) for r in sent][2:] == [ + ("POST", f"{_CARD}/svc/mcp-email/agentenv/ext/sync_time"), ("GET", "http://gw/svc/email/.well-known/agent-env.json"), + ] + + +@pytest.mark.asyncio +async def test_strict_execute_skips_websites_but_not_mcp_servers_that_do_not_follow_the_clock(monkeypatch): + from types import SimpleNamespace + from unittest.mock import patch + + _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], + website_envs=[SimpleNamespace(environment_name="shop"), SimpleNamespace(environment_name="legacy")]) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT], "shop": []}))]) + + with patch("agent_env.env.env.Env.get", return_value=env): + await _step(tolerate_missing_sync_time=False).execute(context) + + record = context.metadata["clock_configurations"][0] + assert [s["service"] for s in record["synced"]] == ["email"] + assert [(s["service"], s["reason"]) for s in record["skipped"]] == [("shop", "not_advertised"), ("legacy", "no_env_card")] + + unsynced = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [], "shop": [_CLOCK_EXT]}))]) + with patch("agent_env.env.env.Env.get", return_value=env), pytest.raises(RuntimeError, match="email does not advertise"): + await _step(tolerate_missing_sync_time=False).execute(unsynced) + + _CLOCK_EXT = {"uri": EXT_CLOCK_URI, "params": {"endpoint": "/agentenv/ext/sync_time", "methods": {"sync_time": {"method": "POST"}}}} _ZONE_EXT = {"uri": EXT_CLOCK_URI, "params": {"endpoint": "/agentenv/ext/sync_time", "methods": {"sync_time": {"method": "POST", "request": { "type": "object", "properties": {"env_get_time_url": {"type": "string"}, "timezone": {"type": "string"}}, "required": ["env_get_time_url"]}}}}} @@ -226,7 +317,7 @@ def test_without_a_zone_virtual_time_is_passed_through_untouched(): @pytest.mark.asyncio async def test_execute_arms_the_gateway_at_the_local_instant_and_tells_every_server_the_zone(monkeypatch): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_service_names", lambda self, deployed: ["email"]) + monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_ZONE_EXT]}))]) await _zoned().execute(context) @@ -241,7 +332,7 @@ async def test_execute_arms_the_gateway_at_the_local_instant_and_tells_every_ser @pytest.mark.parametrize("tolerate", [True, False]) async def test_a_server_that_cannot_take_the_zone_fails_the_step_before_any_sync_is_sent(monkeypatch, tolerate): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_service_names", lambda self, deployed: ["email"]) + monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_OLD_EXT]}))]) with pytest.raises(RuntimeError, match="takes no timezone"): @@ -264,6 +355,62 @@ async def test_a_server_that_does_not_advertise_clock_is_still_skipped_when_a_zo assert out == {"synced": False, "reason": "not_advertised"} +@pytest.mark.asyncio +async def test_a_zoned_step_tells_a_website_that_takes_the_zone_and_fails_on_one_that_cannot(monkeypatch): + """A website is tolerated for not following the clock, but one that follows it without taking the zone fails the + step, as an MCP server does.""" + from types import SimpleNamespace + from unittest.mock import patch + + env = SimpleNamespace(mcp_server_envs=[], website_envs=[SimpleNamespace(environment_name="shop")]) + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"shop": [_ZONE_EXT]}))]) + with patch("agent_env.env.env.Env.get", return_value=env): + await _zoned().execute(context) + assert json.loads(sent[2].content) == {"env_get_time_url": CLOCK_URL, "timezone": "America/New_York"} + assert context.metadata["clock_configurations"][0]["synced"][0]["website"] is True + + _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + older = TaskStepContext(deployed_envs=[_carded(await _composed_card({"shop": [_OLD_EXT]}))]) + with patch("agent_env.env.env.Env.get", return_value=env), pytest.raises(RuntimeError, match="shop advertises .* takes no timezone"): + await _zoned().execute(older) + + +@pytest.mark.asyncio +async def test_a_website_sharing_an_mcp_servers_name_is_synced_with_the_envs_control_headers(monkeypatch): + """Its card is read live, with no record, and its sync_time still carries the env's control token.""" + import dataclasses + from types import SimpleNamespace + from unittest.mock import patch + + import httpx + + sent, real = [], httpx.AsyncClient + website_card = {"name": "email", "url": "/agentenv", "capabilities": {"extensions": [_CLOCK_EXT]}} + + def handle(request: httpx.Request) -> httpx.Response: + sent.append(request) + if request.url.path.endswith("/clock/state"): + return httpx.Response(200, json={"env_get_time_url": CLOCK_URL}) + if request.url.path == "/svc/email/.well-known/agent-env.json": + return httpx.Response(200, json=website_card) + return httpx.Response(200, json={}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], + website_envs=[SimpleNamespace(environment_name="email")]) + record = dataclasses.replace(_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]})), control_token="ctl") + context = TaskStepContext(deployed_envs=[record]) + + with patch("agent_env.env.env.Env.get", return_value=env): + await _step().execute(context) + + website_sync = [r for r in sent if str(r.url) == "http://gw/svc/email/agentenv/ext/sync_time"] + assert len(website_sync) == 1 + assert {k: website_sync[0].headers.get(k) for k in record.control_headers} == record.control_headers + assert [s["service"] for s in context.metadata["clock_configurations"][0]["synced"]] == ["email", "email"] + + def test_a_padded_zone_is_stored_and_sent_stripped(): s = _zoned(timezone=" America/New_York ") assert s.timezone == "America/New_York" and s.to_dict()["timezone"] == "America/New_York" diff --git a/tst/unit/task_step/test_apply_server_config.py b/tst/unit/task_step/test_apply_server_config.py index f9badb26c..ddeb96806 100644 --- a/tst/unit/task_step/test_apply_server_config.py +++ b/tst/unit/task_step/test_apply_server_config.py @@ -589,6 +589,97 @@ async def test_live_read_strips_a_trailing_slash_from_the_gateway_url(): assert [url for url, _ in gw.post_calls] == [f"{_GATEWAY}/svc/mcp-slack/agentenv/ext/set_errors"] +# --- websites --- + + +def _fake_multi_env(mcp_names, website_names): + return SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name=s) for s in mcp_names], + website_envs=[SimpleNamespace(environment_name=s) for s in website_names]) + + +class _KeyedGateway(_MockGateway): + """A gateway serving each card at /svc//.well-known/agent-env.json, where a website's key is its bare name.""" + + async def get(self, url, *, timeout=None, params=None): + self.get_calls.append(url) + for key, card in self.cards.items(): + if url.endswith(f"/svc/{key}/.well-known/agent-env.json"): + return _resp(200, url, card) + return _resp(404, url, {"error": "no card"}) + + +@pytest.mark.asyncio +async def test_stored_card_broadcast_reaches_websites_and_leaves_a_shared_name_to_the_mcp_server(): + gw = _MockGateway({}) + ext = _ext(_SET_ACTING_USER_URI, "/agentenv/ext/set_acting_user") + ctx = _carded_context(await _composed_card({"mcp-slack": [ext], "shop": [ext]})) + step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}]) + with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop", "slack"])): + out = await step.execute(ctx) + + assert gw.get_calls == [] + assert [url for url, _ in gw.post_calls] == [f"{_CARD}/svc/mcp-slack/agentenv/ext/set_acting_user", + f"{_CARD}/svc/shop/agentenv/ext/set_acting_user"] + assert [c["service"] for c in out.metadata["server_config_changes"]] == ["slack", "shop"] + + +@pytest.mark.asyncio +async def test_live_read_broadcast_reads_a_websites_card_at_its_own_gateway_path(): + ext = _ext(_SET_ACTING_USER_URI, "/agentenv/ext/set_acting_user") + gw = _KeyedGateway({"mcp-slack": _card(ext), "shop": _card(ext)}) + step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}]) + with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop"])): + await step.execute(_make_context()) + + assert gw.get_calls == [f"{_GATEWAY}/svc/mcp-slack/.well-known/agent-env.json", f"{_GATEWAY}/svc/shop/.well-known/agent-env.json"] + assert [url for url, _ in gw.post_calls] == [f"{_GATEWAY}/svc/mcp-slack/agentenv/ext/set_acting_user", + f"{_GATEWAY}/svc/shop/agentenv/ext/set_acting_user"] + + +@pytest.mark.asyncio +async def test_broadcast_reaches_a_single_website_env(): + from agent_env.env.envs.website import WebsiteEnv + + gw = _MockGateway({}) + ctx = _carded_context(await _composed_card({"shop": [_ext(_SET_ACTING_USER_URI, "/agentenv/ext/set_acting_user")]})) + step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}]) + with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=WebsiteEnv(id="env-x", version=1, environment_name="shop")): + await step.execute(ctx) + + assert [url for url, _ in gw.post_calls] == [f"{_CARD}/svc/shop/agentenv/ext/set_acting_user"] + + +@pytest.mark.asyncio +async def test_strict_broadcast_skips_websites_without_a_card_or_the_extension(): + gw = _MockGateway({}) + ext = _ext(_SET_ERRORS_URI, "/agentenv/ext/set_errors") + ctx = _carded_context(await _composed_card({"mcp-slack": [ext], "shop": []})) + step = _step(directives=[{"service": "*", "uri": _SET_ERRORS_URI, "args": {"error_rate": 1.0}}]) + with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop", "legacy"])): + out = await step.execute(ctx) + + assert [url for url, _ in gw.post_calls] == [f"{_CARD}/svc/mcp-slack/agentenv/ext/set_errors"] + assert [c["service"] for c in out.metadata["server_config_changes"]] == ["slack"] + assert [(s["service"], s["reason"]) for s in out.metadata["server_config_skipped"]] == [ + ("shop", "extension_not_advertised"), ("legacy", "no_env_card")] + + +@pytest.mark.asyncio +async def test_strict_broadcast_still_fails_for_an_mcp_server_or_a_named_website_lacking_the_extension(): + ext = _ext(_SET_ERRORS_URI, "/agentenv/ext/set_errors") + broadcast = {"service": "*", "uri": _SET_ERRORS_URI, "args": {"error_rate": 1.0}} + unadvertising_mcp = _carded_context(await _composed_card({"mcp-slack": [], "shop": [ext]})) + with _patch_httpx(_MockGateway({})), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop"])): + with pytest.raises(ApplyServerConfigError, match="not advertised on card for service 'slack'"): + await _step(directives=[broadcast]).execute(unadvertising_mcp) + + unadvertising_shop = _carded_context(await _composed_card({"mcp-slack": [ext], "shop": []})) + named = {"service": "shop", "uri": _SET_ERRORS_URI, "args": {"error_rate": 1.0}} + with _patch_httpx(_MockGateway({})), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop"])): + with pytest.raises(ApplyServerConfigError, match="not advertised on card for service 'shop'"): + await _step(directives=[broadcast, named]).execute(unadvertising_shop) + + # --- schema / validation (pure, no I/O) --- diff --git a/tst/unit/task_step/test_snapshot_env.py b/tst/unit/task_step/test_snapshot_env.py index 877244c37..380690fc2 100644 --- a/tst/unit/task_step/test_snapshot_env.py +++ b/tst/unit/task_step/test_snapshot_env.py @@ -1367,3 +1367,177 @@ def shared_credentials_env(self) -> dict[str, str]: store = _SharingS3Store(client=object(), bucket="bucket") monkeypatch.setattr(snapshot_mod, "get_config", lambda: type("Cfg", (), {"get_object_store": lambda self: store})()) + + +# ── websites ───────────────────────────────────────────────────────────────── + +def _multi_env(mcp_names, website_names): + return SimpleNamespace(id="env-1", version=3, mcp_server_envs=[_FakeServiceEnv(n) for n in mcp_names], + website_envs=[_FakeServiceEnv(n) for n in website_names]) + + +def test_enumerate_includes_websites_and_leaves_a_shared_name_to_the_mcp_server(): + servers = SnapshotEnvTaskStep._enumerate_servers(_multi_env(["slack", "gmail"], ["shop", "slack"])) + assert [(s.environment_name, s.mcp) for s in servers] == [("slack", True), ("gmail", True), ("shop", False)] + + +def test_enumerate_a_single_website_env_and_a_multi_env_of_websites_only(): + from agent_env.env.envs.website import WebsiteEnv + + site = WebsiteEnv(id="site-env", version=1, environment_name="shop") + assert [(s.environment_name, s.mcp) for s in SnapshotEnvTaskStep._enumerate_servers(site)] == [("shop", False)] + assert SnapshotEnvTaskStep._enumerate_environments(_multi_env([], ["shop"])) == ["shop"] + + +@pytest.mark.asyncio +async def test_export_of_a_website_reads_its_data_plane_from_the_stored_card(monkeypatch, tmp_path): + record = _carded(await _card({"mcp-slack": [], "shop": []})) + sent = _mock_http(monkeypatch, rpc_result={"parts": [{"kind": "data", "data": {"orders": 2}}]}) + + out = str(tmp_path / "export-tmp") + suffix = await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", out, 30, record, mcp=False) + + assert suffix == ".json" + assert [(r.method, str(r.url)) for r in sent] == [("POST", f"{_CARD}/svc/shop/agentenv")] + with open(out) as f: + assert json.load(f) == {"orders": 2} + + +@pytest.mark.asyncio +async def test_export_of_a_website_without_a_stored_card_probes_its_own_gateway_path(monkeypatch, tmp_path): + import dataclasses + + record = dataclasses.replace(_carded({}), environment_card=None) + sent = _mock_http(monkeypatch, rpc_result={"parts": [{"kind": "data", "data": {"orders": 2}}]}, + body=b'{"name": "shop", "url": "/agentenv"}') + + suffix = await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", str(tmp_path / "out"), 30, record, mcp=False) + + assert suffix == ".json" + assert [(r.method, str(r.url)) for r in sent] == [ + ("GET", "https://gw-1/svc/shop/.well-known/agent-env.json"), + ("GET", "https://gw-1/svc/shop/.well-known/agent-env.json"), + ("POST", "https://gw-1/svc/shop/agentenv"), + ] + + +def _serve_website_card(monkeypatch, card_response): + """A gateway whose website ``shop`` answers its card read with ``card_response`` (a response or an exception to + raise) and whose MCP server ``slack`` serves a v1 data plane; returns the requests sent.""" + import httpx + + sent, real = [], httpx.AsyncClient + + def handle(request: httpx.Request) -> httpx.Response: + sent.append(request) + if request.url.path == f"/svc/shop{WELL_KNOWN_PATH}": + if isinstance(card_response, Exception): + raise card_response + return card_response + if request.url.path.endswith("/agentenv"): + return httpx.Response(200, json={"jsonrpc": "2.0", "id": 1, "result": {"parts": [{"kind": "data", "data": {}}]}}) + return httpx.Response(200, json={}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + return sent + + +def _card_read_failures(): + import httpx + + return [httpx.Response(500, json={}), httpx.Response(503, json={}), httpx.ReadTimeout("card read timed out"), + httpx.ConnectError("connection refused")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("card_response", _card_read_failures(), ids=["500", "503", "timeout", "connect-error"]) +async def test_export_of_a_website_whose_card_read_fails_raises_rather_than_skipping_it(monkeypatch, tmp_path, card_response): + """Without a stored card, a website card read that fails is no proof the website serves no data plane.""" + import httpx + + sent = _serve_website_card(monkeypatch, card_response) + + with pytest.raises(httpx.HTTPError): + await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", str(tmp_path / "out"), 30, mcp=False) + assert [str(r.url) for r in sent] == [f"https://gw-1/svc/shop{WELL_KNOWN_PATH}"] + + +@pytest.mark.asyncio +async def test_export_of_a_website_without_a_card_is_skipped(monkeypatch, tmp_path): + import httpx + + sent = _serve_website_card(monkeypatch, httpx.Response(404, json={})) + + assert await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", str(tmp_path / "out"), 30, mcp=False) is None + assert [str(r.url) for r in sent] == [f"https://gw-1/svc/shop{WELL_KNOWN_PATH}"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("card_response", _card_read_failures(), ids=["500", "503", "timeout", "connect-error"]) +async def test_core_fails_a_snapshot_whose_website_card_cannot_be_read(monkeypatch, card_response): + """Another server exporting does not let the snapshot through without a website it could not check.""" + real_export = SnapshotEnvTaskStep.__dict__["_export_environment_to_file"] + _install_core_artifact_fakes(monkeypatch, EnvironmentUniverseArtifact) + monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", real_export) + _serve_website_card(monkeypatch, card_response) + + with pytest.raises(RuntimeError, match=r"1/2 services were not exported \(missing=\['shop'\]\)"): + await SnapshotEnvTaskStep.snapshot_env_state( + env=_multi_env(["slack"], ["shop"]), gateway_url="https://gw-1", snapshot_id="snap-x", + ) + + +@pytest.mark.asyncio +async def test_export_of_a_website_serving_no_data_plane_exports_nothing_and_sends_nothing(monkeypatch, tmp_path): + record = _carded(await _card({"mcp-slack": []})) + sent = _mock_http(monkeypatch) + + assert await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", str(tmp_path / "out"), 30, record, mcp=False) is None + assert sent == [] + + +@pytest.mark.asyncio +async def test_core_snapshots_websites_through_their_data_plane_and_leaves_out_those_without_one(monkeypatch): + calls = [] + + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **website): + calls.append((name, website)) + if name == "legacy-site": + return None + with open(tmp_path, "w") as f: + json.dump({"ok": name}, f) + return ".json" + + universe_puts = {} + + class _FakeUniverse: + id = "snap-x" + version = 1 + + @staticmethod + def put(id, *, environment_artifacts, metadata=None): + universe_puts["names"] = [sa.environment_name for sa in environment_artifacts] + return _FakeUniverse() + + _install_core_artifact_fakes(monkeypatch, _FakeUniverse) + monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) + + result = await SnapshotEnvTaskStep.snapshot_env_state( + env=_multi_env(["slack"], ["shop", "slack", "legacy-site"]), gateway_url="https://gw", snapshot_id="snap-x", + ) + + assert sorted(calls) == [("legacy-site", {"mcp": False}), ("shop", {"mcp": False}), ("slack", {})] + assert universe_puts["names"] == ["slack", "shop"] + assert result.environments_snapshotted == ["slack", "shop"] and result.total == 3 + + +@pytest.mark.asyncio +async def test_core_raises_when_no_service_exported_state(monkeypatch): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, **website): + return None + + _install_core_artifact_fakes(monkeypatch, EnvironmentUniverseArtifact) + monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) + + with pytest.raises(RuntimeError, match=r"exported no state: its websites \['shop'\] serve no v1 data plane"): + await SnapshotEnvTaskStep.snapshot_env_state(env=_multi_env([], ["shop"]), gateway_url="https://gw", snapshot_id="snap-x") From b18049d5eeaed225444da4c5da146f8850b4e1ef Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 12:52:03 -0800 Subject: [PATCH 02/19] fix(website): sync_env_clock, snapshot_env and broadcasts reach each server the env card lists, at its own route (FD-4683) The servers behind an env's gateway come from its card's children_environments, each at its own /svc route, through new protocol helpers (ChildEnv, children, child_at, child_named, serves), not from the env's definition: a website sharing an MCP server's name is its own server, and a card without data/get in its operations serves no v1 data plane. The definition is read only for servers that serve no card, and for a record whose card can't be read, logged. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/agentenv-protocol/README.md | 10 ++ .../src/agentenv_protocol/client.py | 80 +++++++++ .../agentenv-protocol/tests/test_client.py | 52 ++++++ src/agent_env/env/legacy_protocol.py | 105 ++++++++++-- .../task_steps/apply_server_config.py | 81 +++++---- .../task_step/task_steps/snapshot_env.py | 86 +++++----- .../task_step/task_steps/sync_env_clock.py | 44 ++--- tst/unit/env/test_legacy_protocol.py | 100 ++++++++++- tst/unit/task/local_run_write_fence_test.py | 2 +- tst/unit/task_step/sync_env_clock_test.py | 80 ++++++--- .../task_step/test_apply_server_config.py | 30 +++- tst/unit/task_step/test_snapshot_env.py | 156 ++++++++++++++---- 12 files changed, 631 insertions(+), 195 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index b6cc8c649..4e3b0de48 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -198,6 +198,16 @@ with any other transport are kept beside the SDK's entry. Card URLs are paths, r address the card was fetched from. On the client side, `client.mcp_path(card)` returns the declared path, or `/mcp` for a card without one. +A composed card, such as a gateway's, nests each backing environment's card under +`children_environments`, its `url` and endpoints rooted at the route the composer serves it at. +`client.children(card)` returns each as a `ChildEnv` (its `name`, `url` and `card`, with its +`extensions`, `operations` and `serves(method)`, and `base_url(address)`, its data plane resolved +against the composed card's address). Two children may share a name, as a website and an MCP +server can, so `client.child_at(card, url)` finds one by its url or route; `client.child_named(card, +name)` raises `AmbiguousChildError` for a shared name, and `client.find_child` returns the first. +`client.serves(card, method)` says whether a card serves a data-plane method by its declared +`operations`, None for a card that declares none. + Extensions are invoked from what the card advertises. `client.find_extension_method(card, uri, method)` returns one advertised method, whose `endpoint` is the method's own or else the extension's, and `client.invoke_extension(base_url, card, uri, params, method=...)` calls it with diff --git a/packages/agentenv-protocol/src/agentenv_protocol/client.py b/packages/agentenv-protocol/src/agentenv_protocol/client.py index dcddaf541..a1e352879 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/client.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/client.py @@ -7,6 +7,7 @@ import logging from collections.abc import Iterator from contextvars import ContextVar +from dataclasses import dataclass, field import httpx @@ -210,6 +211,83 @@ def find_tool(card: dict, name: str) -> dict | None: return None +def serves(card: dict, method: str) -> bool | None: + """Whether a card serves the data-plane ``method`` (``data/get`` say): True or False by its declared + ``capabilities.operations``, or None when it declares none, as an older card doesn't, so the caller decides as it + did before cards declared them (a probe, say).""" + declared = operations(card) + return None if declared is None else method in declared + + +@dataclass(frozen=True) +class ChildEnv: + """One child env a composed card nests under ``children_environments``: its ``name`` and ``url`` as its card gives + them, and that ``card``, which says what it offers. A composer roots each child's ``url`` (and its endpoints) at + the route it serves the child at, so two children sharing a name have distinct urls; ``url`` is None for a child + whose card names none.""" + + name: str | None + url: str | None + card: dict = field(hash=False, repr=False) + + @property + def extensions(self) -> list[dict]: + """The extensions the child's card advertises (``capabilities.extensions``).""" + return list((self.card.get("capabilities") or {}).get("extensions") or []) + + @property + def operations(self) -> list[str] | None: + """The data-plane methods the child's card declares (``operations``), or None when it declares none.""" + return operations(self.card) + + def find_extension(self, uri: str) -> dict | None: + """The extension the child's card advertises under ``uri``, or None (``find_extension``).""" + return find_extension(self.card, uri) + + def serves(self, method: str) -> bool | None: + """Whether the child's card serves the data-plane ``method``, None when it declares no operations (``serves``).""" + return serves(self.card, method) + + def base_url(self, address: str) -> str: + """Where the child's data plane is, its ``url`` without the RPC path, resolved against ``address``, the composed + card's: a path joins onto it, a full URL stands alone, and a child naming no url is at ``address`` itself.""" + url = (self.url or RPC_PATH).removesuffix(RPC_PATH) + return url if "://" in url else f"{address.rstrip('/')}{url}" + + +def children(card: dict) -> list[ChildEnv]: + """Each child env a composed card nests under ``children_environments``, in its order; none for a leaf card, or + one whose ``children_environments`` is missing or null. An entry that isn't an object is skipped.""" + return [ChildEnv(name=child.get("name"), url=child.get("url"), card=child) + for child in card.get("children_environments") or [] if isinstance(child, dict)] + + +def child_at(card: dict, url: str) -> ChildEnv | None: + """The child env of a composed card at ``url``: the one whose ``url`` is ``url``, or whose data plane is there + (``url`` without the RPC path, ``/svc/mcp-slack`` for ``/svc/mcp-slack/agentenv`` say), a trailing slash aside; + None when no child is. A composer serves each child at its own route, so this is the lookup two children sharing a + name can't confuse.""" + wanted = url.rstrip("/") + for child in children(card): + if child.url is not None and wanted in (child.url.rstrip("/"), child.url.removesuffix(RPC_PATH).rstrip("/")): + return child + return None + + +class AmbiguousChildError(LookupError): + """More than one child env of a composed card has the name asked for, so the name can't say which is meant.""" + + +def child_named(card: dict, name: str) -> ChildEnv | None: + """The one child env of a composed card named ``name``, or None when none is; raises ``AmbiguousChildError`` when + several are, as a website and an MCP server sharing a name are, which ``child_at`` tells apart by their urls.""" + found = [child for child in children(card) if child.name == name] + if len(found) > 1: + raise AmbiguousChildError(f"{len(found)} child envs are named {name!r} ({', '.join(str(c.url) for c in found)}); " + "look one up by its url") + return found[0] if found else None + + def find_child(card: dict, name: str) -> dict | None: """Return the nested child EnvironmentCard named `name`, or None. @@ -218,6 +296,8 @@ def find_child(card: dict, name: str) -> dict | None: rewritten to be reachable from the composed card's base_url. Callers resolve a service by name here, then use the child card with `find_extension`/`invoke_extension`. Tolerates a missing/null `children_environments` so single/leaf cards return None rather than raising. + Two children sharing `name` return the first; `child_named` refuses that, and `child_at` + finds a child by its url, which tells them apart. """ for child in card.get("children_environments") or []: if child.get("name") == name: diff --git a/packages/agentenv-protocol/tests/test_client.py b/packages/agentenv-protocol/tests/test_client.py index edcd38798..bfc65e9a7 100644 --- a/packages/agentenv-protocol/tests/test_client.py +++ b/packages/agentenv-protocol/tests/test_client.py @@ -190,6 +190,58 @@ def test_find_child_tolerates_missing_children(): assert protocol_v1.find_child({"children_environments": None}, "slack") is None +_SHARED_NAME = {"name": "gw", "children_environments": [ + {"name": "mail", "url": "/svc/mcp-mail/agentenv", "capabilities": {"operations": ["data/get"], "extensions": [ + {"uri": "urn:agentenv:clock/v1"}]}}, + {"name": "mail", "url": "/svc/mail/agentenv", "capabilities": {"operations": []}}, + {"name": "docs", "capabilities": {}}, + "not a card", +]} + + +def test_children_are_each_nested_card_by_its_name_url_and_card(): + found = protocol_v1.children(_SHARED_NAME) + + assert [(c.name, c.url) for c in found] == [("mail", "/svc/mcp-mail/agentenv"), ("mail", "/svc/mail/agentenv"), ("docs", None)] + assert found[0].card is _SHARED_NAME["children_environments"][0] + assert [c.extensions for c in found] == [[{"uri": "urn:agentenv:clock/v1"}], [], []] + assert [c.operations for c in found] == [["data/get"], [], None] + assert [c.serves("data/get") for c in found] == [True, False, None] + assert found[0].find_extension("urn:agentenv:clock/v1") is not None and found[1].find_extension("urn:agentenv:clock/v1") is None + assert protocol_v1.children({"name": "leaf"}) == [] and protocol_v1.children({"children_environments": None}) == [] + + +def test_a_childs_base_url_is_its_url_without_the_rpc_path_against_the_composed_cards_address(): + mcp, website, docs = protocol_v1.children(_SHARED_NAME) + + assert mcp.base_url("https://env.example/") == "https://env.example/svc/mcp-mail" + assert website.base_url("https://env.example") == "https://env.example/svc/mail" + assert docs.base_url("https://env.example") == "https://env.example" + assert protocol_v1.ChildEnv("x", "https://x.example/agentenv", {}).base_url("https://env.example") == "https://x.example" + + +def test_child_at_finds_a_child_by_its_url_or_data_plane_route_where_two_share_a_name(): + assert protocol_v1.child_at(_SHARED_NAME, "/svc/mail").url == "/svc/mail/agentenv" + assert protocol_v1.child_at(_SHARED_NAME, "/svc/mcp-mail/").url == "/svc/mcp-mail/agentenv" + assert protocol_v1.child_at(_SHARED_NAME, "/svc/mail/agentenv").url == "/svc/mail/agentenv" + assert protocol_v1.child_at(_SHARED_NAME, "/svc/docs") is None + assert protocol_v1.child_at({"name": "leaf"}, "/svc/mail") is None + + +def test_child_named_refuses_a_name_two_children_share(): + assert protocol_v1.child_named(_SHARED_NAME, "docs").name == "docs" + assert protocol_v1.child_named(_SHARED_NAME, "gmail") is None + with pytest.raises(protocol_v1.AmbiguousChildError, match="2 child envs are named 'mail'"): + protocol_v1.child_named(_SHARED_NAME, "mail") + assert protocol_v1.find_child(_SHARED_NAME, "mail")["url"] == "/svc/mcp-mail/agentenv" + + +def test_serves_is_what_a_card_declares_of_a_method_and_none_when_it_declares_no_operations(): + assert protocol_v1.serves({"capabilities": {"operations": ["data/get"]}}, "data/get") is True + assert protocol_v1.serves({"capabilities": {"operations": ["data/add"]}}, "data/get") is False + assert protocol_v1.serves({"capabilities": {}}, "data/get") is None + + def test_find_extension_ignores_children(): # set-errors/v1 is advertised only on the children, never at the top level assert protocol_v1.find_extension(_CARD_WITH_CHILDREN, "urn:agentenv:set-errors/v1") is None diff --git a/src/agent_env/env/legacy_protocol.py b/src/agent_env/env/legacy_protocol.py index 8790157ac..f5ca7a002 100644 --- a/src/agent_env/env/legacy_protocol.py +++ b/src/agent_env/env/legacy_protocol.py @@ -8,12 +8,15 @@ from __future__ import annotations import asyncio +import dataclasses import logging from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional import httpx -from agentenv_protocol import RPC_PATH, DataPart, client as protocol_v1 +from agentenv_protocol import METHOD_GET, RPC_PATH, DataPart, client as protocol_v1 + +from agent_env.env.gateway.constants import EXT_WEBSITE_URI if TYPE_CHECKING: from agent_env.env.env import DeployedEnv @@ -36,16 +39,28 @@ def environment_base_url(gateway_url: Optional[str], environment_name: str, mcp: @dataclass(frozen=True) class BackingServer: - """A server behind an env's gateway, by its environment_name: an MCP server, at ``/svc/mcp-{name}``, or a website, - at ``/svc/{name}``.""" + """A server behind an env's address, by its environment_name: an MCP server or, with ``mcp=False``, a website. + + One the env's card lists (``card_servers``) carries that ``child`` and the ``address`` its paths resolve against. + One the card lists none for, a server serving no card, carries the ``address`` alone: it has no card, and none is + read for it. One named by the env definition of a record whose card can't be read carries neither, so its card is + read live at its gateway path, ``/svc/mcp-{name}`` or a website's ``/svc/{name}`` (``server_card``).""" environment_name: str mcp: bool = True + child: Optional[protocol_v1.ChildEnv] = None + address: Optional[str] = None + + @property + def card(self) -> Optional[dict]: + """The server's card as the env's card lists it, or None.""" + return self.child.card if self.child is not None else None -def backing_servers(env: Any) -> list[BackingServer]: - """The servers behind env's gateway: a MultiEnv's MCP servers then its websites, a WebsiteEnv's website, or else - the env's one MCP server; empty for an env with none.""" +def definition_servers(env: Any) -> list[BackingServer]: + """The servers an env definition names: a MultiEnv's MCP servers then its websites, a WebsiteEnv's website, or + else the env's one MCP server; empty for an env with none. Read only for what the env's card can't say + (``backing_servers``).""" from agent_env.env.envs.website import WebsiteEnv if isinstance(env, WebsiteEnv): @@ -57,16 +72,74 @@ def backing_servers(env: Any) -> list[BackingServer]: return servers -def shared_names(servers: list[BackingServer]) -> set[str]: - """The environment_names an MCP server and a website in ``servers`` share, which a stored env card can't tell apart.""" - return {s.environment_name for s in servers if s.mcp} & {s.environment_name for s in servers if not s.mcp} - - -def mcp_wins(servers: list[BackingServer]) -> list[BackingServer]: - """``servers`` without each website that shares its environment_name with an MCP server, as a MultiEnv routes an - artifact of that name to the MCP server.""" - shared = shared_names(servers) - return [s for s in servers if s.mcp or s.environment_name not in shared] +def card_servers(card: dict, address: str) -> list[BackingServer]: + """The servers ``card``, read at ``address``, lists, each with its own card: a leaf card's env itself; else each + child the composed card nests, after the env itself when its own card serves ``data/get``, as a computer's driver + fronting its apps does. A child is a website when its card carries ``website/v1`` or its url is at a website's + gateway route, ``/svc/{name}`` (an MCP server's is ``/svc/mcp-{name}``), so a website and an MCP server sharing a + name are two servers.""" + own = protocol_v1.ChildEnv(name=card.get("name"), url=card.get("url"), card=card) + if not isinstance(card.get("children_environments"), list): + return [_listed(own, address)] + servers = [_listed(own, address)] if protocol_v1.serves(card, METHOD_GET) else [] + return servers + [_listed(child, address) for child in protocol_v1.children(card)] + + +def _listed(child: protocol_v1.ChildEnv, address: str) -> BackingServer: + route, url = f"/svc/{child.name}", (child.url or "").rstrip("/") + website = child.find_extension(EXT_WEBSITE_URI) is not None or url == route or url.startswith(f"{route}/") + return BackingServer(child.name, mcp=not website, child=child, address=address) + + +async def backing_servers(deployed: Optional[DeployedEnv], gateway_url: Optional[str], env: Any, + timeout: int = 10) -> list[BackingServer]: + """The servers behind an env as its card lists them (``card_servers``), then, behind a gateway, each server its + definition ``env`` names that the card lists none of the same name and kind for: one serving no card, which the + gateway leaves off its card, so it has none. + + The card is ``deployed``'s stored one when ``gateway_url`` is its own gateway or None, else read live at + ``gateway_url`` (an override, or a record that kept no card). A record with no card URL, an older one, or no + record, or a live read that fails, leaves the definition alone: its servers, each read live at its gateway path, + logged once.""" + from agent_env.env.env import gateway_url_of + + gateway = gateway_url.rstrip("/") if gateway_url else None + card, address, why = None, None, "it has no record" + if deployed is not None and deployed.environment_card and (gateway is None or (gateway_url_of(deployed) or "").rstrip("/") == gateway): + card, address = deployed.environment_card, deployed.environment_url + elif deployed is not None and deployed.environment_card_url and gateway: + try: + card, address = await protocol_v1.get_card(gateway, timeout=timeout), gateway + except httpx.HTTPError as e: + why = f"its card at {gateway} could not be read ({type(e).__name__})" + elif deployed is not None: + why = "its record has no card URL" + named = definition_servers(env) + if not isinstance(card, dict): + logger.info(f"env {getattr(env, 'id', None)!r}: {why}, so its servers are the ones its definition names, " + f"each read live at its gateway path: {[(s.environment_name, s.mcp) for s in named]}") + return named + servers = card_servers(card, address) + if not gateway: + return servers + listed = {(s.environment_name, s.mcp) for s in servers} + return servers + [dataclasses.replace(s, address=address) for s in named if (s.environment_name, s.mcp) not in listed] + + +async def server_card(server: BackingServer, gateway_url: Optional[str], timeout: int = 10) -> tuple[Optional[str], Optional[dict]]: + """A server's card and the base its endpoints resolve against: as the env's card lists it, None for a server the + card lists none for, else read live at its gateway path (``child_env_card``).""" + if server.address is not None: + return server.address, server.card + return await child_env_card(None, gateway_url, server.environment_name, timeout=timeout, mcp=server.mcp) + + +async def server_v1_base_url(server: BackingServer, gateway_url: Optional[str]) -> Optional[str]: + """A server's v1 data-plane base: its card's ``url`` resolved against the env's address, None for a server the + env's card lists none for, else as the live probe at its gateway path finds it (``v1_base_url``).""" + if server.address is not None: + return server.child.base_url(server.address) if server.child is not None else None + return await v1_base_url(None, gateway_url, server.environment_name, mcp=server.mcp) async def v1_base_url(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str, mcp: bool = True) -> Optional[str]: diff --git a/src/agent_env/task_step/task_steps/apply_server_config.py b/src/agent_env/task_step/task_steps/apply_server_config.py index c8c8e1f08..91cc24167 100644 --- a/src/agent_env/task_step/task_steps/apply_server_config.py +++ b/src/agent_env/task_step/task_steps/apply_server_config.py @@ -9,14 +9,16 @@ Config lives on the backing server, not on the gateway itself. Each server's card comes from the env card stored on the deployed record, whose extension endpoints the gateway has already rewritten to its ``/svc/mcp-{service}`` (or a website's ``/svc/{service}``) -proxy; a record without a stored card reads it live there. The agent never sees the card or these endpoints; -only this harness step invokes them. +proxy; a record without a stored card reads it live there. A broadcast reaches each server +the env card lists, by that server's own card and route, so a website sharing its name with +an MCP server is reached too (``legacy_protocol.backing_servers``). The agent never sees the +card or these endpoints; only this harness step invokes them. """ from __future__ import annotations import logging -from typing import Any, ClassVar, Optional +from typing import TYPE_CHECKING, Any, ClassVar, Optional import httpx from agentenv_protocol import client as protocol_v1 @@ -25,6 +27,9 @@ from agent_env.entity.refs import EntityRef from agent_env.task_step.task_step import TaskStep, TaskStepDependency +if TYPE_CHECKING: + from agent_env.env.legacy_protocol import BackingServer + logger = logging.getLogger(__name__) @@ -131,51 +136,40 @@ def from_dict(cls, data: dict) -> "ApplyServerConfigStep": tolerate_unadvertised=data.get("tolerate_unadvertised", False), ) - def _expand_directives(self, deployed) -> tuple[list[ConfigDirective], set[str], set[int]]: - """Expand any broadcast directive (service == ``*``) into one directive - per MCP server and website in the env, resolved from the env definition, - and return them with the names that are websites, whose cards a live read - finds at ``/svc/{name}``, and the positions of the directives a broadcast - made for a website. Those are skipped, as under ``tolerate_unadvertised``, - when the website has no card or doesn't advertise the URI, so a website - the task didn't name never fails it; a directive naming a website, and - every MCP server, stays strict. The env's child service names aren't otherwise - enumerable here (the deployed card list carries no roster), so we read - them from ``Env.get`` — matching how deploy_env / load_artifact resolve - the env. A website sharing its name with an MCP server is left to the MCP - server, as the stored env card can't tell the two apart. Non-broadcast - directives pass through untouched.""" + async def _expand_directives(self, deployed) -> list[tuple[ConfigDirective, Optional[BackingServer], bool]]: + """Each directive with the server it targets and whether it is lenient: a broadcast (service ``*``) expanded + into one directive per MCP server and website the env card lists (``legacy_protocol.backing_servers``), each + targeting that server by its own card and route. A directive a broadcast made for a website is lenient: it is + skipped, as under ``tolerate_unadvertised``, when the website has no card or doesn't advertise the URI, so a + website the task didn't name never fails it. A named directive, and every MCP server, stays strict; a named one + targets the server of that name, the MCP server when a website shares it, as a MultiEnv's load does, or, without + a broadcast to list them, is looked up by name (None).""" if not any(d.service == self._BROADCAST for d in self.directives): - return self.directives, set(), set() + return [(d, None, False) for d in self.directives] from agent_env.env import legacy_protocol - from agent_env.env.env import Env + from agent_env.env.env import Env, gateway_url_of - env = Env.get(deployed.env_id, deployed.env_version) - # A single-server env (a bare MCPServerEnv) exposes `environment_name` and no - # `mcp_server_envs`; backing_servers falls back to it, as snapshot_env does, - # else a broadcast silently matches nothing on single-server envs. - servers = legacy_protocol.mcp_wins(legacy_protocol.backing_servers(env)) + servers = await legacy_protocol.backing_servers(deployed, gateway_url_of(deployed), Env.get(deployed.env_id, deployed.env_version), + timeout=self.timeout_seconds) # A broadcast that resolves to zero services would apply nothing while the # step reports success: the exact silent misconfig this step prevents. Fail # loud (tolerate_unadvertised governs advertisement, not missing servers). if not servers: raise ApplyServerConfigError( f"apply_server_config: broadcast directive matched no MCP servers or websites " - f"in env {self.env_id!r} (env exposes neither mcp_server_envs, website_envs nor a " - f"environment_name); nothing to apply, likely a task authoring error" + f"in env {self.env_id!r} (its card lists no child envs, and its definition names neither " + f"mcp_server_envs, website_envs nor an environment_name); nothing to apply, likely a task authoring error" ) - expanded: list[ConfigDirective] = [] - lenient: set[int] = set() + expanded: list[tuple[ConfigDirective, Optional[BackingServer], bool]] = [] for d in self.directives: if d.service != self._BROADCAST: - expanded.append(d) + named = [s for s in servers if s.environment_name == d.service] + expanded.append((d, next((s for s in named if s.mcp), named[0] if named else None), False)) continue for server in servers: - if not server.mcp: - lenient.add(len(expanded)) - expanded.append(ConfigDirective(service=server.environment_name, uri=d.uri, args=d.args)) - return expanded, {server.environment_name for server in servers if not server.mcp}, lenient + expanded.append((ConfigDirective(service=server.environment_name, uri=d.uri, args=d.args), server, not server.mcp)) + return expanded async def execute(self, context: TaskStepContext) -> TaskStepContext: deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) @@ -195,7 +189,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # network), so wrap it to keep the broadcast path's failure contract the # same ApplyServerConfigError as the per-directive HTTP calls below. try: - directives, websites, lenient = self._expand_directives(deployed) + directives = await self._expand_directives(deployed) except ApplyServerConfigError: raise except Exception as e: @@ -210,16 +204,17 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # multiple tools) against the same server. A None card marks a child env with no # card: a server predating env-card support, one whose card name differs, or one # whose card the deploy couldn't read. - cards: dict[str, tuple[str, Optional[dict]]] = {} - for position, directive in enumerate(directives): - tolerate = self.tolerate_unadvertised or position in lenient + cards: dict[Any, tuple[Optional[str], Optional[dict]]] = {} + for directive, server, lenient in directives: + tolerate = self.tolerate_unadvertised or lenient + key = directive.service if server is None else server try: - if directive.service not in cards: - cards[directive.service] = await legacy_protocol.child_env_card( - deployed, gateway_url_of(deployed), directive.service, timeout=self.timeout_seconds, - mcp=directive.service not in websites, - ) - base_url, card = cards[directive.service] + if key not in cards: + cards[key] = (await legacy_protocol.child_env_card(deployed, gateway_url_of(deployed), directive.service, + timeout=self.timeout_seconds) + if server is None else + await legacy_protocol.server_card(server, gateway_url_of(deployed), timeout=self.timeout_seconds)) + base_url, card = cards[key] if card is None: if not tolerate: raise ApplyServerConfigError( diff --git a/src/agent_env/task_step/task_steps/snapshot_env.py b/src/agent_env/task_step/task_steps/snapshot_env.py index 9415dac98..abb5573f3 100644 --- a/src/agent_env/task_step/task_steps/snapshot_env.py +++ b/src/agent_env/task_step/task_steps/snapshot_env.py @@ -475,36 +475,43 @@ def _resolve_loaded_universe_ref(deployed) -> Optional[tuple[str, Optional[int]] return None return (su["id"], su.get("version")) if su and su.get("id") else None - @classmethod - def _enumerate_environments(cls, env) -> list[str]: - """environment names for a multi env, a website env or a single MCP server env.""" - return [server.environment_name for server in cls._enumerate_servers(env)] - @staticmethod - def _enumerate_servers(env) -> list[BackingServer]: - """The servers to snapshot: the env's MCP servers and websites, a website sharing its name with an MCP - server left to it.""" - from agent_env.env import legacy_protocol - - servers = legacy_protocol.mcp_wins(legacy_protocol.backing_servers(env)) + def _snapshot_servers(servers: list[BackingServer], env) -> list[BackingServer]: + """The servers to snapshot of ``servers`` (``legacy_protocol.backing_servers``), one per environment_name: a + universe names each environment by it, and a load hands an environment of a name an MCP server and a website + share to the MCP server, so a server whose name one kept earlier holds (the env itself, else an MCP server) is + left out, and logged.""" if not servers: raise RuntimeError( f"Env '{env.id}' v{env.version} has no MCP services or websites to snapshot " - "(expected mcp_server_envs, website_envs or environment_name)" + "(its card lists no child envs, and its definition names no mcp_server_envs, website_envs or environment_name)" ) - return servers + if not all(server.environment_name for server in servers): + raise RuntimeError(f"Env '{env.id}' v{env.version} has a card naming no environment to snapshot") + kept: dict[str, BackingServer] = {} + for server in sorted(servers, key=lambda s: not s.mcp): + if (holder := kept.get(server.environment_name)) is not None: + logger.warning(f"snapshot_env: {'website' if not server.mcp else 'server'} {server.environment_name!r} " + f"of {env.id} is not snapshotted: it shares its name with the {'MCP server' if holder.mcp else 'website'} " + "kept, and a universe names an environment by its name") + continue + kept[server.environment_name] = server + return list(kept.values()) @staticmethod async def _export_environment_to_file( gateway_url: Optional[str], environment_name: str, tmp_path: str, timeout_seconds: float, deployed=None, - data_get_params: Optional[dict] = None, mcp: bool = True, + data_get_params: Optional[dict] = None, mcp: bool = True, server: Optional[BackingServer] = None, ) -> Optional[str | tuple[GetDataResponse, Optional[_SnapshotUpload]]]: """Export one service, an MCP server or with ``mcp=False`` a website; return the suffix written (".json"/".zip") to ``tmp_path``, or the object url a FilePart names when the service uploaded the bundle itself (nothing written to ``tmp_path`` then — export_one registers it), or, for an environment whose card declares a file tree, its data/get answer and the upload it answered under, which export_one keeps file by file, - or None for a website serving no v1 data plane. ``data_get_params`` go to its data/get. + or None for a service serving no v1 data plane: a website with no card, or a service whose card declares + operations without data/get. ``data_get_params`` go to its data/get. ``server`` is the service as the env's + card lists it (``legacy_protocol.backing_servers``), its card and route; without it, the service is looked up + by name on ``deployed``'s stored card, or probed live at its gateway path. v1 ``get_data``, handed a namespace grant when the service takes the data-objects form: an upload through it → its object url; ``DataPart`` → json; ``FilePart`` → @@ -513,12 +520,22 @@ async def _export_environment_to_file( guard. Errors are caught by the caller (fails just this service).""" from agent_env.env import legacy_protocol - base_url = await legacy_protocol.v1_base_url(deployed, gateway_url, environment_name, mcp=mcp) + if server is not None: + mcp = server.mcp + base_url = await legacy_protocol.server_v1_base_url(server, gateway_url) + else: + base_url = await legacy_protocol.v1_base_url(deployed, gateway_url, environment_name, mcp=mcp) + card_base, card = None, None + if base_url is not None: + card_base, card = (await legacy_protocol.server_card(server, gateway_url) if server is not None else + await legacy_protocol.child_env_card(deployed, gateway_url, environment_name, mcp=mcp)) + if card is not None and protocol_v1.serves(card, METHOD_GET) is False: + logger.info(f"snapshot_env: {environment_name}'s card declares {protocol_v1.operations(card)}, no {METHOD_GET}") + return None if base_url is None and not mcp: return None if base_url is not None: headers = deployed.control_headers_for(base_url) if deployed is not None else None - card_base, card = await legacy_protocol.child_env_card(deployed, gateway_url, environment_name, mcp=mcp) if returns_file_tree(card or {}): upload = await asyncio.to_thread(_snapshot_upload, card or {}, timeout_seconds, getattr(deployed, "sandbox_type", None), FILE_TREE_LIMITS) @@ -625,13 +642,12 @@ async def snapshot_env_state( A classmethod so callers that have no step instance can reuse it.""" from agent_env.artifact import FileArtifact, EnvironmentArtifact, EnvironmentUniverseArtifact + from agent_env.env import legacy_protocol from agent_env.env.env import gateway_url_of - from agent_env.env.legacy_protocol import BackingServer - - servers = cls._enumerate_servers(env) if gateway_url else [BackingServer(name) for name in _card_environments(deployed)] - services = [server.environment_name for server in servers] gateway = gateway_url.rstrip("/") if gateway_url else None + servers = cls._snapshot_servers(await legacy_protocol.backing_servers(deployed, gateway, env), env) + services = [server.environment_name for server in servers] # `gateway` identifies which deployment was exported: env= alone doesn't when one env_id has several. logger.info( f"snapshot_env: env={env.id} v{env.version} snapshot_id={snapshot_id} " @@ -665,7 +681,7 @@ async def snapshot_env_state( no_data_plane: set[str] = set() async def export_one(server: BackingServer): - """Returns EnvironmentArtifact on success, (public_err, log_err) on failure, None for a website serving no data plane.""" + """Returns EnvironmentArtifact on success, (public_err, log_err) on failure, None for a service serving no data plane.""" environment_name = server.environment_name website = {} if server.mcp else {"mcp": False} async with sem: @@ -678,7 +694,8 @@ async def export_one(server: BackingServer): try: result = await cls._export_environment_to_file( gateway, environment_name, tmp_path, export_timeout_seconds, - deployed=card_record, data_get_params=(data_get_params or {}).get(environment_name), **website, + deployed=card_record, data_get_params=(data_get_params or {}).get(environment_name), + server=server, **website, ) except httpx.HTTPStatusError as e: body_snippet = "" @@ -795,12 +812,12 @@ def _register_uploaded_bundle(): without_data_plane = [name for name in services if name in no_data_plane] if without_data_plane: logger.warning( - f"snapshot_env: websites {without_data_plane} of {env.id} serve no v1 data plane; " + f"snapshot_env: {without_data_plane} of {env.id} serve no v1 data plane; " "their state is not in the snapshot" ) empty = [name for name, result in zip(services, results) if result is None and name not in no_data_plane] if not environment_artifacts: - reasons = [f"its websites {without_data_plane} serve no v1 data plane"] if without_data_plane else [] + reasons = [f"{without_data_plane} serve no v1 data plane"] if without_data_plane else [] reasons += [f"{', '.join(map(repr, empty))} returned no files to snapshot"] if empty else [] raise RuntimeError(f"Env '{env.id}' v{env.version} exported no state: {'; '.join(reasons)}") if empty: @@ -883,25 +900,6 @@ def registered_upload(object_url: str, file_id: str, filename: str, content_type id=file_id, description=description, filename=filename, content_type=content_type, object_url=object_url)) -def _card_environments(deployed) -> list[str]: - """The environments a deployment without a gateway serves, as the card it recorded names them: the env itself when - it has no children; else each child its name reaches that exports, and the env too, first, when it exports beside - them, as a computer fronting its apps does.""" - card = deployed.environment_card or {} - children = card.get("children_environments") - if not isinstance(children, list): - names = [card.get("name")] - else: - names = [child.get("name") for child in children if deployed.get_child_env_card(child.get("name")) is child - and ((operations := protocol_v1.operations(child)) is None - or METHOD_GET in operations)] - if METHOD_GET in (protocol_v1.operations(card) or []): - names.insert(0, card.get("name")) - if not all(names): - raise RuntimeError(f"Deployed env '{deployed.env_id}' recorded a card naming no environment to snapshot") - return names - - def _changes_route(deployed, override_url: Optional[str]) -> tuple[str, str]: """The verb and URL of the env's changes: an override keeps `{override}/changes`, else the stored card names them.""" if override_url: diff --git a/src/agent_env/task_step/task_steps/sync_env_clock.py b/src/agent_env/task_step/task_steps/sync_env_clock.py index c80cef8a8..8ae463f56 100644 --- a/src/agent_env/task_step/task_steps/sync_env_clock.py +++ b/src/agent_env/task_step/task_steps/sync_env_clock.py @@ -1,9 +1,10 @@ """Arm a deployed env gateway's virtual clock, then sync its backing servers to it. Arms the clock (set_time) and reads the gateway's server-reachable env_get_time_url (state), both as the -stored env card declares clock/v1, then invokes sync_time on each backing server, MCP server or website, -whose child env card advertises it. Non-advertisers are skipped when tolerate_missing_sync_time (default -True), and websites always. Place right before prompt_agent so the virtual timeline starts at agent-start. +stored env card declares clock/v1, then invokes sync_time on each backing server, MCP server or website, the +env card lists (``legacy_protocol.backing_servers``) whose own card advertises it, at the route that card gives. +Non-advertisers are skipped when tolerate_missing_sync_time (default True), and websites always. Place right +before prompt_agent so the virtual timeline starts at agent-start. """ from __future__ import annotations @@ -110,15 +111,15 @@ def from_dict(cls, data: dict) -> SyncEnvClockTaskStep: timezone=data.get("timezone"), ) - def _backing_servers(self, deployed) -> list[BackingServer]: - """The env's backing MCP servers and websites (``legacy_protocol.backing_servers``). A website sharing its - name with an MCP server has its card read live, since the stored env card can't tell the two apart.""" + async def _backing_servers(self, deployed) -> list[BackingServer]: + """The env's backing MCP servers and websites, as its card lists them (``legacy_protocol.backing_servers``): + a website sharing its name with an MCP server is a server of its own, at its own route.""" from agent_env.env import legacy_protocol - from agent_env.env.env import Env - return legacy_protocol.backing_servers(Env.get(deployed.env_id, deployed.env_version)) + from agent_env.env.env import Env, gateway_url_of + return await legacy_protocol.backing_servers(deployed, gateway_url_of(deployed), Env.get(deployed.env_id, deployed.env_version), + timeout=self.timeout_seconds) async def execute(self, context: TaskStepContext) -> TaskStepContext: - from agent_env.env import legacy_protocol from agent_env.env.env import gateway_url_of deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) @@ -133,15 +134,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: state = await deployed.invoke(EXT_CLOCK_URI, "state", timeout=self.timeout_seconds) env_get_time_url = state.get("env_get_time_url") - servers = self._backing_servers(deployed) - shared = legacy_protocol.shared_names(servers) synced, skipped = [], [] - for server in servers: + for server in await self._backing_servers(deployed): service = server.environment_name - record = None if service in shared and not server.mcp else deployed - outcome = await self._sync_one(record, gateway_url_of(deployed), service, env_get_time_url, mcp=server.mcp, - tolerate=self.tolerate_missing_sync_time or not server.mcp, - headers=deployed.control_headers) + outcome = await self._sync_one(deployed, gateway_url_of(deployed), service, env_get_time_url, mcp=server.mcp, + tolerate=self.tolerate_missing_sync_time or not server.mcp, server=server) entry = {"service": service, "environment": service, **({} if server.mcp else {"website": True}), **outcome} (synced if outcome.get("synced") else skipped).append(entry) @@ -161,17 +158,20 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time_url: Optional[str], mcp: bool = True, - tolerate: Optional[bool] = None, headers: Optional[dict] = None) -> dict: + tolerate: Optional[bool] = None, server: Optional[BackingServer] = None) -> dict: """Invoke sync_time on one server, an MCP server or with ``mcp=False`` a website, if its card advertises clock/v1; skip a server with no card or no clock/v1 when ``tolerate`` (tolerate_missing_sync_time by default), else raise. ``execute`` tolerates every website, so one that doesn't follow the clock never fails the step; one - that follows it but can't take the task's timezone does, as an MCP server does. ``headers`` are the env's control - headers, ``deployed``'s by default; ``execute`` passes them because it hands a website that shares its name with - an MCP server no record, so its card is read live.""" + that follows it but can't take the task's timezone does, as an MCP server does. ``server`` is the server as the + env's card lists it (``legacy_protocol.server_card``); without it, the card is ``service``'s on ``deployed``'s + stored card, or read live at its gateway path.""" tolerate = self.tolerate_missing_sync_time if tolerate is None else tolerate from agent_env.env import legacy_protocol - base_url, card = await legacy_protocol.child_env_card(deployed, gateway_url, service, timeout=self.timeout_seconds, mcp=mcp) + if server is not None: + base_url, card = await legacy_protocol.server_card(server, gateway_url, timeout=self.timeout_seconds) + else: + base_url, card = await legacy_protocol.child_env_card(deployed, gateway_url, service, timeout=self.timeout_seconds, mcp=mcp) if card is None: if tolerate: return {"synced": False, "reason": "no_env_card"} @@ -193,5 +193,5 @@ async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time result = await protocol_v1.invoke_extension(base_url, card, EXT_CLOCK_URI, params=params, timeout=self.timeout_seconds, - headers=deployed.control_headers if headers is None else headers) + headers=deployed.control_headers) return {"synced": True, "result": result} diff --git a/tst/unit/env/test_legacy_protocol.py b/tst/unit/env/test_legacy_protocol.py index 7d107ad34..ef8fee3f8 100644 --- a/tst/unit/env/test_legacy_protocol.py +++ b/tst/unit/env/test_legacy_protocol.py @@ -179,7 +179,7 @@ async def test_a_v1_services_state_is_the_data_it_answers_with_else_its_export_s assert ("GET /svc/mcp-slack/export-state" in asked) is reads_export_state -def test_backing_servers_are_an_envs_mcp_servers_then_its_websites(): +def test_definition_servers_are_an_envs_mcp_servers_then_its_websites(): from types import SimpleNamespace from agent_env.env.envs.website import WebsiteEnv @@ -188,11 +188,93 @@ def test_backing_servers_are_an_envs_mcp_servers_then_its_websites(): named = lambda *names: [SimpleNamespace(environment_name=n) for n in names] multi = SimpleNamespace(mcp_server_envs=named("slack", "email"), website_envs=named("shop", "slack")) - assert legacy_protocol.backing_servers(multi) == [BackingServer("slack"), BackingServer("email"), - BackingServer("shop", mcp=False), BackingServer("slack", mcp=False)] - assert legacy_protocol.shared_names(legacy_protocol.backing_servers(multi)) == {"slack"} - assert legacy_protocol.mcp_wins(legacy_protocol.backing_servers(multi)) == [ - BackingServer("slack"), BackingServer("email"), BackingServer("shop", mcp=False)] - assert legacy_protocol.backing_servers(WebsiteEnv(id="w", version=1, environment_name="shop")) == [BackingServer("shop", mcp=False)] - assert legacy_protocol.backing_servers(SimpleNamespace(environment_name="notes")) == [BackingServer("notes")] - assert legacy_protocol.backing_servers(SimpleNamespace()) == [] + assert legacy_protocol.definition_servers(multi) == [BackingServer("slack"), BackingServer("email"), + BackingServer("shop", mcp=False), BackingServer("slack", mcp=False)] + assert legacy_protocol.definition_servers(WebsiteEnv(id="w", version=1, environment_name="shop")) == [BackingServer("shop", mcp=False)] + assert legacy_protocol.definition_servers(SimpleNamespace(environment_name="notes")) == [BackingServer("notes")] + assert legacy_protocol.definition_servers(SimpleNamespace()) == [] + + +_COMPOSED = {"name": "gw", "url": RPC_PATH, "capabilities": {"extensions": []}, "children_environments": [ + {"name": "slack", "url": "/svc/mcp-slack/agentenv", "capabilities": {"extensions": []}}, + {"name": "slack", "url": "/svc/slack/agentenv", "capabilities": {"extensions": []}}, + {"name": "docs", "url": "/svc/mcp-docs/agentenv", "capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1"}]}}, +]} + + +def test_card_servers_are_the_cards_children_a_website_told_apart_by_its_route_or_its_website_extension(): + servers = legacy_protocol.card_servers(_COMPOSED, "https://env.example") + + assert [(s.environment_name, s.mcp, s.address) for s in servers] == [ + ("slack", True, "https://env.example"), ("slack", False, "https://env.example"), ("docs", False, "https://env.example")] + assert [s.child.base_url(s.address) for s in servers] == [ + "https://env.example/svc/mcp-slack", "https://env.example/svc/slack", "https://env.example/svc/mcp-docs"] + + +def test_card_servers_of_a_leaf_card_is_the_env_itself_and_a_composed_card_serving_data_get_comes_first(): + leaf = {"name": "notes", "url": RPC_PATH} + assert [(s.environment_name, s.card) for s in legacy_protocol.card_servers(leaf, "https://n")] == [("notes", leaf)] + driver = {**_COMPOSED, "name": "desktop", "capabilities": {"operations": ["data/get"]}} + assert [s.environment_name for s in legacy_protocol.card_servers(driver, "https://d")] == ["desktop", "slack", "slack", "docs"] + + +def _record(card: dict | None, card_url: str | None = "https://env.example/.well-known/agent-env.json"): + from agent_env.env.env import DeployedGatewayEnv + + return DeployedGatewayEnv(env_id="e", env_version=1, gateway_url="http://gw", mcp_url="http://gw/mcp", db_web_url=None, + sandbox_id="sb", environment_card_url=card_url, environment_card=card) + + +@pytest.mark.asyncio +async def test_backing_servers_are_the_stored_cards_then_those_the_definition_names_serving_no_card(): + from types import SimpleNamespace + + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name=n) for n in ("slack", "legacy")], + website_envs=[SimpleNamespace(environment_name="slack")]) + + servers = await legacy_protocol.backing_servers(_record(_COMPOSED), "http://gw", env) + + assert [(s.environment_name, s.mcp, s.card is not None, s.address) for s in servers] == [ + ("slack", True, True, "https://env.example"), ("slack", False, True, "https://env.example"), + ("docs", False, True, "https://env.example"), ("legacy", True, False, "https://env.example")] + assert await legacy_protocol.server_card(servers[-1], "http://gw") == ("https://env.example", None) + assert await legacy_protocol.server_v1_base_url(servers[-1], "http://gw") is None + + +@pytest.mark.asyncio +async def test_backing_servers_read_the_card_live_without_a_stored_one_or_for_another_gateway(monkeypatch): + from types import SimpleNamespace + + read = [] + + async def get_card(base_url, **kw): + read.append(base_url) + return _COMPOSED + + monkeypatch.setattr(legacy_protocol.protocol_v1, "get_card", get_card) + env = SimpleNamespace(environment_name="slack") + for record, gateway in ((_record(None), "http://gw/"), (_record(_COMPOSED), "http://elsewhere")): + servers = await legacy_protocol.backing_servers(record, gateway, env) + assert [(s.environment_name, s.address) for s in servers] == [ + ("slack", gateway.rstrip("/")), ("slack", gateway.rstrip("/")), ("docs", gateway.rstrip("/"))] + assert read == ["http://gw", "http://elsewhere"] + + +@pytest.mark.asyncio +async def test_a_record_whose_card_cant_be_read_falls_back_to_its_definition_logged_once(monkeypatch, caplog): + from types import SimpleNamespace + + async def get_card(base_url, **kw): + raise httpx.ConnectError("refused") + + monkeypatch.setattr(legacy_protocol.protocol_v1, "get_card", get_card) + env = SimpleNamespace(id="e", mcp_server_envs=[SimpleNamespace(environment_name="slack")], + website_envs=[SimpleNamespace(environment_name="slack")]) + with caplog.at_level("INFO", logger="agent_env.env.legacy_protocol"): + for record in (_record(None, card_url=None), _record(None), None): + servers = await legacy_protocol.backing_servers(record, "http://gw", env) + assert [(s.environment_name, s.mcp, s.address) for s in servers] == [("slack", True, None), ("slack", False, None)] + + assert [r.getMessage().split(",")[0] for r in caplog.records] == [ + "env 'e': its record has no card URL", "env 'e': its card at http://gw could not be read (ConnectError)", + "env 'e': it has no record"] diff --git a/tst/unit/task/local_run_write_fence_test.py b/tst/unit/task/local_run_write_fence_test.py index d3f60293e..108709d94 100644 --- a/tst/unit/task/local_run_write_fence_test.py +++ b/tst/unit/task/local_run_write_fence_test.py @@ -264,7 +264,7 @@ def __init__(self, id, version=None): self.mcp_server_envs = [SimpleNamespace(environment_name="gmail")] -async def _export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): +async def _export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): Path(tmp_path).write_text(json.dumps({"service": name})) return ".json" diff --git a/tst/unit/task_step/sync_env_clock_test.py b/tst/unit/task_step/sync_env_clock_test.py index 5b83c51d7..a9a109e2a 100644 --- a/tst/unit/task_step/sync_env_clock_test.py +++ b/tst/unit/task_step/sync_env_clock_test.py @@ -1,11 +1,11 @@ """Unit tests for SyncEnvClockTaskStep — round-trip + the per-server sync_time fan-out (mocked).""" import json +from types import SimpleNamespace import pytest from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, EnvCapabilityUnsupported from agent_env.env.gateway.constants import EXT_CLOCK_URI -from agent_env.env.legacy_protocol import BackingServer from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps import sync_env_clock as mod from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep, resolve_virtual_time @@ -95,7 +95,7 @@ async def test_sync_one_skips_or_raises_for_a_child_env_missing_from_the_stored_ @pytest.mark.asyncio async def test_execute_syncs_each_child_env_through_the_stored_card(monkeypatch): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) + monkeypatch.setattr("agent_env.env.env.Env.get", lambda *a, **k: SimpleNamespace(environment_name="email")) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))]) await _step().execute(context) @@ -176,21 +176,64 @@ async def fake_card(base_url, **kw): @pytest.mark.asyncio -async def test_execute_reads_a_website_sharing_an_mcp_servers_name_live(monkeypatch): - from types import SimpleNamespace +async def test_execute_syncs_a_website_sharing_an_mcp_servers_name_at_its_own_route(monkeypatch): + """The card lists the two as children at distinct urls, so each is synced by its own card, with no card read.""" from unittest.mock import patch sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], website_envs=[SimpleNamespace(environment_name="email")]) - context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))]) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT], "email": [_CLOCK_EXT]}))]) with patch("agent_env.env.env.Env.get", return_value=env): await _step().execute(context) assert [(r.method, str(r.url)) for r in sent][2:] == [ - ("POST", f"{_CARD}/svc/mcp-email/agentenv/ext/sync_time"), ("GET", "http://gw/svc/email/.well-known/agent-env.json"), + ("POST", f"{_CARD}/svc/mcp-email/agentenv/ext/sync_time"), ("POST", f"{_CARD}/svc/email/agentenv/ext/sync_time"), ] + record = context.metadata["clock_configurations"][0] + assert [(s["service"], s.get("website")) for s in record["synced"]] == [("email", None), ("email", True)] + + +@pytest.mark.asyncio +async def test_execute_without_a_card_url_syncs_the_servers_the_definition_names_read_live(monkeypatch): + """An older record kept no card URL, so its servers come from its env definition and each card is read live at + its gateway path.""" + from unittest.mock import patch + + import httpx + + sent, real = [], httpx.AsyncClient + cards = {"/svc/mcp-email/.well-known/agent-env.json": {"name": "email", "capabilities": {"extensions": [_CLOCK_EXT]}}, + "/svc/shop/.well-known/agent-env.json": {"name": "shop", "capabilities": {"extensions": []}}} + + def handle(request: httpx.Request) -> httpx.Response: + sent.append(request) + if request.url.path == "/clock/state": + return httpx.Response(200, json={"env_get_time_url": CLOCK_URL}) + return httpx.Response(200, json=cards.get(request.url.path, {})) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + record = _cardless() + + async def invoke(uri, method, params=None, **kw): + return {"env_get_time_url": CLOCK_URL} if method == "state" else {} + + record.invoke = invoke + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], + website_envs=[SimpleNamespace(environment_name="shop")]) + context = TaskStepContext(deployed_envs=[record]) + + with patch("agent_env.env.env.Env.get", return_value=env): + await _step().execute(context) + + assert [(r.method, str(r.url)) for r in sent] == [ + ("GET", "http://gw/svc/mcp-email/.well-known/agent-env.json"), ("POST", "http://gw/svc/mcp-email/agentenv/ext/sync_time"), + ("GET", "http://gw/svc/shop/.well-known/agent-env.json"), + ] + cfg = context.metadata["clock_configurations"][0] + assert [s["service"] for s in cfg["synced"]] == ["email"] + assert [(s["service"], s["reason"]) for s in cfg["skipped"]] == [("shop", "not_advertised")] @pytest.mark.asyncio @@ -317,7 +360,7 @@ def test_without_a_zone_virtual_time_is_passed_through_untouched(): @pytest.mark.asyncio async def test_execute_arms_the_gateway_at_the_local_instant_and_tells_every_server_the_zone(monkeypatch): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) + monkeypatch.setattr("agent_env.env.env.Env.get", lambda *a, **k: SimpleNamespace(environment_name="email")) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_ZONE_EXT]}))]) await _zoned().execute(context) @@ -332,7 +375,7 @@ async def test_execute_arms_the_gateway_at_the_local_instant_and_tells_every_ser @pytest.mark.parametrize("tolerate", [True, False]) async def test_a_server_that_cannot_take_the_zone_fails_the_step_before_any_sync_is_sent(monkeypatch, tolerate): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) + monkeypatch.setattr("agent_env.env.env.Env.get", lambda *a, **k: SimpleNamespace(environment_name="email")) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_OLD_EXT]}))]) with pytest.raises(RuntimeError, match="takes no timezone"): @@ -378,34 +421,19 @@ async def test_a_zoned_step_tells_a_website_that_takes_the_zone_and_fails_on_one @pytest.mark.asyncio async def test_a_website_sharing_an_mcp_servers_name_is_synced_with_the_envs_control_headers(monkeypatch): - """Its card is read live, with no record, and its sync_time still carries the env's control token.""" import dataclasses - from types import SimpleNamespace from unittest.mock import patch - import httpx - - sent, real = [], httpx.AsyncClient - website_card = {"name": "email", "url": "/agentenv", "capabilities": {"extensions": [_CLOCK_EXT]}} - - def handle(request: httpx.Request) -> httpx.Response: - sent.append(request) - if request.url.path.endswith("/clock/state"): - return httpx.Response(200, json={"env_get_time_url": CLOCK_URL}) - if request.url.path == "/svc/email/.well-known/agent-env.json": - return httpx.Response(200, json=website_card) - return httpx.Response(200, json={}) - - monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], website_envs=[SimpleNamespace(environment_name="email")]) - record = dataclasses.replace(_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]})), control_token="ctl") + record = dataclasses.replace(_carded(await _composed_card({"mcp-email": [_CLOCK_EXT], "email": [_CLOCK_EXT]})), control_token="ctl") context = TaskStepContext(deployed_envs=[record]) with patch("agent_env.env.env.Env.get", return_value=env): await _step().execute(context) - website_sync = [r for r in sent if str(r.url) == "http://gw/svc/email/agentenv/ext/sync_time"] + website_sync = [r for r in sent if str(r.url) == f"{_CARD}/svc/email/agentenv/ext/sync_time"] assert len(website_sync) == 1 assert {k: website_sync[0].headers.get(k) for k in record.control_headers} == record.control_headers assert [s["service"] for s in context.metadata["clock_configurations"][0]["synced"]] == ["email", "email"] diff --git a/tst/unit/task_step/test_apply_server_config.py b/tst/unit/task_step/test_apply_server_config.py index ddeb96806..94ed35986 100644 --- a/tst/unit/task_step/test_apply_server_config.py +++ b/tst/unit/task_step/test_apply_server_config.py @@ -609,18 +609,38 @@ async def get(self, url, *, timeout=None, params=None): @pytest.mark.asyncio -async def test_stored_card_broadcast_reaches_websites_and_leaves_a_shared_name_to_the_mcp_server(): +async def test_stored_card_broadcast_reaches_websites_and_a_website_sharing_an_mcp_servers_name(): + """The card lists a website and an MCP server sharing a name as two children at distinct urls, so a broadcast + reaches both, each at its own route; a directive naming the name means the MCP server.""" gw = _MockGateway({}) ext = _ext(_SET_ACTING_USER_URI, "/agentenv/ext/set_acting_user") - ctx = _carded_context(await _composed_card({"mcp-slack": [ext], "shop": [ext]})) - step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}]) + ctx = _carded_context(await _composed_card({"mcp-slack": [ext], "shop": [ext], "slack": [ext]})) + step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}, + {"service": "slack", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "b@x.com"}}]) with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop", "slack"])): out = await step.execute(ctx) + assert gw.get_calls == [] + assert gw.post_calls == [(f"{_CARD}/svc/mcp-slack/agentenv/ext/set_acting_user", {"user_email": "a@x.com"}), + (f"{_CARD}/svc/shop/agentenv/ext/set_acting_user", {"user_email": "a@x.com"}), + (f"{_CARD}/svc/slack/agentenv/ext/set_acting_user", {"user_email": "a@x.com"}), + (f"{_CARD}/svc/mcp-slack/agentenv/ext/set_acting_user", {"user_email": "b@x.com"})] + assert [c["service"] for c in out.metadata["server_config_changes"]] == ["slack", "shop", "slack", "slack"] + + +@pytest.mark.asyncio +async def test_stored_card_broadcast_to_an_mcp_only_env_is_unchanged(): + gw = _MockGateway({}) + ext = _ext(_SET_ACTING_USER_URI, "/agentenv/ext/set_acting_user") + ctx = _carded_context(await _composed_card({"mcp-slack": [ext], "mcp-email": [ext]})) + step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}]) + with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=_fake_env("slack", "email")): + out = await step.execute(ctx) + assert gw.get_calls == [] assert [url for url, _ in gw.post_calls] == [f"{_CARD}/svc/mcp-slack/agentenv/ext/set_acting_user", - f"{_CARD}/svc/shop/agentenv/ext/set_acting_user"] - assert [c["service"] for c in out.metadata["server_config_changes"]] == ["slack", "shop"] + f"{_CARD}/svc/mcp-email/agentenv/ext/set_acting_user"] + assert out.metadata["server_config_skipped"] == [] @pytest.mark.asyncio diff --git a/tst/unit/task_step/test_snapshot_env.py b/tst/unit/task_step/test_snapshot_env.py index 380690fc2..cf75fb7ca 100644 --- a/tst/unit/task_step/test_snapshot_env.py +++ b/tst/unit/task_step/test_snapshot_env.py @@ -14,6 +14,7 @@ from agent_env.artifact import EnvironmentUniverseArtifact from agent_env.artifact.store import reset_artifact_store from agent_env.config import set_object_store +from agent_env.env import legacy_protocol from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, DeployedSandboxEnv from agent_env.env.gateway.constants import EXT_STATE_URI, GATEWAY_EXTENSIONS, WELL_KNOWN_PATH from agent_env.task_step.context import TaskStepContext @@ -33,7 +34,7 @@ def _deployed(env_id="env-1", instance_id="env-1-abc123", gateway="https://gw-1" sandbox_id="sb-1", instance_id=instance_id, environment_card_url=f"{gateway}{WELL_KNOWN_PATH}", - environment_card={"capabilities": {"extensions": GATEWAY_EXTENSIONS}}, + environment_card={"capabilities": {"extensions": GATEWAY_EXTENSIONS}, "children_environments": []}, ) @@ -565,7 +566,7 @@ def __init__(self, names): @pytest.mark.asyncio async def test_core_all_or_nothing_on_partial_failure(monkeypatch): - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): if name == "slack": raise ValueError("export-state returned non-JSON body (starts with b'<')") with open(tmp_path, "w") as f: @@ -609,7 +610,7 @@ def put(id, *, environment_name, file_artifact): @pytest.mark.asyncio async def test_core_success_creates_universe_with_metadata(monkeypatch): - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): with open(tmp_path, "w") as f: json.dump({"ok": name}, f) return ".json" @@ -673,7 +674,7 @@ async def test_core_zip_export_creates_zip_filename_artifact(monkeypatch): """A FilePart(zip) export flows through as a .zip-suffixed FileArtifact path, so FileArtifact.filename (= basename) ends in .zip — the load side routes a .zip file:// artifact to reset_data() for a lossless round-trip.""" - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): # Simulate the lossless-zip case: write zip bytes and report ".zip". with open(tmp_path, "wb") as f: f.write(b"PK\x03\x04zip-bytes") @@ -911,7 +912,7 @@ async def test_core_registers_a_service_upload_in_place_and_it_loads_back(local_ bundle_file.write_bytes(b"PK\x03\x04bundle") uploaded = store.put_file_at(upload_url, str(bundle_file)) - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): return uploaded monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) @@ -933,7 +934,7 @@ async def test_core_fails_a_service_whose_upload_the_store_cannot_find(local_sto set_object_store(FakeObjectStore()) reset_artifact_store() - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): return "fake://shared/snapshots/missing.zip" monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) @@ -950,7 +951,7 @@ async def test_core_snapshots_under_an_id_longer_than_a_filename_can_be(local_st local store may be the whole id.""" snapshot_id = f"task-{'x' * 250}__snapshot-snap-1" - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): with open(tmp_path, "w") as f: json.dump({"service": name}, f) return ".json" @@ -1044,14 +1045,16 @@ async def test_export_of_a_child_env_missing_from_the_card_streams_legacy_export @pytest.mark.asyncio async def test_snapshot_uses_the_stored_card_only_for_the_deployments_own_gateway(monkeypatch): + """An override's gateway has its card read live, and the record's stored card describes its own gateway only.""" seen = [] - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): - seen.append((gateway_url, deployed)) + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, server=None, **_): + seen.append((gateway_url, deployed, server.address, server.card and server.card["url"])) raise ValueError("stop after the export call") monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) - record = _carded({"name": "gw", "children_environments": []}) + sent = _mock_http(monkeypatch, body=json.dumps(await _card({"mcp-slack": []})).encode()) + record = _carded(await _card({"mcp-slack": []})) record.gateway_url = "https://gw-1/" for gateway_url in ("https://gw-1/", "https://elsewhere"): with pytest.raises(RuntimeError, match="not exported"): @@ -1059,7 +1062,9 @@ async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_ env=_FakeMultiEnv(["slack"]), gateway_url=gateway_url, snapshot_id="snap-x", deployed=record, ) - assert seen == [("https://gw-1", record), ("https://elsewhere", None)] + assert seen == [("https://gw-1", record, _CARD, "/svc/mcp-slack/agentenv"), + ("https://elsewhere", None, "https://elsewhere", "/svc/mcp-slack/agentenv")] + assert [str(r.url) for r in sent] == [f"https://elsewhere{WELL_KNOWN_PATH}"] @pytest.mark.asyncio @@ -1081,19 +1086,26 @@ async def test_export_with_a_record_that_has_no_stored_card_keeps_the_live_probe ] -def test_enumerate_services_single_mcp_server_env(): +@pytest.mark.asyncio +async def test_a_record_without_a_card_snapshots_the_servers_its_definition_names(): env = _FakeServiceEnv("calendar") env.id, env.version = "cal-env", 1 - assert SnapshotEnvTaskStep._enumerate_environments(env) == ["calendar"] + servers = await legacy_protocol.backing_servers(_deployed_without_a_card(), "https://gw-1", env) + assert [(s.environment_name, s.mcp, s.address) for s in SnapshotEnvTaskStep._snapshot_servers(servers, env)] == [ + ("calendar", True, None)] -def test_enumerate_services_no_services_raises(): +def test_snapshot_servers_refuses_an_env_with_none(): env = type("E", (), {"id": "x", "version": 1})() with pytest.raises(RuntimeError, match="no MCP services"): - SnapshotEnvTaskStep._enumerate_environments(env) + SnapshotEnvTaskStep._snapshot_servers(legacy_protocol.definition_servers(env), env) -# ── loaded-universe metadata carry-over ────────────────────────────────────── +def _deployed_without_a_card() -> DeployedEnv: + """An older record: no card URL, so its servers are the ones its definition names.""" + return DeployedGatewayEnv(env_id="env-1", env_version=3, gateway_url="https://gw-1", mcp_url="https://gw-1/mcp", + db_web_url=None, sandbox_id="sb-1") + def test_resolve_loaded_universe_ref_reads_store(monkeypatch): import agent_env.env.store as store_mod @@ -1139,7 +1151,7 @@ def get_environment_universe(self, instance_id): def _install_core_artifact_fakes(monkeypatch, universe_cls): """Wire the export + FileArtifact/EnvironmentArtifact fakes shared by the core carry-over tests; the caller supplies the EnvironmentUniverseArtifact fake.""" - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): with open(tmp_path, "w") as f: json.dump({"ok": name}, f) return ".json" @@ -1199,7 +1211,7 @@ def get_environment_universe(self, instance_id): await SnapshotEnvTaskStep.snapshot_env_state( env=_FakeMultiEnv(["slack"]), - gateway_url="https://gw", + gateway_url="https://gw-1", snapshot_id="snap-x", deployed=_deployed(instance_id="env-1-abc123"), ) @@ -1240,7 +1252,7 @@ def get_environment_universe(self, instance_id): await SnapshotEnvTaskStep.snapshot_env_state( original_universe_artifact_id="explicit-base", env=_FakeMultiEnv(["slack"]), - gateway_url="https://gw", + gateway_url="https://gw-1", snapshot_id="snap-x", deployed=_deployed(), ) @@ -1276,7 +1288,7 @@ def get_environment_universe(self, instance_id): result = await SnapshotEnvTaskStep.snapshot_env_state( env=_FakeMultiEnv(["slack"]), - gateway_url="https://gw", + gateway_url="https://gw-1", snapshot_id="snap-x", deployed=_deployed(), ) @@ -1302,7 +1314,7 @@ def get(artifact_id, version=None): await SnapshotEnvTaskStep.snapshot_env_state( original_universe_artifact_id="bad", env=_FakeMultiEnv(["slack"]), - gateway_url="https://gw", + gateway_url="https://gw-1", snapshot_id="snap-x", deployed=_deployed(), ) @@ -1376,17 +1388,103 @@ def _multi_env(mcp_names, website_names): website_envs=[_FakeServiceEnv(n) for n in website_names]) -def test_enumerate_includes_websites_and_leaves_a_shared_name_to_the_mcp_server(): - servers = SnapshotEnvTaskStep._enumerate_servers(_multi_env(["slack", "gmail"], ["shop", "slack"])) - assert [(s.environment_name, s.mcp) for s in servers] == [("slack", True), ("gmail", True), ("shop", False)] +@pytest.mark.asyncio +async def test_the_servers_snapshotted_are_the_cards_children_one_per_name_and_those_serving_no_card(caplog): + """A website sharing an MCP server's name is its own server on the card, left out of the snapshot (logged) since a + universe names an environment by its name; a server serving no card, which the card leaves off, keeps its legacy + export.""" + record = _carded(await _card({"mcp-slack": [], "mcp-gmail": [], "shop": [], "slack": []})) + env = _multi_env(["slack", "gmail", "legacy"], ["shop", "slack"]) + + listed = await legacy_protocol.backing_servers(record, "https://gw-1", env) + with caplog.at_level("WARNING"): + servers = SnapshotEnvTaskStep._snapshot_servers(listed, env) + + assert [(s.environment_name, s.mcp, s.card and s.card["url"]) for s in listed] == [ + ("slack", True, "/svc/mcp-slack/agentenv"), ("gmail", True, "/svc/mcp-gmail/agentenv"), + ("shop", False, "/svc/shop/agentenv"), ("slack", False, "/svc/slack/agentenv"), ("legacy", True, None)] + assert [(s.environment_name, s.mcp) for s in servers] == [("slack", True), ("gmail", True), ("legacy", True), ("shop", False)] + assert "website 'slack' of env-1 is not snapshotted: it shares its name with the MCP server kept" in caplog.text -def test_enumerate_a_single_website_env_and_a_multi_env_of_websites_only(): +@pytest.mark.asyncio +async def test_a_single_website_env_and_a_multi_env_of_websites_only_are_snapshotted_as_websites(): from agent_env.env.envs.website import WebsiteEnv site = WebsiteEnv(id="site-env", version=1, environment_name="shop") - assert [(s.environment_name, s.mcp) for s in SnapshotEnvTaskStep._enumerate_servers(site)] == [("shop", False)] - assert SnapshotEnvTaskStep._enumerate_environments(_multi_env([], ["shop"])) == ["shop"] + assert [(s.environment_name, s.mcp) for s in legacy_protocol.definition_servers(site)] == [("shop", False)] + record = _carded(await _card({"shop": []})) + assert [(s.environment_name, s.mcp) for s in await legacy_protocol.backing_servers(record, "https://gw-1", _multi_env([], ["shop"]))] == [ + ("shop", False)] + + +@pytest.mark.asyncio +async def test_a_card_only_website_is_left_out_as_serving_no_v1_data_plane(monkeypatch, tmp_path): + """Its card declares no data/get, so it is skipped with no request, not failed on a data/get it doesn't serve.""" + record = _carded(await _card({"mcp-slack": [], "shop": []})) + record.environment_card["children_environments"][1]["capabilities"]["operations"] = [] + (shop,) = [s for s in await legacy_protocol.backing_servers(record, "https://gw-1", _multi_env(["slack"], ["shop"])) + if s.environment_name == "shop"] + sent = _mock_http(monkeypatch) + + assert await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", str(tmp_path / "out"), 30, record, + server=shop) is None + assert sent == [] + + +@pytest.mark.asyncio +async def test_a_card_only_website_is_reported_as_serving_no_v1_data_plane(monkeypatch): + real_export = SnapshotEnvTaskStep.__dict__["_export_environment_to_file"] + _install_core_artifact_fakes(monkeypatch, EnvironmentUniverseArtifact) + monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", real_export) + record = _carded(await _card({"shop": []})) + record.environment_card["children_environments"][0]["capabilities"]["operations"] = ["data/reset"] + sent = _mock_http(monkeypatch) + + with pytest.raises(RuntimeError, match=r"exported no state: \['shop'\] serve no v1 data plane"): + await SnapshotEnvTaskStep.snapshot_env_state(env=_multi_env([], ["shop"]), gateway_url="https://gw-1", + snapshot_id="snap-x", deployed=record) + assert sent == [] + + +@pytest.mark.asyncio +async def test_an_mcp_only_env_snapshots_each_server_at_its_route_and_one_serving_no_card_by_export_state(monkeypatch): + real_export = SnapshotEnvTaskStep.__dict__["_export_environment_to_file"] + universe_puts = {} + + class _FakeUniverse: + id, version = "snap-x", 1 + + @staticmethod + def put(id, *, environment_artifacts, metadata=None): + universe_puts["names"] = [sa.environment_name for sa in environment_artifacts] + return _FakeUniverse() + + _install_core_artifact_fakes(monkeypatch, _FakeUniverse) + monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", real_export) + sent = _mock_http(monkeypatch, rpc_result={"parts": [{"kind": "data", "data": {"rows": 1}}]}, body=b'{"rows": 2}') + record = _carded(await _card({"mcp-slack": [], "mcp-gmail": []})) + + result = await SnapshotEnvTaskStep.snapshot_env_state(env=_FakeMultiEnv(["slack", "gmail", "legacy"]), gateway_url="https://gw-1", + snapshot_id="snap-x", deployed=record) + + assert sorted((r.method, str(r.url)) for r in sent) == [ + ("GET", "https://gw-1/svc/mcp-legacy/export-state"), ("POST", f"{_CARD}/svc/mcp-gmail/agentenv"), + ("POST", f"{_CARD}/svc/mcp-slack/agentenv")] + assert universe_puts["names"] == ["slack", "gmail", "legacy"] and result.total == 3 + + +@pytest.mark.asyncio +async def test_a_website_sharing_an_mcp_servers_name_is_exported_at_its_own_route(monkeypatch, tmp_path): + """The card tells the two apart by their urls, so the website's data/get goes to /svc/slack, not the MCP server.""" + record = _carded(await _card({"mcp-slack": [], "slack": []})) + servers = await legacy_protocol.backing_servers(record, "https://gw-1", _multi_env(["slack"], ["slack"])) + sent = _mock_http(monkeypatch, rpc_result={"parts": [{"kind": "data", "data": {"orders": 2}}]}) + + for server in servers: + assert await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "slack", str(tmp_path / "out"), 30, record, + server=server) == ".json" + assert [str(r.url) for r in sent] == [f"{_CARD}/svc/mcp-slack/agentenv", f"{_CARD}/svc/slack/agentenv"] @pytest.mark.asyncio @@ -1500,7 +1598,7 @@ async def test_export_of_a_website_serving_no_data_plane_exports_nothing_and_sen async def test_core_snapshots_websites_through_their_data_plane_and_leaves_out_those_without_one(monkeypatch): calls = [] - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **website): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, server=None, **website): calls.append((name, website)) if name == "legacy-site": return None @@ -1539,5 +1637,5 @@ async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, **web _install_core_artifact_fakes(monkeypatch, EnvironmentUniverseArtifact) monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) - with pytest.raises(RuntimeError, match=r"exported no state: its websites \['shop'\] serve no v1 data plane"): + with pytest.raises(RuntimeError, match=r"exported no state: \['shop'\] serve no v1 data plane"): await SnapshotEnvTaskStep.snapshot_env_state(env=_multi_env([], ["shop"]), gateway_url="https://gw", snapshot_id="snap-x") From 76145d25ad039e3509f6841220b8594234346e1e Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 13:34:07 -0800 Subject: [PATCH 03/19] feat(website): WebsiteEnvironment serves a site on the website/v1 contract, and a conformance check tests any server (FD-4683) This PR's change, rebased as one commit onto #427. Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 2 +- packages/agentenv-protocol/README.md | 27 + .../src/agentenv_protocol/website/__init__.py | 36 ++ .../src/agentenv_protocol/website/clock.py | 114 ++++ .../agentenv_protocol/website/conformance.py | 351 +++++++++++ .../agentenv_protocol/website/environment.py | 450 ++++++++++++++ .../src/agentenv_protocol/website/errors.py | 39 ++ .../agentenv_protocol/website/readiness.py | 73 +++ .../src/agentenv_protocol/website/routes.py | 369 ++++++++++++ .../src/agentenv_protocol/website/session.py | 60 ++ .../src/agentenv_protocol/website/state.py | 56 ++ .../src/agentenv_protocol/website/store.py | 134 +++++ .../tests/test_website_api.py | 413 +++++++++++++ .../tests/test_website_conformance.py | 265 +++++++++ .../tests/test_website_environment.py | 552 ++++++++++++++++++ .../tests/test_website_readiness.py | 209 +++++++ .../tests/test_website_state.py | 121 ++++ src/agent_env/env/env.py | 59 +- src/agent_env/env/gateway/gateway.py | 85 ++- src/agent_env/env/legacy_protocol.py | 22 +- .../env_providers/env_gateway_provider.py | 21 +- .../task_step/task_steps/add_skills.py | 16 +- .../mcp_cli_builder/build_mcp_cli.py | 11 +- .../verifiers/website_state_verifier.py | 98 +++- tst/unit/env/gateway/changes_test.py | 15 +- .../env/gateway/rest_proxy_stream_test.py | 15 +- tst/unit/env/gateway/triggers_website_test.py | 2 +- .../gateway/website_reserved_paths_test.py | 96 ++- tst/unit/env/test_env.py | 111 +++- tst/unit/env/test_legacy_protocol.py | 15 + .../env_gateway_provider_test.py | 55 ++ tst/unit/task_step/add_skills_env_test.py | 11 + .../task_step/test_manifest_cli_builder.py | 11 + .../test_website_state_verifier_step.py | 74 ++- 34 files changed, 3893 insertions(+), 95 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/clock.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/environment.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/errors.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/routes.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/session.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/state.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/store.py create mode 100644 packages/agentenv-protocol/tests/test_website_api.py create mode 100644 packages/agentenv-protocol/tests/test_website_conformance.py create mode 100644 packages/agentenv-protocol/tests/test_website_environment.py create mode 100644 packages/agentenv-protocol/tests/test_website_readiness.py create mode 100644 packages/agentenv-protocol/tests/test_website_state.py diff --git a/AGENTS.md b/AGENTS.md index aa980e2e8..17a4fa4e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, checked at its `health` path; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `website_state_verifier`, which reads each checked session of a deployed website env's sites, or a multi env's, at the site's `/go?sid=` through `/svc//` and checks one dot path of its `current_state`, `initial_state` or `state_diff` each, optionally reduced by `len`, `keys` or `bool`, with `eq`, `ne`, `ge`, `gt`, `le`, `lt`, `in` or `contains`, its sites checked before a run against the `env_version` it pins, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 4e3b0de48..847455739 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -430,6 +430,33 @@ async with ToolSession(mcp_url=mcp_url, role="alice@example.com") as session: - **Tools:** `list_tools()` returns the tools the role may use as `EnvironmentTool`, the type a card's `capabilities.tools` holds. Each `inputSchema` is the JSON Schema a model API takes as the tool's parameters. +## Website readiness + +`agentenv_protocol.website` is the one definition of whether a website env is healthy. A site's card advertises +`urn:agentenv:website/v1`'s `ready` (`WEBSITE_READY_OP`), a GET answering a `WebsiteReadiness`: `ready`, a boolean, +and `problems`, the names of what keeps it from being ready (a `WebsiteEnvironment`'s are its `@ready_check` methods +not yet true). The model is strict about those two (`"yes"` is no `ready`) and ignores fields it doesn't know, so a +reader keeps reading a newer site. Everything that judges a site reads it through these helpers: agent-env's deploy, +`DeployedEnv.readiness()`, and the conformance checker's `health`, which fails an answer that isn't a +`WebsiteReadiness`. + +```python +from agentenv_protocol.website import env_readiness, website_readiness + +site = await website_readiness(base_url, card) # WebsiteReadiness, or None for a card without ready +env = await env_readiness(gateway_url, env_card) # WebsiteReadiness of every site of the env +print(env.ready, env.problems) # False ['shop: seeded'] +``` + +- **One site:** `website_readiness(base_url, card, *, timeout=READY_TIMEOUT_S, headers=None)` asks the site as its card + advertises `ready`. It raises httpx's error when the site can't be reached, `ExtensionCallError` for a non-2xx + answer and `ValueError` (pydantic's `ValidationError`) for an answer that isn't a `WebsiteReadiness`. +- **An env:** `env_readiness(base_url, card, ...)` asks, at once, the card itself when it advertises `ready` (a single + site's card), and every child under a composed card's `children_environments` that does, its endpoints rooted at + `base_url` as a gateway's are. A child's problems are prefixed with its name (`shop: seeded`); one that can't be + asked, or answers no `WebsiteReadiness`, is a problem itself. A card or child without `ready`, an MCP server's or an + older site's, isn't counted, so an env with no site serving it is ready with no problems. + ## Computer drivers A computer driver is what runs inside a computer an agent uses through screenshots and input. Write diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py new file mode 100644 index 000000000..cfdc1a39e --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py @@ -0,0 +1,36 @@ +"""The website SDK: a site's server, meeting ``urn:agentenv:website/v1``. Subclass WebsiteEnvironment, declare the +site with ``@website(domains=[...], static="dist/")``, mark its API with ``@api.get/post/put/patch/delete(path)``, and +``serve()`` it; it needs only the plain package, and uvicorn to serve. The class is optional: a server in any language +that meets the contract works, which ``python -m agentenv_protocol.website.conformance `` checks. Whether a site +is ready is its ``WebsiteReadiness`` answer to ``ready``, which ``website_readiness`` reads of one site and +``env_readiness`` of every site of an env. What a task left a site in is its ``WebsiteState`` answer to ``state``, +which ``website_state`` reads of one site of those ``website_sites`` names.""" + +from __future__ import annotations + +from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock +from .environment import ( + DEFAULT_HEALTH_PATH, + SESSION_PATH, + STATE_PATH, + WebsiteDeclaration, + WebsiteEnvironment, + ready_check, + website, +) +from .errors import ApiError, BadRequest, Conflict, Forbidden, NotFound, Unauthorized +from .readiness import READY_TIMEOUT_S, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness, env_readiness, website_readiness +from .routes import ROLE_KEY_ENV, ROLE_KEY_HEADER, ApiContext, ApiRoute, api, reserved_path +from .session import SESSION_COOKIE, SessionSigner +from .state import STATE_TIMEOUT_S, WEBSITE_STATE_OP, WebsiteState, advertises_state, website_sites, website_state +from .store import Collections + +__all__ = [ + "WebsiteEnvironment", "WebsiteDeclaration", "website", "api", "ApiContext", "ApiRoute", "ready_check", + "ApiError", "BadRequest", "Unauthorized", "Forbidden", "NotFound", "Conflict", + "Collections", "UniverseClock", "SessionSigner", "reserved_path", + "WEBSITE_EXTENSION_URI", "CLOCK_EXTENSION_URI", "UNIVERSE_NOW_HEADER", "DEFAULT_HEALTH_PATH", "SESSION_PATH", + "STATE_PATH", "SESSION_COOKIE", "ROLE_KEY_ENV", "ROLE_KEY_HEADER", + "WEBSITE_READY_OP", "READY_TIMEOUT_S", "WebsiteReadiness", "website_readiness", "env_readiness", + "WEBSITE_STATE_OP", "STATE_TIMEOUT_S", "WebsiteState", "advertises_state", "website_sites", "website_state", +] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py b/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py new file mode 100644 index 000000000..0d27d86a9 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py @@ -0,0 +1,114 @@ +"""The universe's clock as a site reads it: ``clock/v1``'s ``sync_time`` hands the server the gateway's ``/clock/time`` +URL, as it hands it to MCP servers, and the site's time then follows that clock's ``{"virtual_time": }``, +read at most once a second and extrapolated between reads. The rate it extrapolates at is one two successive intervals +between reads agreed on, so a clock set or advanced between two reads is taken for a jump, not a speed; until two agree +it is real time's. An unsynced site reads wall time. Every answer of a site carries the time it answered at as ``X-Universe-Now``, +RFC3339 in UTC. ``sync_time`` may also name the universe's time zone, an IANA name, which the clock keeps for the site +to show local times in; the time itself stays UTC.""" +from __future__ import annotations + +import asyncio +import datetime +import json +import logging +import time + +import httpx + +logger = logging.getLogger(__name__) + +CLOCK_EXTENSION_URI = "urn:agentenv:clock/v1" +UNIVERSE_NOW_HEADER = "X-Universe-Now" +REFRESH_S = 1.0 +READ_TIMEOUT_S = 5.0 +RATE_AGREEMENT = 0.1 + + +def iso(moment: datetime.datetime) -> str: + return moment.astimezone(datetime.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def parse_time(value: str) -> datetime.datetime: + moment = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + return moment if moment.tzinfo else moment.replace(tzinfo=datetime.timezone.utc) + + +class UniverseClock: + """The time a site answers with: wall time until ``sync`` names a gateway clock, that clock's after.""" + + def __init__(self) -> None: + self.url: str | None = None + self.timezone: str | None = None + self._virtual: datetime.datetime | None = None + self._read_at = 0.0 + self._tried_at = 0.0 + self._rate = 1.0 + self._measured: float | None = None + self._lock: asyncio.Lock | None = None + + @property + def synced(self) -> bool: + return self._virtual is not None + + def now(self) -> datetime.datetime: + if self._virtual is None: + return datetime.datetime.now(datetime.timezone.utc) + return self._virtual + datetime.timedelta(seconds=(time.monotonic() - self._read_at) * self._rate) + + async def sync(self, url: str, timezone: str | None = None) -> datetime.datetime: + """Follow the clock at ``url`` from now on, in ``timezone`` (none when the sync names none); raises when it + can't be read, keeping the clock and zone it followed. It waits for a refresh reading the clock it followed, + so that read never lands on the new clock.""" + async with self._reading(): + virtual = await _read(url) + self.url, self._virtual, self._rate, self._measured = url, virtual, 1.0, None + self.timezone = timezone + self._read_at = self._tried_at = time.monotonic() + return virtual + + async def refresh(self) -> None: + """Read the synced clock again when the last read is a second old; a failed read keeps extrapolating.""" + if self.url is None or time.monotonic() - self._tried_at < REFRESH_S: + return + async with self._reading(): + started = time.monotonic() + if started - self._tried_at < REFRESH_S: + return + self._tried_at = started + try: + virtual = await _read(self.url) + except Exception as e: + logger.debug("reading the universe's clock at %s failed: %r", self.url, e) + return + read_at = time.monotonic() + if self._virtual is not None and read_at > self._read_at: + measured = max(0.0, (virtual - self._virtual).total_seconds() / (read_at - self._read_at)) + if self._measured is not None and abs(measured - self._measured) <= RATE_AGREEMENT * max(measured, self._measured, 1.0): + self._rate = measured + self._measured = measured + self._virtual, self._read_at = virtual, read_at + + def _reading(self) -> asyncio.Lock: + """The lock a sync and a refresh each read the clock under.""" + self._lock = self._lock or asyncio.Lock() + return self._lock + + +async def _read(url: str) -> datetime.datetime: + async with httpx.AsyncClient() as client: + response = await client.get(url, timeout=READ_TIMEOUT_S) + response.raise_for_status() + return parse_time(response.json()["virtual_time"]) + + +def clock_patch_script(now: datetime.datetime) -> bytes: + """An inline script that shifts the page's ``Date`` to the universe's time ``now``, and follows the + ``X-Universe-Now`` header of every ``fetch`` the page makes after. The ``Date`` it puts in place answers called, + with the time's string, as constructed, and keeps ``Date.parse``, ``Date.UTC`` and ``instanceof Date``.""" + return ("" % (int(now.timestamp() * 1000), json.dumps(UNIVERSE_NOW_HEADER))).encode() diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py new file mode 100644 index 000000000..8df978f4b --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py @@ -0,0 +1,351 @@ +"""Check a running site's server against ``urn:agentenv:website/v1``, whatever it's written in: +``python -m agentenv_protocol.website.conformance `` prints each check and exits 1 when one failed, 2 when the +server couldn't be reached. Everything goes over HTTP: the card, ``/`` under a made-up host, readiness by the card's +``website/v1`` ``ready``, whose answer must be a ``WebsiteReadiness`` (by the declared health path, deprecated, for a +card without it), ``X-Universe-Now`` on every answer the other checks got, a data-plane round trip at the card's ``url`` +(data/get, data/reset, data/get, which no longer returns the exported data, data/add of what the first data/get +returned, data/get again), ``clock/v1``'s +``sync_time`` against a clock the checker serves, and the forwarded headers wherever an answer shows them, on ``/`` and +on a GET route with its trailing slash toggled, which a server may redirect. + +The round trip leaves the server's data as it found it, adding the export back when a step after the reset fails too. +data/add sets the state a task starts from, so a server whose state shows changes since its initial state, as an SDK +site's does, would have them folded into it: the round trip is skipped for such a server. Its state is its card's +``website/v1`` ``state``, else, for a card without it, what the path its ``params.state.path`` names answers, else +``/go``'s, deprecated. The clock check leaves the +server following a clock that is gone once the check ends, so a server it ran against keeps extrapolating from that +clock's time until it's synced again. The checker's clock starts at ``CLOCK_START``, far enough from wall time that a +synced server shows it, and a synced server's ``X-Universe-Now`` may be ``CLOCK_SLACK_S`` seconds from it.""" +from __future__ import annotations + +import argparse +import contextlib +import dataclasses +import datetime +import json +import sys +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Callable, Iterator, Literal + +import httpx +from pydantic import ValidationError + +from ..client import find_extension, find_extension_method +from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH +from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, iso, parse_time +from .environment import DEFAULT_HEALTH_PATH, STATE_PATH +from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness +from .state import WEBSITE_STATE_OP + +Status = Literal["passed", "failed", "skipped"] +CLOCK_START = datetime.datetime(2031, 5, 6, 7, 8, 9, tzinfo=datetime.timezone.utc) +CLOCK_SLACK_S = 120 + + +@dataclasses.dataclass(frozen=True) +class Result: + name: str + status: Status + detail: str = "" + + @property + def passed(self) -> bool: + return self.status != "failed" + + +def check(base_url: str, *, client: httpx.Client | None = None, clock_host: str = "127.0.0.1", + timeout: float = 30) -> list[Result]: + """Each check's result against the server at ``base_url``. ``clock_host`` is the address the server reaches the + checker's clock at. Raises httpx's error when the server can't be reached at all.""" + own = client is None + client = client or httpx.Client(timeout=timeout) + try: + return _Checker(base_url.rstrip("/"), client, clock_host).run() + finally: + if own: + client.close() + + +class _Checker: + def __init__(self, base_url: str, client: httpx.Client, clock_host: str) -> None: + self.base_url, self.client, self.clock_host = base_url, client, clock_host + self.rpc_url = base_url + RPC_PATH + self.responses: list[httpx.Response] = [] + + def get(self, path: str, **kwargs) -> httpx.Response: + return self._seen(self.client.get(self.base_url + path, **kwargs)) + + def _seen(self, response: httpx.Response) -> httpx.Response: + self.responses.append(response) + return response + + def run(self) -> list[Result]: + response = self.get(WELL_KNOWN_PATH) + try: + card = response.json() if response.status_code == 200 else {} + except ValueError: + card = {} + self.rpc_url = _rpc_url(self.base_url, card) + site = (find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {} + health = site.get("health") if isinstance(site.get("health"), str) else DEFAULT_HEALTH_PATH + steps = (("any_host", self._any_host), ("health", lambda: self._health(card, health)), + ("universe_now", self._universe_now), ("data_plane", lambda: self._data_plane(card)), + ("clock", lambda: self._clock(card, health)), ("forwarded", lambda: self._forwarded(site)), + ("cookies", lambda: self._cookies(site))) + outcomes = {name: _outcome(step) for name, step in steps if name != "universe_now"} + outcomes["universe_now"] = _outcome(self._universe_now) + return [self._card(response, site), *(Result(name, *outcomes[name]) for name, _step in steps)] + + def _card(self, response: httpx.Response, site: dict) -> Result: + if response.status_code != 200: + return Result("card", "failed", f"{WELL_KNOWN_PATH} answered {response.status_code}") + problems = [] + if not site: + problems.append(f"declares no {WEBSITE_EXTENSION_URI} with params") + if not isinstance(site.get("domains"), list): + problems.append("its params have no domains list") + if not (isinstance(site.get("health"), str) and site["health"].startswith("/")): + problems.append("its params have no health path") + routes = site.get("routes") + if not isinstance(routes, list) or not all(isinstance(r, dict) and {"method", "path", "name"} <= set(r) + for r in routes): + problems.append("its params' routes aren't a list of {method, path, name}") + return Result("card", "failed" if problems else "passed", "; ".join(problems)) + + def _any_host(self) -> tuple[Status, str]: + host = f"conformance-{uuid.uuid4().hex[:8]}.invalid" + response = self.get("/", headers={"Host": host}) + kind = response.headers.get("content-type", "") + if response.status_code == 200 and kind.startswith("text/html"): + return "passed", f"/ served HTML for Host {host}" + return "failed", f"/ for Host {host} answered {response.status_code} {kind or 'without a content type'}" + + def _health(self, card: dict, health: str) -> tuple[Status, str]: + ready = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP) + if ready is None: + response = self.get(health) + return (("passed" if response.status_code == 200 else "failed"), + f"{health} answered {response.status_code}; the card has no website/v1 ready, so the health path " + "was read, which is deprecated") + answer = self._seen(self.client.request(ready.get("method", "GET"), self.base_url + ready["endpoint"])).json() + try: + readiness = WebsiteReadiness.model_validate(answer) + except ValidationError as e: + return "failed", (f"website/v1 ready answered {answer!r}, not a WebsiteReadiness: " + f"{'; '.join(_field_error(error) for error in e.errors())}") + if readiness.ready: + return "passed", "website/v1 ready says it is ready" + return "failed", f"website/v1 ready says it is not ready: {', '.join(readiness.problems) or 'no problem named'}" + + def _universe_now(self) -> tuple[Status, str]: + missing = [str(r.url.path) for r in self.responses if UNIVERSE_NOW_HEADER not in r.headers] + if missing: + return "failed", f"no {UNIVERSE_NOW_HEADER} on {', '.join(missing)}" + for response in self.responses: + parse_time(response.headers[UNIVERSE_NOW_HEADER]) + return "passed", f"on all {len(self.responses)} answers" + + def _rpc(self, method: str, params: dict | None = None) -> dict: + response = self._seen(self.client.post(self.rpc_url, json={"jsonrpc": "2.0", "id": method, + "method": method, "params": params or {}})) + body = response.json() + if isinstance(body, dict) and "error" in body: + raise RuntimeError(f"{method} answered {body['error']}") + if not response.is_success or not isinstance(body, dict) or "result" not in body: + raise RuntimeError(f"{method} answered {response.status_code} with no JSON-RPC result: {response.text[:200]}") + return body["result"] or {} + + def _data_plane(self, card: dict) -> tuple[Status, str]: + operations = (card.get("capabilities") or {}).get("operations") + missing = [m for m in (METHOD_GET, METHOD_RESET, METHOD_ADD) if operations is not None and m not in operations] + if missing: + return "failed", f"the card's operations lack {', '.join(missing)}" + exported = self._rpc(METHOD_GET).get("parts") or [] + if where := self._changed_since_initial(card): + return "skipped", (f"data/get answered; {where} shows changes since the initial state, which data/reset " + "and data/add would fold into it, so the round trip was not run") + self._rpc(METHOD_RESET) + added_back = False + try: + cleared = self._rpc(METHOD_GET).get("parts") or [] + if _holds_data(exported) and cleared == exported: + added_back = True + return "failed", "data/get after data/reset returned the data it returned before" + if not exported: + added_back = True + return "passed", "data/get, data/reset and data/get answered; data/get returned nothing to add back" + added_back = True + self._rpc(METHOD_ADD, {"parts": exported}) + again = self._rpc(METHOD_GET).get("parts") or [] + finally: + if not added_back: + with contextlib.suppress(Exception): + self._rpc(METHOD_ADD, {"parts": exported}) + if again != exported: + return "failed", "data/get after data/reset and data/add of its own export returned other data" + return "passed", f"{len(exported)} part(s) round-tripped through reset and add" + + def _changed_since_initial(self, card: dict) -> str | None: + """Where the server's state was read when its ``state_diff`` names changes, None when it names none or can't be + read: the card's ``website/v1`` ``state``, else the path its ``params.state.path`` names, else ``/go``. Read + aside, as no check's answer, since a server without one needn't stamp its 404.""" + op = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) + if op is not None and op.get("endpoint"): + where, request = f"website/v1 {WEBSITE_STATE_OP}", (op["method"], self.base_url + op["endpoint"]) + else: + declared = ((find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {}).get("state") + path = declared.get("path") if isinstance(declared, dict) else None + where = path if isinstance(path, str) and path.startswith("/") else STATE_PATH + request = ("GET", self.base_url + where) + try: + state = self.client.request(*request).json() + except Exception: + return None + return where if isinstance(state, dict) and state.get("state_diff") else None + + def _clock(self, card: dict, health: str) -> tuple[Status, str]: + if find_extension(card, CLOCK_EXTENSION_URI) is None: + return "failed", f"the card declares no {CLOCK_EXTENSION_URI}" + params = find_extension(card, CLOCK_EXTENSION_URI).get("params") or {} + method = find_extension_method(card, CLOCK_EXTENSION_URI, next(iter(params.get("methods") or {}), "sync_time")) + if method is None or not method.get("endpoint"): + return "failed", f"{CLOCK_EXTENSION_URI} advertises no sync_time endpoint" + with _clock_server(self.clock_host) as (url, virtual_now): + response = self._seen(self.client.request(method["method"], self.base_url + method["endpoint"], + json={"env_get_time_url": url})) + if response.status_code >= 400: + return "failed", f"sync_time answered {response.status_code}: {response.text[:200]}" + seen = self.get(health).headers.get(UNIVERSE_NOW_HEADER) + expected = virtual_now() + if seen is None: + return "failed", f"no {UNIVERSE_NOW_HEADER} after sync_time" + off = abs((parse_time(seen) - expected).total_seconds()) + if off > CLOCK_SLACK_S: + return "failed", f"after sync_time {UNIVERSE_NOW_HEADER} is {seen}, {off:.0f}s from the clock's {iso(expected)}" + return "passed", f"{UNIVERSE_NOW_HEADER} followed the clock to {seen}" + + def _forwarded(self, site: dict) -> tuple[Status, str]: + internal = f"internal-{uuid.uuid4().hex[:8]}.invalid" + forwarded = f"site-{uuid.uuid4().hex[:8]}.example" + headers = {"Host": internal, "X-Forwarded-Host": forwarded, "X-Forwarded-Proto": "https"} + paths = ["/", *_slash_toggled(site.get("routes"))] + for path in paths: + response = self.get(path, headers=headers, follow_redirects=False) + location = response.headers.get("location", "") + shown = [where for where, text in (("body", response.text), ("Location", location)) if internal in text] + if shown: + return "failed", f"the {' and '.join(shown)} of {path} named the internal Host {internal}, not {forwarded}" + if location.startswith(f"http://{forwarded}"): + return "failed", f"the Location of {path} is {location}, not https as X-Forwarded-Proto said" + return "passed", f"no answer to {', '.join(paths)} named the internal Host" + + def _cookies(self, site: dict) -> tuple[Status, str]: + if isinstance(site.get("session_path"), str): + self.get(site["session_path"]) + cookies = [c for r in self.responses for c in r.headers.get_list("set-cookie")] + if not cookies: + return "skipped", "no answer set a cookie" + scoped = [c.split("=", 1)[0] for c in cookies if any(a.strip().lower().startswith("domain=") for a in c.split(";"))] + if scoped: + return "failed", f"cookies tied to a domain: {', '.join(sorted(set(scoped)))}" + return "passed", f"{len(cookies)} cookie(s), none tied to a domain" + + +def _rpc_url(base_url: str, card: dict) -> str: + """Where the card says its data plane is: its ``url``, a path joined onto ``base_url`` or a full URL, else + ``RPC_PATH``.""" + url = card.get("url") if isinstance(card.get("url"), str) and card.get("url") else RPC_PATH + return url if "://" in url else base_url + url + + +def _outcome(step: Callable[[], tuple[Status, str]]) -> tuple[Status, str]: + try: + return step() + except Exception as e: + return "failed", f"{type(e).__name__}: {e}" + + +def _field_error(error: dict) -> str: + """One of a ``ValidationError``'s errors as a person reads it: where, then what.""" + where = ".".join(map(str, error.get("loc") or ())) or "the answer" + return f"{where}: {error.get('msg')}" + + +def _holds_data(parts: list) -> bool: + """Whether exported parts carry task data: a part that isn't a data part, or a data part with a value in it.""" + return any(not (isinstance(p, dict) and p.get("kind") == "data") or _filled(p.get("data")) for p in parts) + + +def _filled(value: object) -> bool: + if isinstance(value, (dict, list)): + return any(_filled(v) for v in (value.values() if isinstance(value, dict) else value)) + return value not in (None, "") + + +def _slash_toggled(routes: object) -> list[str]: + """The first GET route without path parameters, with its trailing slash added or taken off: a path a server + may redirect to the route's own, which shows the address the redirect names.""" + for route in routes if isinstance(routes, list) else []: + path = route.get("path") if isinstance(route, dict) and route.get("method") == "GET" else None + if isinstance(path, str) and path.startswith("/") and path != "/" and "{" not in path: + return [path[:-1] if path.endswith("/") else path + "/"] + return [] + + +@contextlib.contextmanager +def _clock_server(host: str) -> Iterator[tuple[str, Callable[[], datetime.datetime]]]: + """A gateway-shaped ``/clock/time`` running from ``CLOCK_START`` at real speed, and its time now.""" + started = time.monotonic() + + def virtual_now() -> datetime.datetime: + return CLOCK_START + datetime.timedelta(seconds=time.monotonic() - started) + + class Clock(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = json.dumps({"virtual_time": iso(virtual_now())}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + return None + + server = ThreadingHTTPServer(("127.0.0.1" if host in ("127.0.0.1", "localhost") else "0.0.0.0", 0), Clock) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + address = f"[{host}]" if ":" in host else host + yield f"http://{address}:{server.server_port}/clock/time", virtual_now + finally: + server.shutdown() + server.server_close() + + +_MARK = {"passed": "PASS", "failed": "FAIL", "skipped": "----"} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m agentenv_protocol.website.conformance", + description="Check a running site's server against urn:agentenv:website/v1.") + parser.add_argument("url", help="where the server answers, such as http://localhost:8000") + parser.add_argument("--clock-host", default="127.0.0.1", help="the address the server reaches this checker at") + parser.add_argument("--timeout", type=float, default=30, help="seconds for each request") + args = parser.parse_args(argv) + try: + results = check(args.url, clock_host=args.clock_host, timeout=args.timeout) + except httpx.TransportError as e: + print(f"couldn't reach {args.url}: {type(e).__name__}: {e}", file=sys.stderr) + return 2 + for result in results: + print(f"{_MARK[result.status]} {result.name}{': ' + result.detail if result.detail else ''}") + failed = sum(not r.passed for r in results) + print(f"{args.url}: {len(results) - failed} passed or skipped, {failed} failed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py new file mode 100644 index 000000000..31663b55a --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -0,0 +1,450 @@ +"""WebsiteEnvironment: an AgentEnvEnvironment that is a site's server, meeting ``urn:agentenv:website/v1``. It serves +plain HTTP on ``PORT`` for any host name: its ``@api`` routes, its static folder at ``/`` with ``index.html`` for the +pages a single-page app routes itself, its health path, and the data plane and card the base class serves. Every answer +carries ``X-Universe-Now``, the universe's time, which ``self.now()`` reads and ``clock/v1``'s ``sync_time`` ties to +the gateway's virtual clock. Without data hooks of its own a site keeps its state in ``self.store``. + +Two routes are built in, each replaced by a site's own ``@api`` route at its very path and answering before one whose +path parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the +acting user in when nobody is (GET), or to sign out (DELETE); and, for a store-backed site, ``/go``, its +``{initial_state, current_state, state_diff}`` at the path ``website/v1``'s ``params.state.path`` names, which a reader +of a card without ``state`` falls back to. A store-backed site answers ``website/v1``'s ``state`` with the same +``WebsiteState``, among agentenv's own routes, which is how the website state verifier reads it.""" +from __future__ import annotations + +import inspect +import json +import os +import re +from pathlib import Path +from typing import Any, ClassVar, NamedTuple, Optional + +from starlette.applications import Starlette +from starlette.datastructures import MutableHeaders +from starlette.middleware import Middleware +from starlette.requests import Request +from starlette.responses import FileResponse, JSONResponse, Response +from starlette.routing import Route + +from ..agent_env_environment import ( + _OP_ATTR, + AgentEnvEnvironment, + AgentEnvStarletteApplication, + _marked_members, + _maybe_await, + add_data, + extension, + get_data, + reset_data, +) +from ..types import DataPart, EnvironmentExtension +from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock, clock_patch_script, iso +from .errors import NotFound +from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness +from .routes import ApiContext, ApiRoute, endpoint, error_response, openapi, openapi_path, request_role, reserved_path, routes_of +from .session import SESSION_COOKIE, SessionSigner, expires_at +from .state import WEBSITE_STATE_OP, WebsiteState +from .store import Collections + +DEFAULT_HEALTH_PATH = "/api/health" +SESSION_PATH = "/api/session" +STATE_PATH = "/go" +_WEBSITE_ATTR = "_aee_website" +_READY_ATTR = "_aee_website_ready" +_DEFAULT_HOOK_ATTR = "_aee_default_store_hook" +_DEFAULT_HOOKS = {"reset_data": "_store_reset", "add_data": "_store_add", "get_data": "_store_get"} +_HEAD = re.compile(rb"]*>", re.IGNORECASE) + + +class WebsiteDeclaration(NamedTuple): + """What ``@website`` declares: the site's domains, its static folder and seed data file (relative to the class's + file), its health path, and whether its HTML pages read the universe's time.""" + + domains: tuple[str, ...] = () + static: Optional[str] = None + health: str = DEFAULT_HEALTH_PATH + clock_patch: bool = False + data: Optional[str] = None + + +def website(_cls: type | None = None, *, domains: list[str] | tuple[str, ...] = (), static: str | None = None, + health: str = DEFAULT_HEALTH_PATH, clock_patch: bool = False, data: str | None = None) -> Any: + """Declare a WebsiteEnvironment's site. ``data`` names a JSON file the default store starts with.""" + if _cls is not None: + raise TypeError("use @website(...) with parentheses") + if not health.startswith("/") or health == "/" or reserved_path(health): + raise ValueError(f"health is a path of its own under '/', not {health!r}") + + def deco(cls: type) -> type: + if not (isinstance(cls, type) and issubclass(cls, WebsiteEnvironment)): + raise TypeError(f"@website declares a WebsiteEnvironment subclass, not {cls!r}") + setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data)) + _check_health(cls) + return cls + + return deco + + +def ready_check(fn: Any) -> Any: + """Mark a method that says whether the site is usable yet: the health path answers 200 once every one returns a + truthy value, and 503 while one returns a falsy one or raises.""" + setattr(fn, _READY_ATTR, True) + return fn + + +class WebsiteEnvironment(AgentEnvEnvironment): + """Subclass it, declare the site with ``@website(...)``, mark its API with ``@api``, and ``serve()`` it. A + subclass's ``__init__`` calls ``super().__init__()``. ``app`` may be a Starlette or FastAPI app answering what + ``@api`` and the static folder don't (streaming, uploads); its lifespan is not run.""" + + app: ClassVar[Any] = None + asgi: Any = None + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + routes_of(cls) + own = {getattr(member, _OP_ATTR) for klass in cls.__mro__ for member in vars(klass).values() + if getattr(member, _OP_ATTR, None) is not None and not getattr(member, _DEFAULT_HOOK_ATTR, False)} + for op, name in _DEFAULT_HOOKS.items(): + if op in own: + setattr(cls, name, None) + + def __init__(self) -> None: + declaration = self.declaration() + self.store = Collections() + if declaration.data is not None: + self.store.load(json.loads(self._site_path(declaration.data).read_text(encoding="utf-8"))) + self.store.mark_initial() + self.clock = UniverseClock() + self._signer = SessionSigner() + + @classmethod + def declaration(cls) -> WebsiteDeclaration: + return getattr(cls, _WEBSITE_ATTR, None) or WebsiteDeclaration() + + @classmethod + def routes(cls) -> list[ApiRoute]: + return routes_of(cls) + + @property + def database_url(self) -> str | None: + """Where the site keeps its task state: its own schema of the env's database.""" + return os.environ.get("DATABASE_URL") + + @property + def uses_store(self) -> bool: + """Whether the site's data plane exports ``self.store``: it declares no ``@get_data`` of its own.""" + return type(self)._store_get is not None + + def now(self) -> Any: + """The universe's time, an aware UTC datetime: the gateway's virtual clock once synced, else wall time.""" + return self._clock().now() + + @property + def timezone(self) -> str | None: + """The universe's time zone, an IANA name such as ``America/Chicago``, as the last ``sync_time`` named it; + None when it named none. ``now()`` is UTC either way.""" + return self._clock().timezone + + def acting_user(self) -> str | None: + """Who a browser with no session is signed in as: the store's ``current_user`` unless a site says else.""" + return self.store.current_user if hasattr(self, "store") else None + + def context(self, request: Request) -> ApiContext: + """The ``ApiContext`` of ``request``, for a mounted ``app``'s routes as for ``@api`` ones.""" + session = self._session_signer().decode(request.cookies.get(SESSION_COOKIE)) + user = session["user"] if session is not None else self.acting_user() + return ApiContext(request, user, request_role(request), session, self._session_signer()) + + def create_app(self) -> Starlette: + if self._mounted: + raise RuntimeError(f"{type(self).__name__} is already mounted") + declaration = self.declaration() + _check_health(type(self)) + api_routes = self.routes() + taken = {openapi_path(r.path) for r in api_routes} + root = self._site_path(declaration.static) if declaration.static is not None else None + asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self)]) + AgentEnvStarletteApplication(self._build_card(), self).add_routes_to_app(asgi) + asgi.router.routes.append(Route(declaration.health, self._health, methods=["GET"])) + if SESSION_PATH not in taken: + asgi.router.routes.append(Route(SESSION_PATH, self._session, methods=["GET", "DELETE"])) + if self._serves_state_path(): + asgi.router.routes.append(Route(STATE_PATH, self._state, methods=["GET"])) + asgi.router.routes.extend(Route(r.path, _unreserved(endpoint(getattr(self, r.fn.__name__), r, self.context)), + methods=[r.method]) for r in api_routes) + asgi.router.default = _Fallback(root, type(self).app) + self.asgi = asgi + self._mounted = True + return asgi + + def mount(self, app: Any) -> Any: + raise TypeError(f"{type(self).__name__} builds its own app with create_app(); set app = ... to serve more") + + def serve(self) -> None: + """Serve the site on ``PORT`` (8000 by default) with uvicorn, of the ``agent`` or ``app`` extra.""" + try: + import uvicorn + except ImportError as e: + raise ImportError("WebsiteEnvironment.serve needs uvicorn: pip install 'uvicorn>=0.30'") from e + uvicorn.run(self.asgi or self.create_app(), host=os.environ.get("HOST", "0.0.0.0"), + port=int(os.environ.get("PORT", "8000"))) + + def extension_params(self, uri: str) -> dict: + if uri == WEBSITE_EXTENSION_URI: + declaration = self.declaration() + return {"domains": list(declaration.domains), "health": declaration.health, + "routes": [r.advertised() for r in self.routes()], "session_cookie": SESSION_COOKIE, + "session_path": SESSION_PATH, **({"state": {"path": STATE_PATH}} if self._serves_state_path() else {})} + return super().extension_params(uri) + + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + """The extensions the base class serves, ``website/v1``'s ``state`` only for a store-backed site: one with data + hooks of its own keeps its state where ``self.store`` can't see it.""" + served = super().served_extensions(extensions) + if self.uses_store: + return served + return [_without_method(e, WEBSITE_STATE_OP) if e.uri == WEBSITE_EXTENSION_URI else e for e in served] + + @extension(CLOCK_EXTENSION_URI, description="Sync this site to the gateway's virtual clock.") + async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) -> dict: + """Follow the gateway's virtual time at ``env_get_time_url`` from now on, in ``timezone`` when the task names + one.""" + virtual = await self._clock().sync(env_get_time_url, timezone.strip() if timezone else None) + return {"env_get_time_url": env_get_time_url, "gateway_virtual_time": iso(virtual), + **({"timezone": self.timezone} if self.timezone else {})} + + @extension(WEBSITE_EXTENSION_URI, op="openapi", method="GET", description=( + "A site's server: domains, its health path and its @api routes by name; openapi returns the OpenAPI document " + "of its routes, ready whether the site is usable now and why not, state the task state it is in (its " + "initial and current state and their diff), and session the cookies a browser adds to start signed in as user, " + "or signed out without one.")) + async def _website_openapi(self) -> dict: + name = self._application.environment_card.name if self._application is not None else type(self).__name__ + return openapi(name, self.routes()) + + @extension(WEBSITE_EXTENSION_URI, op=WEBSITE_READY_OP, method="GET") + async def _website_ready(self) -> dict: + """Whether the site is usable now, a ``WebsiteReadiness`` by its ``@ready_check`` methods: ``problems`` names + each not yet true. A deploy reads it through the card, as ``computer-app/v1``'s ``ready`` of an app.""" + problems = await self._not_ready() + return WebsiteReadiness(ready=not problems, problems=problems).model_dump() + + @extension(WEBSITE_EXTENSION_URI, op=WEBSITE_STATE_OP, method="GET") + async def _website_state(self, sid: Optional[str] = None) -> dict: + """The site's task state, a ``WebsiteState`` of ``self.store``: its state when the task started, now, and what + changed between them. A store-backed site keeps one session, so ``sid`` names no other.""" + return WebsiteState.model_validate(self.store.state()).model_dump() + + @extension(WEBSITE_EXTENSION_URI, op="session") + async def _website_session(self, user: Optional[str] = None) -> dict: + cookie = self._session_signer().cookie(user) + return {"user": user, "cookies": [cookie], "expires_at": expires_at(cookie["expires"])} + + @reset_data + async def _store_reset(self) -> None: + self.store.reset() + + @add_data + async def _store_add(self, parts: list) -> None: + self.store.load_parts(parts) + self.store.mark_initial() + + @get_data + async def _store_get(self) -> list: + return [DataPart(data=self.store.export())] + + async def _not_ready(self) -> list[str]: + return [check.__name__ for check, _marker in _marked_members(self, _READY_ATTR) if not await _ready(check)] + + async def _health(self, request: Request) -> Response: + waiting = await self._not_ready() + if waiting: + return JSONResponse({"status": "starting", "waiting_for": waiting}, status_code=503) + return JSONResponse({"status": "ok"}) + + async def _session(self, request: Request) -> Response: + context = self.context(request) + if request.method == "DELETE": + context.sign_out() + elif context.session is None and context.user is not None: + context.sign_in(context.user) + return context.apply(JSONResponse({"user": context.user, "role": context.role, + "signed_in": context.user is not None})) + + def _serves_state_path(self) -> bool: + """Whether the site serves its state at ``STATE_PATH``: it is store-backed and no ``@api`` route takes the path.""" + return self.uses_store and STATE_PATH not in {openapi_path(r.path) for r in self.routes()} + + async def _state(self, request: Request) -> Response: + return JSONResponse(self.store.state()) + + def _clock(self) -> UniverseClock: + if not hasattr(self, "clock"): + raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") + return self.clock + + def _session_signer(self) -> SessionSigner: + if not hasattr(self, "_signer"): + raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") + return self._signer + + @classmethod + def _site_path(cls, path: str) -> Path: + given = Path(path) + return given if given.is_absolute() else Path(inspect.getfile(cls)).resolve().parent / given + + +for _hook in _DEFAULT_HOOKS.values(): + setattr(getattr(WebsiteEnvironment, _hook), _DEFAULT_HOOK_ATTR, True) + + +def _without_method(extension: EnvironmentExtension, method: str) -> EnvironmentExtension: + params = dict(extension.params or {}) + params["methods"] = {name: entry for name, entry in (params.get("methods") or {}).items() if name != method} + return extension.model_copy(update={"params": params}) + + +async def _ready(check: Any) -> bool: + try: + return bool(await _maybe_await(check())) + except Exception: + return False + + +def _check_health(cls: type) -> None: + health = cls.declaration().health + clash = [r.name for r in routes_of(cls) if r.method == "GET" and openapi_path(r.path) == health] + if clash: + raise ValueError(f"{cls.__name__}'s health path {health} is served by @api {clash[0]}; use @ready_check") + + +class _Forwarded: + """Puts the address the browser reached the site at in the scope, from ``X-Forwarded-Host`` and + ``X-Forwarded-Proto``, before anything routes it, so redirects, ``request.url`` and a mounted app's ``url_for`` + name that address and not the internal one the proxy reached the server at.""" + + def __init__(self, app: Any) -> None: + self.app = app + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] in ("http", "websocket"): + headers = dict(scope["headers"]) + host = _first(headers.get(b"x-forwarded-host")) + proto = _first(headers.get(b"x-forwarded-proto")).lower() + scope = dict(scope) + if host: + scope["headers"] = [(k, v) for k, v in scope["headers"] if k != b"host"] + [(b"host", host.encode())] + if proto in ("http", "https"): + scope["scheme"] = proto if scope["type"] == "http" else {"http": "ws", "https": "wss"}[proto] + await self.app(scope, receive, send) + + +def _first(value: bytes | None) -> str: + return value.decode("latin-1").split(",")[0].strip() if value else "" + + +class _Universe: + """Stamps ``X-Universe-Now`` on every answer, and with ``clock_patch`` puts the clock script in its HTML.""" + + def __init__(self, app: Any, env: WebsiteEnvironment) -> None: + self.app, self.env = app, env + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] != "http": + return await self.app(scope, receive, send) + clock = self.env._clock() + await clock.refresh() + patch = self.env.declaration().clock_patch and scope["method"] != "HEAD" + held: dict = {} + + async def stamped(message: dict) -> None: + if message["type"] == "http.response.start": + headers = MutableHeaders(scope=message) + now = clock.now() + headers[UNIVERSE_NOW_HEADER] = iso(now) + if patch and headers.get("content-type", "").startswith("text/html") and "content-encoding" not in headers: + held.update(start=message, body=[], now=now) + return + elif held and message["type"] == "http.response.body": + held["body"].append(message.get("body", b"")) + if message.get("more_body"): + return + body = _with_script(b"".join(held["body"]), clock_patch_script(held["now"])) + MutableHeaders(scope=held["start"])["content-length"] = str(len(body)) + await send(held["start"]) + message = {"type": "http.response.body", "body": body} + await send(message) + + await self.app(scope, receive, stamped) + + +def _with_script(html: bytes, script: bytes) -> bytes: + head = _HEAD.search(html) + return html[:head.end()] + script + html[head.end():] if head else script + html + + +def _unreserved(handler: Any) -> Any: + """``handler``, an ``@api`` route's endpoint, answering 404 at the data plane's and card's paths, so a broad route + (``/{rest:path}``) never serves a page there; the routes agentenv registers at those paths answer before it.""" + async def answer(request: Request) -> Response: + if reserved_path(request.scope["path"]): + return error_response(NotFound("not found")) + return await handler(request) + return answer + + +class _Fallback: + """What no route answers: a file of the static folder, then the mounted app, then for a page's path the folder's + ``index.html``; the data plane's and card's paths answer 404 however they're spelled.""" + + def __init__(self, root: Path | None, app: Any) -> None: + self.root = root.resolve() if root is not None else None + self.app = app + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] != "http": + if self.app is not None: + await self.app(scope, receive, send) + return + path, reading = scope["path"], scope["method"] in ("GET", "HEAD") + if reserved_path(path): + return await error_response(NotFound("not found"))(scope, receive, send) + if reading and (file := self._file(path)) is not None: + return await FileResponse(file)(scope, receive, send) + index = self._file("/") if reading and _page(path) else None + if self.app is not None: + return await self._app_or(index, scope, receive, send) + if index is not None: + return await FileResponse(index)(scope, receive, send) + await error_response(NotFound("not found"))(scope, receive, send) + + def _file(self, path: str) -> Path | None: + if self.root is None: + return None + candidate = (self.root / path.lstrip("/")).resolve() + if candidate.is_dir(): + candidate = (candidate / "index.html").resolve() + if not candidate.is_relative_to(self.root): + return None + return candidate if candidate.is_file() else None + + async def _app_or(self, index: Path | None, scope: dict, receive: Any, send: Any) -> None: + """The mounted app's answer, or ``index`` instead of the app's 404 when there is one.""" + missed = False + + async def unless_missed(message: dict) -> None: + nonlocal missed + if message["type"] == "http.response.start" and message["status"] == 404 and index is not None: + missed = True + if not missed: + await send(message) + + await self.app(scope, receive, unless_missed) + if missed: + await FileResponse(index)(scope, receive, send) + + +def _page(path: str) -> bool: + """Whether ``path`` is a page a single-page app routes itself: not the API's, and not a file's.""" + return not (path == "/api" or path.startswith("/api/")) and "." not in path.rsplit("/", 1)[-1] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/errors.py b/packages/agentenv-protocol/src/agentenv_protocol/website/errors.py new file mode 100644 index 000000000..13e349b88 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/errors.py @@ -0,0 +1,39 @@ +"""Errors an ``@api`` method raises to answer with an HTTP status: the body is ``{"error": message, "code": code}``.""" +from __future__ import annotations + + +class ApiError(Exception): + """An answer other than success, ``status`` with ``{"error": message, "code": code}``; a subclass sets both + defaults, and either may be given per raise.""" + + status: int = 500 + code: str = "error" + + def __init__(self, message: str, *, code: str | None = None, status: int | None = None) -> None: + super().__init__(message) + self.message = message + self.code = code or type(self).code + self.status = status or type(self).status + + def body(self) -> dict: + return {"error": self.message, "code": self.code} + + +class BadRequest(ApiError): + status, code = 400, "bad_request" + + +class Unauthorized(ApiError): + status, code = 401, "unauthorized" + + +class Forbidden(ApiError): + status, code = 403, "forbidden" + + +class NotFound(ApiError): + status, code = 404, "not_found" + + +class Conflict(ApiError): + status, code = 409, "conflict" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py b/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py new file mode 100644 index 000000000..958f828e7 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py @@ -0,0 +1,73 @@ +"""How a website env's health is determined, the one definition everything that judges it reads: a site says whether +it is usable now through its card's ``urn:agentenv:website/v1`` ``ready`` (``WEBSITE_READY_OP``), answering a +``WebsiteReadiness``. ``website_readiness`` asks one site as its card advertises it; ``env_readiness`` asks every site +of an env, a composed card's children or a single site's own card, and answers for the env as a whole; asking one +site may take ``READY_TIMEOUT_S`` by default.""" +from __future__ import annotations + +import asyncio + +import httpx +from pydantic import BaseModel, ConfigDict + +from ..client import find_extension_method, invoke_extension + +WEBSITE_EXTENSION_URI = "urn:agentenv:website/v1" +WEBSITE_READY_OP = "ready" +READY_TIMEOUT_S = 10.0 + + +class WebsiteReadiness(BaseModel): + """What a site says of its readiness, ``website/v1``'s ``ready``: whether it is ``ready``, and the ``problems`` + that keep it from being so, each a name a person reads, such as a ``@ready_check`` method's. Fields it doesn't + name are ignored rather than refused, so a reader keeps reading a newer site that says more, as ``ready`` and + ``problems`` keep their meaning. Its types are strict, as JSON's are: ``"yes"`` or ``1`` is no ``ready``.""" + + model_config = ConfigDict(extra="ignore", strict=True) + + ready: bool + problems: list[str] = [] + + +def _advertises_ready(card: dict) -> bool: + return find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP) is not None + + +async def website_readiness(base_url: str, card: dict, *, timeout: float = READY_TIMEOUT_S, + headers: dict[str, str] | None = None) -> WebsiteReadiness | None: + """What the site whose ``card`` is read from ``base_url`` says of its readiness, None when the card doesn't + advertise ``website/v1``'s ``ready``. Raises httpx's error when it can't be asked, ``ExtensionCallError`` for a + non-2xx answer, and ``ValueError`` (pydantic's ``ValidationError`` among them) for an answer that isn't a + ``WebsiteReadiness``.""" + if not _advertises_ready(card): + return None + answer = await invoke_extension(base_url, card, WEBSITE_EXTENSION_URI, timeout=timeout, method=WEBSITE_READY_OP, + headers=headers) + return WebsiteReadiness.model_validate(answer) + + +async def env_readiness(base_url: str, card: dict, *, timeout: float = READY_TIMEOUT_S, + headers: dict[str, str] | None = None) -> WebsiteReadiness: + """What the env whose ``card`` is read from ``base_url`` says of its readiness, from every site it has, asked at + once: the card itself when it advertises ``website/v1``'s ``ready``, as a single site's does, its problems as the + site names them, and each child under a composed card's ``children_environments`` that advertises it, its endpoints + rooted at ``base_url`` as a gateway's are, its problems prefixed with its name. A site that can't be asked, or + answers something that isn't a ``WebsiteReadiness``, is a problem of its own, and one not ready that names no + problem is one too. A child, or a card, that doesn't advertise ``ready`` isn't counted, so an env with no site + serving it, as one of MCP servers alone, is ready with no problems.""" + sites = [("", card)] if _advertises_ready(card) else [] + sites += [(f"{child.get('name') or 'child'}: ", child) for child in card.get("children_environments") or [] + if isinstance(child, dict) and _advertises_ready(child)] + answers = await asyncio.gather(*(_problems(base_url, site, timeout, headers) for _prefix, site in sites)) + problems = [prefix + problem for (prefix, _site), found in zip(sites, answers) for problem in found] + return WebsiteReadiness(ready=not problems, problems=problems) + + +async def _problems(base_url: str, card: dict, timeout: float, headers: dict[str, str] | None) -> list[str]: + try: + readiness = await website_readiness(base_url, card, timeout=timeout, headers=headers) + except (httpx.HTTPError, ValueError) as e: + return [f"{WEBSITE_READY_OP} could not be read: {type(e).__name__}: {e}"] + if readiness.ready: + return [] + return readiness.problems or [f"{WEBSITE_READY_OP} says not ready"] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py new file mode 100644 index 000000000..221f5e1b2 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py @@ -0,0 +1,369 @@ +"""A site's API: ``@api.get/post/put/patch/delete(path)`` methods, each bound from its type hints. A parameter named in +the path is a path parameter, one annotated with a pydantic model or ``dict`` is the JSON body, one annotated +``ApiContext`` receives the request's context, and every other one is a query parameter; path and query values are +coerced to their annotated types (int, float, bool, datetime, UUID, enum, Literal). What the method returns is the +JSON answer, a Starlette ``Response`` as it is; an ``ApiError`` it raises answers with its status, and a request its +parameters can't be bound from answers 400.""" +from __future__ import annotations + +import dataclasses +import datetime +import hmac +import inspect +import json +import logging +import os +import posixpath +import re +from typing import Any, Callable, Literal, NamedTuple, Optional, get_args, get_origin, get_type_hints + +from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic_core import to_jsonable_python +from starlette.requests import Request +from starlette.responses import JSONResponse, Response + +from ..agent_env_environment import _coerce_scalar, _maybe_await, _schema_for_annotation, _unwrap_annotation +from ..tool_context import DEFAULT_ROLE, caller_from_request +from ..types import RPC_PATH, WELL_KNOWN_PATH +from .errors import ApiError, BadRequest, Forbidden +from .session import SESSION_COOKIE, SessionSigner + +logger = logging.getLogger(__name__) + +_ROUTE_ATTR = "_aee_api_route" +_PATH_PARAM = re.compile(r"{(\w+)(?::\w+)?}") +Kind = Literal["path", "query", "body", "context"] + + +def reserved_path(path: str) -> bool: + """Whether ``path`` is the data plane's or the card's, however it's spelled, as the gateway decides it.""" + normalized = posixpath.normpath("/" + path.lstrip("/")) + return normalized == RPC_PATH or normalized.startswith(RPC_PATH + "/") or normalized == WELL_KNOWN_PATH + + +class ApiRoute(NamedTuple): + """One ``@api`` method: its verb, its Starlette path, its name on the card and in the OpenAPI document, the roles + it answers (any when empty), the status it succeeds with, and its function.""" + + method: str + path: str + name: str + roles: tuple[str, ...] + status: int + fn: Callable + + def advertised(self) -> dict: + return {"method": self.method, "path": openapi_path(self.path), "name": self.name, + "roles": list(self.roles) or None} + + +class _Api: + """``api.get(path, name=..., roles=[...])`` and its siblings mark a method as one route of the site's API.""" + + def get(self, path: str, **options: Any) -> Callable: + return self.route("GET", path, **options) + + def post(self, path: str, **options: Any) -> Callable: + return self.route("POST", path, **options) + + def put(self, path: str, **options: Any) -> Callable: + return self.route("PUT", path, **options) + + def patch(self, path: str, **options: Any) -> Callable: + return self.route("PATCH", path, **options) + + def delete(self, path: str, **options: Any) -> Callable: + return self.route("DELETE", path, **options) + + def route(self, method: str, path: str, *, name: str | None = None, roles: list[str] | None = None, + status: int = 200) -> Callable: + if not path.startswith("/"): + raise ValueError(f"an @api path starts with '/', not {path!r}") + if reserved_path(path): + raise ValueError(f"@api path {path!r} is under the data plane's {RPC_PATH} or the card's {WELL_KNOWN_PATH}") + + def mark(fn: Callable) -> Callable: + missing = sorted(set(_PATH_PARAM.findall(path)) - set(inspect.signature(fn).parameters)) + if missing: + raise TypeError(f"{fn.__qualname__} takes no parameter for the path's {missing}") + setattr(fn, _ROUTE_ATTR, ApiRoute(method.upper(), path, name or fn.__name__, tuple(roles or ()), status, fn)) + return fn + + return mark + + +api = _Api() + + +def routes_of(cls: type) -> list[ApiRoute]: + """The class's ``@api`` routes in the order they were first defined, each as the class resolves its name; + a name or a verb and path served twice is refused.""" + names = dict.fromkeys(name for klass in reversed(cls.__mro__) for name in vars(klass)) + routes = [route for name in names if (route := getattr(getattr(cls, name, None), _ROUTE_ATTR, None)) is not None] + for label, keys in (("name", [r.name for r in routes]), ("route", [f"{r.method} {r.path}" for r in routes])): + if twice := sorted({k for k in keys if keys.count(k) > 1}): + raise ValueError(f"{cls.__name__} serves the @api {label} {twice} twice") + return routes + + +def openapi_path(path: str) -> str: + return _PATH_PARAM.sub(r"{\1}", path) + + +@dataclasses.dataclass +class ApiContext: + """The request an ``@api`` method answers: its headers, cookies and query, the user it's signed in as (the + session cookie's, else the site's acting user), the caller's role (``request_role``), and the site's own address as the browser reached it. ``sign_in`` and ``sign_out`` set the + session cookie on the answer.""" + + request: Request + user: Optional[str] + role: str + session: Optional[dict] + _signer: SessionSigner = dataclasses.field(repr=False, default_factory=SessionSigner) + _set_cookie: list = dataclasses.field(repr=False, default_factory=list) + + @property + def headers(self) -> Any: + return self.request.headers + + @property + def cookies(self) -> dict[str, str]: + return self.request.cookies + + @property + def query(self) -> Any: + return self.request.query_params + + @property + def base_url(self) -> str: + return base_url(self.request) + + def sign_in(self, user: str) -> None: + self._set_session(user) + + def sign_out(self) -> None: + self._set_session(None) + + def _set_session(self, user: str | None) -> None: + value, expires = self._signer.encode(user) + self._set_cookie.append((value, expires)) + self.user, self.session = user, {"user": user} + + def apply(self, response: Response) -> Response: + for value, expires in self._set_cookie: + at = datetime.datetime.fromtimestamp(expires, datetime.timezone.utc) + response.set_cookie(SESSION_COOKIE, value, expires=at, path="/", httponly=True, samesite="lax") + return response + + +def base_url(request: Request) -> str: + """The site's address as the browser reached it: ``X-Forwarded-Proto`` and ``X-Forwarded-Host`` when a proxy + sent them, the request's own scheme and ``Host`` otherwise.""" + proto = request.headers.get("x-forwarded-proto", "").split(",")[0].strip() or request.url.scheme + host = (request.headers.get("x-forwarded-host", "").split(",")[0].strip() or request.headers.get("host") + or request.url.netloc) + return f"{proto}://{host}" + + +ROLE_KEY_ENV = "WEBSITE_ROLE_KEY" +ROLE_KEY_HEADER = "AgentEnv-Role-Key" + + +def request_role(request: Request) -> str: + """The role ``AgentEnv-Role`` names, believed only from a caller that also sends ``AgentEnv-Role-Key`` equal to + ``WEBSITE_ROLE_KEY``; every other caller, and every caller of a site with no key set, is ``default``. The gateway + forwards whatever role a browser sends, so without the key a page could claim any role.""" + claimed = caller_from_request(None, request.headers).role or DEFAULT_ROLE + key = os.environ.get(ROLE_KEY_ENV, "") + if claimed == DEFAULT_ROLE or not key: + return DEFAULT_ROLE + given = request.headers.get(ROLE_KEY_HEADER, "") + return claimed if hmac.compare_digest(given.encode(), key.encode()) else DEFAULT_ROLE + + +class _Param(NamedTuple): + name: str + kind: Kind + annotation: Any + default: Any + + +def plan(route: ApiRoute) -> list[_Param]: + """How each of the route's parameters is bound; at most one is the JSON body, and never on a GET.""" + hints = _hints(route.fn) + path_params = set(_PATH_PARAM.findall(route.path)) + params = [] + for name, param in list(inspect.signature(route.fn).parameters.items())[1:]: + if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + continue + annotation = hints.get(name, param.annotation) + kind: Kind = ("context" if _is_context(annotation) else "path" if name in path_params + else "body" if _body_type(annotation) is not None else "query") + params.append(_Param(name, kind, annotation, param.default)) + bodies = [p.name for p in params if p.kind == "body"] + if len(bodies) > 1 or (bodies and route.method == "GET"): + raise TypeError(f"{route.fn.__qualname__}: a route takes at most one JSON body and a GET route none, " + f"not {bodies}") + return params + + +def _is_context(annotation: Any) -> bool: + if isinstance(annotation, str): + return annotation.strip("'\"").rsplit(".", 1)[-1] in ("ApiContext", "Optional[ApiContext]") + return any(t is ApiContext for t in _unwrap_annotation(annotation)) + + +def _body_type(annotation: Any) -> type | None: + for t in _unwrap_annotation(annotation): + base = get_origin(t) or t + if base is dict or (isinstance(base, type) and issubclass(base, BaseModel)): + return t + return None + + +async def bind(params: list[_Param], request: Request, context: ApiContext) -> dict: + """The method's arguments from ``request``; raises BadRequest for one that is missing or doesn't fit.""" + kwargs: dict = {} + for p in params: + if p.kind == "context": + kwargs[p.name] = context + elif p.kind == "body": + kwargs[p.name] = await _body(p, request) + else: + source = request.path_params if p.kind == "path" else request.query_params + if p.name not in source: + if p.default is inspect.Parameter.empty: + raise BadRequest(f"missing {p.kind} parameter {p.name!r}", code="invalid_params") + kwargs[p.name] = p.default + continue + many = _list_type(p.annotation) if p.kind == "query" else None + try: + kwargs[p.name] = ([_coerce_scalar(v, get_args(many)[0] if get_args(many) else None) + for v in source.getlist(p.name)] if many is not None + else _coerce_scalar(source[p.name], p.annotation)) + except (TypeError, ValueError) as e: + raise BadRequest(f"{p.kind} parameter {p.name!r}: {e}", code="invalid_params") from None + return kwargs + + +def _list_type(annotation: Any) -> Any: + """The ``list[T]`` an annotation names, Optional or Annotated or not; None when it names no list.""" + inner = _unwrap_annotation(annotation) + return inner[0] if len(inner) == 1 and (inner[0] is list or get_origin(inner[0]) is list) else None + + +async def _body(p: _Param, request: Request) -> Any: + raw = await request.body() + if not raw: + if p.default is not inspect.Parameter.empty: + return p.default + raise BadRequest("this route takes a JSON body", code="invalid_params") + try: + value = json.loads(raw) + except ValueError: + raise BadRequest("the request body is not JSON", code="invalid_params") from None + try: + return TypeAdapter(p.annotation).validate_python(value) + except ValidationError as e: + problems = "; ".join(f"{'.'.join(map(str, err['loc'])) or 'body'}: {err['msg']}" for err in e.errors()) + raise BadRequest(f"invalid body: {problems}", code="invalid_params") from None + + +def endpoint(fn: Callable, route: ApiRoute, context_for: Callable[[Request], ApiContext]) -> Callable: + """The Starlette endpoint of one route, calling the bound method ``fn``.""" + params = plan(route) + + async def handle(request: Request) -> Response: + context = None + try: + context = context_for(request) + if route.roles and context.role not in route.roles: + raise Forbidden(f"role {context.role!r} may not call {route.name}") + result = await _maybe_await(fn(**await bind(params, request, context))) + if isinstance(result, Response): + response = result + elif not _has_body(route.status): + response = Response(status_code=route.status) + else: + response = JSONResponse(to_jsonable_python(result), status_code=route.status) + except ApiError as e: + response = error_response(e) + except Exception: + logger.exception("@api %s failed", route.name) + response = error_response(ApiError("internal error", code="internal")) + return context.apply(response) if context is not None else response + + return handle + + +def _has_body(status: int) -> bool: + """Whether an answer with ``status`` may carry a body: not a 1xx, 204 or 304.""" + return status >= 200 and status not in (204, 304) + + +def error_response(error: ApiError) -> Response: + return JSONResponse(error.body(), status_code=error.status) + + +def openapi(title: str, routes: list[ApiRoute]) -> dict: + """The OpenAPI 3.1 document of ``routes``, each operation's id its route's name; the models their bodies and + answers name, nested ones too, are its ``components.schemas``, which their schemas reference. Bodies are + described as they are validated and answers as they are serialized; a model the two modes see differently is in + ``components.schemas`` twice, as ``-Input`` and ``-Output``.""" + paths: dict = {} + schemas: list = [] + for route in routes: + doc = inspect.getdoc(route.fn) or "" + operation: dict = {"operationId": route.name, "parameters": [], + "responses": {str(route.status): {"description": "Success"}}} + if doc: + operation["summary"], operation["description"] = doc.splitlines()[0], doc + if route.roles: + operation["x-agentenv-roles"] = list(route.roles) + for p in plan(route): + if p.kind in ("path", "query"): + schema = _schema_for_annotation(p.annotation) + if p.default is not inspect.Parameter.empty: + schema["default"] = to_jsonable_python(p.default) + operation["parameters"].append({"name": p.name, "in": p.kind, "schema": schema, + "required": p.kind == "path" or p.default is inspect.Parameter.empty}) + elif p.kind == "body": + media: dict = {} + operation["requestBody"] = {"required": p.default is inspect.Parameter.empty, + "content": {"application/json": media}} + schemas.append((media, p.annotation, "validation")) + returns = _hints(route.fn).get("return") + if _has_body(route.status) and _adapter(returns) is not None: + media = {} + operation["responses"][str(route.status)]["content"] = {"application/json": media} + schemas.append((media, returns, "serialization")) + paths.setdefault(openapi_path(route.path), {})[route.method.lower()] = operation + document: dict = {"openapi": "3.1.0", "info": {"title": title, "version": "1.0"}, "paths": paths} + adapters = [(i, mode, adapter) for i, (_media, annotation, mode) in enumerate(schemas) + if (adapter := _adapter(annotation)) is not None] + found, definitions = TypeAdapter.json_schemas(adapters, ref_template="#/components/schemas/{model}") + for i, (media, _annotation, mode) in enumerate(schemas): + media["schema"] = found.get((i, mode), {}) + if definitions.get("$defs"): + document["components"] = {"schemas": definitions["$defs"]} + return document + + +def _hints(fn: Callable) -> dict: + try: + return get_type_hints(fn) + except Exception: + return {} + + +def _adapter(annotation: Any) -> TypeAdapter | None: + """The adapter of a type a JSON schema can describe; None for no type, None itself, or one it can't describe.""" + if annotation is None or annotation is type(None) or annotation is inspect.Parameter.empty: + return None + try: + adapter = TypeAdapter(annotation) + adapter.json_schema() + except Exception: + return None + return adapter diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py new file mode 100644 index 000000000..20e4ad36f --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py @@ -0,0 +1,60 @@ +"""A site's signed session cookie: who the browser is signed in as, or that it signed out, signed with HMAC-SHA256 +under ``WEBSITE_SESSION_SECRET``, or a secret of the process's own when that isn't set. The cookie names no domain, +so it holds under whatever host the site is reached at.""" +from __future__ import annotations + +import base64 +import binascii +import datetime +import hashlib +import hmac +import json +import os +import secrets +import time + +SESSION_COOKIE = "agentenv_session" +SECRET_ENV = "WEBSITE_SESSION_SECRET" +SESSION_TTL_S = 7 * 24 * 3600 +SECRET_BYTES = 32 + + +class SessionSigner: + def __init__(self, secret: str | bytes | None = None) -> None: + secret = secret if secret is not None else os.environ.get(SECRET_ENV) + self._key = (secret.encode() if isinstance(secret, str) else secret) or secrets.token_bytes(SECRET_BYTES) + + def encode(self, user: str | None, ttl_s: int = SESSION_TTL_S) -> tuple[str, int]: + """The cookie value for ``user`` signed in, or signed out when None, and when it expires, in epoch + seconds.""" + expires = int(time.time()) + ttl_s + payload = base64.urlsafe_b64encode(json.dumps({"u": user, "exp": expires}).encode()).decode().rstrip("=") + return f"{payload}.{self._sign(payload)}", expires + + def decode(self, value: str | None) -> dict | None: + """The session a cookie value carries, ``{"user": ...}``; None for one that is absent, forged or expired.""" + payload, _, signature = (value or "").partition(".") + if not payload or not hmac.compare_digest(signature.encode("utf-8", "surrogateescape"), + self._sign(payload).encode()): + return None + try: + data = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + except (ValueError, binascii.Error): + return None + if not isinstance(data, dict) or not isinstance(data.get("exp"), int) or data["exp"] < time.time(): + return None + return {"user": data.get("u")} + + def cookie(self, user: str | None, ttl_s: int = SESSION_TTL_S) -> dict: + """The cookie a browser sets to be signed in as ``user``, as a browser automation adds one; its domain is + left to whoever adds it.""" + value, expires = self.encode(user, ttl_s) + return {"name": SESSION_COOKIE, "value": value, "path": "/", "expires": expires, "httpOnly": True, + "sameSite": "Lax"} + + def _sign(self, payload: str) -> str: + return hmac.new(self._key, payload.encode("utf-8", "surrogateescape"), hashlib.sha256).hexdigest() + + +def expires_at(epoch: int) -> str: + return datetime.datetime.fromtimestamp(epoch, datetime.timezone.utc).isoformat().replace("+00:00", "Z") diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/state.py b/packages/agentenv-protocol/src/agentenv_protocol/website/state.py new file mode 100644 index 000000000..77e0b5558 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/state.py @@ -0,0 +1,56 @@ +"""What a website env's task state is, the one definition everything that grades a site reads: a site gives the state a +task left it in through its card's ``urn:agentenv:website/v1`` ``state`` (``WEBSITE_STATE_OP``), answering a +``WebsiteState``, its state when the task started, its state now and what changed between them. ``website_state`` +asks one site as its card advertises it, of one session when the site keeps several; ``website_sites`` names every +site of an env, a single site's own card or a composed card's website children. The op is one of agentenv's own +routes, under the data plane's path, so a browser on the site's route never reads what grades it.""" +from __future__ import annotations + +from typing import Any, Optional + +from pydantic import BaseModel, ConfigDict + +from ..client import ChildEnv, children, find_extension, find_extension_method, invoke_extension +from .readiness import WEBSITE_EXTENSION_URI + +WEBSITE_STATE_OP = "state" +STATE_TIMEOUT_S = 30.0 + + +class WebsiteState(BaseModel): + """What a site says of its task state, ``website/v1``'s ``state``: ``initial_state``, the state the task started + from, ``current_state``, the state now, and ``state_diff``, what changed between them, each a JSON object or None + when the site gives none. Fields it doesn't name are ignored rather than refused, so a reader keeps reading a newer + site that says more.""" + + model_config = ConfigDict(extra="ignore") + + initial_state: Optional[dict[str, Any]] = None + current_state: Optional[dict[str, Any]] = None + state_diff: Optional[dict[str, Any]] = None + + +def advertises_state(card: dict) -> bool: + """Whether ``card`` advertises ``website/v1``'s ``state``.""" + return find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) is not None + + +def website_sites(card: dict) -> list[ChildEnv]: + """Every site of the env whose card is ``card``: the card itself when it carries ``website/v1``, as a single site's + does, else each child of a composed card that carries it, in its order; none for an env with no site.""" + if find_extension(card, WEBSITE_EXTENSION_URI) is not None: + return [ChildEnv(name=card.get("name"), url=card.get("url"), card=card)] + return [child for child in children(card) if child.find_extension(WEBSITE_EXTENSION_URI) is not None] + + +async def website_state(base_url: str, card: dict, *, sid: str | None = None, timeout: float = STATE_TIMEOUT_S, + headers: dict[str, str] | None = None) -> WebsiteState | None: + """What the site whose ``card`` is read from ``base_url`` says of its task state, of session ``sid`` when it's + given, for a site that keeps several; None when the card doesn't advertise ``website/v1``'s ``state``. Raises + httpx's error when it can't be asked, ``ExtensionCallError`` for a non-2xx answer, and ``ValueError`` (pydantic's + ``ValidationError`` among them) for an answer that isn't a ``WebsiteState``.""" + if not advertises_state(card): + return None + answer = await invoke_extension(base_url, card, WEBSITE_EXTENSION_URI, {"sid": sid} if sid is not None else None, + timeout=timeout, method=WEBSITE_STATE_OP, headers=headers) + return WebsiteState.model_validate(answer) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py new file mode 100644 index 000000000..ce21ad84c --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py @@ -0,0 +1,134 @@ +"""The default state store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON +object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as. +It remembers the state it was last loaded with, so a verifier can read the initial state, the current one and the +difference between them.""" +from __future__ import annotations + +import base64 +import copy +import json +from pathlib import Path +from typing import Any, Iterable +from urllib.parse import unquote, urlparse + +from ..types import DataPart, FilePart, Part, TextPart + +CURRENT_USER = "current_user" + + +class Collections: + """Named lists of JSON objects; ``store["orders"]`` is the list itself, created empty on first use.""" + + def __init__(self, data: dict | None = None) -> None: + self._collections: dict[str, list[dict]] = {} + self.current_user: str | None = None + self.initial: dict = {} + if data: + self.load(data) + self.mark_initial() + + def __getitem__(self, name: str) -> list[dict]: + return self._collections.setdefault(name, []) + + def names(self) -> list[str]: + return list(self._collections) + + def get(self, name: str, id: Any) -> dict | None: + return next((doc for doc in self[name] if doc.get("id") == id), None) + + def find(self, name: str, **match: Any) -> list[dict]: + return [doc for doc in self[name] if all(doc.get(k) == v for k, v in match.items())] + + def insert(self, name: str, doc: dict) -> dict: + """Add ``doc``, giving it the next integer ``id`` of its collection when it has none.""" + doc = dict(doc) + if doc.get("id") is None: + doc["id"] = 1 + max((d["id"] for d in self[name] if isinstance(d.get("id"), int)), default=0) + self[name].append(doc) + return doc + + def update(self, name: str, id: Any, **fields: Any) -> dict | None: + doc = self.get(name, id) + if doc is not None: + doc.update(fields) + return doc + + def delete(self, name: str, id: Any) -> bool: + doc = self.get(name, id) + if doc is not None: + self[name].remove(doc) + return doc is not None + + def load(self, data: dict) -> None: + """Add ``data``'s collections to the store's, and take its ``current_user`` when it names one.""" + if not isinstance(data, dict): + raise ValueError("a store's data is a JSON object of collections by name") + for name, docs in data.items(): + if name == CURRENT_USER: + self.current_user = docs + elif isinstance(docs, list) and all(isinstance(d, dict) for d in docs): + self[name].extend(copy.deepcopy(docs)) + else: + raise ValueError(f"collection {name!r} is not a list of JSON objects") + + def load_parts(self, parts: Iterable[Part]) -> None: + for part in parts: + self.load(_part_data(part)) + + def reset(self) -> None: + self._collections.clear() + self.current_user = None + self.initial = {} + + def export(self) -> dict: + data: dict = copy.deepcopy(self._collections) + if self.current_user is not None: + data[CURRENT_USER] = self.current_user + return data + + def mark_initial(self) -> None: + self.initial = self.export() + + def diff(self) -> dict: + """Each collection that changed since the initial state, as its ``added``, ``removed`` and ``changed`` + objects, matched by ``id``; a changed one as its ``id``, ``before`` and ``after``.""" + current = self.export() + changes: dict = {} + for name in dict.fromkeys([*self.initial, *current]): + if name == CURRENT_USER: + continue + before = {_key(d): d for d in self.initial.get(name, [])} + after = {_key(d): d for d in current.get(name, [])} + entry = { + "added": [d for k, d in after.items() if k not in before], + "removed": [d for k, d in before.items() if k not in after], + "changed": [{"id": d.get("id"), "before": before[k], "after": d} + for k, d in after.items() if k in before and before[k] != d], + } + if any(entry.values()): + changes[name] = entry + return changes + + def state(self) -> dict: + return {"initial_state": copy.deepcopy(self.initial), "current_state": self.export(), "state_diff": self.diff()} + + +def _key(doc: dict) -> str: + return json.dumps(doc.get("id")) if doc.get("id") is not None else json.dumps(doc, sort_keys=True) + + +def _part_data(part: Part) -> dict: + """A part's JSON: a data part's data, a text part's text, or a file part's bytes or ``file://`` file.""" + if isinstance(part, DataPart): + return part.data + if isinstance(part, TextPart): + return json.loads(part.text) + if isinstance(part, FilePart): + file = part.file + if getattr(file, "bytes", None) is not None: + return json.loads(base64.b64decode(file.bytes)) + uri = urlparse(file.uri) + if uri.scheme == "file": + return json.loads(Path(unquote(uri.path)).read_text(encoding="utf-8")) + raise ValueError(f"the store reads file parts by their bytes or a file:// URI, not {file.uri!r}") + raise ValueError(f"the store can't read a {type(part).__name__}") diff --git a/packages/agentenv-protocol/tests/test_website_api.py b/packages/agentenv-protocol/tests/test_website_api.py new file mode 100644 index 000000000..21b73530c --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_api.py @@ -0,0 +1,413 @@ +"""The website SDK's @api routes: binding from type hints, JSON answers, errors, roles, and what a class may not +declare.""" + +from __future__ import annotations + +import datetime +import enum +from typing import Optional, Union + +import pytest +from pydantic import BaseModel, computed_field +from starlette.responses import PlainTextResponse +from starlette.testclient import TestClient + +from agentenv_protocol.website import ( + ROLE_KEY_ENV, + ROLE_KEY_HEADER, + ApiContext, + BadRequest, + Conflict, + NotFound, + WebsiteEnvironment, + api, + reserved_path, + website, +) + + +class Color(enum.Enum): + RED = "red" + BLUE = "blue" + + +class NewItem(BaseModel): + title: str + price: float = 0 + + +@website(domains=["shop.example"]) +class Shop(WebsiteEnvironment): + def __init__(self) -> None: + super().__init__() + self.items: dict[int, dict] = {1: {"id": 1, "title": "mug", "price": 3.5}} + + @api.get("/api/items/{item_id}", name="get_item") + async def get_item(self, item_id: int) -> dict: + """Read one item.""" + if item_id not in self.items: + raise NotFound(f"no item {item_id}") + return self.items[item_id] + + @api.get("/api/items") + def list_items(self, limit: int = 10, cheap: bool = False, color: Optional[Color] = None, + since: Optional[datetime.date] = None, tags: list[str] = []) -> dict: + return {"limit": limit, "cheap": cheap, "color": color.value if color else None, + "since": since.isoformat() if since else None, "tags": tags} + + @api.post("/api/items", name="create_item", status=201) + async def create_item(self, item: NewItem) -> NewItem: + if any(i["title"] == item.title for i in self.items.values()): + raise Conflict(f"{item.title} exists") + self.items[len(self.items) + 1] = {"id": len(self.items) + 1, **item.model_dump()} + return item + + @api.patch("/api/items/{item_id}") + async def patch_item(self, item_id: int, changes: dict) -> dict: + self.items[item_id].update(changes) + return self.items[item_id] + + @api.delete("/api/items/{item_id}", roles=["admin"]) + async def delete_item(self, item_id: int) -> None: + self.items.pop(item_id, None) + + @api.put("/api/teapot") + async def teapot(self) -> None: + raise BadRequest("short and stout", code="teapot", status=418) + + @api.get("/api/whoami") + async def whoami(self, ctx: ApiContext) -> dict: + return {"user": ctx.user, "role": ctx.role, "base_url": ctx.base_url, "q": ctx.query.get("q"), + "header": ctx.headers.get("x-probe"), "cookie": ctx.cookies.get("c")} + + @api.get("/api/plain") + async def plain(self) -> PlainTextResponse: + return PlainTextResponse("as is") + + @api.get("/api/boom") + async def boom(self) -> dict: + raise RuntimeError("secret detail") + + @api.get("/api/picked") + def picked(self, ids: Optional[list[int]] = None) -> dict: + return {"ids": ids} + + @api.delete("/api/cart", status=204) + async def empty_cart(self) -> None: + return None + + +@pytest.fixture +def client() -> TestClient: + return TestClient(Shop().create_app()) + + +def test_path_query_and_body_parameters_are_bound_from_type_hints(client: TestClient) -> None: + assert client.get("/api/items/1").json() == {"id": 1, "title": "mug", "price": 3.5} + assert client.get("/api/items").json() == {"limit": 10, "cheap": False, "color": None, "since": None, "tags": []} + listed = client.get("/api/items?limit=3&cheap=yes&color=blue&since=2030-01-02&tags=a&tags=b").json() + assert listed == {"limit": 3, "cheap": True, "color": "blue", "since": "2030-01-02", "tags": ["a", "b"]} + created = client.post("/api/items", json={"title": "pen", "price": 1}) + assert created.status_code == 201 and created.json() == {"title": "pen", "price": 1.0} + assert client.patch("/api/items/1", json={"price": 4}).json()["price"] == 4 + + +def test_a_request_its_parameters_cannot_be_bound_from_answers_400(client: TestClient) -> None: + for response in (client.get("/api/items/abc"), client.get("/api/items?limit=many"), + client.get("/api/items?color=green"), client.post("/api/items", json={"price": 2}), + client.post("/api/items", content=b"not json"), client.post("/api/items"), + client.patch("/api/items/1", json=[1, 2])): + assert response.status_code == 400, response.text + assert response.json()["code"] == "invalid_params" + assert "title" in client.post("/api/items", json={"price": 2}).json()["error"] + + +class Pin(BaseModel): + lat: float + + +class Place(BaseModel): + name: str + pin: Pin + + +class Note(BaseModel): + text: str + + +@website(domains=["map.example"]) +class Map(WebsiteEnvironment): + @api.post("/api/places") + def add_place(self, place: Place) -> list[Place]: + return [place] + + @api.post("/api/marks") + def mark(self, mark: Union[Note, Pin]) -> dict: + return {"kind": type(mark).__name__} + + @api.put("/api/settings") + def settings(self, settings: Optional[dict]) -> dict: + return {"settings": settings} + + +def test_a_body_is_validated_against_its_whole_annotation() -> None: + client = TestClient(Map().create_app()) + assert client.put("/api/settings", content=b"null", headers={"content-type": "application/json"}).json() == { + "settings": None} + assert client.post("/api/marks", json={"lat": 1.5}).json() == {"kind": "Pin"} + assert client.post("/api/marks", json={"text": "hi"}).json() == {"kind": "Note"} + assert client.post("/api/marks", json={"other": 1}).status_code == 400 + + +def test_nested_models_are_the_openapi_document_s_components_its_schemas_reference() -> None: + document = TestClient(Map().create_app()).get("/agentenv/ext/website/openapi").json() + schemas = document["components"]["schemas"] + assert set(schemas) == {"Pin", "Place", "Note"} + assert schemas["Place"]["properties"]["pin"] == {"$ref": "#/components/schemas/Pin"} + body = document["paths"]["/api/places"]["post"]["requestBody"]["content"]["application/json"]["schema"] + answer = document["paths"]["/api/places"]["post"]["responses"]["200"]["content"]["application/json"]["schema"] + assert body == {"$ref": "#/components/schemas/Place"} + assert answer == {"type": "array", "items": {"$ref": "#/components/schemas/Place"}} + refs = [v for v in _walk(document) if isinstance(v, str) and v.startswith("#/")] + assert refs and all(ref.split("/")[-1] in schemas and ref.startswith("#/components/schemas/") for ref in refs) + + +class Cart(BaseModel): + prices: list[float] + + @computed_field + @property + def total(self) -> float: + return sum(self.prices) + + +@website(domains=["till.example"]) +class Till(WebsiteEnvironment): + @api.post("/api/carts") + def price(self, cart: Cart) -> Cart: + return cart + + +def test_an_answer_s_schema_describes_it_as_serialized_and_a_body_s_as_validated() -> None: + client = TestClient(Till().create_app()) + assert client.post("/api/carts", json={"prices": [1, 2]}).json() == {"prices": [1, 2], "total": 3} + document = client.get("/agentenv/ext/website/openapi").json() + schemas = document["components"]["schemas"] + operation = document["paths"]["/api/carts"]["post"] + body = operation["requestBody"]["content"]["application/json"]["schema"] + answer = operation["responses"]["200"]["content"]["application/json"]["schema"] + assert body == {"$ref": "#/components/schemas/Cart-Input"} + assert answer == {"$ref": "#/components/schemas/Cart-Output"} + assert "total" not in schemas["Cart-Input"]["properties"] and "total" in schemas["Cart-Output"]["properties"] + + +def _walk(value: object): + if isinstance(value, dict): + for v in value.values(): + yield from _walk(v) + elif isinstance(value, list): + for v in value: + yield from _walk(v) + else: + yield value + + +def test_api_errors_answer_with_their_status_and_code(client: TestClient) -> None: + missing = client.get("/api/items/9") + assert (missing.status_code, missing.json()) == (404, {"error": "no item 9", "code": "not_found"}) + assert client.post("/api/items", json={"title": "mug"}).json() == {"error": "mug exists", "code": "conflict"} + teapot = client.put("/api/teapot") + assert (teapot.status_code, teapot.json()["code"]) == (418, "teapot") + boom = client.get("/api/boom") + assert (boom.status_code, boom.json()) == (500, {"error": "internal error", "code": "internal"}) + + +def test_responses_pass_through_and_none_is_json_null(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + assert client.get("/api/plain").text == "as is" + deleted = client.delete("/api/items/1", headers={"AgentEnv-Role": "admin", ROLE_KEY_HEADER: "k3y"}) + assert (deleted.status_code, deleted.json()) == (200, None) + + +def test_an_optional_list_query_parameter_is_a_list_of_its_coerced_values(client: TestClient) -> None: + assert client.get("/api/picked?ids=1&ids=2").json() == {"ids": [1, 2]} + assert client.get("/api/picked").json() == {"ids": None} + assert client.get("/api/picked?ids=x").status_code == 400 + + +def test_a_route_declared_204_answers_with_no_body(client: TestClient) -> None: + emptied = client.delete("/api/cart") + assert (emptied.status_code, emptied.content) == (204, b"") + assert "content-length" not in emptied.headers and "content-type" not in emptied.headers + + +def test_a_204_answer_keeps_a_real_server_connection_usable() -> None: + import socket + import threading + import time + + uvicorn = pytest.importorskip("uvicorn") + server = uvicorn.Server(uvicorn.Config(Shop().create_app(), host="127.0.0.1", port=0, log_level="error", + ws="none")) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + while not server.started: + time.sleep(0.01) + port = server.servers[0].sockets[0].getsockname()[1] + try: + with socket.create_connection(("127.0.0.1", port), timeout=5) as conn: + conn.sendall(b"DELETE /api/cart HTTP/1.1\r\nHost: x\r\n\r\nGET /api/items/1 HTTP/1.1\r\nHost: x\r\n\r\n") + received = b"" + while b"mug" not in received: + chunk = conn.recv(65536) + if not chunk: + break + received += chunk + assert received.startswith(b"HTTP/1.1 204") and b"HTTP/1.1 200" in received and b"mug" in received + finally: + server.should_exit = True + thread.join(5) + + +def test_a_session_cookie_that_is_not_ascii_is_no_session(client: TestClient) -> None: + for path in ("/api/whoami", "/api/session"): + response = client.get(path, headers={"Cookie": "agentenv_session=abc.\xe9".encode("latin-1")}) + assert response.status_code == 200, response.text + assert response.json()["user"] is None + + +def test_roles_refuse_other_callers_with_403(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + for headers in ({}, {"AgentEnv-Role": "default"}, {"AgentEnv-Role": "agent", ROLE_KEY_HEADER: "k3y"}): + refused = client.delete("/api/items/1", headers=headers) + assert refused.status_code == 403 and refused.json()["code"] == "forbidden" + assert client.get("/api/items/1").status_code == 200 + assert client.delete("/api/items/1", headers={"AgentEnv-Role": "admin", ROLE_KEY_HEADER: "k3y"}).status_code == 200 + + +def test_a_role_is_believed_only_with_the_site_s_role_key(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(ROLE_KEY_ENV, raising=False) + claim = {"AgentEnv-Role": "admin"} + assert client.delete("/api/items/1", headers=claim).status_code == 403 + assert client.delete("/api/items/1", headers={**claim, ROLE_KEY_HEADER: ""}).status_code == 403 + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + for key in (None, "", "wrong", "k3y "): + headers = claim if key is None else {**claim, ROLE_KEY_HEADER: key} + assert client.delete("/api/items/1", headers=headers).status_code == 403 + assert client.get("/api/whoami", headers=headers).json()["role"] == "default" + assert client.get("/api/whoami", headers={**claim, ROLE_KEY_HEADER: "k3y"}).json()["role"] == "admin" + + +def test_the_context_carries_the_request_role_and_forwarded_address(client: TestClient, + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + client.cookies.set("c", "crumb") + seen = client.get("/api/whoami?q=1", headers={"x-probe": "p", "AgentEnv-Role": "agent", ROLE_KEY_HEADER: "k3y"}).json() + assert seen == {"user": None, "role": "agent", "base_url": "http://testserver", "q": "1", "header": "p", + "cookie": "crumb"} + forwarded = client.get("/api/whoami", headers={"X-Forwarded-Host": "shop.example, proxy", + "X-Forwarded-Proto": "https"}).json() + assert (forwarded["base_url"], forwarded["role"]) == ("https://shop.example", "default") + + +def test_a_redirect_names_the_forwarded_address(client: TestClient) -> None: + redirected = client.get("/api/items/", follow_redirects=False, + headers={"Host": "internal-x.invalid", "X-Forwarded-Host": "shop.example", + "X-Forwarded-Proto": "https"}) + assert (redirected.status_code, redirected.headers["location"]) == (307, "https://shop.example/api/items") + assert client.get("/api/items/", follow_redirects=False).headers["location"] == "http://testserver/api/items" + + +def test_parameterized_routes_never_take_the_data_plane_or_card() -> None: + @website(domains=["wiki.example"]) + class Wiki(WebsiteEnvironment): + @api.get("/{slug}") + def page(self, slug: str) -> dict: + return {"slug": slug} + + @api.post("/{collection}") + def add(self, collection: str) -> dict: + return {"captured": collection} + + @api.get("/.well-known/{name}") + def well_known(self, name: str) -> dict: + return {"captured": name} + + @api.get("/files/{rest:path}") + def files(self, rest: str) -> dict: + return {"rest": rest} + + client = TestClient(Wiki().create_app()) + assert client.get("/hello").json() == {"slug": "hello"} + assert client.post("/notes").json() == {"captured": "notes"} + assert client.get("/.well-known/other.json").json() == {"captured": "other.json"} + assert client.get("/files/a/b.txt").json() == {"rest": "a/b.txt"} + reset = client.post("/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/get", "params": {}}).json() + assert reset["id"] == 1 and "result" in reset + assert client.get("/.well-known/agent-env.json").json()["name"] == "Wiki" + assert client.get("/api/health").json() == {"status": "ok"} + + +@pytest.mark.parametrize("path", ["/agentenv", "/agentenv/ext/x", "/.well-known/agent-env.json", + "/api/../agentenv/x", "//agentenv"]) +def test_api_paths_under_the_reserved_ones_are_refused(path: str) -> None: + assert reserved_path(path) + with pytest.raises(ValueError, match="data plane"): + api.get(path) + + +def test_a_class_serving_a_name_or_route_twice_is_refused() -> None: + with pytest.raises(ValueError, match="name"): + class Twice(WebsiteEnvironment): + @api.get("/a", name="same") + def a(self) -> None: ... + + @api.get("/b", name="same") + def b(self) -> None: ... + + with pytest.raises(ValueError, match="route"): + class Again(WebsiteEnvironment): + @api.post("/a") + def a(self) -> None: ... + + @api.post("/a") + def b(self) -> None: ... + + +def test_a_route_must_take_its_path_parameters_and_at_most_one_body() -> None: + with pytest.raises(TypeError, match="item_id"): + api.get("/api/{item_id}")(lambda self: None) + + class Bodies(WebsiteEnvironment): + @api.post("/api/x") + def x(self, a: dict, b: NewItem) -> None: ... + + with pytest.raises(TypeError, match="one JSON body"): + Bodies().create_app() + + +def test_the_health_path_is_not_an_api_route() -> None: + with pytest.raises(ValueError, match="ready_check"): + @website(health="/api/up") + class Clash(WebsiteEnvironment): + @api.get("/api/up") + def up(self) -> dict: + return {} + + with pytest.raises(ValueError, match="health"): + website(health="/agentenv/health") + + +@website(domains=["shelf.example"]) +class Shelf(WebsiteEnvironment): + @api.delete("/api/books/{book_id}", status=204) + def drop(self, book_id: int) -> Pin: + return Pin(lat=1.0) + + +def test_a_bodyless_status_publishes_no_body_and_answers_none() -> None: + client = TestClient(Shelf().create_app()) + answered = client.delete("/api/books/1") + assert answered.status_code == 204 and answered.content == b"" + responses = client.get("/agentenv/ext/website/openapi").json()["paths"]["/api/books/{book_id}"]["delete"]["responses"] + assert "content" not in responses["204"] diff --git a/packages/agentenv-protocol/tests/test_website_conformance.py b/packages/agentenv-protocol/tests/test_website_conformance.py new file mode 100644 index 000000000..50f981b54 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_conformance.py @@ -0,0 +1,265 @@ +"""The website/v1 conformance checker against an SDK site, a hand-written server that meets the contract, and servers +that don't.""" + +from __future__ import annotations + +import datetime +from pathlib import Path + +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import HTMLResponse, JSONResponse, Response +from starlette.routing import Route +from starlette.testclient import TestClient + +from agentenv_protocol.website import WebsiteEnvironment, api, website +from agentenv_protocol.website import conformance +from agentenv_protocol.website.clock import UNIVERSE_NOW_HEADER, iso + +CHECKS = ["card", "any_host", "health", "universe_now", "data_plane", "clock", "forwarded", "cookies"] + + +def statuses(results: list[conformance.Result]) -> dict[str, str]: + return {r.name: r.status for r in results} + + +@pytest.fixture +def notes(tmp_path: Path) -> WebsiteEnvironment: + (tmp_path / "index.html").write_text("hi") + + @website(domains=["notes.example"], static=str(tmp_path)) + class Notes(WebsiteEnvironment): + @api.get("/api/notes") + def notes(self) -> list: + return self.store["notes"] + + env = Notes() + env.store.load({"current_user": "ann", "notes": [{"id": 1, "text": "hello"}]}) + env.store.mark_initial() + return env + + +@pytest.fixture +def sdk_site(notes: WebsiteEnvironment) -> TestClient: + return TestClient(notes.create_app()) + + +def test_an_sdk_site_passes_every_check(sdk_site: TestClient) -> None: + results = conformance.check("http://testserver", client=sdk_site) + assert [r.name for r in results] == CHECKS + assert all(r.status == "passed" for r in results), results + assert sdk_site.post("/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/get"}).json()["result"][ + "parts"][0]["data"] == {"current_user": "ann", "notes": [{"id": 1, "text": "hello"}]} + + +def _handwritten(*, leak_host: bool = False, domain_cookie: bool = False, clock: bool = True, + leak_redirect: bool = False, stamp_rpc: bool = True, reset_keeps: bool = False, + get_fails_after_reset: bool = False, rpc_reply: tuple[int, dict] | None = None, + ready_answer: object = None, rpc_path: str | None = None, state_path: str | None = None, + changed: bool = False) -> Starlette: + """A server written without the SDK: the contract by hand, optionally broken. Its state is its ``state.data``; with + ``ready_answer`` its card advertises ``website/v1``'s ``ready``, which answers it. With ``rpc_path`` its card's + ``url`` names its data plane there, not at ``/agentenv``, and with ``state_path`` its ``params.state.path`` names + where it serves its state, which shows changes since the initial state when ``changed``.""" + state = {"items": ["a"], "url": None, "at": None, "reset": False} + + def now() -> datetime.datetime: + return state["at"] or datetime.datetime.now(datetime.timezone.utc) + + def stamp(response: Response) -> Response: + response.headers[UNIVERSE_NOW_HEADER] = iso(now()) + return response + + async def page(request: Request) -> Response: + host = request.headers["host"] if leak_host else request.headers.get("x-forwarded-host", request.headers["host"]) + response = stamp(HTMLResponse(f'x')) + response.set_cookie("s", "1", domain="example.com" if domain_cookie else None) + return response + + async def health(request: Request) -> Response: + return stamp(JSONResponse({"ok": True})) + + async def ready(request: Request) -> Response: + return stamp(JSONResponse(ready_answer)) + + async def items(request: Request) -> Response: + return stamp(JSONResponse(state["items"])) + + async def items_slash(request: Request) -> Response: + host = request.headers["host"] if leak_redirect else request.headers.get("x-forwarded-host", request.headers["host"]) + return stamp(Response(status_code=307, headers={"Location": f"https://{host}/items"})) + + async def card(request: Request) -> Response: + extensions = [{"uri": "urn:agentenv:website/v1", + "params": {"domains": ["h.example"], "health": "/healthz", + "routes": [{"method": "GET", "path": "/items", "name": "items"}]}}] + if ready_answer is not None: + extensions[0]["params"]["methods"] = {"ready": {"method": "GET", "endpoint": "/ready"}} + if state_path is not None: + extensions[0]["params"]["state"] = {"path": state_path} + if clock: + extensions.append({"uri": "urn:agentenv:clock/v1", "params": { + "endpoint": "/agentenv/ext/sync_time", "methods": {"sync_time": {"method": "POST"}}}}) + return stamp(JSONResponse({"name": "hand", **({"url": rpc_path} if rpc_path else {}), + "capabilities": {"extensions": extensions, + "operations": ["data/reset", "data/add", "data/get"]}})) + + async def site_state(request: Request) -> Response: + return JSONResponse({"state_diff": {"items": {"added": ["b"]}} if changed else {}}) + + async def rpc(request: Request) -> Response: + body = await request.json() + method = body["method"] + if rpc_reply is not None: + return stamp(JSONResponse(rpc_reply[1], status_code=rpc_reply[0])) + if method == "data/reset": + state["reset"] = True + if not reset_keeps: + state["items"] = [] + elif method == "data/add": + state["reset"] = False + added = [item for part in body["params"]["parts"] for item in part["data"]["items"]] + state["items"] = added if reset_keeps else state["items"] + added + if method == "data/get" and get_fails_after_reset and state["reset"]: + answer = {"jsonrpc": "2.0", "id": body["id"], "error": {"code": -32603, "message": "down"}} + else: + result = {"parts": [{"kind": "data", "data": {"items": state["items"]}}]} if method == "data/get" else {} + answer = {"jsonrpc": "2.0", "id": body["id"], "result": result} + return stamp(JSONResponse(answer)) if stamp_rpc else JSONResponse(answer) + + async def sync_time(request: Request) -> Response: + import httpx + + url = (await request.json())["env_get_time_url"] + async with httpx.AsyncClient() as client: + virtual = (await client.get(url)).json()["virtual_time"] + state["at"] = datetime.datetime.fromisoformat(virtual.replace("Z", "+00:00")) + return stamp(JSONResponse({"gateway_virtual_time": virtual})) + + app = Starlette(routes=[Route("/", page), Route("/healthz", health), Route("/ready", ready), Route("/items", items), + Route("/items/", items_slash), + Route("/.well-known/agent-env.json", card), Route(rpc_path or "/agentenv", rpc, methods=["POST"]), + Route("/agentenv/ext/sync_time", sync_time, methods=["POST"]), + *([Route(state_path, site_state)] if state_path else [])]) + app.state.data = state + return app + + +def test_a_handwritten_server_meeting_the_contract_passes() -> None: + results = conformance.check("http://testserver", client=TestClient(_handwritten())) + assert statuses(results) == dict.fromkeys(CHECKS, "passed"), results + + +@pytest.mark.parametrize(("answer", "status", "detail"), [ + ({"ready": True, "problems": [], "since": "boot"}, "passed", "website/v1 ready says it is ready"), + ({"ready": False, "problems": ["seeded", "indexed"]}, "failed", "website/v1 ready says it is not ready: seeded, indexed"), + ({"ready": False}, "failed", "website/v1 ready says it is not ready: no problem named"), +]) +def test_health_reads_the_cards_ready_as_a_website_readiness(answer: dict, status: str, detail: str) -> None: + results = {r.name: r for r in conformance.check("http://testserver", client=TestClient(_handwritten(ready_answer=answer)))} + assert (results["health"].status, results["health"].detail) == (status, detail) + + +@pytest.mark.parametrize(("answer", "why"), [ + ({"ok": True}, "ready: Field required"), + ({"ready": "soon", "problems": []}, "ready: Input should be a valid boolean"), + ({"ready": False, "problems": "seeded"}, "problems: Input should be a valid list"), + (["ready"], "the answer: Input should be a valid dictionary or instance of WebsiteReadiness"), +]) +def test_an_answer_to_ready_that_isnt_a_website_readiness_fails_health_saying_why(answer: object, why: str) -> None: + health = next(r for r in conformance.check("http://testserver", client=TestClient(_handwritten(ready_answer=answer))) + if r.name == "health") + assert health.status == "failed" + assert health.detail.startswith(f"website/v1 ready answered {answer!r}, not a WebsiteReadiness: ") and why in health.detail + + +def test_a_server_breaking_the_contract_fails_those_checks() -> None: + results = statuses(conformance.check("http://testserver", + client=TestClient(_handwritten(leak_host=True, domain_cookie=True, clock=False)))) + assert results["forwarded"] == results["cookies"] == results["clock"] == "failed" + assert results["card"] == results["health"] == results["data_plane"] == "passed" + + +def test_the_forwarded_check_follows_no_redirect_and_reads_its_location(sdk_site: TestClient) -> None: + forwarded = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["forwarded"] + assert forwarded.status == "passed" and "/api/notes/" in forwarded.detail + leaked = {r.name: r for r in conformance.check("http://testserver", + client=TestClient(_handwritten(leak_redirect=True)))}["forwarded"] + assert leaked.status == "failed" and "Location of /items/" in leaked.detail + + +def test_universe_now_is_checked_on_the_answers_of_every_check() -> None: + result = {r.name: r for r in conformance.check("http://testserver", + client=TestClient(_handwritten(stamp_rpc=False)))}["universe_now"] + assert result.status == "failed" and "/agentenv" in result.detail + + +def test_a_reset_that_clears_nothing_fails_the_data_plane_and_leaves_the_data() -> None: + app = _handwritten(reset_keeps=True) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "failed" and "data/reset" in result.detail + assert app.state.data["items"] == ["a"] + + +def test_the_data_is_added_back_when_a_step_after_the_reset_fails() -> None: + app = _handwritten(get_fails_after_reset=True) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "failed" and "down" in result.detail + assert app.state.data["items"] == ["a"] + + +@pytest.mark.parametrize("reply", [(500, {}), (200, {}), (500, {"jsonrpc": "2.0", "id": "x", "result": {}})]) +def test_a_data_request_without_a_2xx_json_rpc_result_fails_the_data_plane(reply: tuple[int, dict]) -> None: + app = _handwritten(rpc_reply=reply) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "failed" and "no JSON-RPC result" in result.detail + + +def test_an_sdk_site_keeps_its_initial_state_and_one_changed_since_skips_the_round_trip( + notes: WebsiteEnvironment, sdk_site: TestClient) -> None: + before = sdk_site.get("/go").json() + conformance.check("http://testserver", client=sdk_site) + assert sdk_site.get("/go").json() == before + notes.store.insert("notes", {"text": "new"}) + changed = sdk_site.get("/go").json() + assert changed["state_diff"] + result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] + assert result.status == "skipped" and "website/v1 state shows changes" in result.detail + assert sdk_site.get("/go").json() == changed + + +def test_the_data_plane_is_reached_at_the_cards_url() -> None: + app = _handwritten(rpc_path="/rpc") + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "passed", result + assert app.state.data["items"] == ["a"] + + +@pytest.mark.parametrize(("changed", "status"), [(True, "skipped"), (False, "passed")]) +def test_a_card_without_state_is_read_at_the_state_path_it_names(changed: bool, status: str) -> None: + app = _handwritten(state_path="/grading/state", changed=changed) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == status, result + assert ("/grading/state shows changes" in result.detail) is changed + + +def test_a_server_with_no_card_health_or_header_fails_them() -> None: + bare = Starlette(routes=[Route("/", lambda request: JSONResponse({"not": "html"}))]) + results = statuses(conformance.check("http://testserver", client=TestClient(bare))) + assert results["card"] == results["any_host"] == results["health"] == results["universe_now"] == "failed" + assert results["data_plane"] == results["clock"] == "failed" + assert results["cookies"] == "skipped" + + +def test_main_prints_each_check_and_exits_by_outcome(monkeypatch, capsys) -> None: + monkeypatch.setattr(conformance, "check", lambda url, **options: [conformance.Result("card", "passed"), + conformance.Result("clock", "failed", "late")]) + assert conformance.main(["http://site"]) == 1 + printed = capsys.readouterr().out + assert "PASS card" in printed and "FAIL clock: late" in printed + + +def test_main_exits_2_when_the_server_cannot_be_reached(capsys) -> None: + assert conformance.main(["http://127.0.0.1:9", "--timeout", "2"]) == 2 + assert "couldn't reach" in capsys.readouterr().err diff --git a/packages/agentenv-protocol/tests/test_website_environment.py b/packages/agentenv-protocol/tests/test_website_environment.py new file mode 100644 index 000000000..6f28c4ba1 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_environment.py @@ -0,0 +1,552 @@ +"""WebsiteEnvironment as a site's server: static pages, health, the card and its OpenAPI document, the default store +and its data plane, sessions, the universe's clock, and a mounted app.""" + +from __future__ import annotations + +import asyncio +import base64 +import datetime +import json +import shutil +import subprocess +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +from starlette.applications import Starlette +from starlette.responses import PlainTextResponse +from starlette.routing import Route +from starlette.testclient import TestClient + +from agentenv_protocol import DataPart, get_data +from agentenv_protocol.website import ( + SESSION_COOKIE, + UNIVERSE_NOW_HEADER, + WEBSITE_EXTENSION_URI, + WEBSITE_READY_OP, + ApiContext, + Collections, + WebsiteEnvironment, + WebsiteReadiness, + api, + ready_check, + website, +) +from agentenv_protocol.website import clock as clock_module +from agentenv_protocol.website.clock import parse_time + +RPC = "/agentenv" + + +def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}).json() + assert "error" not in body, body + return body["result"] + + +@pytest.fixture +def dist(tmp_path: Path) -> Path: + root = tmp_path / "dist" + (root / "assets").mkdir(parents=True) + (root / "index.html").write_text("Shophome") + (root / "assets" / "app.js").write_text("console.log(1)") + (tmp_path / "secret.txt").write_text("outside") + return root + + +def site(dist: Path, **declared) -> type[WebsiteEnvironment]: + @website(domains=["shop.example", "www.shop.example"], static=str(dist), **declared) + class Shop(WebsiteEnvironment): + @api.get("/api/orders", name="list_orders") + def list_orders(self, ctx: ApiContext) -> list: + """List the signed-in user's orders.""" + return self.store.find("orders", user=ctx.user) + + @api.post("/api/orders", name="place_order", roles=["default", "agent"]) + def place_order(self, order: dict, ctx: ApiContext) -> dict: + return self.store.insert("orders", {**order, "user": ctx.user, "at": self.now().isoformat()}) + + @api.post("/api/sign-in") + def sign_in(self, body: dict, ctx: ApiContext) -> dict: + ctx.sign_in(body["user"]) + return {"user": ctx.user} + + return Shop + + +SEED = {"current_user": "alice", "users": [{"id": "alice"}, {"id": "bob"}], "orders": [{"id": 1, "user": "bob"}]} + + +@pytest.fixture +def shop(dist: Path) -> WebsiteEnvironment: + env = site(dist)() + env.store.load(SEED) + env.store.mark_initial() + return env + + +@pytest.fixture +def client(shop: WebsiteEnvironment) -> TestClient: + return TestClient(shop.create_app()) + + +def test_static_files_and_index_fallback_answer_for_any_host(client: TestClient) -> None: + for host in ("testserver", "shop.example", "10.0.0.7:8000", "anything.invalid"): + page = client.get("/", headers={"Host": host}) + assert page.status_code == 200 and "home" in page.text and page.headers["content-type"].startswith("text/html") + assert client.get("/assets/app.js").text == "console.log(1)" + assert "home" in client.get("/orders/42/edit").text + assert client.get("/assets/missing.js").status_code == 404 + assert client.get("/api/nothing-here").json() == {"error": "not found", "code": "not_found"} + assert client.get("/../secret.txt").status_code == 404 + assert client.post("/orders").status_code == 404 + + +def test_a_link_out_of_the_static_folder_is_never_served(dist: Path) -> None: + secret = dist.parent / "secret.txt" + (dist / "leak.txt").symlink_to(secret) + (dist / "linked").mkdir() + (dist / "linked" / "index.html").symlink_to(secret) + client = TestClient(site(dist)().create_app()) + assert "outside" not in client.get("/leak.txt").text + assert "outside" not in client.get("/linked").text + assert "outside" not in client.get("/linked/").text + + +def test_the_fallback_never_shadows_api_health_or_reserved_paths(client: TestClient) -> None: + assert client.get("/api/orders").json() == [] + assert client.put("/api/orders").status_code == 405 + assert client.get("/api/health").json() == {"status": "ok"} + for path in ("/agentenv", "/agentenv/anything", "/agentenv/ext/website/nope", "/agentenv/../agentenv/x"): + assert client.get(path).status_code in (404, 405), path + assert client.get("/.well-known/agent-env.json").json()["name"] == "Shop" + + +def test_health_waits_for_every_ready_check(dist: Path) -> None: + class Slow(site(dist)): + ready = False + + @ready_check + def database_up(self) -> bool: + return self.ready + + @ready_check + async def cache_warm(self) -> bool: + return True + + env = Slow() + client = TestClient(env.create_app()) + waiting = client.get("/api/health") + assert waiting.status_code == 503 and waiting.json() == {"status": "starting", "waiting_for": ["database_up"]} + env.ready = True + assert client.get("/api/health").status_code == 200 + + +def test_the_cards_website_v1_ready_says_whether_the_site_is_usable_by_its_ready_checks(dist: Path) -> None: + class Slow(site(dist)): + ready = False + + @ready_check + def database_up(self) -> bool: + return self.ready + + env = Slow() + client = TestClient(env.create_app()) + card = client.get("/.well-known/agent-env.json").json() + ready = _extension(card, WEBSITE_EXTENSION_URI)["params"]["methods"][WEBSITE_READY_OP] + assert (ready["method"], ready["endpoint"]) == ("GET", "/agentenv/ext/website/ready") + assert client.get(ready["endpoint"]).json() == {"ready": False, "problems": ["database_up"]} + assert WebsiteReadiness.model_validate(client.get(ready["endpoint"]).json()) == WebsiteReadiness( + ready=False, problems=["database_up"]) + env.ready = True + assert client.get(ready["endpoint"]).json() == {"ready": True, "problems": []} + + + +def test_a_declared_health_path_is_served_there(dist: Path) -> None: + client = TestClient(site(dist, health="/healthz")().create_app()) + assert client.get("/healthz").json() == {"status": "ok"} + card = client.get("/.well-known/agent-env.json").json() + assert _extension(card, WEBSITE_EXTENSION_URI)["params"]["health"] == "/healthz" + + +def _extension(card: dict, uri: str) -> dict: + return next(e for e in card["capabilities"]["extensions"] if e["uri"] == uri) + + +def test_the_card_declares_website_v1_its_routes_clock_and_data_plane(client: TestClient) -> None: + card = client.get("/.well-known/agent-env.json").json() + assert card["capabilities"]["operations"] == ["data/reset", "data/add", "data/get"] + params = _extension(card, WEBSITE_EXTENSION_URI)["params"] + assert params["domains"] == ["shop.example", "www.shop.example"] + assert params["health"] == "/api/health" + assert params["routes"] == [ + {"method": "GET", "path": "/api/orders", "name": "list_orders", "roles": None}, + {"method": "POST", "path": "/api/orders", "name": "place_order", "roles": ["default", "agent"]}, + {"method": "POST", "path": "/api/sign-in", "name": "sign_in", "roles": None}, + ] + assert params["methods"]["openapi"] == {"method": "GET", "endpoint": "/agentenv/ext/website/openapi", + "request": {"type": "object", "properties": {}}} + assert params["methods"]["session"]["endpoint"] == "/agentenv/ext/website/session" + clock = _extension(card, "urn:agentenv:clock/v1")["params"] + assert clock["endpoint"] == "/agentenv/ext/sync_time" + + +def test_the_openapi_document_names_each_route(client: TestClient) -> None: + document = client.get("/agentenv/ext/website/openapi").json() + assert document["openapi"].startswith("3.") and document["info"]["title"] == "Shop" + listed = document["paths"]["/api/orders"]["get"] + assert listed["operationId"] == "list_orders" and listed["summary"] == "List the signed-in user's orders." + placed = document["paths"]["/api/orders"]["post"] + assert placed["x-agentenv-roles"] == ["default", "agent"] + assert placed["requestBody"]["content"]["application/json"]["schema"]["type"] == "object" + assert listed["parameters"] == [] + + +def test_the_default_store_round_trips_through_the_data_plane(client: TestClient, shop: WebsiteEnvironment) -> None: + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + rpc(client, "data/reset") + assert rpc(client, "data/get")["parts"][0]["data"] == {} + as_file = base64.b64encode(json.dumps({"orders": [{"id": 7, "user": "carol"}]}).encode()).decode() + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"current_user": "carol", "users": [{"id": "carol"}]}}, + {"kind": "file", "file": {"bytes": as_file, "name": "orders.json"}}, + {"kind": "text", "text": json.dumps({"notes": [{"id": "n1"}]})}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == { + "current_user": "carol", "users": [{"id": "carol"}], "orders": [{"id": 7, "user": "carol"}], + "notes": [{"id": "n1"}]} + assert shop.acting_user() == "carol" + + +def test_go_reports_initial_current_and_diff_state(client: TestClient) -> None: + client.post("/api/orders", json={"sku": "mug"}) + state = client.get("/go", params={"sid": "default"}).json() + assert state["initial_state"] == SEED + assert state["current_state"]["orders"][-1]["sku"] == "mug" + assert state["state_diff"] == {"orders": {"added": [state["current_state"]["orders"][-1]], "removed": [], + "changed": []}} + + +def test_broad_routes_leave_the_built_in_ones_answering_their_paths(dist: Path) -> None: + class Broad(site(dist)): + @api.get("/{slug}") + def page(self, slug: str) -> dict: + return {"page": slug} + + @api.get("/api/{name}") + def named(self, name: str) -> dict: + return {"named": name} + + env = Broad() + env.store.load(SEED) + env.store.mark_initial() + client = TestClient(env.create_app()) + assert client.get("/go").json()["initial_state"] == SEED + assert client.get("/api/session").json()["user"] == "alice" + assert client.get("/api/health").json() == {"status": "ok"} + assert client.get("/about").json() == {"page": "about"} and client.get("/api/other").json() == {"named": "other"} + + +def test_a_broad_route_never_serves_a_page_at_the_data_planes_or_cards_paths(dist: Path) -> None: + class Catchall(site(dist)): + @api.get("/{rest:path}") + def page(self, rest: str) -> dict: + return {"page": rest} + + client = TestClient(Catchall().create_app()) + for path in ("/agentenv/anything", "/agentenv/", "/x/../agentenv/y", "/.well-known/agent-env.json/"): + assert client.get(path).status_code == 404, path + assert client.get("/.well-known/agent-env.json").status_code == 200 + assert client.get("/docs/a").json() == {"page": "docs/a"} + + +def test_a_site_with_its_own_hooks_keeps_them_and_serves_no_go(dist: Path) -> None: + class Own(site(dist)): + @get_data + def export(self) -> list: + return [DataPart(data={"mine": True})] + + client = TestClient(Own().create_app()) + assert rpc(client, "data/get")["parts"][0]["data"] == {"mine": True} + assert client.get("/.well-known/agent-env.json").json()["capabilities"]["operations"] == ["data/reset", "data/add", "data/get"] + assert "home" in client.get("/go").text + + +def test_a_seed_data_file_is_the_initial_state(tmp_path: Path, dist: Path) -> None: + (tmp_path / "seed.json").write_text(json.dumps(SEED)) + env = site(dist, data=str(tmp_path / "seed.json"))() + assert env.store.export() == SEED and env.store.initial == SEED and env.acting_user() == "alice" + + +def test_the_acting_user_is_signed_in_until_a_session_says_else(client: TestClient) -> None: + assert client.get("/api/orders").json() == [] + first = client.get("/api/session") + assert first.json() == {"user": "alice", "role": "default", "signed_in": True} + cookie = first.headers["set-cookie"] + assert cookie.startswith(f"{SESSION_COOKIE}=") and "domain" not in cookie.lower() and "httponly" in cookie.lower() + client.post("/api/sign-in", json={"user": "bob"}) + assert client.get("/api/orders").json() == [{"id": 1, "user": "bob"}] + assert client.get("/api/session").json()["user"] == "bob" + assert client.delete("/api/session").json() == {"user": None, "role": "default", "signed_in": False} + assert client.get("/api/session").json()["user"] is None + client.cookies.set(SESSION_COOKIE, "forged.signature") + assert client.get("/api/session").json()["user"] == "alice" + + +def test_the_session_cookie_expires_when_the_session_does(client: TestClient) -> None: + from email.utils import parsedate_to_datetime + + cookie = client.get("/api/session").headers["set-cookie"] + expires = next(a.split("=", 1)[1] for a in cookie.split(";") if a.strip().lower().startswith("expires=")) + ahead = parsedate_to_datetime(expires) - datetime.datetime.now(datetime.timezone.utc) + assert datetime.timedelta(days=7) - datetime.timedelta(minutes=1) < ahead <= datetime.timedelta(days=7) + + +def test_the_session_operation_mints_cookies_for_a_user_or_signed_out(shop: WebsiteEnvironment) -> None: + client = TestClient(shop.create_app()) + minted = client.post("/agentenv/ext/website/session", json={"user": "bob"}).json() + assert minted["user"] == "bob" and minted["expires_at"].endswith("Z") + (cookie,) = minted["cookies"] + assert cookie["name"] == SESSION_COOKIE and "domain" not in cookie and cookie["path"] == "/" + browser = TestClient(shop.asgi, cookies={SESSION_COOKIE: cookie["value"]}) + assert browser.get("/api/session").json()["user"] == "bob" + signed_out = client.post("/agentenv/ext/website/session", json={}).json()["cookies"][0]["value"] + assert TestClient(shop.asgi, cookies={SESSION_COOKIE: signed_out}).get("/api/session").json()["user"] is None + + +def test_every_answer_carries_the_universe_time(client: TestClient) -> None: + for response in (client.get("/"), client.get("/api/orders"), client.get("/nope.js"), + client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": "data/get"}), + client.get("/.well-known/agent-env.json")): + now = parse_time(response.headers[UNIVERSE_NOW_HEADER]) + assert abs((now - datetime.datetime.now(datetime.timezone.utc)).total_seconds()) < 60 + + +class _Clock: + """A gateway-shaped /clock/time whose time a test sets.""" + + def __init__(self, virtual: str) -> None: + self.virtual = virtual + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = json.dumps({"virtual_time": outer.virtual}).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + return None + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"http://127.0.0.1:{self.server.server_port}/clock/time" + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +@pytest.fixture +def gateway_clock(): + clock = _Clock("2031-01-01T00:00:00Z") + yield clock + clock.close() + + +def test_sync_time_ties_now_and_the_header_to_the_gateway_clock(client: TestClient, shop: WebsiteEnvironment, + gateway_clock: _Clock, monkeypatch) -> None: + synced = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}).json() + assert synced == {"env_get_time_url": gateway_clock.url, "gateway_virtual_time": "2031-01-01T00:00:00.000Z"} + assert shop.now().year == 2031 + assert client.get("/").headers[UNIVERSE_NOW_HEADER].startswith("2031-01-01T00:00:0") + order = client.post("/api/orders", json={"sku": "pen"}).json() + assert order["at"].startswith("2031-01-01") + gateway_clock.virtual = "2031-06-01T12:00:00Z" + monkeypatch.setattr(clock_module, "REFRESH_S", 0.0) + assert client.get("/").headers[UNIVERSE_NOW_HEADER].startswith("2031-06-01T12:00") + + +def test_sync_time_takes_the_universe_s_time_zone_and_a_later_sync_without_one_drops_it( + client: TestClient, shop: WebsiteEnvironment, gateway_clock: _Clock) -> None: + method = _extension(client.get("/.well-known/agent-env.json").json(), "urn:agentenv:clock/v1")["params"]["methods"]["sync_time"] + assert "timezone" in method["request"]["properties"] and method["request"]["required"] == ["env_get_time_url"] + zoned = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url, "timezone": " America/Chicago "}) + assert zoned.json()["timezone"] == "America/Chicago" == shop.timezone + assert shop.now().tzinfo == datetime.timezone.utc + unzoned = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}).json() + assert "timezone" not in unzoned and shop.timezone is None + + +def test_a_clock_that_cannot_be_read_is_refused_and_a_lost_one_extrapolates(client: TestClient, + shop: WebsiteEnvironment, + gateway_clock: _Clock, monkeypatch) -> None: + refused = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": "http://127.0.0.1:9/clock/time"}) + assert refused.status_code == 500 and shop.now().year == datetime.datetime.now().year + client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}) + gateway_clock.close() + monkeypatch.setattr(clock_module, "REFRESH_S", 0.0) + assert client.get("/").headers[UNIVERSE_NOW_HEADER].startswith("2031-01-01") + + +def test_clock_patch_puts_the_clock_script_in_html_only(dist: Path, gateway_clock: _Clock) -> None: + client = TestClient(site(dist, clock_patch=True)().create_app()) + client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}) + page = client.get("/") + head = page.text.index("") + len("") + assert page.text[head:].startswith(" + + diff --git a/packages/agentenv-protocol/examples/website/notes/web/package-lock.json b/packages/agentenv-protocol/examples/website/notes/web/package-lock.json new file mode 100644 index 000000000..938f9c2cd --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/package-lock.json @@ -0,0 +1,1020 @@ +{ + "name": "notes-site-web", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "notes-site-web", + "version": "0.0.0", + "devDependencies": { + "vite": "5.4.10" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.67.2.tgz", + "integrity": "sha512-qB7Gzl1QI45zP1zXgqLKzjFmOBGQw2s2MLtdw0eB4xdFGPNE8LhmOnItFgyYJXXNSW8Go9c2ovvXrUUzw5sAMQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.67.2.tgz", + "integrity": "sha512-8+hXNO1J+GsYCGHwyJuBlnhyozivEvIEm/P6NkJeuoVUN+60TDvYYXk38UTm5bKRTLhIpqEQYx4d9CeN6kjh0Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-x64/-/rollup-android-x64-4.67.2.tgz", + "integrity": "sha512-/uEZY+tcWDE8DXa/rajU3Bwv6IfvQavvzjmbkn39GgQenT+UYtAxfEMdB7VKr74y8aonncm0eQa1EbiNfg+Cjg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.67.2.tgz", + "integrity": "sha512-3tCLEgakEvb5M3/KyYdhPjoteyWYziMTeznoXfcs/31eeFu+Vgs0VNRlb6Zc+YC07/sLa84OwTDF5zHOMFk92Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.67.2.tgz", + "integrity": "sha512-z31ODzkdkX7OOJCssvpz++hhtkQKWKw2L0eO2HAxFP7RMSXXedzkrTjesqCUxs4XLckRShjNfxsNxjGY5DQCGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.67.2.tgz", + "integrity": "sha512-CTvrb0KTAQqcrpcj7rg8FgAI0+jRWnCl+En6exqXDZztCo7vspxlRUX8z9Hp/J/ja2XRnQ1JDbexkBtRyD9jSw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.67.2.tgz", + "integrity": "sha512-SJdblZpBkbOh1e4JwMM5pJJ2PdVmD3WWbMTXvncMSRLA3DdZuwU+P93RaSChOAXN4Mu53tBoFp1ADPzi3dIVAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.67.2.tgz", + "integrity": "sha512-pUBEd7uI9dAu3TpbPcvi9eU2aJslsjIomormeTaQHmzWNhyqNx3mmq+ekC6blGNZu8F3uTHngTibKp7tbblF0A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.67.2.tgz", + "integrity": "sha512-wW6ARw2Ah0/teOBTgTeSJIpVdPKaxQmEgZ/irkUfr3F9sgimqpApCv/NbQzjm0Nfi5EFilHh1H8kz01uQLaq5Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.67.2.tgz", + "integrity": "sha512-xcFYMHNL42fRu2MFipvxznr+zd+DRqwL98bhNvDWbMPRcFtMYf34NAz4xVOlnEhzgpNRE5htSemcqAsAxf2eQg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.67.2.tgz", + "integrity": "sha512-U9HrnJ9PFXqA//P/oEGaXBIJraEqzYMS9yuQOdOcPCvHZIC1FFOlnd8ocAaeDDPDpY8xOrhfxMRtr6KqKi3gyg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.67.2.tgz", + "integrity": "sha512-2hZN8CFGfsbDw+DNtImkPRitpLNjVM2XEbi4ZQ5igLxMFD4gI/P6Zo9czGxuvQXtRyZtZyZp8Kr4ERsAipplBw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.67.2.tgz", + "integrity": "sha512-HZO7UfWEsReAnkdVByyQaFsNXJCtvdfun/LFb2Xd4qnhgF1ZfhikpbgqrTRtNMmB5OgNnSwFLiPobOT8oNT9iw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.67.2.tgz", + "integrity": "sha512-AdYIihuLkJ2TScy9BpSPiDex3zOMjkNJLsFlk9NeJ0U3WEtOx4W8z0nOkPEhDS2S4gwsRWBa6ZZexJ9e6OxyzQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.67.2.tgz", + "integrity": "sha512-iZE0NXcAGqLoCWeHRPkBxJe93JT4Q981z1ZqR+y7qAiIksdRlWhw8hmqHFo1B4ZnU66He31NUb52Iyv+vwMEBQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.67.2.tgz", + "integrity": "sha512-vC7EAtJpSqSrrbg8huMfgxtwlR8fUsFCe5rStDk1WA8POky/6EEp52dxENMmqLKjSfdM3TT1WWPcZ01hiYZcdw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.67.2.tgz", + "integrity": "sha512-DdqG6x72MHpQOlkK/cqrru9b2QMJHLkNojCB3bteB5fmkE/7XEAkReJMANp8wgef8eiEC99SvdP/BH7Ho/BpLA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.67.2.tgz", + "integrity": "sha512-F10mjdnRmGg9K+LGdqQVeUYFr08f3fVBOh7YNE5p1KIpTzhtKKOLqcQEsqZG6yEM2eouXaMo2AZzeDAcseVTrQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.67.2.tgz", + "integrity": "sha512-ChzK9f3iE4N5GInBquiZtzWOqh+CYS5nlLlXGJxeG3d2T9rk8L1Hxg0s+v2Eh6vmAc/1T6IhOVkQU0ORP2eV7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.67.2.tgz", + "integrity": "sha512-95WwzC+KH9hPHtbgwvwwW6MkKME6vMAdAQLS2diCvoi/iUiYdC0Y1RIPYVESx0iWqkN5ZaZq6qD1BflLcyTMEQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.67.2.tgz", + "integrity": "sha512-eKbpwlolQmDKd/dbFjvqB1+2qdYQLrgxi3Mtbijp7/3yBBvO9OoHtQ3ecYIaCdW46ijSjy5/yydyrrxHI416/g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.67.2.tgz", + "integrity": "sha512-is9lnlv6khapl+H9Hjp0gIVsUVtDxQga2nwMHpQbhDXMCMGcNmlel4/q9ihf/3FvvbL4GdGOPOGvWoei2RCggg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.67.2.tgz", + "integrity": "sha512-yTq+7YjsgbBcMdlk7Wq+VoJilGQtdtRc4PyTvN7vhO958atCO2nQeIjCF/YANSD/Z/fnrCr27fYsGgC6PFTndQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.67.2.tgz", + "integrity": "sha512-nAvZib4Xz1zx8OKhtRuyCsSmCmbfegv1AGbfpG+CowQKxCBW1pLlfYWjcr1+GI3BzRKQbBl7l9640GwWAwOF7g==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.67.2.tgz", + "integrity": "sha512-R3CVbEbNa1OXjpldD1tOLarb5rB08xjQiRMpe7PZKtT8eHCZhrV4ClSPfLLbSgRMM1RfLtmAfouBRf14f47lMA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.67.2.tgz", + "integrity": "sha512-DeuVFilvaGtoS9TFIbTIQ0Kt4CBqP3xnjJ8SFqWWbXlFbrpmCdivEW7Rm38trRA8rZzQQ8lhs73nEetWotJpTg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/estree": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz", + "integrity": "sha512-lTyOMnOcmRBzdCv6/4zUZCrEhV0xrqQiUZBZtVcbmLHBXKLnN8q89v/wKRF/Q0HbwVsqbIG3o5HDoFl9sduM+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.5.32", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.32.tgz", + "integrity": "sha512-LJlKCCAUFKOuZxipUdYw1NzIITEsPcgyOG9LkvRJvuKj/bRzbuRY0ARXvL0pWGpsZvn8jku+LMep2eU0seHp8g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.20", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rollup": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.67.2.tgz", + "integrity": "sha512-sFZk5BeRUQzkjmuClrJI41BowIP4yv/40d5gM9vtjxVLEt+8JG9FGlBps6cliTP2KhFiw22oqNGo0G9pVpPZlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.10" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.67.2", + "@rollup/rollup-android-arm64": "4.67.2", + "@rollup/rollup-android-x64": "4.67.2", + "@rollup/rollup-darwin-arm64": "4.67.2", + "@rollup/rollup-darwin-x64": "4.67.2", + "@rollup/rollup-freebsd-arm64": "4.67.2", + "@rollup/rollup-freebsd-x64": "4.67.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.67.2", + "@rollup/rollup-linux-arm-musleabihf": "4.67.2", + "@rollup/rollup-linux-arm64-gnu": "4.67.2", + "@rollup/rollup-linux-arm64-musl": "4.67.2", + "@rollup/rollup-linux-loong64-gnu": "4.67.2", + "@rollup/rollup-linux-loong64-musl": "4.67.2", + "@rollup/rollup-linux-ppc64-gnu": "4.67.2", + "@rollup/rollup-linux-ppc64-musl": "4.67.2", + "@rollup/rollup-linux-riscv64-gnu": "4.67.2", + "@rollup/rollup-linux-riscv64-musl": "4.67.2", + "@rollup/rollup-linux-s390x-gnu": "4.67.2", + "@rollup/rollup-linux-x64-gnu": "4.67.2", + "@rollup/rollup-linux-x64-musl": "4.67.2", + "@rollup/rollup-openbsd-x64": "4.67.2", + "@rollup/rollup-openharmony-arm64": "4.67.2", + "@rollup/rollup-win32-arm64-msvc": "4.67.2", + "@rollup/rollup-win32-ia32-msvc": "4.67.2", + "@rollup/rollup-win32-x64-gnu": "4.67.2", + "@rollup/rollup-win32-x64-msvc": "4.67.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/vite": { + "version": "5.4.10", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.10.tgz", + "integrity": "sha512-1hvaPshuPUtxeQ0hsVH3Mud0ZanOLwVTneA1EgbAM5LhaZEqyPWGRQ7BtaMvUrTDeEaC8pxtj6a6jku3x4z6SQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + } + } +} diff --git a/packages/agentenv-protocol/examples/website/notes/web/package.json b/packages/agentenv-protocol/examples/website/notes/web/package.json new file mode 100644 index 000000000..c741095dd --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/package.json @@ -0,0 +1,14 @@ +{ + "name": "notes-site-web", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "devDependencies": { + "vite": "5.4.10" + } +} diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/main.js b/packages/agentenv-protocol/examples/website/notes/web/src/main.js new file mode 100644 index 000000000..f4318fabf --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/src/main.js @@ -0,0 +1,210 @@ +import "./style.css"; + +let offsetMs = 0; +let filter = "all"; +let renders = 0; +const saving = new Map(); +const lastSent = new Map(); +const landed = new Map(); +const CLOCK_REFRESH_MS = 5000; + +export function universeNow() { + return new Date(Date.now() + offsetMs); +} + +function followUniverse(response) { + const at = Date.parse(response.headers.get("X-Universe-Now") || ""); + if (!Number.isNaN(at)) offsetMs = at - Date.now(); +} + +async function api(path, { method = "GET", body } = {}) { + const response = await fetch(path, { + method, + headers: body === undefined ? {} : { "Content-Type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + followUniverse(response); + const data = await response.json().catch(() => null); + if (!response.ok) throw new Error(data?.error || `${method} ${path}: ${response.status}`); + return data; +} + +const $ = (id) => document.getElementById(id); + +function queueSave(id, fields) { + lastSent.set(id, fields); + const queued = saving.get(id); + if (queued) { + queued.next = fields; + return; + } + const state = { next: fields }; + saving.set(id, state); + run(async () => { + try { + while (state.next) { + const next = state.next; + state.next = null; + await api(`api/notes/${id}`, { method: "PATCH", body: next }); + } + landed.set(id, renders); + } catch (error) { + lastSent.set(id, {}); + landed.delete(id); + throw error; + } finally { + saving.delete(id); + } + }); +} + +function showError(error) { + $("error").textContent = error ? error.message : ""; + $("error").hidden = !error; +} + +function formatTime(value) { + return new Date(value).toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short", timeZone: "UTC" }); +} + +function tick() { + $("universe-now").textContent = `${formatTime(universeNow())} UTC`; + $("universe-now").dateTime = universeNow().toISOString(); +} + +function noteItem(note) { + const item = document.createElement("li"); + item.className = `note card${note.done ? " done" : ""}`; + item.dataset.id = note.id; + + const done = document.createElement("input"); + done.type = "checkbox"; + done.checked = note.done; + done.setAttribute("aria-label", `Mark "${note.title}" ${note.done ? "not done" : "done"}`); + done.addEventListener("change", () => run(() => api(`api/notes/${note.id}/toggle`, { method: "POST" }))); + + const title = document.createElement("input"); + title.className = "title"; + title.defaultValue = note.title; + title.setAttribute("aria-label", "Title"); + + const body = document.createElement("textarea"); + body.className = "body"; + body.rows = 2; + body.defaultValue = note.body; + body.placeholder = "Details"; + body.setAttribute("aria-label", "Details"); + + const save = () => { + const sent = lastSent.get(note.id) ?? { title: note.title, body: note.body }; + if (title.value === sent.title && body.value === sent.body) return; + queueSave(note.id, { title: title.value, body: body.value }); + }; + for (const field of [title, body]) { + field.addEventListener("change", save); + field.addEventListener("blur", save); + } + + const meta = document.createElement("small"); + meta.textContent = `Created ${formatTime(note.created_at)} · updated ${formatTime(note.updated_at)}`; + + const remove = document.createElement("button"); + remove.className = "delete"; + remove.textContent = "Delete"; + remove.addEventListener("click", () => run(() => api(`api/notes/${note.id}`, { method: "DELETE" }))); + + const text = document.createElement("div"); + text.className = "text"; + text.append(title, body, meta); + item.append(done, text, remove); + return item; +} + +function editing() { + const kept = []; + for (const field of document.querySelectorAll("#notes .title, #notes .body")) { + const focused = field === document.activeElement; + if (focused || field.value !== field.defaultValue) { + kept.push({ id: field.closest("li").dataset.id, kind: field.className, value: field.value, focused, + start: field.selectionStart, end: field.selectionEnd }); + } + } + return kept; +} + +function restore(kept) { + for (const { id, kind, value, focused, start, end } of kept) { + const field = document.querySelector(`#notes li[data-id="${id}"] .${kind}`); + if (!field) continue; + field.value = value; + if (focused) { + field.focus(); + field.setSelectionRange(start, end); + } + } +} + +async function render() { + const query = filter === "all" ? "" : `?done=${filter === "done"}`; + const request = ++renders; + const notes = await api(`api/notes${query}`); + if (request !== renders) return; + for (const [id, after] of landed) { + if (request > after && !saving.has(id)) { + lastSent.delete(id); + landed.delete(id); + } + } + const kept = editing(); + $("notes").replaceChildren(...notes.map(noteItem)); + restore(kept); + if (!notes.length) { + const empty = document.createElement("li"); + empty.className = "empty"; + empty.textContent = "No notes here yet."; + $("notes").append(empty); + } +} + +async function run(action) { + try { + await action(); + showError(null); + await render(); + } catch (error) { + showError(error); + } +} + +async function start() { + const [session, users] = await Promise.all([api("api/session"), api("api/users")]); + $("user").replaceChildren(...users.map((user) => new Option(user.name || user.id, user.id))); + $("user").value = session.user ?? ""; + $("user").addEventListener("change", (event) => + run(() => api("api/sign-in", { method: "POST", body: { user: event.target.value } }))); + + $("new-note").addEventListener("submit", (event) => { + event.preventDefault(); + const body = { title: $("new-title").value, body: $("new-body").value }; + run(async () => { + await api("api/notes", { method: "POST", body }); + if ($("new-title").value === body.title) $("new-title").value = ""; + if ($("new-body").value === body.body) $("new-body").value = ""; + }); + }); + + for (const button of document.querySelectorAll("[data-filter]")) { + button.addEventListener("click", () => { + filter = button.dataset.filter; + document.querySelectorAll("[data-filter]").forEach((b) => b.classList.toggle("active", b === button)); + run(async () => {}); + }); + } + + tick(); + setInterval(tick, 1000); + setInterval(() => fetch("healthz").then(followUniverse).catch(() => {}), CLOCK_REFRESH_MS); + await render(); +} + +start().catch(showError); diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/style.css b/packages/agentenv-protocol/examples/website/notes/web/src/style.css new file mode 100644 index 000000000..8ecd48b6a --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/src/style.css @@ -0,0 +1,57 @@ +:root { + --bg: #f6f6f3; + --card: #ffffff; + --text: #1d1d1b; + --muted: #6b6b66; + --line: #deded8; + --accent: #2f5bd3; + font-family: system-ui, -apple-system, "Segoe UI", sans-serif; + color: var(--text); + background: var(--bg); +} + +* { box-sizing: border-box; } +body { margin: 0; } + +.bar { + display: flex; + flex-wrap: wrap; + gap: 12px; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-bottom: 1px solid var(--line); + background: var(--card); +} +.bar h1 { margin: 0; font-size: 1.25rem; } +.who { display: flex; flex-wrap: wrap; gap: 12px; align-items: center; color: var(--muted); font-size: 0.9rem; } +.who time { font-variant-numeric: tabular-nums; } + +main { max-width: 720px; margin: 0 auto; padding: 16px; } + +.card { background: var(--card); border: 1px solid var(--line); border-radius: 8px; padding: 12px; } + +input, textarea, select, button { font: inherit; color: inherit; } +input, textarea { width: 100%; border: 1px solid var(--line); border-radius: 6px; padding: 6px 8px; background: transparent; } +textarea { resize: vertical; } +button { border: 1px solid var(--line); border-radius: 6px; padding: 6px 12px; background: var(--card); cursor: pointer; } +button[type="submit"], .filters .active { background: var(--accent); border-color: var(--accent); color: #fff; } + +#new-note { display: grid; gap: 8px; } +#new-note button { justify-self: end; } + +.filters { display: flex; gap: 8px; margin: 16px 0; } + +#error { color: #b3261e; margin: 0 0 12px; } + +#notes { list-style: none; margin: 0; padding: 0; display: grid; gap: 8px; } +.note { display: grid; grid-template-columns: auto 1fr auto; gap: 12px; align-items: start; } +.note .text { display: grid; gap: 6px; min-width: 0; } +.note .title { font-weight: 600; border-color: transparent; } +.note .body { border-color: transparent; color: var(--muted); } +.note .title:focus, .note .body:focus { border-color: var(--line); } +.note small { color: var(--muted); } +.note.done .title { text-decoration: line-through; color: var(--muted); } +.note input[type="checkbox"] { width: auto; margin-top: 10px; } +.delete { color: #b3261e; } +.empty { color: var(--muted); text-align: center; padding: 24px; } diff --git a/packages/agentenv-protocol/examples/website/notes/web/vite.config.js b/packages/agentenv-protocol/examples/website/notes/web/vite.config.js new file mode 100644 index 000000000..d07ecd2c2 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/vite.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from "vite"; + +export default defineConfig({ + base: "./", + server: { + proxy: { + "/api": "http://127.0.0.1:8000", + "/healthz": "http://127.0.0.1:8000", + }, + }, +}); diff --git a/packages/agentenv-protocol/tests/test_website_notes_example.py b/packages/agentenv-protocol/tests/test_website_notes_example.py new file mode 100644 index 000000000..e02147b10 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_notes_example.py @@ -0,0 +1,237 @@ +"""The Notes example site (examples/website/notes) as website/v1's reference site: it passes the conformance checker, +and its own behavior holds: the acting user's auto sign-in and signing in as another user, notes kept per user, the +admin-only route, timestamps from the universe's clock, and the store's seed loaded, exported and reset through the +data plane.""" + +from __future__ import annotations + +import importlib.util +import json +import re +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import urljoin, urlsplit + +import pytest +from starlette.responses import PlainTextResponse +from starlette.testclient import TestClient + +from agentenv_protocol.website import (ROLE_KEY_ENV, ROLE_KEY_HEADER, SESSION_COOKIE, UNIVERSE_NOW_HEADER, WebsiteEnvironment, + conformance, website) + +NOTES_DIR = Path(__file__).resolve().parents[1] / "examples" / "website" / "notes" +SEED = json.loads((NOTES_DIR / "data" / "seed.json").read_text(encoding="utf-8")) +RPC = "/agentenv" + + +def _load_site(): + """The example's ``site.py``, imported under a name of its own: ``site`` is the standard library's.""" + spec = importlib.util.spec_from_file_location("notes_example_site", NOTES_DIR / "site.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +notes_site = _load_site() + + +def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}).json() + assert "error" not in body, body + return body["result"] + + +@pytest.fixture +def notes_class(tmp_path: Path) -> type[WebsiteEnvironment]: + """The example's class, serving a stand-in page instead of ``web/dist``, which only a Vite build makes.""" + (tmp_path / "index.html").write_text("Notes") + declared = notes_site.Notes.declaration()._replace(static=str(tmp_path), data=str(NOTES_DIR / "data" / "seed.json")) + return website(**declared._asdict())(type("Notes", (notes_site.Notes,), {})) + + +@pytest.fixture +def env(notes_class: type[WebsiteEnvironment]) -> WebsiteEnvironment: + return notes_class() + + +@pytest.fixture +def app(env: WebsiteEnvironment): + return env.create_app() + + +@pytest.fixture +def client(app) -> TestClient: + return TestClient(app) + + +class _Clock: + """A gateway-shaped /clock/time whose time a test sets.""" + + def __init__(self, virtual: str) -> None: + self.virtual = virtual + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = json.dumps({"virtual_time": outer.virtual}).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + return None + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"http://127.0.0.1:{self.server.server_port}/clock/time" + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +@pytest.fixture +def gateway_clock(): + clock = _Clock("2040-02-03T04:05:06Z") + yield clock + clock.close() + + +def test_the_example_passes_every_conformance_check(client: TestClient) -> None: + results = conformance.check("http://testserver", client=client) + assert {r.name: r.status for r in results} == dict.fromkeys( + ["card", "any_host", "health", "universe_now", "data_plane", "clock", "forwarded", "cookies"], "passed"), results + + +def test_the_card_declares_the_site_its_health_path_and_its_routes_by_name(client: TestClient) -> None: + card = client.get("/.well-known/agent-env.json").json() + site = next(e for e in card["capabilities"]["extensions"] if e["uri"] == "urn:agentenv:website/v1")["params"] + assert card["name"] == "notes" + assert (site["domains"], site["health"]) == (["notes.test"], "/healthz") + assert [r["name"] for r in site["routes"]] == ["list_users", "sign_in", "list_notes", "create_note", "get_note", + "edit_note", "toggle_note", "delete_note", "all_notes"] + assert next(r for r in site["routes"] if r["name"] == "all_notes")["roles"] == ["admin"] + assert client.get("/healthz").json() == {"status": "ok"} + + +def test_a_browser_is_signed_in_as_the_acting_user_and_can_sign_in_as_another(client: TestClient) -> None: + session = client.get("/api/session") + assert session.json() == {"user": "ann", "role": "default", "signed_in": True} + assert SESSION_COOKIE in session.cookies + assert [n["title"] for n in client.get("/api/notes").json()] == ["Buy milk", "Book dentist"] + assert client.post("/api/sign-in", json={"user": "ben"}).json() == {"user": "ben"} + assert client.get("/api/session").json()["user"] == "ben" + assert [n["title"] for n in client.get("/api/notes").json()] == ["Draft the quarterly report"] + refused = client.post("/api/sign-in", json={"user": "mallory"}) + assert refused.status_code == 404 and refused.json()["code"] == "not_found" + + +def test_notes_are_created_edited_toggled_and_deleted_by_their_owner_only(app, client: TestClient) -> None: + created = client.post("/api/notes", json={"title": " Call Sam ", "body": "about the trip"}) + assert created.status_code == 201 + note = created.json() + assert (note["owner"], note["title"], note["done"]) == ("ann", "Call Sam", False) + assert note["created_at"] == note["updated_at"] + + ben = TestClient(app) + ben.post("/api/sign-in", json={"user": "ben"}) + assert "Call Sam" not in [n["title"] for n in ben.get("/api/notes").json()] + for response in (ben.get(f"/api/notes/{note['id']}"), ben.patch(f"/api/notes/{note['id']}", json={"done": True}), + ben.post(f"/api/notes/{note['id']}/toggle"), ben.delete(f"/api/notes/{note['id']}")): + assert response.status_code == 404, response.text + + edited = client.patch(f"/api/notes/{note['id']}", json={"body": "about the trip, before Friday"}).json() + assert (edited["title"], edited["body"]) == ("Call Sam", "about the trip, before Friday") + assert client.post(f"/api/notes/{note['id']}/toggle").json()["done"] is True + assert [n["title"] for n in client.get("/api/notes", params={"done": "true"}).json()] == ["Call Sam", "Book dentist"] + assert client.patch(f"/api/notes/{note['id']}", json={"title": " "}).json()["code"] == "empty_title" + assert client.post("/api/notes", json={"body": "no title"}).status_code == 400 + assert client.delete(f"/api/notes/{note['id']}").json() == {"deleted": note["id"]} + assert client.get(f"/api/notes/{note['id']}").status_code == 404 + + +def test_every_users_notes_are_listed_for_an_admin_caller_only(client: TestClient, + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + for headers in ({}, {"AgentEnv-Role": "agent", ROLE_KEY_HEADER: "k3y"}, {"AgentEnv-Role": "admin"}, + {"AgentEnv-Role": "admin", ROLE_KEY_HEADER: "guess"}): + refused = client.get("/api/admin/notes", headers=headers) + assert refused.status_code == 403, headers + every = client.get("/api/admin/notes", headers={"AgentEnv-Role": "admin", ROLE_KEY_HEADER: "k3y"}).json() + assert {n["owner"] for n in every} == {"ann", "ben"} + + +def test_timestamps_follow_the_universes_clock_once_synced(client: TestClient, gateway_clock: _Clock) -> None: + before = client.post("/api/notes", json={"title": "before the sync"}).json() + assert not before["created_at"].startswith("2040") + client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}) + after = client.post("/api/notes", json={"title": "after the sync"}) + assert after.json()["created_at"].startswith("2040-02-03T04:05") + assert after.headers[UNIVERSE_NOW_HEADER].startswith("2040-02-03T04:05") + toggled = client.post(f"/api/notes/{before['id']}/toggle").json() + assert toggled["created_at"] == before["created_at"] and toggled["updated_at"].startswith("2040-02-03T04:05") + + +def test_the_seed_loads_exports_and_resets_through_the_data_plane(client: TestClient) -> None: + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + rpc(client, "data/reset") + assert rpc(client, "data/get")["parts"][0]["data"] == {} + assert client.get("/api/notes").status_code == 401 + assert client.get("/healthz").status_code == 503 + + seed_file = {"kind": "file", "file": {"uri": (NOTES_DIR / "data" / "seed.json").as_uri(), + "mimeType": "application/json"}} + rpc(client, "data/add", {"parts": [seed_file]}) + assert client.get("/healthz").status_code == 200 + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + assert client.get("/api/session").json()["user"] == "ann" + + client.post("/api/notes/1/toggle") + state = client.get("/go").json() + assert state["initial_state"] == SEED + assert state["state_diff"]["notes"]["changed"][0]["after"]["done"] is True + + +GATEWAY_PREFIX = "/website/notes" + + +def _behind_gateway_prefix(app): + """The gateway's ``/website//`` route: a request under the prefix reaches the site with the prefix taken off, + and any other one answers the gateway's own 404.""" + + async def gateway(scope, receive, send): + if scope["type"] == "http" and scope["path"].startswith(GATEWAY_PREFIX + "/"): + path = scope["path"][len(GATEWAY_PREFIX):] + return await app({**scope, "path": path, "raw_path": path.encode()}, receive, send) + if scope["type"] != "http": + return await app(scope, receive, send) + await PlainTextResponse("no gateway route", status_code=404)(scope, receive, send) + + return gateway + + +@pytest.mark.skipif(not (NOTES_DIR / "web" / "dist" / "index.html").is_file(), reason="web/dist needs a Vite build") +def test_the_built_page_loads_its_assets_and_api_under_the_gateways_prefix() -> None: + page_url = f"http://gateway.test{GATEWAY_PREFIX}/" + browser = TestClient(_behind_gateway_prefix(notes_site.Notes().create_app()), base_url="http://gateway.test") + page = browser.get(page_url) + assert page.status_code == 200 and page.headers["content-type"].startswith("text/html") + + assets = [urljoin(page_url, ref) for ref in re.findall(r'(?:src|href)="([^"]+)"', page.text)] + assert assets and all(urlsplit(url).path.startswith(GATEWAY_PREFIX + "/") for url in assets), assets + scripts = [] + for url in assets: + fetched = browser.get(url) + assert fetched.status_code == 200, url + if url.endswith(".js"): + scripts.append(fetched.text) + + called = {path for script in scripts for path in re.findall(r'["`](/?api/[\w-]+)', script)} + assert {"api/session", "api/users", "api/notes", "api/sign-in"} <= called + assert not [path for path in called if path.startswith("/")], called + assert browser.get(urljoin(page_url, "api/session")).json()["user"] == "ann" + assert [u["id"] for u in browser.get(urljoin(page_url, "api/users")).json()] == ["ann", "ben"] From 4ee1be2eda14b9e81161edadb85f902efb737ca1 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 14:22:37 -0800 Subject: [PATCH 09/19] fix(website): the Notes page saves queued edits before switching users, and adds a note once per submit (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/notes/web/src/main.js | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/main.js b/packages/agentenv-protocol/examples/website/notes/web/src/main.js index f4318fabf..d061a6cea 100644 --- a/packages/agentenv-protocol/examples/website/notes/web/src/main.js +++ b/packages/agentenv-protocol/examples/website/notes/web/src/main.js @@ -40,7 +40,7 @@ function queueSave(id, fields) { } const state = { next: fields }; saving.set(id, state); - run(async () => { + state.done = run(async () => { try { while (state.next) { const next = state.next; @@ -180,14 +180,31 @@ async function start() { const [session, users] = await Promise.all([api("api/session"), api("api/users")]); $("user").replaceChildren(...users.map((user) => new Option(user.name || user.id, user.id))); $("user").value = session.user ?? ""; - $("user").addEventListener("change", (event) => - run(() => api("api/sign-in", { method: "POST", body: { user: event.target.value } }))); + $("user").addEventListener("change", (event) => { + const user = event.target.value; + $("user").disabled = true; + run(async () => { + try { + while (saving.size) await Promise.all([...saving.values()].map((state) => state.done)); + await api("api/sign-in", { method: "POST", body: { user } }); + } finally { + $("user").disabled = false; + } + }); + }); + const add = $("new-note").querySelector("button[type=submit]"); $("new-note").addEventListener("submit", (event) => { event.preventDefault(); + if (add.disabled) return; const body = { title: $("new-title").value, body: $("new-body").value }; + add.disabled = true; run(async () => { - await api("api/notes", { method: "POST", body }); + try { + await api("api/notes", { method: "POST", body }); + } finally { + add.disabled = false; + } if ($("new-title").value === body.title) $("new-title").value = ""; if ($("new-body").value === body.body) $("new-body").value = ""; }); From 687c4a98852671eff1ab3589ac4fde5bb069ba7d Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 15:01:02 -0800 Subject: [PATCH 10/19] fix(website): the Notes page keeps the signed-in user when a save or sign-in fails, and reloads itself at its slash-ending address (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/notes/README.md | 2 +- .../examples/website/notes/web/index.html | 3 +++ .../examples/website/notes/web/src/main.js | 13 +++++++++++-- .../tests/test_website_notes_example.py | 1 + 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/examples/website/notes/README.md b/packages/agentenv-protocol/examples/website/notes/README.md index 3311112b0..67bac1ae3 100644 --- a/packages/agentenv-protocol/examples/website/notes/README.md +++ b/packages/agentenv-protocol/examples/website/notes/README.md @@ -8,7 +8,7 @@ them. The page shows the universe's time. | --- | --- | | `site.py` | The server: the `Notes` WebsiteEnvironment, its `@api` routes, and `serve()` on `PORT` (8000). | | `data/seed.json` | The seed data: two users (`ann`, `ben`), their notes, and `current_user`, the acting user. | -| `web/` | A minimal Vite app (`index.html`, `src/main.js`, `src/style.css`). Its assets (`base: "./"`) and API calls are relative to the page, so it works at `/` and under the gateway's `/website/notes/` alike. | +| `web/` | A minimal Vite app (`index.html`, `src/main.js`, `src/style.css`). Its assets (`base: "./"`) and API calls are relative to the page, so it works at `/` and under the gateway's `/website/notes/` alike; opened at a path without its final slash, such as `/website/notes`, the page first reloads itself with it. | | `Dockerfile` | The server image: Node builds `web/`, then Python installs this repo's `packages/agentenv-protocol` with uvicorn. It installs `curl`, which the gateway's compose health check runs. | | `env.toml` | The website env as a bundle entry. | diff --git a/packages/agentenv-protocol/examples/website/notes/web/index.html b/packages/agentenv-protocol/examples/website/notes/web/index.html index 1e498d3c1..468ef6385 100644 --- a/packages/agentenv-protocol/examples/website/notes/web/index.html +++ b/packages/agentenv-protocol/examples/website/notes/web/index.html @@ -4,6 +4,9 @@ Notes +
diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/main.js b/packages/agentenv-protocol/examples/website/notes/web/src/main.js index d061a6cea..fafffe38f 100644 --- a/packages/agentenv-protocol/examples/website/notes/web/src/main.js +++ b/packages/agentenv-protocol/examples/website/notes/web/src/main.js @@ -49,6 +49,7 @@ function queueSave(id, fields) { } landed.set(id, renders); } catch (error) { + state.error = error; lastSent.set(id, {}); landed.delete(id); throw error; @@ -179,15 +180,23 @@ async function run(action) { async function start() { const [session, users] = await Promise.all([api("api/session"), api("api/users")]); $("user").replaceChildren(...users.map((user) => new Option(user.name || user.id, user.id))); - $("user").value = session.user ?? ""; + let signedInAs = session.user ?? ""; + $("user").value = signedInAs; $("user").addEventListener("change", (event) => { const user = event.target.value; $("user").disabled = true; run(async () => { try { - while (saving.size) await Promise.all([...saving.values()].map((state) => state.done)); + while (saving.size) { + const pending = [...saving.values()]; + await Promise.all(pending.map((state) => state.done)); + const failed = pending.find((state) => state.error); + if (failed) throw failed.error; + } await api("api/sign-in", { method: "POST", body: { user } }); + signedInAs = user; } finally { + $("user").value = signedInAs; $("user").disabled = false; } }); diff --git a/packages/agentenv-protocol/tests/test_website_notes_example.py b/packages/agentenv-protocol/tests/test_website_notes_example.py index e02147b10..c7ae6212d 100644 --- a/packages/agentenv-protocol/tests/test_website_notes_example.py +++ b/packages/agentenv-protocol/tests/test_website_notes_example.py @@ -220,6 +220,7 @@ def test_the_built_page_loads_its_assets_and_api_under_the_gateways_prefix() -> browser = TestClient(_behind_gateway_prefix(notes_site.Notes().create_app()), base_url="http://gateway.test") page = browser.get(page_url) assert page.status_code == 200 and page.headers["content-type"].startswith("text/html") + assert page.text.index('location.pathname.endsWith("/")') < page.text.index(" diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/main.js b/packages/agentenv-protocol/examples/website/notes/web/src/main.js index fafffe38f..61ead9d31 100644 --- a/packages/agentenv-protocol/examples/website/notes/web/src/main.js +++ b/packages/agentenv-protocol/examples/website/notes/web/src/main.js @@ -6,6 +6,7 @@ let renders = 0; const saving = new Map(); const lastSent = new Map(); const landed = new Map(); +const drafts = new Map(); const CLOCK_REFRESH_MS = 5000; export function universeNow() { @@ -31,6 +32,21 @@ async function api(path, { method = "GET", body } = {}) { const $ = (id) => document.getElementById(id); +function setDraft(id, kind, value, saved) { + const draft = { ...drafts.get(id) }; + if (value === saved) delete draft[kind]; + else draft[kind] = value; + if (Object.keys(draft).length) drafts.set(id, draft); + else drafts.delete(id); +} + +function settleDraft(id, sent) { + const draft = drafts.get(id); + if (!draft) return; + for (const kind of Object.keys(sent)) if (draft[kind] === sent[kind]) delete draft[kind]; + if (!Object.keys(draft).length) drafts.delete(id); +} + function queueSave(id, fields) { lastSent.set(id, fields); const queued = saving.get(id); @@ -46,6 +62,7 @@ function queueSave(id, fields) { const next = state.next; state.next = null; await api(`api/notes/${id}`, { method: "PATCH", body: next }); + settleDraft(id, next); } landed.set(id, renders); } catch (error) { @@ -84,15 +101,18 @@ function noteItem(note) { done.setAttribute("aria-label", `Mark "${note.title}" ${note.done ? "not done" : "done"}`); done.addEventListener("change", () => run(() => api(`api/notes/${note.id}/toggle`, { method: "POST" }))); + const draft = drafts.get(note.id) ?? {}; const title = document.createElement("input"); title.className = "title"; title.defaultValue = note.title; + title.value = draft.title ?? note.title; title.setAttribute("aria-label", "Title"); const body = document.createElement("textarea"); body.className = "body"; body.rows = 2; body.defaultValue = note.body; + body.value = draft.body ?? note.body; body.placeholder = "Details"; body.setAttribute("aria-label", "Details"); @@ -101,7 +121,8 @@ function noteItem(note) { if (title.value === sent.title && body.value === sent.body) return; queueSave(note.id, { title: title.value, body: body.value }); }; - for (const field of [title, body]) { + for (const [kind, field] of [["title", title], ["body", body]]) { + field.addEventListener("input", () => setDraft(note.id, kind, field.value, note[kind])); field.addEventListener("change", save); field.addEventListener("blur", save); } @@ -112,7 +133,10 @@ function noteItem(note) { const remove = document.createElement("button"); remove.className = "delete"; remove.textContent = "Delete"; - remove.addEventListener("click", () => run(() => api(`api/notes/${note.id}`, { method: "DELETE" }))); + remove.addEventListener("click", () => run(async () => { + await api(`api/notes/${note.id}`, { method: "DELETE" }); + drafts.delete(note.id); + })); const text = document.createElement("div"); text.className = "text"; @@ -121,28 +145,19 @@ function noteItem(note) { return item; } -function editing() { - const kept = []; - for (const field of document.querySelectorAll("#notes .title, #notes .body")) { - const focused = field === document.activeElement; - if (focused || field.value !== field.defaultValue) { - kept.push({ id: field.closest("li").dataset.id, kind: field.className, value: field.value, focused, - start: field.selectionStart, end: field.selectionEnd }); - } - } - return kept; +function focused() { + const field = document.activeElement; + if (!field?.matches?.("#notes .title, #notes .body")) return null; + return { id: field.closest("li").dataset.id, kind: field.className, start: field.selectionStart, + end: field.selectionEnd }; } -function restore(kept) { - for (const { id, kind, value, focused, start, end } of kept) { - const field = document.querySelector(`#notes li[data-id="${id}"] .${kind}`); - if (!field) continue; - field.value = value; - if (focused) { - field.focus(); - field.setSelectionRange(start, end); - } - } +function refocus(kept) { + if (!kept) return; + const field = document.querySelector(`#notes li[data-id="${kept.id}"] .${kept.kind}`); + if (!field) return; + field.focus(); + field.setSelectionRange(kept.start, kept.end); } async function render() { @@ -156,9 +171,9 @@ async function render() { landed.delete(id); } } - const kept = editing(); + const kept = focused(); $("notes").replaceChildren(...notes.map(noteItem)); - restore(kept); + refocus(kept); if (!notes.length) { const empty = document.createElement("li"); empty.className = "empty"; @@ -195,6 +210,7 @@ async function start() { } await api("api/sign-in", { method: "POST", body: { user } }); signedInAs = user; + drafts.clear(); } finally { $("user").value = signedInAs; $("user").disabled = false; From 8389f76db9419e5c087ae048ef536b7c2a39a5f7 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 13:34:07 -0800 Subject: [PATCH 13/19] feat(gateway): a keyed deploy's role key, forwarded headers, and triggers by route (FD-4683) This PR's change, rebased as one commit onto #429. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/notes/README.md | 6 +- .../agentenv_protocol/website/environment.py | 21 +- .../src/agentenv_protocol/website/routes.py | 24 +- .../tests/test_website_api.py | 13 ++ .../tests/test_website_environment.py | 12 + src/agent_env/a2a_agent/a2a_agent.py | 12 +- src/agent_env/cli/env/deploy.py | 7 +- src/agent_env/env/env.py | 9 +- src/agent_env/env/envs/mcp_server.py | 4 +- src/agent_env/env/envs/multi_env.py | 7 +- src/agent_env/env/envs/website.py | 4 +- src/agent_env/env/gateway/__init__.py | 2 + src/agent_env/env/gateway/caller.py | 3 +- src/agent_env/env/gateway/constants.py | 6 +- src/agent_env/env/gateway/entrypoint.py | 3 +- src/agent_env/env/gateway/gateway.py | 139 +++++++++--- src/agent_env/env/gateway/rest_proxy.py | 129 ++++++++++- src/agent_env/env/gateway/triggers.py | 36 ++- .../env_providers/env_gateway_provider.py | 39 +++- src/agent_env/task/episode/run.py | 9 +- src/agent_env/task/episode/steps.py | 5 +- .../task_step/task_steps/deploy_agent.py | 13 +- .../task_step/task_steps/deploy_env.py | 11 +- .../task_step/task_steps/load_artifact.py | 3 +- .../task_steps/mcp_cli_builder/codegen.py | 17 +- .../task_step/task_steps/post_turn_hooks.py | 3 +- .../task_steps/register_env_triggers.py | 6 +- tst/integration/env/gateway/gateway_test.py | 44 ++-- .../test_next_env_event_hook_local.py | 3 +- .../test_register_env_triggers_local.py | 25 ++- .../envs/test_mcp_server_plugin_provider.py | 2 +- tst/unit/env/envs/test_multi_env_name.py | 2 +- .../env/gateway/rest_proxy_stream_test.py | 212 ++++++++++++++++++ tst/unit/env/gateway/triggers_website_test.py | 188 +++++++++++++++- tst/unit/env/test_env.py | 10 +- .../env_gateway_provider_test.py | 52 ++++- tst/unit/task_step/deploy_env_test.py | 36 +++ .../task_step/local_container_urls_test.py | 32 ++- .../test_deploy_agent_role_delivery.py | 26 ++- tst/unit/task_step/test_load_artifact_file.py | 16 ++ tst/unit/task_step/test_mcp_cli_builder.py | 10 + .../test_prompt_agent_post_turn_hooks.py | 22 ++ .../task_step/test_register_env_triggers.py | 6 +- 43 files changed, 1074 insertions(+), 155 deletions(-) diff --git a/packages/agentenv-protocol/examples/website/notes/README.md b/packages/agentenv-protocol/examples/website/notes/README.md index 67d54de43..eade87bde 100644 --- a/packages/agentenv-protocol/examples/website/notes/README.md +++ b/packages/agentenv-protocol/examples/website/notes/README.md @@ -20,8 +20,10 @@ them. The page shows the universe's time. `edit_note`, `toggle_note` and `delete_note`. Errors are raised as `NotFound`, `BadRequest` and `Unauthorized`. - **Roles.** `all_notes` (`GET /api/admin/notes`) is declared with `roles=["admin"]`. It answers only a caller that sends `AgentEnv-Role: admin` with `AgentEnv-Role-Key` equal to the container's `WEBSITE_ROLE_KEY`; any other caller - gets 403. The gateway passes on whatever role a browser sends, so without the key a page could claim `admin`; with no - `WEBSITE_ROLE_KEY` set, every caller is `default` and the route is closed. + gets 403. A deploy with `keyed_roles` gives the site its own role key as `WEBSITE_ROLE_KEY` and its gateway sends it + on only beside a role it verified with the same key, so a page cannot claim `admin` through it. Any other deploy sets + no key, and its gateway passes on whatever role a browser sends; with no `WEBSITE_ROLE_KEY` set, every caller is + `default` and the route is closed. - **Users.** `ApiContext.user` is the user in the signed session cookie. With no cookie, it is the acting user (the store's `current_user`), and `GET /api/session` signs that user in. `POST /api/sign-in` switches to another seeded user. A user's notes are invisible to every other user, and another user's note answers 404. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py index e9eaee9fa..ac05ba6be 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -16,12 +16,14 @@ import re from pathlib import Path from typing import Any, ClassVar, NamedTuple, Optional +from urllib.parse import unquote from starlette.applications import Starlette from starlette.datastructures import MutableHeaders from starlette.middleware import Middleware from starlette.requests import Request from starlette.responses import FileResponse, JSONResponse, Response +from starlette._utils import get_route_path from starlette.routing import Route from ..agent_env_environment import ( @@ -39,7 +41,8 @@ from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock, clock_patch_script, iso from .errors import NotFound from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness -from .routes import ApiContext, ApiRoute, endpoint, error_response, openapi, openapi_path, request_role, reserved_path, routes_of +from .routes import (ApiContext, ApiRoute, endpoint, error_response, forwarded_prefix, openapi, openapi_path, request_role, + reserved_path, routes_of) from .session import SESSION_COOKIE, SessionSigner, expires_at from .store import Collections @@ -284,9 +287,11 @@ def _check_health(cls: type) -> None: class _Forwarded: - """Puts the address the browser reached the site at in the scope, from ``X-Forwarded-Host`` and - ``X-Forwarded-Proto``, before anything routes it, so redirects, ``request.url`` and a mounted app's ``url_for`` - name that address and not the internal one the proxy reached the server at.""" + """Puts the address the browser reached the site at in the scope, from ``X-Forwarded-Host``, + ``X-Forwarded-Proto`` and ``X-Forwarded-Prefix``, before anything routes it, so redirects, ``request.url`` and a + mounted app's ``url_for`` name that address and not the internal one the proxy reached the server at. A prefix + becomes the scope's ``root_path`` and leads its ``path``, as ASGI has a mounted app's, so routing still sees the + site's own path.""" def __init__(self, app: Any) -> None: self.app = app @@ -301,6 +306,12 @@ async def __call__(self, scope: dict, receive: Any, send: Any) -> None: scope["headers"] = [(k, v) for k, v in scope["headers"] if k != b"host"] + [(b"host", host.encode())] if proto in ("http", "https"): scope["scheme"] = proto if scope["type"] == "http" else {"http": "ws", "https": "wss"}[proto] + prefix = forwarded_prefix(_first(headers.get(b"x-forwarded-prefix"))) + if prefix: + scope["root_path"] = unquote(prefix) + scope.get("root_path", "") + scope["path"] = unquote(prefix) + scope["path"] + if scope.get("raw_path") is not None: + scope["raw_path"] = prefix.encode("latin-1") + scope["raw_path"] await self.app(scope, receive, send) @@ -381,7 +392,7 @@ async def __call__(self, scope: dict, receive: Any, send: Any) -> None: if self.app is not None: await self.app(scope, receive, send) return - path, reading = scope["path"], scope["method"] in ("GET", "HEAD") + path, reading = get_route_path(scope) or "/", scope["method"] in ("GET", "HEAD") if reserved_path(path): return await error_response(NotFound("not found"))(scope, receive, send) if reading and (file := self._file(path)) is not None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py index 3059e78bb..939803342 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py @@ -53,8 +53,9 @@ class ApiRoute(NamedTuple): fn: Callable def advertised(self) -> dict: - return {"method": self.method, "path": openapi_path(self.path), "name": self.name, - "roles": list(self.roles) or None} + """The route as the card names it: its path is the Starlette path it is served at, converters and all + (``/files/{rest:path}``), so a gateway matches the requests it answers, nested paths included.""" + return {"method": self.method, "path": self.path, "name": self.name, "roles": list(self.roles) or None} class _Api: @@ -159,11 +160,23 @@ def apply(self, response: Response) -> Response: def base_url(request: Request) -> str: """The site's address as the browser reached it: ``X-Forwarded-Proto`` and ``X-Forwarded-Host`` when a proxy - sent them, the request's own scheme and ``Host`` otherwise.""" + sent them, the request's own scheme and ``Host`` otherwise, under the path ``X-Forwarded-Prefix`` names when the + proxy serves the site below one (the gateway's ``/website/``).""" proto = request.headers.get("x-forwarded-proto", "").split(",")[0].strip() or request.url.scheme host = (request.headers.get("x-forwarded-host", "").split(",")[0].strip() or request.headers.get("host") or request.url.netloc) - return f"{proto}://{host}" + return f"{proto}://{host}{forwarded_prefix(request.headers.get('x-forwarded-prefix', ''))}" + + +def forwarded_prefix(value: str) -> str: + """The path an ``X-Forwarded-Prefix`` value names, without its trailing slash; empty for none, and for one that is + not an absolute path or would make the site's URLs name another host (``//host``). + + >>> [forwarded_prefix(v) for v in ("/website/notes/", "/a, /b", "", "/", "website", "//evil.example", "/a//b")] + ['/website/notes', '/a', '', '', '', '', ''] + """ + prefix = value.split(",")[0].strip().rstrip("/") + return prefix if prefix.startswith("/") and "//" not in prefix and prefix.isprintable() else "" ROLE_KEY_ENV = "WEBSITE_ROLE_KEY" @@ -172,7 +185,8 @@ def base_url(request: Request) -> str: def request_role(request: Request) -> str: """The role ``AgentEnv-Role`` names, believed only from a caller that also sends ``AgentEnv-Role-Key`` equal to - ``WEBSITE_ROLE_KEY``; every other caller, and every caller of a site with no key set, is ``default``. The gateway + ``WEBSITE_ROLE_KEY``; every other caller, and every caller of a site with no key set, is ``default``. A deploy with + ``keyed_roles`` sets the key and its gateway sends it on beside a role it verified the same way; any other gateway forwards whatever role a browser sends, so without the key a page could claim any role.""" claimed = caller_from_request(None, request.headers).role or DEFAULT_ROLE key = os.environ.get(ROLE_KEY_ENV, "") diff --git a/packages/agentenv-protocol/tests/test_website_api.py b/packages/agentenv-protocol/tests/test_website_api.py index b7754cf9d..e461ce097 100644 --- a/packages/agentenv-protocol/tests/test_website_api.py +++ b/packages/agentenv-protocol/tests/test_website_api.py @@ -320,6 +320,19 @@ def test_a_redirect_names_the_forwarded_address(client: TestClient) -> None: assert client.get("/api/items/", follow_redirects=False).headers["location"] == "http://testserver/api/items" +def test_a_site_behind_a_forwarded_prefix_names_it_in_its_address_and_redirects_and_routes_its_own_paths( + client: TestClient) -> None: + behind = {"Host": "internal-x.invalid", "X-Forwarded-Host": "gw.example", "X-Forwarded-Proto": "https", + "X-Forwarded-Prefix": "/website/shop/"} + assert client.get("/api/whoami", headers=behind).json()["base_url"] == "https://gw.example/website/shop" + assert client.get("/api/items/1", headers=behind).json()["title"] == "mug" + redirected = client.get("/api/items/", follow_redirects=False, headers=behind) + assert (redirected.status_code, redirected.headers["location"]) == (307, "https://gw.example/website/shop/api/items") + for unusable in ("//evil.example", "website", ""): + seen = client.get("/api/whoami", headers={**behind, "X-Forwarded-Prefix": unusable}).json() + assert seen["base_url"] == "https://gw.example", unusable + + def test_parameterized_routes_never_take_the_data_plane_or_card() -> None: @website(domains=["wiki.example"]) class Wiki(WebsiteEnvironment): diff --git a/packages/agentenv-protocol/tests/test_website_environment.py b/packages/agentenv-protocol/tests/test_website_environment.py index 5920b3995..05c118591 100644 --- a/packages/agentenv-protocol/tests/test_website_environment.py +++ b/packages/agentenv-protocol/tests/test_website_environment.py @@ -102,6 +102,18 @@ def test_static_files_and_index_fallback_answer_for_any_host(client: TestClient) assert client.post("/orders").status_code == 404 +def test_behind_a_forwarded_prefix_the_site_serves_its_own_paths_and_the_fallback_still_guards_reserved_ones( + client: TestClient) -> None: + behind = {"X-Forwarded-Prefix": "/website/shop"} + assert "home" in client.get("/", headers=behind).text + assert client.get("/assets/app.js", headers=behind).text == "console.log(1)" + assert "home" in client.get("/orders/42/edit", headers=behind).text + assert client.get("/api/orders", headers=behind).json() == [] + assert client.get("/.well-known/agent-env.json", headers=behind).json()["name"] == "Shop" + assert client.get("/website/shop/assets/app.js", headers=behind).status_code == 404 + assert client.post("/agentenv/x", headers=behind).status_code in (404, 405) + + def test_a_link_out_of_the_static_folder_is_never_served(dist: Path) -> None: secret = dist.parent / "secret.txt" (dist / "leak.txt").symlink_to(secret) diff --git a/src/agent_env/a2a_agent/a2a_agent.py b/src/agent_env/a2a_agent/a2a_agent.py index 24c643be7..baa2ccf8f 100644 --- a/src/agent_env/a2a_agent/a2a_agent.py +++ b/src/agent_env/a2a_agent/a2a_agent.py @@ -275,21 +275,25 @@ def _skill_extension(deployed: "DeployedA2AAgent") -> dict: @staticmethod async def install_cli( deployed: "DeployedA2AAgent", cli_artifact: "CliArtifact", gateway_url: str, container: str | None = None, - on_host: bool = False, + on_host: bool = False, role_key: str | None = None, ) -> str: """Install a CliArtifact bundle into the agent's filesystem at /opt/cli//, in ``container`` when install_agent put the agent in one, or on the VM host, ``on_host``, when it put it there. Writes a sidecar - bin/.env with the gateway URL and chmod +x's the entrypoint. Idempotent. Returns the path to the entrypoint.""" + bin/.env with the gateway URL, and with ``role_key``, the key of that gateway's deploy, when it has one, so + each installed CLI sends its own deploy's key beside its role; then chmod +x's the entrypoint. Idempotent. + Returns the path to the entrypoint.""" from agent_env.providers.sandbox_providers.sandbox import VmSandbox, into_container, stage_files_into_container + from agent_env.task_step.task_steps.mcp_cli_builder.codegen import GATEWAY_URL_ENV_VAR, ROLE_KEY_ENV_VAR from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider, get_agent_sandbox_provider provider = build_sandbox_provider(deployed.sandbox_type) if deployed.sandbox_type else get_agent_sandbox_provider() sandbox = await provider.get_sandbox(deployed.sandbox_id) target_root = f"/opt/cli/{cli_artifact.command_name}" entrypoint_path = f"{target_root}/{cli_artifact.entrypoint}" env_dir = posixpath.dirname(entrypoint_path) + sidecar = f"{GATEWAY_URL_ENV_VAR}={gateway_url}\n" + (f"{ROLE_KEY_ENV_VAR}={role_key}\n" if role_key else "") if on_host: - await sandbox.write_host_file(f"AGENT_ENV_GATEWAY_URL={gateway_url}\n".encode(), f"{env_dir}/{GATEWAY_ENV_FILE}") + await sandbox.write_host_file(sidecar.encode(), f"{env_dir}/{GATEWAY_ENV_FILE}") await stage_files_into_container( sandbox, cli_artifact.get_cli_files().get_file_artifacts(), target_root, on_host=True) await sandbox.exec_script(f"chmod +x {shlex.quote(entrypoint_path)}") @@ -297,7 +301,7 @@ async def install_cli( return entrypoint_path # write .env first — write_file_from_text creates the parent dir; write_file_from_object does not. - await sandbox.write_file_from_text(f"AGENT_ENV_GATEWAY_URL={gateway_url}\n", f"{env_dir}/{GATEWAY_ENV_FILE}", + await sandbox.write_file_from_text(sidecar, f"{env_dir}/{GATEWAY_ENV_FILE}", **into_container(container)) for rel_path, file_artifact in cli_artifact.get_cli_files().get_file_artifacts().items(): diff --git a/src/agent_env/cli/env/deploy.py b/src/agent_env/cli/env/deploy.py index fe1984c28..f7afb1040 100644 --- a/src/agent_env/cli/env/deploy.py +++ b/src/agent_env/cli/env/deploy.py @@ -48,7 +48,10 @@ @click.option("--env-state-instance-id", default=None, help="Attach against an EXISTING env state instance instead of creating a fresh store " "(its own type selects the provider).") -def deploy(env_id: str, env_version: int | None, ttl_seconds: int, gateway_mode: str, sandbox: str, service_db_env_id: str | None, gateway_env_id: str | None, website_browser_env_id: str | None, disk_size_gb: float, cpu: float | None, memory_mb: int | None, env_state_type: str | None, env_state_instance_id: str | None): +@click.option("--keyed-roles", is_flag=True, default=False, + help="Draw a role key for the deploy: its gateway then believes a role other than default only from a " + "caller that sends the key beside it.") +def deploy(env_id: str, env_version: int | None, ttl_seconds: int, gateway_mode: str, sandbox: str, service_db_env_id: str | None, gateway_env_id: str | None, website_browser_env_id: str | None, disk_size_gb: float, cpu: float | None, memory_mb: int | None, env_state_type: str | None, env_state_instance_id: str | None, keyed_roles: bool): """Deploy an MCP server environment.""" from agent_env.providers import build_sandbox_provider from agent_env.config import get_config @@ -89,6 +92,8 @@ def deploy(env_id: str, env_version: int | None, ttl_seconds: int, gateway_mode: if env_state_instance_id: deploy_kwargs["env_state_instance_id"] = env_state_instance_id click.echo(f"Env state instance id: {env_state_instance_id}") + if keyed_roles: + deploy_kwargs["keyed_roles"] = True try: with run_scope(env.id): deployed_env = asyncio.run(env.deploy(**deploy_kwargs)) diff --git a/src/agent_env/env/env.py b/src/agent_env/env/env.py index ef3373e2c..9e50ebabf 100644 --- a/src/agent_env/env/env.py +++ b/src/agent_env/env/env.py @@ -17,7 +17,7 @@ from agent_env.store.routing import namespace_routing_enabled from agent_env.utils.paths import validate_relative_filename -from .gateway import GatewayMode +from .gateway import AGENT_ENV_ROLE_HEADER, AGENT_ENV_ROLE_KEY_HEADER, GatewayMode from .gateway.constants import WELL_KNOWN_PATH from .store import get_env_store @@ -83,6 +83,7 @@ class DeployedEnv: # The provider that made the record, so the class it loads as (see from_dict); capabilities still come from the card. env_provider_type: Optional[str] = None environment_card_url: Optional[str] = None + role_key: Optional[str] = None # The env card as last read, and when (ISO 8601, UTC); written at deploy from the readiness probe. environment_card: Optional[dict] = None environment_card_read_at_utc: Optional[str] = None @@ -122,6 +123,11 @@ def control_headers_for(self, url: str) -> dict[str, str]: """What a call to ``url`` sends of the env's control token: ``control_headers`` when ``url`` is the env's address or under it, as a step's grant gives them, and none anywhere else, an override's address say.""" return protocol_v1.granted_headers(url, {self.environment_url: self.control_headers}) + def role_headers(self, role: str) -> dict[str, str]: + """The headers a request to the env sends to act as `role`: the role, and the deploy's `role_key` beside it, + without which a gateway that has one takes the request for `default`'s. A record from before the key, of a deploy without + `keyed_roles`, or of one with no gateway, has none, and its role header alone is believed.""" + return {AGENT_ENV_ROLE_HEADER: role, **({AGENT_ENV_ROLE_KEY_HEADER: self.role_key} if self.role_key else {})} def supports(self, uri: str, method: str) -> bool: """Whether the stored env card offers `method` on the extension at `uri`; no network.""" @@ -200,6 +206,7 @@ def _fields_from(cls, data: dict) -> dict: created_at_utc=data.get("created_at_utc"), expires_at_utc=data.get("expires_at_utc"), control_token=data.get("control_token"), + role_key=data.get("role_key"), ) diff --git a/src/agent_env/env/envs/mcp_server.py b/src/agent_env/env/envs/mcp_server.py index 2f950f77a..96fad5ec0 100644 --- a/src/agent_env/env/envs/mcp_server.py +++ b/src/agent_env/env/envs/mcp_server.py @@ -107,13 +107,13 @@ def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: @environment_card when it isn't set. Raises BundleError listing every problem.""" return ctx.accept_env(data, cls, named_by="image") - async def deploy(self, ttl_seconds: int = 10800, disk_size_gb: float = 10, gateway_mode: GatewayMode = GatewayMode.PERFORMANCE, cpu: float | None = None, memory_mb: int | None = None, sandbox_type: str | None = None, env_state_type: str | None = None, env_state_instance_id: str | None = None, artifact_id: str | None = None, artifact_version: int | None = None, *, attribution: Optional[Attribution] = None) -> DeployedEnv: + async def deploy(self, ttl_seconds: int = 10800, disk_size_gb: float = 10, gateway_mode: GatewayMode = GatewayMode.PERFORMANCE, cpu: float | None = None, memory_mb: int | None = None, sandbox_type: str | None = None, env_state_type: str | None = None, env_state_instance_id: str | None = None, artifact_id: str | None = None, artifact_version: int | None = None, *, attribution: Optional[Attribution] = None, keyed_roles: bool = False) -> DeployedEnv: # In container mode the server runs in its own container, so loads stage there, as for a MultiEnv child. return await deploy_through_provider( self, environment_name=self.environment_name, ttl_seconds=ttl_seconds, sandbox_type=sandbox_type, disk_size_gb=disk_size_gb, gateway_mode=gateway_mode, cpu=cpu, memory_mb=memory_mb, env_state_type=env_state_type, env_state_instance_id=env_state_instance_id, attribution=attribution, - artifact_id=artifact_id, artifact_version=artifact_version, + artifact_id=artifact_id, artifact_version=artifact_version, **({"keyed_roles": True} if keyed_roles else {}), ) def deploy_refusal(self, *, sandbox_type: str | None = None, **options) -> str | None: diff --git a/src/agent_env/env/envs/multi_env.py b/src/agent_env/env/envs/multi_env.py index e193771d2..d1da07009 100644 --- a/src/agent_env/env/envs/multi_env.py +++ b/src/agent_env/env/envs/multi_env.py @@ -91,6 +91,7 @@ def __init__(self, id: str, version: Optional[int], mcp_server_envs: list[MCPSer self._gateway_mode = GatewayMode.PERFORMANCE self._env_provider = None self._mcp_server_name: Optional[str] = None + self._role_key: Optional[str] = None self._deployed: Optional[DeployedEnv] = None self._replaced: list[tuple] = [] # the provider and sandbox of each deployment a later deploy replaced, for close() @@ -156,15 +157,16 @@ def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: ctx.refuse(problems) return fields - async def deploy(self, ttl_seconds: int = 10800, disk_size_gb: float = 10, gateway_mode: GatewayMode = GatewayMode.PERFORMANCE, cpu: float | None = None, memory_mb: int | None = None, sandbox_type: str | None = None, env_state_type: str | None = None, env_state_instance_id: str | None = None, artifact_id: str | None = None, artifact_version: int | None = None, *, attribution: Optional[Attribution] = None) -> DeployedEnv: + async def deploy(self, ttl_seconds: int = 10800, disk_size_gb: float = 10, gateway_mode: GatewayMode = GatewayMode.PERFORMANCE, cpu: float | None = None, memory_mb: int | None = None, sandbox_type: str | None = None, env_state_type: str | None = None, env_state_instance_id: str | None = None, artifact_id: str | None = None, artifact_version: int | None = None, *, attribution: Optional[Attribution] = None, keyed_roles: bool = False) -> DeployedEnv: deployed_env = await deploy_through_provider( self, environment_name=None, ttl_seconds=ttl_seconds, sandbox_type=sandbox_type, disk_size_gb=disk_size_gb, gateway_mode=gateway_mode, cpu=cpu, memory_mb=memory_mb, env_state_type=env_state_type, env_state_instance_id=env_state_instance_id, attribution=attribution, - artifact_id=artifact_id, artifact_version=artifact_version, + artifact_id=artifact_id, artifact_version=artifact_version, **({"keyed_roles": True} if keyed_roles else {}), ) self._gateway_mode = gateway_mode self._mcp_server_name = deployed_env.mcp_server_name + self._role_key = deployed_env.role_key _hand_provider_to_child_envs(self) _hand_record_to_child_envs(self, deployed_env) return deployed_env @@ -190,6 +192,7 @@ async def from_deployed_env(cls, deployed: DeployedEnv) -> MultiEnv: _hand_record_to_child_envs(env, deployed) env._instance_id = deployed.instance_id env._mcp_server_name = deployed.mcp_server_name + env._role_key = deployed.role_key return env async def load_environment_artifact(self, environment_artifact: EnvironmentArtifact) -> None: diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index 64d4b2d5c..702dd4c61 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -169,12 +169,12 @@ def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: ctx.refuse([ctx.config_problem(problem)]) return ctx.accept_env(data, cls, named_by="backend_image") - async def deploy(self, ttl_seconds: int = 10800, disk_size_gb: float = 10, gateway_mode: GatewayMode = GatewayMode.PERFORMANCE, cpu: float | None = None, memory_mb: int | None = None, sandbox_type: str | None = None, env_state_type: str | None = None, env_state_instance_id: str | None = None, artifact_id: str | None = None, artifact_version: int | None = None, *, attribution: Optional[Attribution] = None) -> DeployedEnv: + async def deploy(self, ttl_seconds: int = 10800, disk_size_gb: float = 10, gateway_mode: GatewayMode = GatewayMode.PERFORMANCE, cpu: float | None = None, memory_mb: int | None = None, sandbox_type: str | None = None, env_state_type: str | None = None, env_state_instance_id: str | None = None, artifact_id: str | None = None, artifact_version: int | None = None, *, attribution: Optional[Attribution] = None, keyed_roles: bool = False) -> DeployedEnv: return await deploy_through_provider( self, environment_name=self.environment_name, ttl_seconds=ttl_seconds, sandbox_type=sandbox_type, disk_size_gb=disk_size_gb, gateway_mode=gateway_mode, cpu=cpu, memory_mb=memory_mb, env_state_type=env_state_type, env_state_instance_id=env_state_instance_id, attribution=attribution, - artifact_id=artifact_id, artifact_version=artifact_version, + artifact_id=artifact_id, artifact_version=artifact_version, **({"keyed_roles": True} if keyed_roles else {}), ) def deploy_refusal(self, *, sandbox_type: str | None = None, **options) -> str | None: diff --git a/src/agent_env/env/gateway/__init__.py b/src/agent_env/env/gateway/__init__.py index 6cd59de6f..cb94520d2 100644 --- a/src/agent_env/env/gateway/__init__.py +++ b/src/agent_env/env/gateway/__init__.py @@ -6,6 +6,7 @@ from .constants import ( AGENT_ENV_HIDDEN_BY_DEFAULT_META_KEY, AGENT_ENV_ROLE_HEADER, + AGENT_ENV_ROLE_KEY_HEADER, AGENT_ENV_ROLE_META_KEY, AGENT_ENV_SESSION_META_KEY, DEFAULT_ROLE, @@ -31,6 +32,7 @@ class InternalMCPServer: "AGENT_ENV_GATEWAY_MCP_PORT", "AGENT_ENV_HIDDEN_BY_DEFAULT_META_KEY", "AGENT_ENV_ROLE_HEADER", + "AGENT_ENV_ROLE_KEY_HEADER", "AGENT_ENV_ROLE_META_KEY", "AGENT_ENV_SESSION_META_KEY", "DEFAULT_ROLE", diff --git a/src/agent_env/env/gateway/caller.py b/src/agent_env/env/gateway/caller.py index 4c5c50edb..1546fe2c9 100644 --- a/src/agent_env/env/gateway/caller.py +++ b/src/agent_env/env/gateway/caller.py @@ -1,6 +1,7 @@ """Who the gateway acts for on one inbound request. -A ``Caller`` is the role the request's ``AgentEnv-Role`` header named (``default`` when it named none) and the key of +A ``Caller`` is the role the request's ``AgentEnv-Role`` header named (``default`` when it named none, or named another +without the deploy's ``AgentEnv-Role-Key`` when the deploy has one) and the key of the MCP session the request belongs to (empty on the REST and harness paths, which have no session). It is bound in three scopes, which read in order in ``gateway.py``: per HTTP request by the served app (the role), per inbound MCP session by the session lifespan (the key, kept for every tools/call of that session), and never per process. Every diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 30bd00031..90f5a609d 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -126,9 +126,10 @@ def sources(names: list[str]) -> str: TRIGGER_EXECUTOR_HOURS_TIME_ZONE = "executor_hours_time_zone" TRIGGER_NEXT_EVENT_AT = "next_event_at" TRIGGER_NOTIFY = "notify" +TRIGGER_WEBSITE_OP = "website_op" TRIGGER_FEATURES = (TRIGGER_NAMED_EXECUTORS, TRIGGER_PROVOKING_ROLES, TRIGGER_ACTION_DELAY, TRIGGER_EXECUTOR_HOURS, TRIGGER_UNLESS, TRIGGER_WEBSITE, TRIGGER_EXECUTOR_HOURS_TIME_ZONE, TRIGGER_NEXT_EVENT_AT, - TRIGGER_NOTIFY) + TRIGGER_NOTIFY, TRIGGER_WEBSITE_OP) # The one `source` a /step call may name instead of step_api, with the id of the hook artifact that made it. STEP_SOURCE_POST_TURN_HOOK = "post_turn_hook" @@ -142,6 +143,9 @@ def sources(names: list[str]) -> str: ) AGENT_ENV_ROLE_HEADER = "AgentEnv-Role" +AGENT_ENV_ROLE_KEY_HEADER = "AgentEnv-Role-Key" +GATEWAY_ROLE_KEY_ENV = "AGENTENV_ROLE_KEY" +WEBSITE_ROLE_KEY_ENV = "WEBSITE_ROLE_KEY" # MCP request `_meta` keys the gateway stamps on every child tools/call: the caller's role, and on the MCP path a # key naming the inbound session, so a child can keep per-caller state behind the gateway's one shared session. AGENT_ENV_ROLE_META_KEY = "agentenv.io/role" diff --git a/src/agent_env/env/gateway/entrypoint.py b/src/agent_env/env/gateway/entrypoint.py index b0b936ae9..b6d469a17 100644 --- a/src/agent_env/env/gateway/entrypoint.py +++ b/src/agent_env/env/gateway/entrypoint.py @@ -7,7 +7,7 @@ import httpx from . import AGENT_ENV_GATEWAY_MCP_PORT, GatewayMode, InternalMCPServer -from .constants import DEFAULT_MCP_SERVER_NAME +from .constants import DEFAULT_MCP_SERVER_NAME, GATEWAY_ROLE_KEY_ENV from .gateway import Gateway logger = logging.getLogger(__name__) @@ -152,5 +152,6 @@ def parse_name_url_map(env_str: str) -> dict[str, str]: rest_proxy_urls=rest_proxy_urls, gateway_mode=gateway_mode, service_db_url=service_db_url, + role_key=os.environ.get(GATEWAY_ROLE_KEY_ENV) or None, ) gateway.run() diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index b7dd22091..94dfd2366 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -4,6 +4,7 @@ import asyncio import atexit import contextlib +import hmac import json import logging import os @@ -41,6 +42,7 @@ from .constants import ( AGENT_ENV_HIDDEN_BY_DEFAULT_META_KEY, AGENT_ENV_ROLE_HEADER, + AGENT_ENV_ROLE_KEY_HEADER, AGENT_ENV_ROLE_META_KEY, # noqa: F401 (the server module serves the whole vocabulary, used in this file or not) AGENT_ENV_SESSION_META_KEY, # noqa: F401 CARD_FETCH_TIMEOUT_S, @@ -72,7 +74,19 @@ from .child_transport import child_http_client from .clock import Clock, ClockError, _iso from .get_time import GET_TIME_TOOL_NAME, build_get_time_tool -from .rest_proxy import ROLE_HEADER_BYTES, UpstreamResponse, fetch_child_card, forward, operations, rewrite_child_card, upstream_url +from .rest_proxy import ( + ROLE_HEADER_BYTES, + ROLE_KEY_HEADER_BYTES, + UpstreamResponse, + WebsiteRoute, + fetch_child_card, + forward, + operations, + rewrite_child_card, + route_name, + upstream_url, + website_routes, +) from .triggers import TriggerEngine, TriggerError from .notifications import CHECK_TOOL_NAMES, NotificationError, Notifications, build_check_tools @@ -119,7 +133,10 @@ def __init__( rest_proxy_urls: dict[str, str] | None = None, gateway_mode: GatewayMode = GatewayMode.PERFORMANCE, service_db_url: str | None = None, + role_key: str | None = None, ): + """`role_key` is the deploy's secret: with one, a request is believed to act as a role other than `default` + only when it carries the key beside the role; without one, as before it, the role header alone is believed.""" self.host = host self.port = port self.server_name = server_name @@ -129,6 +146,8 @@ def __init__( self._site_reserved: dict[str, frozenset[str]] = {} self.gateway_mode = gateway_mode self._service_db_url = service_db_url + self._role_key = role_key or None + self._website_routes: dict[str, list[WebsiteRoute] | None] = {} self._db_conn: psycopg2.extensions.connection | None = None if service_db_url: self._db_conn = psycopg2.connect(service_db_url) @@ -240,13 +259,15 @@ async def get_state(request: Request) -> Response: async def step_endpoint(request: Request) -> Response: return await self._handle_step(request) - async def _trigger_write(request: Request, method) -> Response: - """Shared body for the trigger write routes: parse JSON, invoke the engine method, - map a TriggerError (bad config) to a 400.""" + async def _trigger_write(request: Request, method, prepare=None) -> Response: + """Shared body for the trigger write routes: parse JSON, await `prepare` with it, invoke the engine + method, map a TriggerError (bad config) to a 400.""" try: body = await request.json() except Exception: return self._json_response({"ok": False, "error": "Invalid JSON body"}, 400) + if prepare is not None: + await prepare(body) try: return self._json_response(method(body), 200) except TriggerError as e: @@ -254,7 +275,7 @@ async def _trigger_write(request: Request, method) -> Response: @self._mcp.custom_route("/triggers/register", methods=["POST"]) async def triggers_register(request: Request) -> Response: - return await _trigger_write(request, self._trigger_engine.register) + return await _trigger_write(request, self._trigger_engine.register, prepare=self._read_trigger_website_routes) @self._mcp.custom_route("/triggers/remove", methods=["POST"]) async def triggers_remove(request: Request) -> Response: @@ -408,7 +429,7 @@ def run(self) -> None: anyio.run(server.serve) # The three scopes a Caller is bound in, innermost first. Request: the served app binds the role the - # AgentEnv-Role header named. Session: mcp's Server.run enters the session lifespan once per inbound MCP + # AgentEnv-Role header named, as `_attributed_role` believes it. Session: mcp's Server.run enters the session lifespan once per inbound MCP # session, in the session's server task before any handler task is spawned, so the key bound there is inherited # by every tools/call of that session (and the role is the one its initialize request carried, which that task # inherited). Process: child sessions and the time-trigger driver, outer to the MCP session manager's lifespan. @@ -423,11 +444,13 @@ async def app(scope, receive, send): if scope["type"] != "http": await inner(scope, receive, send) return - role = DEFAULT_ROLE + claimed, given_key = None, None for name, value in scope.get("headers", ()): - if name == ROLE_HEADER_BYTES: - role = value.decode(errors="ignore").strip() or DEFAULT_ROLE - break + if name == ROLE_HEADER_BYTES and claimed is None: + claimed = value.decode(errors="ignore") + elif name == ROLE_KEY_HEADER_BYTES and given_key is None: + given_key = value.decode(errors="ignore") + role = self._attributed_role(claimed or "", given_key or "") token = Caller(role=role, session=Caller.current().session).bind() try: await inner(scope, receive, send) @@ -603,7 +626,8 @@ async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | role = self._request_role(request) mcp_server = url_map is self.rest_proxy_urls and service_name.startswith("mcp-") if request.method in _SAFE_METHODS: - return await forward(request, base_url, role=role, timeout=self.REST_PROXY_TIMEOUT, mcp_server=mcp_server) + return await forward(request, base_url, role=role, timeout=self.REST_PROXY_TIMEOUT, mcp_server=mcp_server, + role_key=self._role_key) changelog_before = self._query_changelog_id() attribution = self._attribution(role) # at the start: a fire may be gone by the time the response ends @@ -611,12 +635,16 @@ async def log_write(status: int) -> None: changelog_after = self._query_changelog_id() if changelog_after is None or changelog_after <= (changelog_before or 0): return # only a request that wrote changelog rows is a write window + path = "/" + request.path_params.get("path", "") + op = (self._website_op(service_name, request.method, site_request_path(request) or path) + if url_map is self.website_urls else None) await self._log_event({ "event_type": "http_request", "route": "website" if url_map is self.website_urls else "svc", "service": service_name, "method": request.method, - "path": "/" + request.path_params.get("path", ""), + "path": path, + **({"op": op} if op else {}), "status": status, **attribution, "changelog_id_before": changelog_before, @@ -624,7 +652,7 @@ async def log_write(status: int) -> None: }) return await forward(request, base_url, role=role, timeout=self.REST_PROXY_TIMEOUT, mcp_server=mcp_server, - on_done=log_write) + role_key=self._role_key, on_done=log_write) async def _proxy_website_request(self, request: Request) -> Response: """Proxy a website request; with a website or state trigger registered, feed it to the engine and hold barriers. @@ -640,9 +668,10 @@ async def _proxy_website_request(self, request: Request) -> Response: app = request.path_params["service_name"] engine = self._trigger_engine if app in self.website_urls and (engine.has_website_triggers or engine.has_state_triggers): + path = site_request_path(request) or "/" + request.path_params.get("path", "") if isinstance(response, UpstreamResponse): - if not engine.may_hold_website_request(self._request_role(request), app, request.method, - "/" + request.path_params.get("path", ""), response.status_code): + if not engine.may_hold_website_request(self._request_role(request), app, request.method, path, + response.status_code, self._website_op(app, request.method, path)): response.after_body(lambda status: self._fire_website_triggers(request, app, status)) # answered at its last byte return response await response.read_ahead_if_sized(self.BARRIER_READ_AHEAD_MAX_BYTES) # its window ends before its triggers run @@ -656,16 +685,55 @@ async def _fire_website_triggers(self, request: Request, app: str, try: # Only a website trigger reads the request's args; a state re-check needs just the method and status. args = _request_args(request, await request.body()) if self._trigger_engine.has_website_triggers else {} - return self._trigger_engine.on_website_request(self._request_role(request), app, request.method, - "/" + request.path_params.get("path", ""), status, args) + path = site_request_path(request) or "/" + request.path_params.get("path", "") + return self._trigger_engine.on_website_request(self._request_role(request), app, request.method, path, + status, args, self._website_op(app, request.method, path)) except Exception: logger.exception("Trigger evaluation failed (fail-open)") return [], None - @staticmethod - def _request_role(request: Request) -> str: - """The role the gateway attributes a REST request to: the first AgentEnv-Role header, else the default role.""" - return request.headers.get(AGENT_ENV_ROLE_HEADER, "").strip() or DEFAULT_ROLE + def _request_role(self, request: Request) -> str: + """The role the gateway attributes a REST request to: the first AgentEnv-Role header, as `_attributed_role` + believes it, else the default role.""" + return self._attributed_role(request.headers.get(AGENT_ENV_ROLE_HEADER, ""), + request.headers.get(AGENT_ENV_ROLE_KEY_HEADER, "")) + + def _attributed_role(self, claimed: str, given_key: str) -> str: + """The role a request claiming `claimed` acts as: `default` when it claims none, and when the deploy has a + role key the request did not send. Without a key (a deploy without keyed_roles) the claim alone is believed.""" + role = claimed.strip() or DEFAULT_ROLE + if self._role_key is None or role == DEFAULT_ROLE: + return role + return role if hmac.compare_digest(given_key.encode(), self._role_key.encode()) else DEFAULT_ROLE + + async def _read_trigger_website_routes(self, body: Any) -> None: + """Read the routes of each website a website trigger in a register `body` names an `op` of, so registration + can check the op and requests can be matched by it; one whose card could not be read before is read again.""" + triggers = body.get("triggers") if isinstance(body, dict) else None + apps = {trigger["when"]["app"] for trigger in triggers if isinstance(trigger, dict) + and isinstance(trigger.get("when"), dict) and trigger["when"].get("type") == "website" + and trigger["when"].get("op") is not None and isinstance(trigger["when"].get("app"), str) + and trigger["when"]["app"] in self.website_urls} if isinstance(triggers, list) else set() + await asyncio.gather(*(self._read_website_routes(app, again=True) for app in apps)) + + async def _read_website_routes(self, app: str, *, again: bool = False) -> None: + """Read website `app`'s routes from its card once (`again`: once more, when its card could not be read). The + env card's composition reads them too, so a deploy, which reads that card, leaves them read.""" + if app in self._website_routes and not (again and self._website_routes[app] is None): + return + async with httpx.AsyncClient() as client: + card = await fetch_child_card(client, app, self.rest_proxy_urls.get(app) or self.website_urls[app]) + self._website_routes[app] = website_routes(card) if card is not None else None + + def website_route_names(self, app: str) -> set[str] | None: + """The names of the routes website `app`'s card names, as last read; None when it has not been read.""" + routes = self._website_routes.get(app) + return None if routes is None else {route.name for route in routes if route.name is not None} + + def _website_op(self, app: str, method: str, path: str) -> str | None: + """The name of the route of website `app` a request reached, by the routes last read from its card; never a + read of its own, so answering a request waits on no card.""" + return route_name(self._website_routes.get(app) or [], method, path) async def _site_reserved_paths(self, app: str) -> frozenset[str] | None: """The paths ``app``'s card advertises for agentenv (``site_reserved_paths``), read once from its server, the one @@ -702,8 +770,11 @@ def _rewrite_child_card(self, key: str, card: dict) -> dict: return rewrite_child_card(key, card, self._gateway_extensions()) async def _fetch_child_card(self, client: httpx.AsyncClient, key: str, base_url: str) -> dict | None: - """Fetch + rewrite one backing server's card; None (omit) on 404 / error / timeout / malformed JSON.""" + """Fetch + rewrite one backing server's card; None (omit) on 404 / error / timeout / malformed JSON. A + website's card read here also refreshes the routes its requests are named by.""" child = await fetch_child_card(client, key, base_url) + if child is not None and key in self.website_urls: + self._website_routes[key] = website_routes(child) return None if child is None else self._rewrite_child_card(key, child) def _gateway_extensions(self) -> list[dict]: @@ -1563,16 +1634,22 @@ def site_reserved_paths(card: dict) -> frozenset[str]: return frozenset(_normalized(p) for p in paths if isinstance(p, str) and p.startswith("/") and _normalized(p) != "/") +def site_request_path(request: Request) -> str | None: + """The path a site's server receives for a request on its route: the raw path forwarded, its dot segments removed + by httpx and then decoded. It differs from the path as routed here when an encoded slash sits beside a dot segment + (``public%2Fx/../api`` reaches ``/api``). None for a request-target the proxy can't build a URL for: ``forward()`` + refuses it.""" + try: + return httpx.URL(upstream_url("http://site", request)).path + except (ValueError, httpx.InvalidURL): + return None + + def reserved_site_request(request: Request, reserved: frozenset[str] = frozenset()) -> bool: """Whether a request on a site's route reaches its data plane or env card, or one of the paths ``reserved`` names - (``site_reserved_paths``): by the path as routed here, or by the one its server receives, which is the raw path - forwarded, its dot segments removed by httpx and then decoded. The two differ when an encoded slash sits beside a - dot segment (``public%2Fx/../agentenv``). A request-target the proxy can't build a URL for isn't reserved here: - ``forward()`` refuses it.""" + (``site_reserved_paths``): by the path as routed here, or by the one its server receives (``site_request_path``). + A request-target the proxy can't build a URL for isn't reserved here: ``forward()`` refuses it.""" if reserved_site_path("/" + request.path_params.get("path", ""), reserved): return True - try: - sent = httpx.URL(upstream_url("http://site", request)).path - except (ValueError, httpx.InvalidURL): - return False - return reserved_site_path(sent, reserved) + sent = site_request_path(request) + return sent is not None and reserved_site_path(sent, reserved) diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 0812440ec..10a3b3ba2 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -10,12 +10,17 @@ round-trip through str), minus the hop-by-hop set, in both directions; - the role header is the gateway's to write: the client's copies are dropped and the role the gateway attributed the request to goes in their place, and a POST to a backing MCP server's own endpoint gets the same treatment - inside its JSON-RPC ``params._meta``; + inside its JSON-RPC ``params._meta``; so is the role key, which goes upstream only beside a role other than + ``default`` the key verified, so a site holding the same key believes the role; +- the upstream learns the address the client reached it at: ``X-Forwarded-Host`` and ``X-Forwarded-Proto`` (a front's + own, when one before the gateway sent them, else the request's ``Host`` and scheme) and ``X-Forwarded-Prefix``, the + route prefix (``/svc/{service}`` or ``/website/{service}``) the gateway strips, which a client never sets; - the upstream response and the client it came through are closed when the response ends, iterated or not. A backing server's env card names its endpoints relative to itself; :func:`rewrite_child_card` roots them under ``/svc/{key}`` so they resolve against the env's address, keeping the gateway's own routes where the gateway offers -the same operation. +the same operation. A website's card names its routes on ``urn:agentenv:website/v1``; :func:`website_routes` reads +them so :func:`route_name` can name the route a proxied request reached. """ from __future__ import annotations @@ -23,8 +28,9 @@ import json import logging import posixpath +import re from collections.abc import AsyncIterator, Awaitable, Callable -from typing import Any +from typing import Any, NamedTuple from urllib.parse import unquote import httpx @@ -35,9 +41,12 @@ # Relative import — gateway runs as a standalone container with no agent_env.* from .constants import ( AGENT_ENV_ROLE_HEADER, + AGENT_ENV_ROLE_KEY_HEADER, AGENT_ENV_ROLE_META_KEY, AGENT_ENV_SESSION_META_KEY, CARD_FETCH_TIMEOUT_S, + DEFAULT_ROLE, + EXT_WEBSITE_URI, WELL_KNOWN_PATH, ) @@ -51,11 +60,16 @@ # The role header as it appears in the raw ASGI header list (lower-cased by the server). ROLE_HEADER_BYTES = AGENT_ENV_ROLE_HEADER.lower().encode() +ROLE_KEY_HEADER_BYTES = AGENT_ENV_ROLE_KEY_HEADER.lower().encode() + +FORWARDED_HOST, FORWARDED_PROTO, FORWARDED_PREFIX = "x-forwarded-host", "x-forwarded-proto", "x-forwarded-prefix" +_FORWARDED = frozenset(name.encode() for name in (FORWARDED_HOST, FORWARDED_PROTO, FORWARDED_PREFIX)) async def forward(request: Request, base_url: str, *, role: str, timeout: httpx.Timeout, mcp_server: bool, - on_done: Callable[[int], Awaitable[None]] | None = None) -> Response: - """Forward `request` to the service at `base_url` as `role` and stream the response through. `mcp_server` says + role_key: str | None = None, on_done: Callable[[int], Awaitable[None]] | None = None) -> Response: + """Forward `request` to the service at `base_url` as `role`, vouched for by `role_key` when the deploy has one, + and stream the response through. `mcp_server` says the service is a backing MCP server, whose own endpoint takes the role in its JSON-RPC `_meta` as well as in the header. A 400 names a request-target the proxy cannot route; a 502 names an upstream that never answered. `on_done` is awaited with the status once the response has ended, after its last byte when streamed.""" @@ -64,7 +78,7 @@ async def forward(request: Request, base_url: str, *, role: str, timeout: httpx. except ValueError as e: # an escaped slash inside the route prefix; refuse rather than forward a wrong path return await _done(_json_response({"error": str(e)}, 400), on_done) content = await request.body() # /svc and /website request bodies are small (JSON-RPC, forms) - headers = upstream_headers(request, role) + headers = upstream_headers(request, role, role_key) if mcp_server and request.method == "POST" and is_child_mcp_path(request.path_params.get("path", "")): # The gateway is the trust boundary for identity: a child's proxied MCP endpoint stays reachable, but the # role in its JSON-RPC `_meta` is the gateway's to write, like the header above. @@ -101,18 +115,49 @@ def upstream_url(base_url: str, request: Request) -> str: return url -def upstream_headers(request: Request, role: str) -> httpx.Headers: +def upstream_headers(request: Request, role: str, role_key: str | None = None) -> httpx.Headers: """One entry per raw header, so repeated fields (cookie, x-forwarded-for) keep their multiplicity. The role - header is the gateway's to set: the client's copies are dropped and the role the gateway attributed the - request to goes in their place, so the upstream never reads a value the gateway did not write.""" + and role key headers are the gateway's to set: the client's copies are dropped and the role the gateway + attributed the request to goes in their place, with `role_key` beside any role but `default`, so the upstream + never reads a value the gateway did not write. So are the forwarded-address headers (`forwarded_address`).""" headers = httpx.Headers([(k, v) for k, v in request.headers.raw - if k.lower() not in HOP_BY_HOP and k.lower() != ROLE_HEADER_BYTES]) + if k.lower() not in HOP_BY_HOP and k.lower() not in (ROLE_HEADER_BYTES, ROLE_KEY_HEADER_BYTES) + and k.lower() not in _FORWARDED]) headers[AGENT_ENV_ROLE_HEADER] = role + if role_key and role != DEFAULT_ROLE: + headers[AGENT_ENV_ROLE_KEY_HEADER] = role_key + headers.update(forwarded_address(request)) # Bytes are forwarded undecoded, so only ask upstream for encodings the caller can take. headers.setdefault("accept-encoding", "identity") return headers +def forwarded_address(request: Request) -> dict[str, str]: + """Where the client reached the proxied service: the host and scheme it addressed (as a front before the gateway + names them in its own ``X-Forwarded-Host``/``X-Forwarded-Proto``, else the request's ``Host`` and scheme; the + client could set ``Host`` anyway, so believing its front names nothing it couldn't), and the route prefix as sent. + A header with nothing to say is left out.""" + host = _first(request.headers.get(FORWARDED_HOST, "")) or request.headers.get("host", "") + proto = _first(request.headers.get(FORWARDED_PROTO, "")).lower() + address = {FORWARDED_HOST: host, FORWARDED_PROTO: proto if proto in ("http", "https") else request.url.scheme, + FORWARDED_PREFIX: route_prefix(request)} + return {name: value for name, value in address.items() if value} + + +def _first(value: str) -> str: + return value.split(",")[0].strip() + + +def route_prefix(request: Request) -> str: + """The path the client reached the proxied service under, as it sent it: the request-target up to the `{path}` + remainder, without its trailing slash, e.g. ``/website/notes``, or under a root path ``/env/website/notes``.""" + raw_path = request.scope.get("raw_path") or request.scope["path"].encode() + decoded_rest = request.path_params.get("path", "") + if decoded_rest: + raw_path = raw_path[:-len(raw_remainder(raw_path, request.scope["path"], decoded_rest))] + return raw_path.decode("latin-1").rstrip("/") + + def is_child_mcp_path(decoded_rest: str) -> bool: """Whether a /svc/{service}/{path} remainder reaches the backing server's MCP endpoint (`mcp` or anything under it), judged after dot-segment normalisation, since httpx collapses `x/../mcp` to `/mcp` on the way upstream. @@ -326,5 +371,69 @@ def operations(params: dict) -> list[tuple[str | None, dict | None, dict, str, s if isinstance(holder.get("endpoint"), str) and holder["endpoint"]] +class WebsiteRoute(NamedTuple): + """One route a website's card names: its verb, the pattern its path matches (a Starlette converter's own, one + path segment for any other converter or none), and its name, None for a route the site builds in.""" + + method: str + pattern: re.Pattern + name: str | None + + +_CONVERTERS = {"path": ".*", "int": "[0-9]+", "float": r"[0-9]+(?:\.[0-9]+)?", + "uuid": "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"} +_PATH_PARAM = re.compile(r"{\w+(?::(\w+))?}") + + +def website_routes(card: dict | None) -> list[WebsiteRoute]: + """The routes a website's card names on ``urn:agentenv:website/v1``, in its order: each one's method, its path + (``/api/notes/{note_id}``, with or without a Starlette converter such as ``{note_id:int}``) and its name. An entry + missing any of the three is skipped; a card naming none, or no card, has none. + + Before them come, unnamed, the built-in routes the card names, which answer before any of its routes would: a GET + to its ``health`` path, and a GET or DELETE to its ``session_path`` unless one of its routes is at that very path + (then the site serves its own there). A request one of them answers is named by no route. + + >>> [(r.method, r.pattern.pattern, r.name) for r in website_routes({"capabilities": {"extensions": [ + ... {"uri": "urn:agentenv:website/v1", "params": {"health": "/healthz", "routes": [ + ... {"method": "get", "path": "/api/notes/{note_id:int}", "name": "get_note"}, {"path": "/x"}]}}]}})] + [('GET', '/healthz', None), ('GET', '/api/notes/(?:[0-9]+)', 'get_note')] + """ + found: list[WebsiteRoute] = [] + for ext in ((card or {}).get("capabilities") or {}).get("extensions") or []: + params = ext.get("params") if isinstance(ext, dict) and ext.get("uri") == EXT_WEBSITE_URI else None + routes = params.get("routes") if isinstance(params, dict) else None + named = [route for route in (routes if isinstance(routes, list) else []) + if isinstance(route, dict) and all(isinstance(route.get(k), str) and route[k] + for k in ("method", "path", "name"))] + health, session = (params.get(k) if isinstance(params, dict) else None for k in ("health", "session_path")) + if isinstance(health, str) and health: + found.append(WebsiteRoute("GET", _path_pattern(health), None)) + if isinstance(session, str) and session and all(route["path"] != session for route in named): + found += [WebsiteRoute(verb, _path_pattern(session), None) for verb in ("GET", "DELETE")] + found += [WebsiteRoute(route["method"].upper(), _path_pattern(route["path"]), route["name"]) for route in named] + return found + + +def _path_pattern(path: str) -> re.Pattern: + parts, end = [], 0 + for param in _PATH_PARAM.finditer(path): + parts += [re.escape(path[end:param.start()]), f"(?:{_CONVERTERS.get(param.group(1) or '', '[^/]+')})"] + end = param.end() + return re.compile("".join(parts) + re.escape(path[end:])) + + +def route_name(routes: list[WebsiteRoute], method: str, path: str) -> str | None: + """The name of the first of `routes` a `method` request to `path` reaches (a HEAD reaches a GET route), else None, + as it is when that first one is a built-in route. + + >>> route_name(website_routes({"capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1", "params": { + ... "routes": [{"method": "GET", "path": "/api/notes/{note_id}", "name": "get_note"}]}}]}}), "HEAD", "/api/notes/7") + 'get_note' + """ + verbs = {method.upper(), "GET"} if method.upper() == "HEAD" else {method.upper()} + return next((route.name for route in routes if route.method in verbs and route.pattern.fullmatch(path)), None) + + def _json_response(data: dict | list, status_code: int) -> Response: return Response(content=json.dumps(data), status_code=status_code, media_type="application/json") diff --git a/src/agent_env/env/gateway/triggers.py b/src/agent_env/env/gateway/triggers.py index 1017d3ef3..69dbf0d20 100644 --- a/src/agent_env/env/gateway/triggers.py +++ b/src/agent_env/env/gateway/triggers.py @@ -713,7 +713,8 @@ def _normalize_action_when(when: dict, tid: str) -> None: _normalize_provoked_when(when, tid) def _normalize_website_when(self, when: dict, tid: str) -> None: - """Validate a website `when`: a known app, upper-cased method, `path`/`status` predicates, `where`, `repeat`.""" + """Validate a website `when`: a known app, upper-cased method, the `op` (a route the site's card names), + `path`/`status` predicates, `where`, `repeat`.""" w = f"trigger '{tid}'" app = when.get("app") if not isinstance(app, str) or not app: @@ -726,6 +727,17 @@ def _normalize_website_when(self, when: dict, tid: str) -> None: raise TriggerError(f"{w}: when.method must be a non-empty HTTP method, e.g. 'POST'") else: when["method"] = when["method"].strip().upper() + if when.get("op") is None: + when.pop("op", None) + elif not isinstance(when["op"], str) or not when["op"]: + raise TriggerError(f"{w}: when.op must be a non-empty string naming one of the website's routes") + else: + names = self._gw.website_route_names(app) + if names is None: + raise TriggerError(f"{w}: when.op names a route of website {app!r}, whose card the gateway could not " + "read; match its path instead") + if when["op"] not in names: + raise TriggerError(f"{w}: when.op names no route of website {app!r} (routes: {sorted(names)})") for key in ("path", "status"): if when.get(key) is None: when.pop(key, None) @@ -921,17 +933,18 @@ def on_tool_call(self, role: str, name: str, arguments: dict, return pending, timeout_s def on_website_request(self, role: str, app: str, method: str, path: str, status: int, - args: dict) -> tuple[list[asyncio.Task], float | None]: - """Detection hook for an answered website request, before its response goes out; returns what on_tool_call does.""" + args: dict, op: str | None = None) -> tuple[list[asyncio.Task], float | None]: + """Detection hook for an answered website request, before its response goes out, `op` naming the route of + the site's card it reached; returns what on_tool_call does.""" provokable = [(tid, trig) for tid, trig in self._triggers.items() if self._provoked_by(trig, role)] if not provokable: return [], None if self._has_time_triggers: self._eval_time_triggers() - request = {"app": app, "method": method, "path": path, "status": status} + request = {"app": app, "method": method, "path": path, "status": status, **({"op": op} if op else {})} provoking = {"source": "website", **request, "role": role} matched = [(tid, trig) for tid, trig in provokable - if _website_matches(trig["spec"]["when"], app, method, path, status)] + if _website_matches(trig["spec"]["when"], app, method, path, status, op)] pending, timeout_s = self._detect(matched, provoking, {"args": args, "request": request}) if method not in _SAFE_METHODS and status < 400: self._reevaluate_states(provokable, provoking) @@ -963,10 +976,12 @@ def _reevaluate_states(self, provokable: list[tuple[str, dict]], provoking: dict trig["evaluating"] = True self._schedule(self._eval_state(tid, trig, provoking)) - def may_hold_website_request(self, role: str, app: str, method: str, path: str, status: int) -> bool: + def may_hold_website_request(self, role: str, app: str, method: str, path: str, status: int, + op: str | None = None) -> bool: """Whether a `provoking_call` barrier could hold this website request: a matcher check, `where` and status aside.""" return any((trig["spec"].get("barrier") or {}).get("at") == "provoking_call" and self._provoked_by(trig, role) - and _website_matches(trig["spec"]["when"], app, method, path, status) for trig in self._triggers.values()) + and _website_matches(trig["spec"]["when"], app, method, path, status, op) + for trig in self._triggers.values()) @property def has_website_triggers(self) -> bool: @@ -1608,10 +1623,13 @@ def _normalize_provoked_when(when: dict, tid: str) -> None: raise TriggerError(f"trigger '{tid}': when.repeat must be a boolean") -def _website_matches(when: dict, app: str, method: str, path: str, status: int) -> bool: - """Whether a request matches a website `when` (`where` aside); without `status`, only a response below 400 does.""" +def _website_matches(when: dict, app: str, method: str, path: str, status: int, op: str | None = None) -> bool: + """Whether a request matches a website `when` (`where` aside); with `op`, only one reaching that route does, and + without `status`, only a response below 400 does.""" if when["type"] != "website" or when["app"] != app or when.get("method", method) != method: return False + if "op" in when and when["op"] != op: + return False if "path" in when and not _eval_predicate(when["path"], path): return False return _eval_predicate(when["status"], status) if "status" in when else status < 400 diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index 03913adac..319329d45 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -7,6 +7,7 @@ import json import logging import re +import secrets import shlex import time from collections import Counter @@ -23,7 +24,9 @@ from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT, GatewayMode from agent_env.env.gateway.constants import ( DEFAULT_MCP_SERVER_NAME, + GATEWAY_ROLE_KEY_ENV, STATE_TOOL_NAME_CLASHES_KEY, + WEBSITE_ROLE_KEY_ENV, WELL_KNOWN_PATH, random_mcp_server_name, tool_name_clashes_text, @@ -117,6 +120,7 @@ class DeployedGateway: # The env card the readiness probe read, and when (ISO 8601, UTC). environment_card: dict | None = None environment_card_read_at_utc: str | None = None + role_key: str | None = None @dataclass @@ -281,6 +285,7 @@ def create_docker_compose( host_ips: tuple[str, ...] = (), extra_hosts: tuple[str, ...] = (), mcp_server_name: str | None = None, + role_key: str | None = None, ) -> str: """Generate docker-compose.yml content for a gateway deployment onto a VM/laptop/arbitrary machine. @@ -292,6 +297,8 @@ def create_docker_compose( gateway_image: Docker image for the gateway. gateway_port: Port to expose for the gateway. website_configs: Optional list of website configurations (frontend + backend pairs). + role_key: The deploy's role key, given to the gateway as AGENTENV_ROLE_KEY and to each website server as + WEBSITE_ROLE_KEY; without one, both believe the AgentEnv-Role header as before it. Returns: docker-compose.yml content as a string. @@ -489,6 +496,8 @@ def create_docker_compose( gateway_env.append(f" - REST_PROXY_URLS={','.join(rest_proxy_urls)}") gateway_env.append(f" - GATEWAY_MODE={gateway_mode.value}") gateway_env.append(f" - SERVICE_DB_URL={state_provider.base_url(state_instance)}") + if role_key: + gateway_env.append(f" - {GATEWAY_ROLE_KEY_ENV}={role_key}") publish = host_port or (lambda port: port) exposed_gateway_ports = [] if expose_gateway_port: @@ -534,6 +543,7 @@ def create_docker_compose( f" - ENVIRONMENT_NAME={wc.environment_name}", f" - PORT={AGENT_ENV_WEBSITE_BACKEND_PORT}", " - PYTHONUNBUFFERED=1", + *([f" - {WEBSITE_ROLE_KEY_ENV}={role_key}"] if role_key else []), " healthcheck:", f' test: ["CMD-SHELL", "curl -sf -o /dev/null http://localhost:{AGENT_ENV_WEBSITE_BACKEND_PORT}{WELL_KNOWN_PATH}' f' || curl -sf -o /dev/null http://localhost:{AGENT_ENV_WEBSITE_BACKEND_PORT}{wc.health_path}"]', @@ -601,10 +611,12 @@ async def deploy( attribution: Optional[Attribution] = None, artifact_id: str | None = None, artifact_version: int | None = None, + keyed_roles: bool = False, ) -> DeployedGatewayEnv: """A gateway in front of the env, with the state store it acquires (local Postgres by default); returns the record. A MultiEnv's gateway fronts its MCP servers and websites, a WebsiteEnv's its website, and any other env's its one MCP server. + With ``keyed_roles``, the deploy draws a role key (see ``create_gateway``). """ from agent_env.config import get_config, service_db_env_id from agent_env.env.env import Env @@ -623,6 +635,7 @@ async def deploy( mcp_server_images=topology.mcp_server_images, website_configs=topology.website_configs, website_images=topology.website_images, mcp_server_name=topology.mcp_server_name, gateway_mode=gateway_mode, ttl_seconds=ttl_seconds, disk_size_gb=disk_size_gb, cpu=cpu, memory_mb=memory_mb, attribution=attribution, + keyed_roles=keyed_roles, ) return DeployedGatewayEnv( env_id=env.id, @@ -639,6 +652,7 @@ async def deploy( website_frontend_urls=deployed.website_frontend_urls, gateway_mode=gateway_mode.value, env_state_instance_ids=deployed.env_state_instance_ids, + role_key=deployed.role_key, ) async def create_gateway( @@ -661,6 +675,7 @@ async def create_gateway( mcp_server_name: str | None = None, *, attribution: Optional[Attribution] = None, + keyed_roles: bool = False, ) -> DeployedGateway: """Create and start a gateway from explicit topology, for a caller that builds its own record rather than calling ``deploy()``. @@ -674,20 +689,26 @@ async def create_gateway( ``mcp_server_name`` is the name the gateway presents on its card and as its MCP server; drawn as ``env`` + 4 random digits when the env declares none; the record takes it from the gateway's card. + + ``keyed_roles`` draws a secret role key for the deploy, given to the gateway and each website server and kept on + the record: the gateway then believes a role other than ``default`` only beside it, so every caller acting as one + must send it. Off, as before the key, the role header alone is believed. """ return await self._launch( sandbox_provider, env_id, state_instance=state_instance, mcp_servers=mcp_servers, mcp_server_images=mcp_server_images, gateway_port=gateway_port, website_configs=website_configs, website_images=website_images, gateway_mode=gateway_mode, ttl_seconds=ttl_seconds, disk_size_gb=disk_size_gb, cpu=cpu, memory_mb=memory_mb, existing_sandbox=existing_sandbox, sidecars=sidecars, mcp_server_name=mcp_server_name, - attribution=attribution, + attribution=attribution, keyed_roles=keyed_roles, ) - async def _launch(self, sandbox_provider: SandboxProvider, env_id: str | None, *, mcp_server_name: str | None = None, **gateway) -> DeployedGateway: + async def _launch(self, sandbox_provider: SandboxProvider, env_id: str | None, *, mcp_server_name: str | None = None, + keyed_roles: bool = False, **gateway) -> DeployedGateway: """One gateway, deployed on the sandbox provider or each chained member in turn, then its tools probed.""" attempt = functools.partial( self._deploy_gateway, **gateway, mcp_server_name=mcp_server_name or random_mcp_server_name(), # drawn once, so every chained attempt presents the same name + role_key=secrets.token_urlsafe(32) if keyed_roles else None, ) earlier_sandbox = self._sandbox result = await self._run(sandbox_provider, env_id, attempt) @@ -739,6 +760,7 @@ async def _deploy_via_vm( existing_sandbox: Sandbox | None = None, sidecars: list[SidecarConfig] | None = None, mcp_server_name: str | None = None, + role_key: str | None = None, ) -> DeployedGateway: from agent_env.env.env import Env from agent_env.config import gateway_env_id, get_config, website_browser_env_id @@ -830,6 +852,7 @@ async def _stand_up(_spec): host_ips=sandbox.host_ips, extra_hosts=sandbox.extra_hosts, mcp_server_name=mcp_server_name, + role_key=role_key, ) logger.info(f"Generated docker-compose.yml:\n{_redact_compose_secrets(compose_content)}") @@ -923,7 +946,7 @@ async def _stand_up(_spec): gateway_url=gateway_url, mcp_url=mcp_url, db_web_url=db_web_url, db_mcp_url=db_mcp_url, website_frontend_urls=website_frontend_urls, mcp_server_name=mcp_server_name, env_state_instance_ids=[state_instance.instance_id], - environment_card=card, environment_card_read_at_utc=card_read_at_utc, + environment_card=card, environment_card_read_at_utc=card_read_at_utc, role_key=role_key, ) async def _deploy_via_containers( @@ -940,6 +963,7 @@ async def _deploy_via_containers( memory_mb: int | None = None, attribution: Optional[Attribution] = None, mcp_server_name: str | None = None, + role_key: str | None = None, ) -> DeployedGateway: from agent_env.env.env import Env from agent_env.config import gateway_env_id, get_config @@ -1036,6 +1060,7 @@ async def _deploy_via_containers( "GATEWAY_MODE": gateway_mode.value, "SERVICE_DB_URL": self._state_provider.base_url(state_instance), "MCP_SERVER_NAME": mcp_server_name or DEFAULT_MCP_SERVER_NAME, + **({GATEWAY_ROLE_KEY_ENV: role_key} if role_key else {}), } gateway_sb = await sandbox_provider.create_container( image_name=gateway_env.docker_image_artifact.image_name, @@ -1069,6 +1094,7 @@ async def _deploy_via_containers( env_state_instance_ids=[state_instance.instance_id], environment_card=card, environment_card_read_at_utc=card_read_at_utc, + role_key=role_key, ) except Exception: await self.close() @@ -1370,6 +1396,7 @@ async def _deploy_gateway( sidecars: list[SidecarConfig] | None = None, mcp_server_name: str | None = None, attribution: Optional[Attribution] = None, + role_key: str | None = None, ) -> DeployedGateway: """One gateway deploy on one (unchained) sandbox provider: a VM on one that creates VMs, else a container per server on its private network.""" @@ -1394,7 +1421,7 @@ async def _deploy_gateway( sandbox_provider, mcp_servers, mcp_server_images, gateway_port=gateway_port, website_configs=website_configs, gateway_mode=gateway_mode, ttl_seconds=ttl_seconds, disk_size_gb=disk_size_gb, - cpu=cpu, memory_mb=memory_mb, mcp_server_name=mcp_server_name, **deploy_kwargs, + cpu=cpu, memory_mb=memory_mb, mcp_server_name=mcp_server_name, role_key=role_key, **deploy_kwargs, ) earlier_sandbox = self._sandbox try: @@ -1403,7 +1430,7 @@ async def _deploy_gateway( gateway_port=gateway_port, website_configs=website_configs, website_images=website_images, gateway_mode=gateway_mode, ttl_seconds=ttl_seconds, disk_size_gb=disk_size_gb, cpu=cpu, memory_mb=memory_mb, existing_sandbox=existing_sandbox, sidecars=sidecars, - mcp_server_name=mcp_server_name, **deploy_kwargs, + mcp_server_name=mcp_server_name, role_key=role_key, **deploy_kwargs, ) except BaseException: await self._terminate_new_sandbox(earlier_sandbox=earlier_sandbox, existing_sandbox=existing_sandbox) @@ -1566,7 +1593,7 @@ def _compose_name_problems(mcp_servers: list[MCPServerConfig], website_configs: # Compose `environment:` keys whose values are secrets and must be masked before the # rendered compose is logged. Matches the key to the left of the first '=' on an # ` - KEY=VALUE` env line. -_SECRET_ENV_KEY_RE = re.compile(r"^(\s*-\s*)([A-Za-z0-9_]*(?:SECRET|PASSWORD|TOKEN|API_KEY)[A-Za-z0-9_]*)=.*$") +_SECRET_ENV_KEY_RE = re.compile(r"^(\s*-\s*)([A-Za-z0-9_]*(?:SECRET|PASSWORD|TOKEN|API_KEY|ROLE_KEY)[A-Za-z0-9_]*)=.*$") def _compose_ps_rows(text: str) -> list[dict]: diff --git a/src/agent_env/task/episode/run.py b/src/agent_env/task/episode/run.py index 2b082cd69..5ee2a6214 100644 --- a/src/agent_env/task/episode/run.py +++ b/src/agent_env/task/episode/run.py @@ -283,9 +283,11 @@ def _raise_unless_completed(self) -> None: if (error := self._run.exception()) is not None: raise error - async def _serve_in_process(self, agent: str, role: str | None, mcp_urls: dict[str, str]) -> DeployedAgent: + async def _serve_in_process(self, agent: str, role: str | None, mcp_urls: dict[str, str], + role_keys: dict[str, str | None] | None = None) -> DeployedAgent: """A proxy served in this process for ``agent``, which works in no sandbox, with each env server registered as - ``deploy_agent`` registers them with any agent, at the URL this process reaches: played as the others are.""" + ``deploy_agent`` registers them with any agent, at the URL this process reaches, with its deploy's role key + from ``role_keys``: played as the others are.""" server = InProcessProxy() self._in_process.append(server) url = await server.start() @@ -295,7 +297,8 @@ async def _serve_in_process(self, agent: str, role: str | None, mcp_urls: dict[s card = response.json() mcp_config = A2AAgent.find_extension(card, A2AAgent.EXT_MCP_CONFIG) for env_id, mcp_url in mcp_urls.items(): - body = _mcp_add_body(mcp_config, mcp_url, sandbox_request_headers_for_url(mcp_url), env_id, role) + body = _mcp_add_body(mcp_config, mcp_url, sandbox_request_headers_for_url(mcp_url), env_id, role, + (role_keys or {}).get(env_id)) (await client.post(url + mcp_config["params"]["endpoint"], json=body)).raise_for_status() deployed = DeployedAgent(agent_name=agent, api_url=url, a2a_url=url, a2a_card=card, role=role) await self._attach(agent, deployed) diff --git a/src/agent_env/task/episode/steps.py b/src/agent_env/task/episode/steps.py index 9bcc85816..7b8c12e51 100644 --- a/src/agent_env/task/episode/steps.py +++ b/src/agent_env/task/episode/steps.py @@ -45,11 +45,12 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: episode = _episode_of(context) agent = self.step.agent_name if not self.sandboxed: - mcp_urls = {} + mcp_urls, role_keys = {}, {} for env_id in getattr(self.step, "env_ids", None) or []: deployed = bound_deployment(context, env_id, env_step_id=self.step.env_step_id, step_id=self.id) mcp_urls[env_id] = deployed.mcp_url if deployed is not None else live_mcp_url(env_id) - deployed = await episode._serve_in_process(agent, getattr(self.step, "role", None), mcp_urls) + role_keys[env_id] = deployed.role_key if deployed is not None else None + deployed = await episode._serve_in_process(agent, getattr(self.step, "role", None), mcp_urls, role_keys) context.deployed_agents.append(deployed) if system_prompt := getattr(self.step, "system_prompt", None): config = A2AAgent.negotiate_agent_config(deployed.a2a_card, {"system_prompt": system_prompt}) diff --git a/src/agent_env/task_step/task_steps/deploy_agent.py b/src/agent_env/task_step/task_steps/deploy_agent.py index cbacdc804..1f37a3847 100644 --- a/src/agent_env/task_step/task_steps/deploy_agent.py +++ b/src/agent_env/task_step/task_steps/deploy_agent.py @@ -31,7 +31,7 @@ from agent_env.a2a_agent.staging import StagedObjectStore, transfer_store from agent_env.config import get_config from agent_env.env.env import DeployedEnv, DeployedSandboxEnv, Env -from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER +from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER, AGENT_ENV_ROLE_KEY_HEADER from agent_env.providers.sandbox_providers.local_docker.provider import host_url_for from agent_env.a2a_agent.a2a_agent import DEFAULT_A2A_PORT, unwritten_local_agent from agent_env.providers.sandbox_providers.sandbox_provider import ( @@ -70,9 +70,11 @@ def _mcp_add_request(mcp_ext: dict) -> dict: return request if isinstance(request, dict) else {} -def _mcp_add_body(mcp_ext: dict, url: str, headers: dict | None, card_name: str | None, role: str | None = None) -> dict: +def _mcp_add_body(mcp_ext: dict, url: str, headers: dict | None, card_name: str | None, role: str | None = None, + role_key: str | None = None) -> dict: """Body for the agent's mcp-config `add`. The card name and the role header ride along only when the card's `add` - takes them, judged on the exact field set sent; what an env does with the role is the env's business.""" + takes them, judged on the exact field set sent, the deploy's `role_key` beside the role so its gateway believes + it; what an env does with the role is the env's business.""" request = _mcp_add_request(mcp_ext) body: dict = {"url": url} if headers: @@ -81,6 +83,8 @@ def _mcp_add_body(mcp_ext: dict, url: str, headers: dict | None, card_name: str body["name"] = card_name if role and card_request_accepts(request, {*body, "headers"}): body.setdefault("headers", {})[AGENT_ENV_ROLE_HEADER] = role + if role_key: + body["headers"][AGENT_ENV_ROLE_KEY_HEADER] = role_key return body @@ -433,7 +437,8 @@ async def _execute(self, context: TaskStepContext, deployed_agents: list) -> Tas url = host_url_for(live_mcp_url(env_id), deployed.sandbox_type) card_name = None if mcp_ext: - body = _mcp_add_body(mcp_ext, url, sandbox_request_headers_for_url(url), card_name, self.role) + body = _mcp_add_body(mcp_ext, url, sandbox_request_headers_for_url(url), card_name, self.role, + deployed_env.role_key if deployed_env is not None else None) registrations.append((env_id, body)) identity_payload: dict[str, str] = {"name": self.agent_name} diff --git a/src/agent_env/task_step/task_steps/deploy_env.py b/src/agent_env/task_step/task_steps/deploy_env.py index 375a4c515..55d605b0a 100644 --- a/src/agent_env/task_step/task_steps/deploy_env.py +++ b/src/agent_env/task_step/task_steps/deploy_env.py @@ -48,6 +48,7 @@ def __init__( artifact_id: Optional[str] = None, artifact_version: Optional[int] = None, env_params: Optional[dict] = None, + keyed_roles: bool = False, ): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) self.env_id = env_id @@ -66,6 +67,7 @@ def __init__( self.metadata = metadata or {} # The env type's own deploy options, which this step passes to its deploy() without reading them. self.env_params = env_params + self.keyed_roles = keyed_roles def run_state(self, user_overrides: Mapping[str, Any]) -> tuple[str | None, str | None]: """The env state type and attached state instance a run with ``user_overrides`` deploys on: each the run's, @@ -89,6 +91,7 @@ def to_dict(self) -> dict: base["env_state_instance_id"] = self.env_state_instance_id base["metadata"] = self.metadata base["env_params"] = self.env_params + base["keyed_roles"] = self.keyed_roles return base @classmethod @@ -109,6 +112,7 @@ def from_dict(cls, data: dict) -> DeployEnvTaskStep: env_state_instance_id=data.get("env_state_instance_id"), metadata=dict(data.get("metadata") or {}), env_params=data.get("env_params"), + keyed_roles=bool(data.get("keyed_roles", False)), ) def preflight(self) -> list[str]: @@ -133,7 +137,8 @@ def preflight(self) -> list[str]: return [f"deploy_env '{self.id}': {refusal}"] if refusal else [] def _env_params_option(self) -> dict: - return {"env_params": self.env_params} if self.env_params else {} + return {**({"env_params": self.env_params} if self.env_params else {}), + **({"keyed_roles": True} if self.keyed_roles else {})} async def execute(self, context: TaskStepContext) -> TaskStepContext: from agent_env.env.env import Env @@ -188,6 +193,10 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if "env_params" not in params and not accepts_kwargs: raise ValueError(f"deploy_env '{self.id}': {env.type} envs take no env_params") deploy_kwargs["env_params"] = self.env_params + if self.keyed_roles: + if "keyed_roles" not in params and not accepts_kwargs: + raise ValueError(f"deploy_env '{self.id}': {env.type} envs take no keyed_roles") + deploy_kwargs["keyed_roles"] = True deployed_env = await env.deploy(**deploy_kwargs) # create_instance runs inside env.deploy(), so an annotation set after it # returns has to be persisted explicitly to reach the instance record. diff --git a/src/agent_env/task_step/task_steps/load_artifact.py b/src/agent_env/task_step/task_steps/load_artifact.py index fdecb9e75..c79d6153c 100644 --- a/src/agent_env/task_step/task_steps/load_artifact.py +++ b/src/agent_env/task_step/task_steps/load_artifact.py @@ -645,7 +645,8 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if not (agent.on_host and agent.sandbox_type == LocalSandbox.type): gateway_url = reachable_url(gateway_url, from_sandbox_type=deployed.sandbox_type, to_sandbox_type=agent.sandbox_type) - install_path = await A2AAgent.install_cli(deployed_agent, artifact, gateway_url, **_agent_files(agent)) + install_path = await A2AAgent.install_cli(deployed_agent, artifact, gateway_url, role_key=deployed.role_key, + **_agent_files(agent)) agent_clis = context.metadata.setdefault("installed_clis", {}).setdefault(self.agent_name, {}) agent_clis[artifact.id] = { "install_path": install_path, diff --git a/src/agent_env/task_step/task_steps/mcp_cli_builder/codegen.py b/src/agent_env/task_step/task_steps/mcp_cli_builder/codegen.py index 4be0384b0..9853400f2 100644 --- a/src/agent_env/task_step/task_steps/mcp_cli_builder/codegen.py +++ b/src/agent_env/task_step/task_steps/mcp_cli_builder/codegen.py @@ -14,10 +14,11 @@ from agentenv_protocol import MANIFEST_VERSION, manifest_compatible -from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER +from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER, AGENT_ENV_ROLE_KEY_HEADER GATEWAY_URL_ENV_VAR = "AGENT_ENV_GATEWAY_URL" ROLE_ENV_VAR = "AGENT_ENV_ROLE" +ROLE_KEY_ENV_VAR = "AGENT_ENV_ROLE_KEY" DEFAULT_CLI_ROLE = "cli" _COMMAND_NAME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9_-]*$") _WORD_RE = re.compile(r"[A-Z]+(?=[A-Z][a-z]|$)|[A-Z]?[a-z]+|[0-9]+") @@ -166,6 +167,8 @@ def generate_cli_script( role_env_var=ROLE_ENV_VAR, default_role=DEFAULT_CLI_ROLE, role_header=AGENT_ENV_ROLE_HEADER, + role_key_env_var=ROLE_KEY_ENV_VAR, + role_key_header=AGENT_ENV_ROLE_KEY_HEADER, ) @@ -186,6 +189,8 @@ def generate_cli_script( ROLE_ENV = "{role_env_var}" DEFAULT_ROLE = "{default_role}" ROLE_HEADER = "{role_header}" +ROLE_KEY_ENV = "{role_key_env_var}" +ROLE_KEY_HEADER = "{role_key_header}" SERVICE_NAME = {environment_name} INTERFACE_MANIFEST = {interface_manifest_literal} _CONFIG_FILE_NAME = ".env" @@ -226,11 +231,17 @@ def _role(): return os.environ.get(ROLE_ENV, DEFAULT_ROLE) +def _role_headers(): + """The role header, with the deploy's role key beside it when one is set, so the gateway believes the role.""" + key = os.environ.get(ROLE_KEY_ENV) + return {{ROLE_HEADER: _role(), **({{ROLE_KEY_HEADER: key}} if key else {{}})}} + + def _post_step(payload): response = httpx.post( f"{{_gateway_url()}}/step", json=payload, - headers={{ROLE_HEADER: _role()}}, + headers=_role_headers(), timeout=120, ) response.raise_for_status() @@ -248,7 +259,7 @@ def _list_tools(): allowed_names = {{t.get("name") for t in (role_filtered.get("tools") or []) if t.get("name")}} state_resp = httpx.get( f"{{_gateway_url()}}/state", - headers={{ROLE_HEADER: _role()}}, + headers=_role_headers(), timeout=120, ) state_resp.raise_for_status() diff --git a/src/agent_env/task_step/task_steps/post_turn_hooks.py b/src/agent_env/task_step/task_steps/post_turn_hooks.py index 0afb54b71..cca67b1fd 100644 --- a/src/agent_env/task_step/task_steps/post_turn_hooks.py +++ b/src/agent_env/task_step/task_steps/post_turn_hooks.py @@ -22,7 +22,6 @@ from agent_env.artifact import FileArtifact from agent_env.env.env import DeployedEnv, EnvCapabilityUnsupported from agent_env.env.gateway.constants import ( - AGENT_ENV_ROLE_HEADER, EXT_CLOCK_URI, EXT_NOTIFICATIONS_URI, EXT_STEP_URI, @@ -152,7 +151,7 @@ async def call_tool(self, tool: str, args: Optional[dict], role: str) -> dict: "source": STEP_SOURCE_POST_TURN_HOOK, "hook_artifact_id": self._hook_artifact_id} async with httpx.AsyncClient() as client: resp = await client.request(op["method"], f"{self._deployed.environment_url}{op['endpoint']}", json=body, - headers={AGENT_ENV_ROLE_HEADER: role, **self._deployed.control_headers}, + headers={**self._deployed.role_headers(role), **self._deployed.control_headers}, timeout=_TOOL_CALL_TIMEOUT_S) if resp.status_code >= 400: raise RuntimeError(f"env '{self.env_id}': /step call_tool {tool!r} failed (HTTP {resp.status_code}): " diff --git a/src/agent_env/task_step/task_steps/register_env_triggers.py b/src/agent_env/task_step/task_steps/register_env_triggers.py index 156d17fb9..237f77259 100644 --- a/src/agent_env/task_step/task_steps/register_env_triggers.py +++ b/src/agent_env/task_step/task_steps/register_env_triggers.py @@ -21,6 +21,7 @@ TRIGGER_PROVOKING_ROLES, TRIGGER_UNLESS, TRIGGER_WEBSITE, + TRIGGER_WEBSITE_OP, ) from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef @@ -146,12 +147,15 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: f"redeploy it on a gateway image that does.") # An older gateway ignores what it does not declare: named actions would go to the default executor, a # trigger's when.roles would leave it to the watched roles, delays and hours would not wait, a deadline - # would fire even after its `unless` call, and hours in a time zone would be read as UTC. + # would fire even after its `unless` call, hours in a time zone would be read as UTC, and a website trigger + # naming an `op` would fire on every request to the site. used = {TRIGGER_NAMED_EXECUTORS: bool(named), TRIGGER_PROVOKING_ROLES: _when_has(self.triggers, "roles"), TRIGGER_ACTION_DELAY: _delays(self.triggers), TRIGGER_EXECUTOR_HOURS: bool(self.executor_hours), TRIGGER_UNLESS: _when_has(self.triggers, "unless"), TRIGGER_WEBSITE: any(isinstance(t.get("when"), dict) and t["when"].get("type") == "website" for t in self.triggers), + TRIGGER_WEBSITE_OP: any(isinstance(t.get("when"), dict) and t["when"].get("type") == "website" + and t["when"].get("op") is not None for t in self.triggers), TRIGGER_EXECUTOR_HOURS_TIME_ZONE: _hours_in_a_time_zone(self.executor_hours), TRIGGER_NOTIFY: any(a.get("type") == "notify" for t in self.triggers for a in t.get("actions") or [] if isinstance(a, dict))} diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index ab4057715..eefe02c96 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -780,7 +780,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) cli_list = await client.post( f"{result.gateway_url}/step", json={"action": "list_tools"}, - headers={AGENT_ENV_ROLE_HEADER: "cli"}, + headers=result.role_headers("cli"), timeout=30, ) cli_list.raise_for_status() @@ -797,7 +797,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) cli_call = await client.post( f"{result.gateway_url}/step", json={"action": "call_tool", "tool_name": "list_emails", "arguments": {"folder_name": "INBOX"}}, - headers={AGENT_ENV_ROLE_HEADER: "cli"}, + headers=result.role_headers("cli"), timeout=30, ) assert cli_call.status_code == 200, f"cli call failed: {cli_call.status_code} {cli_call.text}" @@ -814,7 +814,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) async with streamable_http_client( result.mcp_url, - http_client=httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: "cli"}), + http_client=httpx.AsyncClient(headers=result.role_headers("cli")), ) as (read_stream, write_stream, _): async with ClientSession(read_stream, write_stream) as session: await session.initialize() @@ -826,7 +826,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) assert "list_emails" not in [t.name for t in await default_session.list_tools()] hidden = await default_session.call_tool("list_emails", {"folder_name": "INBOX"}) assert hidden.isError and "is disabled for role 'default'" in hidden.text, hidden - async with ToolSession(mcp_url=result.mcp_url, role="cli") as cli_session: + async with ToolSession(mcp_url=result.mcp_url, headers=result.role_headers("cli")) as cli_session: assert "list_emails" in [t.name for t in await cli_session.list_tools()] assert "alex.chen@techcorp.com" in (await cli_session.call_tool("list_emails", {"folder_name": "INBOX"})).text @@ -869,7 +869,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) assert state_resp.json()["roles"]["*"] == {"disabled": ["list_emails"], "allowed": []} for role in (None, "cli", "marketing"): - headers = {AGENT_ENV_ROLE_HEADER: role} if role else {} + headers = result.role_headers(role) if role else {} lst = await client.post( f"{result.gateway_url}/step", json={"action": "list_tools"}, headers=headers, timeout=30, ) @@ -879,14 +879,14 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) blocked = await client.post( f"{result.gateway_url}/step", json={"action": "call_tool", "tool_name": "list_emails", "arguments": {"folder_name": "INBOX"}}, - headers={AGENT_ENV_ROLE_HEADER: "marketing"}, + headers=result.role_headers("marketing"), timeout=30, ) assert blocked.status_code == 403, f"expected 403 for never-seen role under global disable, got {blocked.status_code} {blocked.text}" async with streamable_http_client( result.mcp_url, - http_client=httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: "cli"}), + http_client=httpx.AsyncClient(headers=result.role_headers("cli")), ) as (read_stream, write_stream, _): async with ClientSession(read_stream, write_stream) as session: await session.initialize() @@ -926,7 +926,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) mkt_lst = await client.post( f"{result.gateway_url}/step", json={"action": "list_tools"}, - headers={AGENT_ENV_ROLE_HEADER: "marketing"}, + headers=result.role_headers("marketing"), timeout=30, ) mkt_lst.raise_for_status() @@ -944,7 +944,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) cli_lst = await client.post( f"{result.gateway_url}/step", json={"action": "list_tools"}, - headers={AGENT_ENV_ROLE_HEADER: "cli"}, + headers=result.role_headers("cli"), timeout=30, ) cli_lst.raise_for_status() @@ -955,7 +955,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) mkt_lst = await client.post( f"{result.gateway_url}/step", json={"action": "list_tools"}, - headers={AGENT_ENV_ROLE_HEADER: "marketing"}, + headers=result.role_headers("marketing"), timeout=30, ) mkt_lst.raise_for_status() @@ -970,7 +970,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) cli_lst_after_reset = await client.post( f"{result.gateway_url}/step", json={"action": "list_tools"}, - headers={AGENT_ENV_ROLE_HEADER: "cli"}, + headers=result.role_headers("cli"), timeout=30, ) cli_lst_after_reset.raise_for_status() @@ -1002,7 +1002,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) blocked = await client.post( f"{result.gateway_url}/step", json={"action": "call_tool", "tool_name": "list_emails", "arguments": {"folder_name": "INBOX"}}, - headers={AGENT_ENV_ROLE_HEADER: "step-role"}, + headers=result.role_headers("step-role"), timeout=30, ) assert blocked.status_code == 403, f"expected 403 after step-driven disable, got {blocked.status_code} {blocked.text}" @@ -1020,7 +1020,7 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) unblocked = await client.post( f"{result.gateway_url}/step", json={"action": "call_tool", "tool_name": "list_emails", "arguments": {"folder_name": "INBOX"}}, - headers={AGENT_ENV_ROLE_HEADER: "step-role"}, + headers=result.role_headers("step-role"), timeout=30, ) assert unblocked.status_code == 200, f"expected 200 after step-driven enable, got {unblocked.status_code} {unblocked.text}" @@ -1816,7 +1816,7 @@ async def test_gateway_changes_name_the_role_behind_each_write(mcp_server_envs, seen = (await client.get(f"{gw}/changes", params={"limit": 10000}, timeout=30)).json()["changes"] after_id = seen[-1]["id"] if seen else 0 for role, subject in (("default", "first"), ("alice@example.com", "second")): - resp = await client.post(f"{gw}/step", headers={AGENT_ENV_ROLE_HEADER: role}, timeout=60, json={ + resp = await client.post(f"{gw}/step", headers=result.role_headers(role), timeout=60, json={ "action": "call_tool", "tool_name": "send_email", "arguments": {"recipients": ["dana@example.com"], "subject": subject, "content": "body"}}) assert resp.status_code == 200, resp.text @@ -1850,7 +1850,7 @@ async def test_gateway_changes_name_the_request_behind_a_write_through_the_rest_ async with httpx.AsyncClient() as client: seen = (await client.get(f"{gw}/changes", params={"limit": 10000}, timeout=30)).json()["changes"] after_id = seen[-1]["id"] if seen else 0 - async with httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: "carol@example.com"}, timeout=60) as as_carol: + async with httpx.AsyncClient(headers=result.role_headers("carol@example.com"), timeout=60) as as_carol: async with streamable_http_client(f"{gw}/svc/mcp-email/mcp", http_client=as_carol) as (read, write, _): async with ClientSession(read, write) as session: await session.initialize() @@ -2170,7 +2170,7 @@ async def assert_tool_sees_items(session, tools_result, tool_names): await _verify_list_tools(mcp_url=result.mcp_url, tool_verifier_callback=assert_tool_sees_items) # The role a ToolSession sends reaches the child's tool as its caller, with the gateway's key for each session - async with (ToolSession(mcp_url=result.mcp_url, role="alice@example.com") as alice, + async with (ToolSession(mcp_url=result.mcp_url, headers=result.role_headers("alice@example.com")) as alice, ToolSession(mcp_url=result.mcp_url) as anonymous): seen = [json.loads((await session.call_tool("items_whoami")).text) for session in (alice, alice, anonymous)] assert [caller["role"] for caller in seen] == ["alice@example.com", "alice@example.com", None] @@ -2548,7 +2548,7 @@ async def _poll(client, gw, want_fired, timeout_s=120): hr = await client.post(f"{gw}/step", json={"action": "call_tool", "tool_name": "search_emails", "arguments": {"query": "alex.chen", "folder_name": "INBOX"}}, - headers={AGENT_ENV_ROLE_HEADER: "harness"}, timeout=60) + headers=result.role_headers("harness"), timeout=60) assert hr.status_code == 200, f"harness /step failed: {hr.text}" await asyncio.sleep(4) assert {t["id"]: t["status"] for t in (await _state(client, gw))["triggers"]}["decoy"] == "armed", ( @@ -3064,7 +3064,7 @@ async def test_gateway_triggers_nl_executor(sandbox_provider, mcp_server_envs, e # The one-tool constraint really held (determinism lever, verified live). lt = await client.post(f"{gw}/step", json={"action": "list_tools"}, - headers={AGENT_ENV_ROLE_HEADER: "executor"}, timeout=30) + headers=result.role_headers("executor"), timeout=30) exec_tools = [t["name"] for t in lt.json()["tools"]] assert exec_tools == ["send_email"], f"executor tool surface not locked to one tool: {exec_tools}" @@ -3105,7 +3105,7 @@ async def _state(): se = await client.post(f"{gw}/step", json={"action": "call_tool", "tool_name": "search_emails", "arguments": {"query": marker, "folder_name": "SENT"}}, - headers={AGENT_ENV_ROLE_HEADER: "harness"}, timeout=30) + headers=result.role_headers("harness"), timeout=30) assert marker in se.text, f"marker email not found in SENT: {se.text[:400]}" # Cascade-safety: nlact was detected exactly once (from the provoke), NOT from the @@ -3351,17 +3351,17 @@ async def _rate_check(client, gw, rate, sleep_s=5.0): await client.post(f"{gw}/tools/{TOOL_DISABLE_ACTION}", json={"role": "locked", "tools": ["*"]}, timeout=30) locked = await client.post(f"{gw}/step", json={"action": "list_tools"}, - headers={AGENT_ENV_ROLE_HEADER: "locked"}, timeout=30) + headers=result.role_headers("locked"), timeout=30) assert GET_TIME not in [t["name"] for t in locked.json()["tools"]], "wildcard deny did not hide get_time" denied = await client.post(f"{gw}/step", json={"action": "call_tool", "tool_name": GET_TIME, "arguments": {}}, - headers={AGENT_ENV_ROLE_HEADER: "locked"}, timeout=30) + headers=result.role_headers("locked"), timeout=30) assert denied.status_code == 403, denied.text assert GET_TIME in (await _get_time_surfaces(client, gw, result.mcp_url)).step, "default role lost it" await client.post(f"{gw}/tools/{TOOL_ENABLE_ACTION}", json={"role": "locked", "tools": [GET_TIME]}, timeout=30) relocked = await client.post(f"{gw}/step", json={"action": "list_tools"}, - headers={AGENT_ENV_ROLE_HEADER: "locked"}, timeout=30) + headers=result.role_headers("locked"), timeout=30) assert GET_TIME in [t["name"] for t in relocked.json()["tools"]], "re-enable did not restore it" finally: await env.close() diff --git a/tst/integration/task_step/test_next_env_event_hook_local.py b/tst/integration/task_step/test_next_env_event_hook_local.py index de63dc0d7..103127090 100644 --- a/tst/integration/task_step/test_next_env_event_hook_local.py +++ b/tst/integration/task_step/test_next_env_event_hook_local.py @@ -12,7 +12,6 @@ import agent_env from agent_env.a2a_agent import conversation_store from agent_env.artifact import FileArtifact -from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.register_env_triggers import RegisterEnvTriggersStep @@ -65,7 +64,7 @@ async def _run(label: str, triggers: list[dict], params: dict) -> tuple: state = (await client.get("/triggers/state")).raise_for_status().json() clock = (await client.get("/clock/state")).raise_for_status().json() stored = await client.post("/step", json={"action": "call_tool", "tool_name": "list_items", "arguments": {}}, - headers={AGENT_ENV_ROLE_HEADER: "harness"}) + headers=deployed.role_headers("harness")) finally: await env.close() items = json.loads(stored.json()["content"][0]["text"])["items"] diff --git a/tst/integration/task_step/test_register_env_triggers_local.py b/tst/integration/task_step/test_register_env_triggers_local.py index 0a695d011..7c6964cca 100644 --- a/tst/integration/task_step/test_register_env_triggers_local.py +++ b/tst/integration/task_step/test_register_env_triggers_local.py @@ -17,9 +17,9 @@ import pytest import pytest_asyncio +from agent_env.env.env import DeployedEnv from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv -from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER from agent_env.env.gateway.constants import METHOD_RESET, RPC_PATH, TRIGGER_IN_FLIGHT_STATUSES from agent_env.task_step.context import DeployedAgent, TaskStepContext from agent_env.task_step.task_steps.register_env_triggers import RegisterEnvTriggersStep @@ -66,8 +66,8 @@ async def test_each_nl_action_reaches_the_executor_it_names(items_env): {"id": "on-add", "when": {"type": "action", "tool": "items_add_item", "repeat": True}, "actions": []}, ] try: - alice = _Executor("alice", "alice@example.com", deployed.gateway_url, host, started) - bob = _Executor("bob", "bob@example.com", deployed.gateway_url, host, started) + alice = _Executor("alice", "alice@example.com", deployed, host, started) + bob = _Executor("bob", "bob@example.com", deployed, host, started) context = TaskStepContext(deployed_envs=[deployed], deployed_agents=[alice.agent, bob.agent], metadata={}) await RegisterEnvTriggersStep(id="reg", version=None, env_id=env.id, triggers=triggers, watch_roles=["default"], executor_timeout_seconds=60).execute(context) @@ -76,7 +76,7 @@ async def test_each_nl_action_reaches_the_executor_it_names(items_env): assert provoke.status_code == 200, provoke.text state = await _quiet(client, {"to-alice": 1, "to-bob": 1, "on-add": 0}) stored = await client.post("/step", json={"action": "call_tool", "tool_name": "list_items", "arguments": {}}, - headers={AGENT_ENV_ROLE_HEADER: "harness"}) + headers=deployed.role_headers("harness")) finally: for executor in started: executor.stop() @@ -108,8 +108,8 @@ async def test_personas_react_to_each_other_until_one_stays_quiet(items_env): "actions": [{"type": "nl", "executor": "denise", "instruction": "DENISE-TO-BOB"}]}, ] try: - denise = _Executor("denise", "denise@example.com", deployed.gateway_url, host, started) - bob = _Executor("bob", "bob@example.com", deployed.gateway_url, host, started, replies=1) + denise = _Executor("denise", "denise@example.com", deployed, host, started) + bob = _Executor("bob", "bob@example.com", deployed, host, started, replies=1) context = TaskStepContext(deployed_envs=[deployed], deployed_agents=[denise.agent, bob.agent], metadata={}) await RegisterEnvTriggersStep(id="reg", version=None, env_id=env.id, triggers=triggers, watch_roles=["default"], executor_timeout_seconds=60).execute(context) @@ -118,7 +118,7 @@ async def test_personas_react_to_each_other_until_one_stays_quiet(items_env): assert provoke.status_code == 200, provoke.text state = await _quiet(client, {"denise-hears-solver": 1, "bob-hears-denise": 2, "denise-hears-bob": 1}) stored = await client.post("/step", json={"action": "call_tool", "tool_name": "list_items", "arguments": {}}, - headers={AGENT_ENV_ROLE_HEADER: "harness"}) + headers=deployed.role_headers("harness")) finally: for executor in started: executor.stop() @@ -140,7 +140,7 @@ async def test_a_reply_waits_its_delay_and_hours_and_its_writes_name_the_fire(it triggers = [{"id": "bob-hears-solver", "when": {"type": "action", "tool": "list_items"}, "actions": [{"type": "nl", "executor": "bob", "instruction": "BOB-ACTS", "delay": "PT2M"}]}] try: - bob = _Executor("bob", "bob@example.com", deployed.gateway_url, host, started) + bob = _Executor("bob", "bob@example.com", deployed, host, started) context = TaskStepContext(deployed_envs=[deployed], deployed_agents=[bob.agent], metadata={}) async with httpx.AsyncClient(base_url=deployed.gateway_url, timeout=60) as client: # The env is shared, so its trajectory holds the earlier tests' calls: read this test's, after them. @@ -185,7 +185,7 @@ async def test_a_deadline_fires_on_silence_and_a_reply_cancels_it(items_env): "where": {"item": {"regex": f"^REPLY-{n}"}}}}, "actions": [{"type": "nl", "executor": "bob", "instruction": f"ESCALATE-{n}"}]}] try: - bob = _Executor("bob", "bob@example.com", deployed.gateway_url, host, started) + bob = _Executor("bob", "bob@example.com", deployed, host, started) context = TaskStepContext(deployed_envs=[deployed], deployed_agents=[bob.agent], metadata={}) async with httpx.AsyncClient(base_url=deployed.gateway_url, timeout=60) as client: await _arm_clock(client, "2026-10-14T10:00:00Z", 60) @@ -239,9 +239,10 @@ class _Executor: """An A2A executor on this machine that acts on each instruction through the gateway, under its own role; with ``replies``, it stops acting after that many.""" - def __init__(self, name: str, role: str, gateway_url: str, host: str, started: list, + def __init__(self, name: str, role: str, deployed: DeployedEnv, host: str, started: list, replies: int | None = None) -> None: - self.role, self.gateway_url, self.replies = role, gateway_url, replies + self.role, self.gateway_url, self.replies = role, deployed.gateway_url, replies + self.headers = deployed.role_headers(role) self.instructions: list[str] = [] self.acted_as: list[str] = [] self._server = ThreadingHTTPServer((host, 0), functools.partial(_ExecutorHandler, self)) @@ -265,7 +266,7 @@ def stop(self) -> None: def _call(self, tool: str, arguments: dict) -> dict: response = httpx.post(f"{self.gateway_url}/step", json={"action": "call_tool", "tool_name": tool, "arguments": arguments}, - headers={AGENT_ENV_ROLE_HEADER: self.role}, timeout=60) + headers=self.headers, timeout=60) response.raise_for_status() return response.json() diff --git a/tst/unit/env/envs/test_mcp_server_plugin_provider.py b/tst/unit/env/envs/test_mcp_server_plugin_provider.py index b37528070..a1d482d0d 100644 --- a/tst/unit/env/envs/test_mcp_server_plugin_provider.py +++ b/tst/unit/env/envs/test_mcp_server_plugin_provider.py @@ -230,7 +230,7 @@ async def test_a_multi_env_child_of_a_plugin_type_is_still_the_gateways(): multi = MultiEnv(id="multi", version=1, mcp_server_envs=[child]) record = SimpleNamespace(env_id="multi", env_version=1, sandbox_id="gw", sandbox_type="modal", sandbox_ids={}, env_state_instance_ids=[], instance_id="i", mcp_server_name="m", environment_card=None, - environment_card_url=None, gateway_url="https://gw.example") + environment_card_url=None, gateway_url="https://gw.example", role_key=None) gw = MagicMock(sandbox_id="gw", mode="vm", type="modal") with patch("agent_env.env.env.Env.get", return_value=multi), \ patch("agent_env.providers.build_sandbox_provider", return_value=MagicMock(get_sandbox=AsyncMock(return_value=gw))), \ diff --git a/tst/unit/env/envs/test_multi_env_name.py b/tst/unit/env/envs/test_multi_env_name.py index 0055bef90..70a9ea224 100644 --- a/tst/unit/env/envs/test_multi_env_name.py +++ b/tst/unit/env/envs/test_multi_env_name.py @@ -67,7 +67,7 @@ async def test_a_deploy_records_the_mcp_url_and_name_its_gateway_card_gives(): card = {"name": "crm", "additionalInterfaces": [{"url": "/mcp", "transport": "mcp"}]} result = SimpleNamespace(gateway_url="https://gw.example", mcp_url="https://gw.example/mcp", db_web_url=None, db_mcp_url=None, website_frontend_urls=None, environment_card=card, environment_card_read_at_utc="t", - env_state_instance_ids=[], mcp_server_name="crm") + env_state_instance_ids=[], mcp_server_name="crm", role_key=None) async def deploy_gateway(self, sandbox_provider, **kwargs): self._sandbox = SimpleNamespace(sandbox_id="gw", type="modal_vm") diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index f2378150f..79e8b5fb6 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -13,10 +13,13 @@ import httpx import pytest +from starlette.testclient import TestClient +from agentenv_protocol.website import ROLE_KEY_ENV, ApiContext, WebsiteEnvironment, api, website from agent_env.env.gateway import AGENT_ENV_ROLE_META_KEY, AGENT_ENV_SESSION_META_KEY from agent_env.env.gateway import gateway as gateway_module from agent_env.env.gateway import rest_proxy as rest_proxy_module +from agent_env.env.gateway.caller import Caller from agent_env.env.gateway.gateway import Gateway UPSTREAM = "http://upstream" @@ -479,3 +482,212 @@ async def test_bodies_that_are_not_a_child_mcp_request_are_forwarded_untouched(m status, _, _ = await _call(gw, path, method=method, body=body, headers={"AgentEnv-Role": "alice@example.com"}) assert status == 200, path assert [r.content for r in up.requests] == [body for _, _, body in cases] + + +ROLE_KEY = "deploy-secret" + + +def _role_and_key(request: httpx.Request) -> tuple[list[str], list[str]]: + return request.headers.get_list("agentenv-role"), request.headers.get_list("agentenv-role-key") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/svc/mcp-svc/x", "/website/web/x"]) +async def test_with_a_role_key_a_role_is_believed_only_beside_it_and_the_key_goes_upstream_only_with_it(monkeypatch, path): + up = _Upstream([b"{}"]) + up.install(monkeypatch) + gw = _gateway(website_urls={"web": "http://web"}, role_key=ROLE_KEY) + for headers in ({"AgentEnv-Role": "admin"}, {"AgentEnv-Role": "admin", "AgentEnv-Role-Key": "guess"}, + {"AgentEnv-Role": "admin", "AgentEnv-Role-Key": ROLE_KEY}, {"AgentEnv-Role-Key": ROLE_KEY}): + status, _, _ = await _call(gw, path, headers=headers) + assert status == 200 + assert [_role_and_key(r) for r in up.forwarded] == [ + (["default"], []), (["default"], []), (["admin"], [ROLE_KEY]), (["default"], [])] + + +@pytest.mark.asyncio +async def test_without_a_role_key_the_role_header_alone_is_believed_and_a_client_s_key_never_goes_upstream(monkeypatch): + up = _Upstream([b"{}"]) + up.install(monkeypatch) + gw = _gateway(website_urls={"web": "http://web"}) + await _call(gw, "/website/web/x", headers={"AgentEnv-Role": "admin"}) + await _call(gw, "/svc/mcp-svc/x", headers={"AgentEnv-Role": "admin", "AgentEnv-Role-Key": "forged"}) + assert [_role_and_key(r) for r in up.forwarded] == [(["admin"], []), (["admin"], [])] + + +@pytest.mark.asyncio +async def test_with_a_role_key_step_attributes_a_role_sent_without_it_to_default(monkeypatch): + gw = _gateway(role_key=ROLE_KEY) + seen: list[str] = [] + monkeypatch.setattr(gw, "_step_list_tools", lambda role: (seen.append(role), gw._json_response({"tools": []}, 200))[1]) + try: + for headers in ({"AgentEnv-Role": "admin"}, {"AgentEnv-Role": "admin", "AgentEnv-Role-Key": ROLE_KEY}): + status, _, _ = await _call(gw, "/step", method="POST", body=b'{"action": "list_tools"}', headers=headers) + assert status == 200 + finally: + await gw._close_child_sessions() + assert seen == ["default", "admin"] + + +class _CallerRecorder: + """Stands in for FastMCP's app under the gateway's ASGI wrapper: records the caller bound for each request.""" + + def __init__(self) -> None: + self.router = type("Router", (), {"lifespan_context": None})() + self.roles: list[str] = [] + + async def __call__(self, scope, receive, send) -> None: + self.roles.append(Caller.current().role) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role_key, expected", [ + (ROLE_KEY, ["default", "default", "admin", "default"]), + (None, ["admin", "admin", "admin", "default"]), +], ids=["keyed", "legacy"]) +async def test_the_mcp_path_binds_the_role_the_key_vouches_for(monkeypatch, role_key, expected): + gw = _gateway(role_key=role_key) + recorder = _CallerRecorder() + monkeypatch.setattr(gw._mcp, "streamable_http_app", lambda: recorder) + app = gw._asgi_app() + for headers in ([(b"agentenv-role", b"admin")], [(b"agentenv-role", b"admin"), (b"agentenv-role-key", b"guess")], + [(b"agentenv-role-key", ROLE_KEY.encode()), (b"agentenv-role", b"admin")], []): + await app({"type": "http", "method": "POST", "path": "/mcp", "headers": headers}, None, None) + assert recorder.roles == expected + + +def _forwarded(request: httpx.Request) -> tuple[list[str], list[str], list[str]]: + return tuple(request.headers.get_list(name) for name in ("x-forwarded-host", "x-forwarded-proto", "x-forwarded-prefix")) + + +@pytest.mark.asyncio +async def test_the_upstream_learns_the_host_scheme_and_route_prefix_the_client_reached_it_at(monkeypatch): + up = _Upstream([b"ok"]) + up.install(monkeypatch) + gw = _gateway(website_urls={"web": "http://web"}) + await _call(gw, "/website/web/api/x/", headers={"host": "gw.example", "x-forwarded-prefix": "/elsewhere"}) + await _call(gw, "/website/web", headers={"host": "gw.example"}) + await _call(gw, "/website/web/", headers={"host": "gw.example"}) + await _call(gw, "/svc/mcp-svc/x%2Fy", headers={"host": "gw.example"}) + await _call(gw, "/gw%20x/svc/mcp-svc/x", headers={"host": "gw.example"}, root_path="/gw x") + assert [_forwarded(r) for r in up.forwarded] == [ + (["gw.example"], ["http"], ["/website/web"]), + (["gw.example"], ["http"], ["/website/web"]), + (["gw.example"], ["http"], ["/website/web"]), + (["gw.example"], ["http"], ["/svc/mcp-svc"]), + (["gw.example"], ["http"], ["/gw%20x/svc/mcp-svc"]), + ] + + +@pytest.mark.asyncio +async def test_a_front_s_own_forwarded_host_and_scheme_win_and_reach_the_upstream_once(monkeypatch): + up = _Upstream([b"ok"]) + up.install(monkeypatch) + gw = _gateway(website_urls={"web": "http://web"}) + await _call(gw, "/website/web/x", raw_headers=[ + (b"host", b"gateway.internal"), (b"x-forwarded-host", b"env.example, proxy"), (b"x-forwarded-proto", b"https"), + (b"x-forwarded-proto", b"http"), (b"x-forwarded-for", b"10.0.0.1")]) + await _call(gw, "/website/web/x", headers={"host": "gw.example", "x-forwarded-proto": "gopher"}) + assert [_forwarded(r) for r in up.forwarded] == [ + (["env.example"], ["https"], ["/website/web"]), + (["gw.example"], ["http"], ["/website/web"]), + ] + assert up.forwarded[0].headers.get_list("x-forwarded-for") == ["10.0.0.1"] + + +@pytest.mark.asyncio +async def test_an_sdk_site_behind_the_gateway_believes_only_a_keyed_role_and_names_the_gateway_s_address(monkeypatch): + @website(domains=["notes.example"]) + class Notes(WebsiteEnvironment): + @api.get("/api/whoami") + def whoami(self, ctx: ApiContext) -> dict: + return {"role": ctx.role, "base_url": ctx.base_url} + + monkeypatch.setenv(ROLE_KEY_ENV, ROLE_KEY) + transport = httpx.ASGITransport(app=Notes().create_app()) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(website_urls={"notes": "http://notes"}, role_key=ROLE_KEY) + + async def whoami(path: str, **headers) -> tuple[int, dict, bytes]: + status, answered, chunks = await _call(gw, path, headers={"host": "gw.example", **headers}) + return status, answered, b"".join(chunks) + + _, _, spoofed = await whoami("/website/notes/api/whoami", **{"AgentEnv-Role": "admin"}) + _, _, keyed = await whoami("/website/notes/api/whoami", **{"AgentEnv-Role": "admin", "AgentEnv-Role-Key": ROLE_KEY}) + status, redirect, _ = await whoami("/website/notes/api/whoami/") + assert json.loads(spoofed) == {"role": "default", "base_url": "http://gw.example/website/notes"} + assert json.loads(keyed)["role"] == "admin" + assert (status, redirect["location"]) == (307, "http://gw.example/website/notes/api/whoami") + + +def test_the_route_an_sdk_card_names_reaches_its_nested_and_typed_paths(): + @website(domains=["files.example"]) + class Files(WebsiteEnvironment): + @api.get("/files/{rest:path}") + def files(self, rest: str) -> dict: + return {"rest": rest} + + @api.get("/api/notes/{note_id:int}") + def get_note(self, note_id: int) -> dict: + return {"id": note_id} + + card = TestClient(Files().create_app()).get("/.well-known/agent-env.json").json() + routes = rest_proxy_module.website_routes(card) + assert rest_proxy_module.route_name(routes, "GET", "/files/a/b.txt") == "files" + assert rest_proxy_module.route_name(routes, "GET", "/api/notes/7") == "get_note" + assert rest_proxy_module.route_name(routes, "GET", "/api/notes/seven") is None + + +def test_a_request_a_built_in_route_answers_is_named_by_no_card_route(): + @website(domains=["any.example"]) + class AnyApi(WebsiteEnvironment): + @api.get("/api/{rest:path}") + def any_api(self, rest: str) -> dict: + return {"rest": rest} + + @api.delete("/{rest:path}") + def any_delete(self, rest: str) -> dict: + return {"rest": rest} + + card = TestClient(AnyApi().create_app()).get("/.well-known/agent-env.json").json() + routes = rest_proxy_module.website_routes(card) + assert rest_proxy_module.route_name(routes, "GET", "/api/session") is None + assert rest_proxy_module.route_name(routes, "DELETE", "/api/session") is None + assert rest_proxy_module.route_name(routes, "GET", "/api/health") is None + assert rest_proxy_module.route_name(routes, "GET", "/api/notes") == "any_api" + assert rest_proxy_module.route_name(routes, "DELETE", "/api/health") == "any_delete" + + +def test_a_site_route_at_the_session_path_replaces_the_built_in_one(): + @website(domains=["own.example"]) + class OwnSession(WebsiteEnvironment): + @api.post("/api/session") + def sign_in(self) -> dict: + return {} + + @api.get("/api/{rest:path}") + def any_api(self, rest: str) -> dict: + return {"rest": rest} + + card = TestClient(OwnSession().create_app()).get("/.well-known/agent-env.json").json() + routes = rest_proxy_module.website_routes(card) + assert rest_proxy_module.route_name(routes, "POST", "/api/session") == "sign_in" + assert rest_proxy_module.route_name(routes, "GET", "/api/session") == "any_api" + + +@pytest.mark.parametrize("path, method, target, expected", [ + ("/api/notes/{note_id:int}", "GET", "/api/notes/7", "route"), + ("/api/notes/{note_id:int}", "GET", "/api/notes/seven", None), + ("/api/notes/{note_id}", "GET", "/api/notes/seven", "route"), + ("/api/notes/{note_id}", "GET", "/api/notes/7/extra", None), + ("/files/{rest:path}", "GET", "/files/a/b.txt", "route"), + ("/api/price/{p:float}", "GET", "/api/price/3.50", "route"), + ("/api/x/{id:uuid}", "GET", "/api/x/123e4567-e89b-12d3-a456-426614174000", "route"), + ("/api/a.b", "GET", "/api/aXb", None), + ("/api/notes", "HEAD", "/api/notes", "route"), + ("/api/notes", "POST", "/api/notes", None), +]) +def test_a_request_is_named_by_the_card_route_its_method_and_path_reach(path, method, target, expected): + card = {"capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1", "params": { + "routes": [{"method": "GET", "path": path, "name": "route"}]}}]}} + assert rest_proxy_module.route_name(rest_proxy_module.website_routes(card), method, target) == expected diff --git a/tst/unit/env/gateway/triggers_website_test.py b/tst/unit/env/gateway/triggers_website_test.py index f885d541c..6571b0ed1 100644 --- a/tst/unit/env/gateway/triggers_website_test.py +++ b/tst/unit/env/gateway/triggers_website_test.py @@ -12,7 +12,7 @@ from agent_env.env.gateway import gateway as gateway_module from agent_env.env.gateway import rest_proxy as rest_proxy_module from agent_env.env.gateway.clock import Clock -from agent_env.env.gateway.constants import EXT_TRIGGERS_URI, GATEWAY_EXTENSIONS, TRIGGER_WEBSITE +from agent_env.env.gateway.constants import EXT_TRIGGERS_URI, GATEWAY_EXTENSIONS, TRIGGER_WEBSITE, TRIGGER_WEBSITE_OP from agent_env.env.gateway.gateway import GATEWAY_TRAJECTORY_FILE, Gateway from agent_env.env.gateway.triggers import TriggerEngine, TriggerError @@ -27,12 +27,16 @@ def __init__(self, responses: dict) -> None: self.events: list[dict] = [] self.rules: list[tuple[str, str, bool]] = [] self.website_urls = {"mail": "http://mail-frontend", "shop": "http://shop-frontend"} + self.route_names = {"mail": {"send_message", "list_messages"}} self._role_rules_lock = asyncio.Lock() self._server_tools: dict = {} self._tool_server_urls: dict = {} self._clock = Clock() self.TRIGGER_BARRIER_TIMEOUT_S = 30.0 + def website_route_names(self, app): + return self.route_names.get(app) + def _apply_rule(self, role, tool, value): self.rules.append((role, tool, value)) @@ -98,6 +102,10 @@ def _detected(engine) -> list[tuple[str, dict]]: "a website request carries args, not a result"), ({"type": "website", "app": "mail", "unless": {"tool": "t"}}, [], "when.unless is only valid on time triggers"), ({"type": "website", "app": "mail", "roles": []}, [], "when.roles must be a non-empty list"), + ({"type": "website", "app": "mail", "op": ""}, [], "when.op must be a non-empty string"), + ({"type": "website", "app": "mail", "op": "archive"}, [], + "when.op names no route of website 'mail' (routes: ['list_messages', 'send_message'])"), + ({"type": "website", "app": "shop", "op": "checkout"}, [], "whose card the gateway could not read"), ]) def test_registration_fails_loud(monkeypatch, when, actions, fragment): engine = _make_engine(monkeypatch) @@ -121,7 +129,7 @@ def test_registration_normalizes_the_method_and_state_echoes_the_request_matcher def test_the_card_declares_website_triggers(): triggers = next(e for e in GATEWAY_EXTENSIONS if e["uri"] == EXT_TRIGGERS_URI) - assert TRIGGER_WEBSITE in triggers["params"]["features"] + assert {TRIGGER_WEBSITE, TRIGGER_WEBSITE_OP} <= set(triggers["params"]["features"]) # ---------------------------------------------------------------- detection @@ -158,6 +166,21 @@ async def test_only_a_request_matching_app_method_path_status_and_where_provokes assert len(engine._test_gw.calls) == 1 +@pytest.mark.asyncio +async def test_a_trigger_naming_an_op_is_provoked_only_by_a_request_that_reached_that_route(monkeypatch): + engine = _make_engine(monkeypatch) + by_op = {**REPLY, "when": {"type": "website", "app": "mail", "op": "send_message", "repeat": True}} + engine.register({"triggers": [by_op]}) + assert engine._triggers["reply"]["spec"]["when"]["op"] == "send_message" + engine.on_website_request("default", "mail", "POST", "/api/messages", 201, dict(MESSAGE)) + engine.on_website_request("default", "mail", "POST", "/api/messages", 201, dict(MESSAGE), "list_messages") + engine.on_website_request("default", "mail", "POST", "/api/messages", 201, dict(MESSAGE), "send_message") + await _settle(engine) + request = {"app": "mail", "method": "POST", "path": "/api/messages", "status": 201, "op": "send_message"} + assert _detected(engine) == [("reply", {"source": "website", **request, "role": "default"})] + assert len(engine._test_gw.calls) == 1 + + @pytest.mark.asyncio async def test_a_status_predicate_can_match_a_failed_request(monkeypatch): engine = _make_engine(monkeypatch) @@ -335,12 +358,12 @@ async def test_the_proxy_hands_the_engine_the_role_path_status_and_args_of_each_ assert await _request(gw, "GET", "/website/calendar/") == 404 assert seen == [ - ("default", "mail", "GET", "/", 200, {}), - ("carol@example.com", "mail", "GET", "/inbox", 200, {"folder": "work", "tag": ["a", "b"]}), - ("default", "mail", "POST", "/api/messages", 200, {"draft": "0", "to": "dana@example.com"}), - ("default", "mail", "POST", "/compose", 200, {"subject": "Hi there", "to": "dana@example.com"}), - ("default", "mail", "POST", "/api/batch", 200, {"op": "send"}), - ("default", "mail", "PUT", "/api/drafts/a/b", 200, {}), + ("default", "mail", "GET", "/", 200, {}, None), + ("carol@example.com", "mail", "GET", "/inbox", 200, {"folder": "work", "tag": ["a", "b"]}, None), + ("default", "mail", "POST", "/api/messages", 200, {"draft": "0", "to": "dana@example.com"}, None), + ("default", "mail", "POST", "/compose", 200, {"subject": "Hi there", "to": "dana@example.com"}, None), + ("default", "mail", "POST", "/api/batch", 200, {"op": "send"}, None), + ("default", "mail", "PUT", "/api/drafts/a/b", 200, {}, None), ] assert [r.content for r in website.requests if r.url.path != "/.well-known/agent-env.json"][2:4] == [b'{"to": "dana@example.com", "draft": "0"}', b"subject=Hi+there&to=dana%40example.com"] @@ -419,3 +442,152 @@ def explode(*_): gw._trigger_engine.on_website_request = explode assert await _request(gw, "GET", "/website/mail/inbox") == 200 + + +@pytest.mark.asyncio +async def test_with_a_role_key_the_engine_sees_a_role_only_beside_it_so_a_page_cannot_pose_as_the_executor(monkeypatch): + _Website(status=200).install(monkeypatch) + gw = Gateway(host="127.0.0.1", port=0, server_name="t", internal_mcp_servers=[], + website_urls={"mail": "http://mail-frontend"}, role_key="deploy-secret") + gw._trigger_engine.register({"triggers": [REPLY]}) + seen: list[str] = [] + gw._trigger_engine.on_website_request = lambda role, *a: seen.append(role) or ([], None) + + await _request(gw, "GET", "/website/mail/inbox", headers={"AgentEnv-Role": "executor"}) + await _request(gw, "GET", "/website/mail/inbox", headers={"AgentEnv-Role": "executor", "AgentEnv-Role-Key": "deploy-secret"}) + + assert seen == ["default", "executor"] + + +NOTES_CARD = {"name": "notes", "capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1", "params": {"routes": [ + {"method": "GET", "path": "/api/notes", "name": "list_notes", "roles": None}, + {"method": "POST", "path": "/api/notes", "name": "create_note", "roles": None}, + {"method": "PATCH", "path": "/api/notes/{note_id}", "name": "edit_note", "roles": None}, +]}}]}} + + +class _NotesSite(_Website): + """A website whose server serves `NOTES_CARD` at the card's path, counting how often the gateway reads it.""" + + def install(self, monkeypatch) -> None: + self.card_reads = 0 + + async def body(): + yield b'{"ok": true}' + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/.well-known/agent-env.json": + self.card_reads += 1 + return httpx.Response(200, json=NOTES_CARD) + self.requests.append(request) + return httpx.Response(self.status, headers={"content-type": "application/json"}, content=body()) + + transport = httpx.MockTransport(handler) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + + +def _notes_gateway() -> Gateway: + return Gateway(host="127.0.0.1", port=0, server_name="t", internal_mcp_servers=[], + website_urls={"notes": "http://notes-website-backend:8000"}, + rest_proxy_urls={"notes": "http://notes-website-backend:8000"}) + + +async def _register(gw: Gateway, body: dict) -> tuple[int, dict]: + sent = json.dumps(body).encode() + answered: list[bytes] = [] + statuses: list[int] = [] + + async def receive(): + return {"type": "http.request", "body": sent, "more_body": False} + + async def send(message): + if message["type"] == "http.response.start": + statuses.append(message["status"]) + elif message["type"] == "http.response.body": + answered.append(message.get("body", b"")) + + scope = {"type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", "method": "POST", "scheme": "http", + "path": "/triggers/register", "raw_path": b"/triggers/register", "query_string": b"", "root_path": "", + "server": ("gateway", 80), "client": ("c", 1), "headers": [(b"content-type", b"application/json")]} + await gw._mcp.streamable_http_app()(scope, receive, send) + return statuses[0], json.loads(b"".join(answered)) + + +EDITED = {"id": "edited", "when": {"type": "website", "app": "notes", "op": "edit_note", "repeat": True}, "actions": []} + + +@pytest.mark.asyncio +async def test_registering_an_op_trigger_reads_the_site_s_routes_and_refuses_a_route_it_does_not_name(monkeypatch): + site = _NotesSite(status=200) + site.install(monkeypatch) + gw = _notes_gateway() + + status, answer = await _register(gw, {"triggers": [{**EDITED, "id": "typo", "when": {**EDITED["when"], "op": "edit"}}]}) + assert status == 400 and "routes: ['create_note', 'edit_note', 'list_notes']" in answer["error"] + status, _ = await _register(gw, {"triggers": [EDITED]}) + assert status == 200 and site.card_reads == 1 + + +@pytest.mark.asyncio +async def test_registering_an_op_trigger_whose_app_is_not_a_name_is_refused_as_bad_config(monkeypatch): + site = _NotesSite(status=200) + site.install(monkeypatch) + gw = _notes_gateway() + + status, answer = await _register(gw, {"triggers": [{**EDITED, "when": {**EDITED["when"], "app": []}}]}) + assert status == 400 and answer["ok"] is False and site.card_reads == 0 + + +@pytest.mark.asyncio +async def test_the_proxy_names_the_route_a_request_reached_to_the_engine_and_the_trajectory(monkeypatch): + site = _NotesSite(status=200) + site.install(monkeypatch) + gw = _notes_gateway() + assert (await _register(gw, {"triggers": [EDITED]}))[0] == 200 + changelog = iter(range(100)) + monkeypatch.setattr(gw, "_query_changelog_id", lambda: next(changelog)) + before = len(_trajectory()) + + await _request(gw, "PATCH", "/website/notes/api/notes/7", headers={"content-type": "application/json"}, body=b"{}") + await _request(gw, "PATCH", "/website/notes/api/notes/7/extra", headers={"content-type": "application/json"}, body=b"{}") + await _request(gw, "POST", "/website/notes/api/notes", headers={"content-type": "application/json"}, body=b"{}") + await asyncio.sleep(0.05) + + detected = [p for _, p in _detected(gw._trigger_engine)] + assert [(p["path"], p.get("op")) for p in detected] == [("/api/notes/7", "edit_note")] + writes = [e for e in _trajectory()[before:] if e["event_type"] == "http_request"] + assert [(w["method"], w["path"], w.get("op")) for w in writes] == [ + ("PATCH", "/api/notes/7", "edit_note"), ("PATCH", "/api/notes/7/extra", None), ("POST", "/api/notes", "create_note")] + + +@pytest.mark.asyncio +async def test_a_request_is_named_by_the_path_the_site_s_server_receives(monkeypatch): + """An encoded slash beside a dot segment routes here as one path and reaches the site's server as another; the + route, the trigger's match and the write's trajectory event all go by the one the server receives.""" + site = _NotesSite(status=200) + site.install(monkeypatch) + gw = _notes_gateway() + assert (await _register(gw, {"triggers": [EDITED]}))[0] == 200 + changelog = iter(range(100)) + monkeypatch.setattr(gw, "_query_changelog_id", lambda: next(changelog)) + before = len(_trajectory()) + + await _request(gw, "PATCH", "/website/notes/public%2Fx/../api/notes/7", headers={"content-type": "application/json"}, + body=b"{}") + await asyncio.sleep(0.05) + + assert [r.url.path for r in site.requests] == ["/api/notes/7"] + assert [(p["path"], p.get("op")) for _, p in _detected(gw._trigger_engine)] == [("/api/notes/7", "edit_note")] + writes = [e for e in _trajectory()[before:] if e["event_type"] == "http_request"] + assert [w.get("op") for w in writes] == ["edit_note"] + + +@pytest.mark.asyncio +async def test_composing_the_env_card_reads_each_site_s_routes(monkeypatch): + site = _NotesSite(status=200) + site.install(monkeypatch) + gw = _notes_gateway() + assert gw.website_route_names("notes") is None + await gw._serve_env_card() + assert gw.website_route_names("notes") == {"list_notes", "create_note", "edit_note"} and site.card_reads == 1 + assert gw._website_op("notes", "GET", "/api/notes") == "list_notes" diff --git a/tst/unit/env/test_env.py b/tst/unit/env/test_env.py index 02947560b..1498888bf 100644 --- a/tst/unit/env/test_env.py +++ b/tst/unit/env/test_env.py @@ -284,7 +284,7 @@ def test_a_gateway_record_is_filled_the_way_it_always_was(): sandbox_type="modal_vm", sandbox_ids={"gateway_server": "gw"}, db_web_url="https://pg/", db_mcp_url="https://dm/mcp", website_frontend_urls={"shop": "https://gw/shop/"}, vnc_url="https://vnc", env_state_instance_ids=["st-1"], environment_card_url=_CARD_URL, environment_card=_CARD, environment_card_read_at_utc="2026-09-28T00:00:00+00:00", - metadata={"k": "v"}, instance_id="i1", created_at_utc="c", expires_at_utc="x"), + metadata={"k": "v"}, instance_id="i1", created_at_utc="c", expires_at_utc="x", role_key="deploy-secret"), ], ids=["base", "sandbox", "gateway"]) def test_a_record_round_trips_through_its_stored_form(record): assert DeployedEnv.from_dict(dataclasses.asdict(record)) == record @@ -295,6 +295,14 @@ def test_each_record_class_loads_every_one_of_its_fields(cls): assert set(cls._fields_from({"env_id": "e", "env_version": 1})) == {f.name for f in dataclasses.fields(cls)} +def test_a_record_s_role_headers_carry_its_role_key_beside_the_role_and_an_older_record_s_only_the_role(): + keyed = DeployedEnv.from_dict({"env_id": "e", "env_version": 1, "gateway_url": "https://gw", "sandbox_id": "sb", + "role_key": "deploy-secret"}) + older = DeployedEnv.from_dict({"env_id": "e", "env_version": 1, "gateway_url": "https://gw", "sandbox_id": "sb"}) + assert keyed.role_headers("admin") == {"AgentEnv-Role": "admin", "AgentEnv-Role-Key": "deploy-secret"} + assert (older.role_key, older.role_headers("admin")) == (None, {"AgentEnv-Role": "admin"}) + + def test_the_gateway_only_fields_are_the_gateway_classs_own(): own = {f.name for f in dataclasses.fields(DeployedGatewayEnv)} - {f.name for f in dataclasses.fields(DeployedSandboxEnv)} assert set(_GATEWAY_ONLY_FIELDS) == own - {"gateway_mode", "env_provider_type"} diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index 4345c8fde..7902db84a 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -375,6 +375,7 @@ async def _spy_acquire(self, ctx): ttl_seconds=60, disk_size_gb=10, mcp_server_name="crm", + role_key="deploy-secret", ) # Last create_container call is the gateway. Inspect its env var. @@ -395,6 +396,7 @@ async def _spy_acquire(self, ctx): assert re.search(r"^postgresql://agentenv:agentenv@\[fdaa::fake:[0-9a-f]+\]:5432/agentenv", env["SERVICE_DB_URL"]), env["SERVICE_DB_URL"] assert env["MCP_SERVER_NAME"] == "crm" assert result.mcp_server_name == "crm" + assert env["AGENTENV_ROLE_KEY"] == result.role_key == "deploy-secret" # Each MCP got DATABASE_URL with its scoped search_path mcp_calls = [c for c in create_calls if c["image_name"].startswith("mcp-")] @@ -1191,6 +1193,30 @@ def test_redact_compose_secrets_masks_only_secret_env_lines(): assert "s3cr3t-token-value" in compose +@pytest.mark.parametrize("role_key", ["deploy-secret", None], ids=["keyed", "unkeyed"]) +def test_compose_gives_the_role_key_to_the_gateway_and_every_website_server_and_masks_it_in_the_log(role_key): + """The gateway believes a role only beside the deploy's key, and a site holds the same key to believe the role the + gateway forwards; without a key neither is set, as before it.""" + from agent_env.env.envs.service_db import ServiceDBConfig + + compose = EnvironmentGatewayProvider().create_docker_compose( + mcp_servers=[MCPServerConfig(image="mcp-slack", environment_name="slack")], + gateway_image="agent-gateway", + website_configs=[WebsiteConfig(backend_image="notes-be", frontend_image=None, environment_name="notes")], + state_provider=LocalPostgresStateProvider(service_db_config=ServiceDBConfig()), + state_instance=LocalPostgresStateProvider.default_instance(), + role_key=role_key, + ) + + services = yaml.safe_load(compose)["services"] + keyed = {name: [entry for entry in service.get("environment") or [] if "ROLE_KEY" in entry] + for name, service in services.items()} + expected = {GATEWAY_SERVICE_NAME: [f"AGENTENV_ROLE_KEY={role_key}"], "notes-website-backend": [f"WEBSITE_ROLE_KEY={role_key}"]} + assert {name: entries for name, entries in keyed.items() if entries} == (expected if role_key else {}) + if role_key: + assert role_key not in _redact_compose_secrets(compose) and " - AGENTENV_ROLE_KEY=***" in _redact_compose_secrets(compose) + + @pytest.mark.asyncio async def test_acquire_state_for_deploy_local_returns_none(): """Local Postgres cannot be pre-provisioned (its container's host doesn't exist yet), so the @@ -1953,12 +1979,28 @@ async def test_create_gateway_hands_the_deploy_path_every_argument(attribution, [call] = vm.await_args_list assert call.args == (provider, args.pop("mcp_servers"), args.pop("mcp_server_images")) - assert call.kwargs == {**args, "attribution": forwarded} + assert call.kwargs == {**args, "attribution": forwarded, "role_key": None} assert call.kwargs["attribution"] is not attribution @pytest.mark.asyncio -async def test_deploy_returns_the_gateway_record_for_the_env(caplog): +@pytest.mark.parametrize("keyed_roles", [True, False], ids=["keyed", "unkeyed"]) +async def test_create_gateway_draws_a_role_key_only_for_a_deploy_with_keyed_roles(keyed_roles): + """A keyed deploy draws a fresh secret for its gateway; any other deploy draws none, so its callers' role header + alone is believed, as before the key.""" + gp, provider = EnvironmentGatewayProvider(), _vm_provider() + with patch.object(gp, "_deploy_via_vm", AsyncMock(return_value=_probed())) as vm, \ + patch.object(env_gateway_provider, "_tool_names", AsyncMock(return_value=set())): + await gp.create_gateway(sandbox_provider=provider, mcp_servers=[MagicMock()], mcp_server_images=[MagicMock()], + env_id="crm-env", keyed_roles=keyed_roles) + + role_key = vm.await_args.kwargs["role_key"] + assert (isinstance(role_key, str) and len(role_key) >= 32) if keyed_roles else role_key is None, role_key + + +@pytest.mark.asyncio +@pytest.mark.parametrize("keyed_roles", [False, True], ids=["unkeyed", "keyed"]) +async def test_deploy_returns_the_gateway_record_for_the_env(caplog, keyed_roles): """deploy(env) on a scripted VM: the env's image behind the gateway, its state acquired, and main's record plus its provider type.""" from agent_env.env.gateway import GatewayMode from agent_env.providers.env_state import DatabaseStateProvider @@ -1984,7 +2026,7 @@ async def test_deploy_returns_the_gateway_record_for_the_env(caplog): patch.object(env_gateway_provider, "_tool_names", AsyncMock(return_value={"slack_send"})), \ patch("agent_env.providers.env_providers.env_gateway_provider.asyncio.sleep", AsyncMock()): record = await gp.deploy(env, _vm_provider(create_vm=AsyncMock(return_value=vm)), gateway_mode=GatewayMode.CONSISTENT, - ttl_seconds=60, env_state_type="remote") + ttl_seconds=60, env_state_type="remote", keyed_roles=keyed_roles) clashes_checked.assert_awaited_once_with("https://vm.example") @@ -1997,7 +2039,9 @@ async def test_deploy_returns_the_gateway_record_for_the_env(caplog): env_id="crm-env", env_version=3, gateway_url="https://vm.example", mcp_url="https://vm.example/mcp", db_web_url=None, sandbox_id="vm-1", sandbox_type="modal_vm", sandbox_ids={}, db_mcp_url=None, environment_card_url="https://vm.example/.well-known/agent-env.json", environment_card=card, - environment_card_read_at_utc=record.environment_card_read_at_utc, gateway_mode="consistent", env_state_instance_ids=["st-ext"])) + environment_card_read_at_utc=record.environment_card_read_at_utc, gateway_mode="consistent", env_state_instance_ids=["st-ext"], + role_key=compose.call_args.kwargs["role_key"])) + assert (record.role_key and len(record.role_key) >= 32) if keyed_roles else record.role_key is None assert record.environment_card_read_at_utc and record.instance_id is None # unregistered: the env registers it assert (record.mcp_url, record.mcp_server_name, record.env_provider_type) == ("https://vm.example/mcp", "env1234", "gateway") messages = [r.getMessage() for r in caplog.records if r.getMessage().startswith(("env_deploy_provider_attempt ", "env_deploy_tools_probe "))] diff --git a/tst/unit/task_step/deploy_env_test.py b/tst/unit/task_step/deploy_env_test.py index 874832e4a..0fe426a6e 100644 --- a/tst/unit/task_step/deploy_env_test.py +++ b/tst/unit/task_step/deploy_env_test.py @@ -242,6 +242,36 @@ async def test_execute_merges_annotations_into_metadata_deploy_set(): +@pytest.mark.parametrize("keyed_roles", [True, False], ids=["keyed", "unkeyed"]) +def test_keyed_roles_roundtrips_and_an_older_step_is_unkeyed(keyed_roles): + step = DeployEnvTaskStep(id="t-1.deploy_env", version=None, env_id="art-1", keyed_roles=keyed_roles) + assert DeployEnvTaskStep.from_dict(step.to_dict()).keyed_roles is keyed_roles + older = {k: v for k, v in step.to_dict().items() if k != "keyed_roles"} + assert DeployEnvTaskStep.from_dict(older).keyed_roles is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("keyed_roles", [True, False], ids=["keyed", "unkeyed"]) +async def test_execute_asks_for_a_role_key_only_for_a_keyed_step(keyed_roles): + """An unkeyed step passes no keyed_roles, so an env type or plugin provider that doesn't name it isn't broken.""" + env = _fake_env(_deployed_env(None)) + step = DeployEnvTaskStep(id="t-1.deploy_env", version=None, env_id="art-1", keyed_roles=keyed_roles) + with patch("agent_env.env.env.Env") as Env: + Env.get.return_value = env + await step.execute(TaskStepContext()) + assert env.deploy.await_args.kwargs.get("keyed_roles") is (True if keyed_roles else None) + + +@pytest.mark.asyncio +async def test_execute_refuses_keyed_roles_for_an_env_whose_deploy_does_not_take_them(): + env = _env_with_fixed_deploy(_deployed_env(None)) + env.type = "fixed" + step = DeployEnvTaskStep(id="d", version=None, env_id="art-1", keyed_roles=True) + with patch("agent_env.env.env.Env") as Env, pytest.raises(ValueError, match="fixed envs take no keyed_roles"): + Env.get.return_value = env + await step.execute(TaskStepContext()) + + # --- preflight: an option the step's env would refuse at deploy is a save-time problem --- @@ -276,6 +306,12 @@ def test_preflight_refuses_env_params_an_env_doesnt_take(kind, refusal): assert _preflight(env, env_params={"form": "container"}) == [f"deploy_env 'd': {refusal}"] +def test_preflight_passes_keyed_roles_for_a_gateway_env_and_refuses_them_for_a_server_env(): + assert _preflight(_mcp_env("gateway"), keyed_roles=True) == [] + assert _preflight(_mcp_env("server"), keyed_roles=True) == [ + "deploy_env 'd': env 'mcp-email' has env_provider_type 'server', which doesn't take keyed_roles"] + + @pytest.mark.parametrize("kind", ["server", "gateway", "other"]) @pytest.mark.parametrize("mode", ["bogus", "Performance", None]) def test_preflight_reports_an_invalid_gateway_mode_for_every_env(kind, mode): diff --git a/tst/unit/task_step/local_container_urls_test.py b/tst/unit/task_step/local_container_urls_test.py index 8320dfcd3..7af4f11e1 100644 --- a/tst/unit/task_step/local_container_urls_test.py +++ b/tst/unit/task_step/local_container_urls_test.py @@ -346,4 +346,34 @@ async def install_cli(deployed, cli_artifact, gateway_url, **placed): await LoadArtifactTaskStep(id="cli", version=None, env_id="env-x", artifact_id=cli.id, artifact_version=cli.version, agent_name="solver").execute(context) - assert installed == [(expected, {"on_host": True} if on_host else {})] + assert installed == [(expected, {"role_key": None, **({"on_host": True} if on_host else {})})] + + +@pytest.mark.asyncio +async def test_each_cli_is_installed_with_the_role_key_of_the_deploy_it_talks_to(monkeypatch, local_stores, tmp_path): + (tmp_path / "cli").mkdir() + (tmp_path / "cli" / "run.sh").write_text("#!/bin/sh\n") + clis = [CliArtifact.put(f"{name}-cli", command_name=name, entrypoint="run.sh", cli_dir=tmp_path / "cli") + for name in ("slack", "crm")] + installed = [] + + async def install_cli(deployed, cli_artifact, gateway_url, **placed): + installed.append((cli_artifact.command_name, gateway_url, placed["role_key"])) + return f"/opt/cli/{cli_artifact.command_name}/run.sh" + + monkeypatch.setattr(A2AAgent, "install_cli", staticmethod(install_cli)) + monkeypatch.setattr(agent_store, "get_a2a_agent_instance_store", + lambda: types.SimpleNamespace(get=lambda instance_id: object())) + context = TaskStepContext( + deployed_envs=[DeployedGatewayEnv(env_id=f"env-{name}", env_version=1, gateway_url=f"https://{name}.gw", + mcp_url="", sandbox_id=f"sb-{name}", sandbox_type="modal_vm", + role_key=f"key-{name}") for name in ("slack", "crm")], + deployed_agents=[DeployedAgent(agent_name="solver", api_url="http://agent", sandbox_id="sb-agent", + sandbox_type="modal_vm", instance_id="inst-1")], + ) + for cli in clis: + await LoadArtifactTaskStep(id=f"load-{cli.command_name}", version=None, env_id=f"env-{cli.command_name}", + artifact_id=cli.id, artifact_version=cli.version, agent_name="solver" + ).execute(context) + + assert installed == [("slack", "https://slack.gw", "key-slack"), ("crm", "https://crm.gw", "key-crm")] diff --git a/tst/unit/task_step/test_deploy_agent_role_delivery.py b/tst/unit/task_step/test_deploy_agent_role_delivery.py index e642fd0de..47bf80c46 100644 --- a/tst/unit/task_step/test_deploy_agent_role_delivery.py +++ b/tst/unit/task_step/test_deploy_agent_role_delivery.py @@ -14,7 +14,7 @@ from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.env.env import DeployedGatewayEnv -from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER +from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER, AGENT_ENV_ROLE_KEY_HEADER from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps import deploy_agent from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep @@ -227,6 +227,30 @@ def test_a_selection_takes_exactly_one_of_disable_and_only(): assert DeployAgentTaskStep.from_dict({**step.to_dict(), "version": 1}).builtin_tools == {"disable": ["Bash"]} +@pytest.mark.asyncio +async def test_a_keyed_deploy_s_registration_carries_its_role_key_and_the_agent_s_environment_does_not(wiring): + wiring.context.deployed_envs[0].role_key = "deploy-secret" + + await wiring.deploy(_card(HEADERS_MCP), role="executor", env_ids=["crm", "ext"]) + + assert wiring.bodies("/ext/mcp-config") == [ + {"url": _GATEWAY_MCP_URL, "headers": {AGENT_ENV_ROLE_HEADER: "executor", AGENT_ENV_ROLE_KEY_HEADER: "deploy-secret"}, + "name": "crm"}, + {"url": _LIVE_MCP_URL, "headers": {AGENT_ENV_ROLE_HEADER: "executor"}}, + ] + assert wiring.env_vars == {ROLE_ENV_VAR: "executor"} + + +@pytest.mark.asyncio +async def test_a_keyed_deploy_gives_its_key_to_no_agent_without_a_role(wiring): + wiring.context.deployed_envs[0].role_key = "deploy-secret" + + await wiring.deploy(_card(HEADERS_MCP), role=None) + + assert wiring.bodies("/ext/mcp-config") == [{"url": _GATEWAY_MCP_URL, "name": "crm"}] + assert wiring.env_vars is None + + class _Wiring: """One deploy_agent run: the requests the agent received, the agent double, and the step's context.""" diff --git a/tst/unit/task_step/test_load_artifact_file.py b/tst/unit/task_step/test_load_artifact_file.py index cde1c353e..c736efa94 100644 --- a/tst/unit/task_step/test_load_artifact_file.py +++ b/tst/unit/task_step/test_load_artifact_file.py @@ -3,6 +3,7 @@ from __future__ import annotations +import base64 import re from types import SimpleNamespace @@ -275,3 +276,18 @@ async def test_a_cli_installs_on_the_vm_host_of_an_agent_installed_there(monkeyp "pull s3://bucket/check.py /opt/cli/tool/bin/tool", "chmod +x /opt/cli/tool/bin/tool"] assert [s.split("\n")[0] for s in sandbox.scripts if s.startswith("base64 -d")] == [ "base64 -d <<'ENDB64' > /opt/cli/tool/bin/.env"] + + +@pytest.mark.asyncio +async def test_each_installed_cli_carries_its_own_deploy_s_role_key_beside_its_gateway(monkeypatch): + async def _sidecar(role_key: str | None) -> str: + sandbox = _Vm() + monkeypatch.setattr(sandbox_provider, "get_agent_sandbox_provider", _provider(sandbox)) + await A2AAgent.install_cli(SimpleNamespace(sandbox_type=None, sandbox_id="sb-agent"), TOOL, "https://gateway", + on_host=True, role_key=role_key) + (written,) = [s for s in sandbox.scripts if s.startswith("base64 -d <<'ENDB64' > /opt/cli/tool/bin/.env")] + return base64.b64decode("".join(written.split("\n")[1:-1])).decode() + + assert await _sidecar("key-a") == "AGENT_ENV_GATEWAY_URL=https://gateway\nAGENT_ENV_ROLE_KEY=key-a\n" + assert await _sidecar("key-b") == "AGENT_ENV_GATEWAY_URL=https://gateway\nAGENT_ENV_ROLE_KEY=key-b\n" + assert await _sidecar(None) == "AGENT_ENV_GATEWAY_URL=https://gateway\n" diff --git a/tst/unit/task_step/test_mcp_cli_builder.py b/tst/unit/task_step/test_mcp_cli_builder.py index af7d6eb74..a2513e5b4 100644 --- a/tst/unit/task_step/test_mcp_cli_builder.py +++ b/tst/unit/task_step/test_mcp_cli_builder.py @@ -52,6 +52,16 @@ def test_codegen_compiles_and_renders_help(): assert "Auto-generated CLI for slack" in group_help.stdout +def test_the_generated_cli_sends_the_role_key_beside_its_role_only_when_one_is_set(monkeypatch, tmp_path): + namespace: dict = {"__name__": "generated_cli", "__file__": str(tmp_path / "svc")} + exec(compile(generate_cli_script("svc"), "svc", "exec"), namespace) + monkeypatch.setenv("AGENT_ENV_ROLE", "admin") + monkeypatch.delenv("AGENT_ENV_ROLE_KEY", raising=False) + assert namespace["_role_headers"]() == {"AgentEnv-Role": "admin"} + monkeypatch.setenv("AGENT_ENV_ROLE_KEY", "deploy-secret") + assert namespace["_role_headers"]() == {"AgentEnv-Role": "admin", "AgentEnv-Role-Key": "deploy-secret"} + + def test_codegen_loads_sibling_env_file(): src = generate_cli_script("svc") diff --git a/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py b/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py index f54d8a200..f485117b9 100644 --- a/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py +++ b/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py @@ -47,6 +47,7 @@ def __init__(self, monkeypatch, replies, *, clock: dict | None = None, engine: T self.clock = clock or {"armed": False} self.engine = engine self.steps: list[tuple[str, dict]] = [] + self.step_keys: list[str | None] = [] def _answer(self, request: httpx.Request) -> httpx.Response: origin = f"{request.url.scheme}://{request.url.host}" @@ -62,6 +63,7 @@ def _answer(self, request: httpx.Request) -> httpx.Response: return httpx.Response(200, json={"to": body.get("to", "default"), "items": [], "until": None}) if origin == ENV and request.url.path == "/step": self.steps.append((request.headers.get("AgentEnv-Role"), json.loads(request.content))) + self.step_keys.append(request.headers.get("AgentEnv-Role-Key")) return httpx.Response(200, json={"content": [{"type": "text", "text": "ok"}], "changelog_id": 7}) return super()._answer(request) @@ -475,6 +477,26 @@ async def post_turn(turn, params): "arguments": {"to": "a@example.com"}, "source": "post_turn_hook", "hook_artifact_id": "mailer"})] assert _recorded(ctx)[0]["result"]["changelog_id"] == 7 + assert peers.step_keys == [None] + + +@pytest.mark.asyncio +async def test_a_hook_s_tool_call_carries_the_deploy_s_role_key_beside_its_role(monkeypatch): + peers = EnvPeers(monkeypatch, {AGENT: [text("one")]}) + ctx = _context() + env = _env(EXT_STEP_URI) + env.role_key = "deploy-secret" + ctx.deployed_envs.append(env) + hook = _hook("keyed", """ + async def post_turn(turn, params): + [env] = turn.envs + await env.call_tool("send_email", {}, role="school") + return None + """) + + await _step(hook, turns=1).execute(ctx) + + assert [role for role, _ in peers.steps] == ["school"] and peers.step_keys == ["deploy-secret"] @pytest.mark.asyncio diff --git a/tst/unit/task_step/test_register_env_triggers.py b/tst/unit/task_step/test_register_env_triggers.py index 1abba6959..78f370aee 100644 --- a/tst/unit/task_step/test_register_env_triggers.py +++ b/tst/unit/task_step/test_register_env_triggers.py @@ -273,11 +273,13 @@ async def test_executor_hours_for_an_agent_no_trigger_uses_raise(monkeypatch): ([{"id": "x", "when": {"type": "action", "tool": "t"}, "actions": [{"type": "nl", "instruction": "x"}]}], {"exec-1": _HOURS}, "executor_hours"), ([{"id": "x", "when": {"type": "website", "app": "mail", "method": "POST"}, "actions": []}], None, "website"), -], ids=["delay", "unless", "hours", "website"]) + ([{"id": "x", "when": {"type": "website", "app": "mail", "op": "send"}, "actions": []}], None, "website_op"), +], ids=["delay", "unless", "hours", "website", "website-op"]) async def test_a_gateway_without_a_feature_refuses_the_triggers_that_use_it(monkeypatch, triggers, hours, missing): sent = _mock_gateway(monkeypatch, {}) context = _context(with_executor=True) - context.deployed_envs = [_record(_with_features("named_executors", "provoking_roles"))] + features = ("named_executors", "provoking_roles", "website") if missing == "website_op" else ("named_executors", "provoking_roles") + context.deployed_envs = [_record(_with_features(*features))] with pytest.raises(RuntimeError, match=rf"does not declare \['{missing}'\]"): await _step(triggers=triggers, executor_agent_name="exec-1", executor_hours=hours).execute(context) assert sent == [] From f5454b6089f7cf45818028bf765727078f366f90 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 13:34:07 -0800 Subject: [PATCH 14/19] feat(website): a session cookie per site, /go?sid=, the clock's rate, conformance levels and models= (FD-4683) This PR's change, rebased as one commit onto #443. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/notes/README.md | 14 +- packages/agentenv-protocol/pyproject.toml | 7 + .../src/agentenv_protocol/website/clock.py | 26 +- .../agentenv_protocol/website/conformance.py | 128 +++++++--- .../src/agentenv_protocol/website/database.py | 228 ++++++++++++++++++ .../agentenv_protocol/website/environment.py | 173 +++++++++++-- .../src/agentenv_protocol/website/routes.py | 4 +- .../src/agentenv_protocol/website/session.py | 42 +++- .../src/agentenv_protocol/website/store.py | 93 ++++++- .../tests/test_website_conformance.py | 134 ++++++++-- .../tests/test_website_database.py | 213 ++++++++++++++++ .../tests/test_website_environment.py | 150 +++++++++++- .../tests/test_website_notes_example.py | 16 +- .../tests/test_website_state.py | 132 ++++++++++ src/agent_env/env/gateway/clock.py | 12 +- tst/unit/env/gateway/clock_advance_test.py | 19 ++ uv.lock | 92 ++++++- 17 files changed, 1359 insertions(+), 124 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/database.py create mode 100644 packages/agentenv-protocol/tests/test_website_database.py create mode 100644 packages/agentenv-protocol/tests/test_website_state.py diff --git a/packages/agentenv-protocol/examples/website/notes/README.md b/packages/agentenv-protocol/examples/website/notes/README.md index eade87bde..fc5cba314 100644 --- a/packages/agentenv-protocol/examples/website/notes/README.md +++ b/packages/agentenv-protocol/examples/website/notes/README.md @@ -24,12 +24,14 @@ them. The page shows the universe's time. on only beside a role it verified with the same key, so a page cannot claim `admin` through it. Any other deploy sets no key, and its gateway passes on whatever role a browser sends; with no `WEBSITE_ROLE_KEY` set, every caller is `default` and the route is closed. -- **Users.** `ApiContext.user` is the user in the signed session cookie. With no cookie, it is the acting user (the +- **Users.** `ApiContext.user` is the user in the signed session cookie, `agentenv_session_notes` (the site's own, so + it doesn't collide with another site's under the same gateway host; the card names it). With no cookie, it is the acting user (the store's `current_user`), and `GET /api/session` signs that user in. `POST /api/sign-in` switches to another seeded user. A user's notes are invisible to every other user, and another user's note answers 404. - **State.** The site writes no data hooks, so it uses the default store. `data/add` loads the seed (as a data part, or - as a file part by bytes or `file://`), `data/get` exports it, and `data/reset` empties it. `data/get` is how - agentenv reads what a task left the site with. + as a file part by bytes or `file://`), `data/get` exports it, `data/reset` empties it, and `/go` returns + `{initial_state, current_state, state_diff}`. A page opened at `?sid=` keeps a state of its own, started + from the seed, which `/go?sid=` reports; without `sid` it is the default session, the data plane's. - **Time.** Notes are stamped with `self.now()`, which follows the gateway's clock once `clock/v1`'s `sync_time` names it. The frontend reads `X-Universe-Now` from every API response into `universeNow()` and shows it in the header. It uses this helper rather than `clock_patch=True`, so the page works the same against a server written without the @@ -66,11 +68,13 @@ The build context is the repo root, because the image installs the protocol pack ```bash docker build -f packages/agentenv-protocol/examples/website/notes/Dockerfile -t notes-site . docker run -d --name notes-site -p 18080:8000 notes-site -python -m agentenv_protocol.website.conformance http://localhost:18080 --clock-host host.docker.internal +python -m agentenv_protocol.website.conformance http://localhost:18080 --clock-host host.docker.internal --strict curl -s http://localhost:18080/api/notes ``` -Use `--clock-host` so the container can reach the checker's clock. The clock check leaves the server following a +Use `--clock-host` so the container can reach the checker's clock. `--strict` fails on a recommended check's warning +too (the card, `X-Universe-Now`, the clock, forwarded headers, cookies); without it only the contract's required checks +fail the run, and Notes passes every check either way. The clock check leaves the server following a clock that is gone once the check ends: later timestamps start from 2031 until the server is synced again, so restart the container before using it. diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index 73559b1c2..17492f651 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -49,6 +49,12 @@ computer = [ "Pillow>=10.4", "uvicorn>=0.30", ] +# A website whose state is an MCP server's database: @website(models=) and self.db, on a +# postgresql:// DATABASE_URL through psycopg 3, SQLAlchemy 2.1's driver for it. +db = [ + "sqlalchemy>=2.1", + "psycopg[binary]>=3.1", +] dev = [ "a2a-sdk[http-server]>=0.3.26,<0.4", "python-multipart>=0.0.13", @@ -59,6 +65,7 @@ dev = [ "fastmcp>=3.4,<4", # the standalone-fastmcp mount path "Pillow>=10.4", # the computer extra's frame crops and encodings "av>=17", # the computer extra's encoder, and the tests' decoder + "sqlalchemy>=2.1", # the db extra's models= site, tested on sqlite ] [project.urls] Homepage = "https://www.agentenvframework.com" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py b/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py index 0d27d86a9..dec301780 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py @@ -1,8 +1,9 @@ """The universe's clock as a site reads it: ``clock/v1``'s ``sync_time`` hands the server the gateway's ``/clock/time`` URL, as it hands it to MCP servers, and the site's time then follows that clock's ``{"virtual_time": }``, -read at most once a second and extrapolated between reads. The rate it extrapolates at is one two successive intervals -between reads agreed on, so a clock set or advanced between two reads is taken for a jump, not a speed; until two agree -it is real time's. An unsynced site reads wall time. Every answer of a site carries the time it answered at as ``X-Universe-Now``, +read at most once a second and extrapolated between reads. The rate it extrapolates at is the one the clock answers +with, ``virtual_seconds_per_real_second``, as the gateway's does; for a clock that answers none, it is one two +successive intervals between reads agreed on, so a clock set or advanced between two reads is taken for a jump, not a +speed, and until two agree it is real time's. An unsynced site reads wall time. Every answer of a site carries the time it answered at as ``X-Universe-Now``, RFC3339 in UTC. ``sync_time`` may also name the universe's time zone, an IANA name, which the clock keeps for the site to show local times in; the time itself stays UTC.""" from __future__ import annotations @@ -11,6 +12,7 @@ import datetime import json import logging +import math import time import httpx @@ -60,8 +62,8 @@ async def sync(self, url: str, timezone: str | None = None) -> datetime.datetime can't be read, keeping the clock and zone it followed. It waits for a refresh reading the clock it followed, so that read never lands on the new clock.""" async with self._reading(): - virtual = await _read(url) - self.url, self._virtual, self._rate, self._measured = url, virtual, 1.0, None + virtual, rate = await _read(url) + self.url, self._virtual, self._rate, self._measured = url, virtual, 1.0 if rate is None else rate, None self.timezone = timezone self._read_at = self._tried_at = time.monotonic() return virtual @@ -76,12 +78,14 @@ async def refresh(self) -> None: return self._tried_at = started try: - virtual = await _read(self.url) + virtual, rate = await _read(self.url) except Exception as e: logger.debug("reading the universe's clock at %s failed: %r", self.url, e) return read_at = time.monotonic() - if self._virtual is not None and read_at > self._read_at: + if rate is not None: + self._rate = rate + elif self._virtual is not None and read_at > self._read_at: measured = max(0.0, (virtual - self._virtual).total_seconds() / (read_at - self._read_at)) if self._measured is not None and abs(measured - self._measured) <= RATE_AGREEMENT * max(measured, self._measured, 1.0): self._rate = measured @@ -94,11 +98,15 @@ def _reading(self) -> asyncio.Lock: return self._lock -async def _read(url: str) -> datetime.datetime: +async def _read(url: str) -> tuple[datetime.datetime, float | None]: + """The clock's time, and the rate it says it runs at: None when it says none, or none that is a finite rate.""" async with httpx.AsyncClient() as client: response = await client.get(url, timeout=READ_TIMEOUT_S) response.raise_for_status() - return parse_time(response.json()["virtual_time"]) + body = response.json() + rate = body.get("virtual_seconds_per_real_second") + ok = isinstance(rate, (int, float)) and not isinstance(rate, bool) and math.isfinite(rate) and rate >= 0 + return parse_time(body["virtual_time"]), float(rate) if ok else None def clock_patch_script(now: datetime.datetime) -> bytes: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py index 14e62c93b..67a8a114a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py @@ -1,18 +1,27 @@ -"""Check a running site's server against ``urn:agentenv:website/v1``, whatever it's written in: -``python -m agentenv_protocol.website.conformance `` prints each check and exits 1 when one failed, 2 when the -server couldn't be reached. Everything goes over HTTP: the card, ``/`` under a made-up host, readiness by the card's -``website/v1`` ``ready``, whose answer must be a ``WebsiteReadiness`` (by the declared health path, deprecated, for a -card without it), ``X-Universe-Now`` on every answer the other checks got, a data-plane round trip at the card's ``url`` -(data/get, data/reset, data/get, which no longer returns the exported data, data/add of what the first data/get -returned, data/get again), ``clock/v1``'s -``sync_time`` against a clock the checker serves, and the forwarded headers wherever an answer shows them, on ``/`` and -on a GET route with its trailing slash toggled, which a server may redirect. +"""Check a running site's server against ``urn:agentenv:website/v1``, whatever it's written in: ``python -m +agentenv_protocol.website.conformance `` prints each check and exits 1 when a required one failed (or, with +``--strict``, when a recommended one warned), 2 when the server couldn't be reached. Everything goes over HTTP. + +The required checks are the contract, what the gateway and a browser rely on of any server: ``/`` serves HTML under a +made-up host, the site is ready, by the card's ``website/v1`` ``ready``, whose answer must be a +``WebsiteReadiness``, or, for a card without it, by the health path (the card's, else ``/api/health``) answering 200, which is deprecated, the data plane and card's paths are not +the site's pages, and, when the card advertises the data plane, a round trip at the card's ``url`` (data/get, +data/reset, data/get, which no longer returns the exported data, data/add of what the first data/get returned, data/get +again) returns what it was given. A failed one is ``failed``. The recommended checks are what an SDK site does and the +gateway uses when a server offers it: a card declaring the site's domains, health path and routes, ``X-Universe-Now`` +on every answer the other checks got, ``clock/v1``'s ``sync_time`` against a clock the checker serves, the forwarded +headers wherever an answer shows them (on ``/`` and on a GET route with its trailing slash toggled, which a server may +redirect), and cookies tied to no domain, the session path's the one the card names. A failed one is a ``warning``. The round trip leaves the server's data as it found it, adding the export back when a step after the reset fails too. -The clock check leaves the -server following a clock that is gone once the check ends, so a server it ran against keeps extrapolating from that -clock's time until it's synced again. The checker's clock starts at ``CLOCK_START``, far enough from wall time that a -synced server shows it, and a synced server's ``X-Universe-Now`` may be ``CLOCK_SLACK_S`` seconds from it.""" +data/add sets the state a task starts from, so a server whose state shows changes since its initial state, as an SDK +site's does, would have them folded into it, and data/reset drops a site's named sessions, which data/get doesn't +export: the round trip is skipped for a server whose state shows changes or ``changed_sessions``. Its state is its +card's ``website/v1`` ``state``, else, for a card without it, what the path its ``params.state.path`` names answers, +else ``/go``'s, deprecated. The clock check leaves the server following a clock that is gone once the check ends, so a +server it ran against keeps extrapolating from that clock's time until it's synced again. The checker's clock starts at +``CLOCK_START``, far enough from wall time that a synced server shows it, and a synced server's ``X-Universe-Now`` may +be ``CLOCK_SLACK_S`` seconds from it.""" from __future__ import annotations import argparse @@ -33,24 +42,34 @@ from ..client import find_extension, find_extension_method from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, iso, parse_time -from .environment import DEFAULT_HEALTH_PATH +from .environment import DEFAULT_HEALTH_PATH, STATE_PATH from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness +from .state import WEBSITE_STATE_OP -Status = Literal["passed", "failed", "skipped"] +Status = Literal["passed", "failed", "warning", "skipped"] CLOCK_START = datetime.datetime(2031, 5, 6, 7, 8, 9, tzinfo=datetime.timezone.utc) CLOCK_SLACK_S = 120 +REQUIRED = ("any_host", "health", "reserved_paths", "data_plane") @dataclasses.dataclass(frozen=True) class Result: + """One check's outcome; ``required`` for a check of the contract, whose failure is ``failed``, and not for a + recommended one, whose failure is a ``warning``.""" + name: str status: Status detail: str = "" + required: bool = True @property def passed(self) -> bool: return self.status != "failed" + def fails(self, strict: bool = False) -> bool: + """Whether the result fails the run: a failed check, or with ``strict`` a warning too.""" + return self.status == "failed" or (strict and self.status == "warning") + def check(base_url: str, *, client: httpx.Client | None = None, clock_host: str = "127.0.0.1", timeout: float = 30) -> list[Result]: @@ -84,20 +103,27 @@ def run(self) -> list[Result]: card = response.json() if response.status_code == 200 else {} except ValueError: card = {} + card = card if isinstance(card, dict) else {} self.rpc_url = _rpc_url(self.base_url, card) site = (find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {} health = site.get("health") if isinstance(site.get("health"), str) else DEFAULT_HEALTH_PATH - steps = (("any_host", self._any_host), ("health", lambda: self._health(card, health)), - ("universe_now", self._universe_now), ("data_plane", lambda: self._data_plane(card)), + steps = (("card", lambda: self._card(response, site)), ("any_host", self._any_host), + ("health", lambda: self._health(card, health)), ("universe_now", self._universe_now), + ("reserved_paths", self._reserved_paths), ("data_plane", lambda: self._data_plane(card)), ("clock", lambda: self._clock(card, health)), ("forwarded", lambda: self._forwarded(site)), ("cookies", lambda: self._cookies(site))) outcomes = {name: _outcome(step) for name, step in steps if name != "universe_now"} outcomes["universe_now"] = _outcome(self._universe_now) - return [self._card(response, site), *(Result(name, *outcomes[name]) for name, _step in steps)] - - def _card(self, response: httpx.Response, site: dict) -> Result: + results = [] + for name, _step in steps: + status, detail = outcomes[name] + required = name in REQUIRED + results.append(Result(name, "warning" if status == "failed" and not required else status, detail, required)) + return results + + def _card(self, response: httpx.Response, site: dict) -> tuple[Status, str]: if response.status_code != 200: - return Result("card", "failed", f"{WELL_KNOWN_PATH} answered {response.status_code}") + return "failed", f"{WELL_KNOWN_PATH} answered {response.status_code}" problems = [] if not site: problems.append(f"declares no {WEBSITE_EXTENSION_URI} with params") @@ -109,7 +135,7 @@ def _card(self, response: httpx.Response, site: dict) -> Result: if not isinstance(routes, list) or not all(isinstance(r, dict) and {"method", "path", "name"} <= set(r) for r in routes): problems.append("its params' routes aren't a list of {method, path, name}") - return Result("card", "failed" if problems else "passed", "; ".join(problems)) + return ("failed" if problems else "passed"), "; ".join(problems) def _any_host(self) -> tuple[Status, str]: host = f"conformance-{uuid.uuid4().hex[:8]}.invalid" @@ -151,6 +177,14 @@ def _universe_now(self) -> tuple[Status, str]: parse_time(response.headers[UNIVERSE_NOW_HEADER]) return "passed", f"on all {len(self.responses)} answers" + def _reserved_paths(self) -> tuple[Status, str]: + paths = [RPC_PATH, WELL_KNOWN_PATH, f"{RPC_PATH}/conformance-{uuid.uuid4().hex[:8]}", f"{RPC_PATH}/", f"{WELL_KNOWN_PATH}/"] + for path in paths: + response = self.get(path) + if response.is_success and response.headers.get("content-type", "").startswith("text/html"): + return "failed", f"{path} answered a page, but the gateway keeps {RPC_PATH} and {WELL_KNOWN_PATH}" + return "passed", f"no page at {', '.join(paths)}" + def _rpc(self, method: str, params: dict | None = None) -> dict: response = self._seen(self.client.post(self.rpc_url, json={"jsonrpc": "2.0", "id": method, "method": method, "params": params or {}})) @@ -163,10 +197,17 @@ def _rpc(self, method: str, params: dict | None = None) -> dict: def _data_plane(self, card: dict) -> tuple[Status, str]: operations = (card.get("capabilities") or {}).get("operations") - missing = [m for m in (METHOD_GET, METHOD_RESET, METHOD_ADD) if operations is not None and m not in operations] + operations = operations if isinstance(operations, list) else [] + missing = [m for m in (METHOD_GET, METHOD_RESET, METHOD_ADD) if m not in operations] + if len(missing) == 3: + return "skipped", "the card advertises no data plane" if missing: return "failed", f"the card's operations lack {', '.join(missing)}" exported = self._rpc(METHOD_GET).get("parts") or [] + if where := self._changed_since_initial(card): + return "skipped", (f"data/get answered; {where} shows changes since the initial state, or named sessions " + "with changes, which data/reset and data/add would fold in or drop, so the round trip " + "was not run") self._rpc(METHOD_RESET) added_back = False try: @@ -188,6 +229,26 @@ def _data_plane(self, card: dict) -> tuple[Status, str]: return "failed", "data/get after data/reset and data/add of its own export returned other data" return "passed", f"{len(exported)} part(s) round-tripped through reset and add" + def _changed_since_initial(self, card: dict) -> str | None: + """Where the server's state was read when its ``state_diff`` names changes or it names ``changed_sessions``, + None when it names neither or can't be read: the card's ``website/v1`` ``state``, else the path its + ``params.state.path`` names, else ``/go``. Read aside, as no check's answer, since a server without one needn't + stamp its 404.""" + op = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) + if op is not None and op.get("endpoint"): + where, request = f"website/v1 {WEBSITE_STATE_OP}", (op["method"], self.base_url + op["endpoint"]) + else: + declared = ((find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {}).get("state") + path = declared.get("path") if isinstance(declared, dict) else None + where = path if isinstance(path, str) and path.startswith("/") else STATE_PATH + request = ("GET", self.base_url + where) + try: + state = self.client.request(*request).json() + except Exception: + return None + changed = isinstance(state, dict) and bool(state.get("state_diff") or state.get("changed_sessions")) + return where if changed else None + def _clock(self, card: dict, health: str) -> tuple[Status, str]: if find_extension(card, CLOCK_EXTENSION_URI) is None: return "failed", f"the card declares no {CLOCK_EXTENSION_URI}" @@ -224,14 +285,18 @@ def _forwarded(self, site: dict) -> tuple[Status, str]: return "passed", f"no answer to {', '.join(paths)} named the internal Host" def _cookies(self, site: dict) -> tuple[Status, str]: + session = [] if isinstance(site.get("session_path"), str): - self.get(site["session_path"]) + session = self.get(site["session_path"]).headers.get_list("set-cookie") cookies = [c for r in self.responses for c in r.headers.get_list("set-cookie")] if not cookies: return "skipped", "no answer set a cookie" scoped = [c.split("=", 1)[0] for c in cookies if any(a.strip().lower().startswith("domain=") for a in c.split(";"))] if scoped: return "failed", f"cookies tied to a domain: {', '.join(sorted(set(scoped)))}" + named = site.get("session_cookie") + if isinstance(named, str) and session and named not in {c.split("=", 1)[0].strip() for c in session}: + return "failed", f"{site['session_path']} set {', '.join(c.split('=', 1)[0] for c in session)}, not the card's {named}" return "passed", f"{len(cookies)} cookie(s), none tied to a domain" @@ -286,7 +351,7 @@ def virtual_now() -> datetime.datetime: class Clock(BaseHTTPRequestHandler): def do_GET(self) -> None: - body = json.dumps({"virtual_time": iso(virtual_now())}).encode() + body = json.dumps({"virtual_time": iso(virtual_now()), "virtual_seconds_per_real_second": 1.0}).encode() self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) @@ -306,7 +371,7 @@ def log_message(self, *args) -> None: server.server_close() -_MARK = {"passed": "PASS", "failed": "FAIL", "skipped": "----"} +_MARK = {"passed": "PASS", "failed": "FAIL", "warning": "WARN", "skipped": "----"} def main(argv: list[str] | None = None) -> int: @@ -315,6 +380,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("url", help="where the server answers, such as http://localhost:8000") parser.add_argument("--clock-host", default="127.0.0.1", help="the address the server reaches this checker at") parser.add_argument("--timeout", type=float, default=30, help="seconds for each request") + parser.add_argument("--strict", action="store_true", help="fail on a recommended check's warning too") args = parser.parse_args(argv) try: results = check(args.url, clock_host=args.clock_host, timeout=args.timeout) @@ -322,10 +388,12 @@ def main(argv: list[str] | None = None) -> int: print(f"couldn't reach {args.url}: {type(e).__name__}: {e}", file=sys.stderr) return 2 for result in results: - print(f"{_MARK[result.status]} {result.name}{': ' + result.detail if result.detail else ''}") - failed = sum(not r.passed for r in results) - print(f"{args.url}: {len(results) - failed} passed or skipped, {failed} failed") - return 1 if failed else 0 + kind = "required" if result.required else "recommended" + print(f"{_MARK[result.status]} {result.name} ({kind}){': ' + result.detail if result.detail else ''}") + counts = {status: sum(r.status == status for r in results) for status in _MARK} + print(f"{args.url}: {counts['passed']} passed, {counts['failed']} failed, {counts['warning']} warning(s), " + f"{counts['skipped']} skipped{' (strict)' if args.strict else ''}") + return 1 if any(r.fails(args.strict) for r in results) else 0 if __name__ == "__main__": diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py new file mode 100644 index 000000000..34373ea47 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -0,0 +1,228 @@ +"""A site whose state is a database's: ``@website(models=...)`` names the data an MCP server keeps, so the site's pages +and the server's tools read and write the same rows under the same constraints (and the same rules, where the site +calls the server's own service code). ``models`` is either an object with +``load(data)``, ``export()`` and ``reset()`` (each may be async), the data hooks themselves, or SQLAlchemy models: a +declarative ``Base``, a ``MetaData``, or a module holding one (its ``Base``, else its one metadata), as the +synthetic_servers kit's ``database.py`` declares them. For models, the site's ``self.db`` is a SQLAlchemy session +factory on ``DATABASE_URL``, which agent-env points at the env's own schema of the service database (its +``search_path``), creating the models' tables there on first use; its data is a JSON object of each table's rows by +table name, which data/add inserts, data/get exports, and data/reset deletes. SQLAlchemy is imported only for models, +from the ``db`` extra, which carries SQLAlchemy 2.1 and its ``postgresql://`` driver, ``psycopg``; a site whose +``DATABASE_URL`` names another driver installs that one, and an engine whose driver isn't installed raises ImportError +naming it. + +Beside the tables, the data's string ``current_user`` names the user a browser starts signed in as, as the default +store's does, and the state data/add last loaded is the initial one ``/go`` reports. A models site keeps one state, +the database's, so ``/go`` answers it for every ``sid``.""" +from __future__ import annotations + +import copy +import datetime +import decimal +import itertools +import json +import types +import uuid +from typing import Any, Callable, Iterable + +from pydantic_core import to_jsonable_python + +from ..agent_env_environment import _maybe_await +from .store import CURRENT_USER, part_data, verifier_state + +HOOKS = ("load", "export", "reset") + + +def is_hooks(models: Any) -> bool: + """Whether ``models`` is data hooks of its own: an object with callable ``load``, ``export`` and ``reset``.""" + return all(callable(getattr(models, name, None)) for name in HOOKS) + + +def metadata_of(models: Any) -> Any: + """The SQLAlchemy ``MetaData`` ``models`` names: itself, its ``metadata``, or a module's ``Base``'s or one + metadata; raises TypeError for anything else.""" + sqlalchemy = _sqlalchemy() + if isinstance(models, sqlalchemy.MetaData): + return models + if isinstance(getattr(models, "metadata", None), sqlalchemy.MetaData): + return models.metadata + if isinstance(models, types.ModuleType): + if isinstance(getattr(getattr(models, "Base", None), "metadata", None), sqlalchemy.MetaData): + return models.Base.metadata + found = {id(m): m for value in vars(models).values() if isinstance( + m := value if isinstance(value, sqlalchemy.MetaData) else getattr(value, "metadata", None), + sqlalchemy.MetaData)} + if len(found) == 1: + return next(iter(found.values())) + if found: + raise TypeError(f"module {models.__name__} holds {len(found)} MetaData; pass models= its Base") + raise TypeError(f"models= is an object with load/export/reset, or SQLAlchemy models (a declarative Base, a " + f"MetaData, or a module holding one), not {models!r}") + + +def _sqlalchemy() -> Any: + try: + import sqlalchemy + except ImportError as e: + raise ImportError("@website(models=...) of SQLAlchemy models needs sqlalchemy: " + "pip install 'agentenv-framework-protocol[db]'") from e + return sqlalchemy + + +class Tables: + """The data hooks of SQLAlchemy ``metadata`` on the database at ``url()``: each table's rows by table name.""" + + def __init__(self, metadata: Any, url: Callable[[], str | None]) -> None: + self.metadata, self._url = metadata, url + self._engine: Any = None + self._sessions: Any = None + + @property + def engine(self) -> Any: + """The engine on ``DATABASE_URL``, made on first use with the models' tables created there; raises + ImportError naming the driver when the URL's isn't installed.""" + if self._engine is None: + url = self._url() + if not url: + raise RuntimeError("a site of SQLAlchemy models keeps its rows in DATABASE_URL, which isn't set") + sqlalchemy = _sqlalchemy() + try: + engine = sqlalchemy.create_engine(url) + except ImportError as e: + scheme = sqlalchemy.engine.make_url(url).drivername + raise ImportError(f"DATABASE_URL's {scheme}:// needs the database driver {e.name or 'it names'}, which " + f"isn't installed; the db extra carries psycopg for postgresql://, and a site installs any " + f"other driver itself (pip install {e.name or 'the driver'})") from e + self.metadata.create_all(engine) + self._engine = engine + return self._engine + + @property + def sessions(self) -> Any: + if self._sessions is None: + from sqlalchemy.orm import sessionmaker + + self._sessions = sessionmaker(bind=self.engine) + return self._sessions + + def export(self) -> dict: + sqlalchemy = _sqlalchemy() + with self.engine.connect() as connection: + return {table.name: [to_jsonable_python(dict(row._mapping)) for row in connection.execute( + sqlalchemy.select(table).order_by(*table.primary_key.columns))] for table in self.metadata.sorted_tables} + + def load(self, data: dict) -> None: + """Insert ``data``'s rows, each table's after the tables it refers to, in their order, each run of rows that name + the same columns as one insert (one insert takes its columns from its first row, so a row that leaves an + optional column out goes in its own); refuses a name that is no table.""" + tables = {table.name: table for table in self.metadata.sorted_tables} + unknown = sorted(set(data) - set(tables)) + if unknown: + raise ValueError(f"no table {', '.join(map(repr, unknown))}; the tables are {sorted(tables)}") + with self.engine.begin() as connection: + for name, table in tables.items(): + rows = data.get(name) or [] + if not (isinstance(rows, list) and all(isinstance(row, dict) for row in rows)): + raise ValueError(f"table {name!r}'s data is not a list of rows") + for _, run in itertools.groupby(rows, key=lambda row: sorted(row)): + connection.execute(table.insert(), [_row(table, row) for row in run]) + if connection.dialect.name == "postgresql": + _advance_sequences(connection, tables.values()) + + def reset(self) -> None: + with self.engine.begin() as connection: + for table in reversed(self.metadata.sorted_tables): + connection.execute(table.delete()) + + def key(self, name: str, row: dict) -> str: + """A row's identity, its primary key's values, which ``/go``'s diff matches rows by.""" + table = next((t for t in self.metadata.sorted_tables if t.name == name), None) + columns = [c.name for c in table.primary_key.columns] if table is not None else [] + return json.dumps([row.get(c) for c in columns] if columns else row, sort_keys=True, default=str) + + +_PARSERS: dict = {datetime.datetime: datetime.datetime.fromisoformat, datetime.date: datetime.date.fromisoformat, + datetime.time: datetime.time.fromisoformat, decimal.Decimal: decimal.Decimal, uuid.UUID: uuid.UUID} + + +def _row(table: Any, row: dict) -> dict: + """``row`` with each JSON string a date, time, decimal or UUID column holds parsed to that type.""" + parsed = dict(row) + for column in table.columns: + value = parsed.get(column.name) + if isinstance(value, str): + kind = _python_type(column) + if kind in _PARSERS: + parsed[column.name] = _PARSERS[kind](value.replace("Z", "+00:00") if kind is datetime.datetime else value) + return parsed + + +def _python_type(column: Any) -> type | None: + try: + return column.type.python_type + except NotImplementedError: + return None + + +def _advance_sequences(connection: Any, tables: Iterable[Any]) -> None: + """Move each serial key's sequence past the rows just loaded, so the next row the site or server adds takes a key + of its own.""" + sqlalchemy = _sqlalchemy() + for table in tables: + keys = list(table.primary_key.columns) + if len(keys) == 1 and keys[0].autoincrement in (True, "auto") and _python_type(keys[0]) is int: + quote = connection.dialect.identifier_preparer + name = quote.format_table(table) + connection.execute(sqlalchemy.text( + f"SELECT setval(pg_get_serial_sequence(:table, :column), COALESCE(MAX({quote.quote(keys[0].name)}), 0) + 1, " + f"false) FROM {name}"), {"table": name, "column": keys[0].name}) + + +class Sessions: + """A site's ``self.db``: ``self.db()`` is a SQLAlchemy ``Session`` on the site's database, and ``self.db.begin()`` + one in a transaction that commits as its ``with`` ends; the engine is made on first use, not at mount.""" + + def __init__(self, tables: Tables) -> None: + self._tables = tables + + def __call__(self, **options: Any) -> Any: + return self._tables.sessions(**options) + + def begin(self) -> Any: + return self._tables.sessions.begin() + + @property + def engine(self) -> Any: + return self._tables.engine + + +class ModelState: + """A models site's data plane and ``/go``: ``hooks`` loads, exports and resets its data, the state data/add last + loaded is the initial one, and the data's ``current_user`` is kept beside it.""" + + def __init__(self, hooks: Any) -> None: + self.hooks = hooks + self.initial: dict = {} + self.current_user: str | None = None + + async def reset(self) -> None: + await _maybe_await(self.hooks.reset()) + self.initial, self.current_user = {}, None + + async def load_parts(self, parts: list) -> None: + for part in parts: + data = dict(part_data(part)) + if CURRENT_USER in data: + self.current_user = data.pop(CURRENT_USER) + await _maybe_await(self.hooks.load(data)) + self.initial = copy.deepcopy(await self.export()) + + async def export(self) -> dict: + data = dict(await _maybe_await(self.hooks.export())) + if self.current_user is not None: + data[CURRENT_USER] = self.current_user + return data + + async def state(self) -> dict: + key = getattr(self.hooks, "key", None) + return verifier_state(copy.deepcopy(self.initial), await self.export(), key) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py index ac05ba6be..9a7a27059 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -2,12 +2,23 @@ plain HTTP on ``PORT`` for any host name: its ``@api`` routes, its static folder at ``/`` with ``index.html`` for the pages a single-page app routes itself, its health path, and the data plane and card the base class serves. Every answer carries ``X-Universe-Now``, the universe's time, which ``self.now()`` reads and ``clock/v1``'s ``sync_time`` ties to -the gateway's virtual clock. Without data hooks of its own a site keeps its data in ``self.store``, which the data -plane loads, exports and resets: ``data/get`` is how agentenv reads what a task left a site with. - -One route is built in, replaced by a site's own ``@api`` route at its very path and answering before one whose path -parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the acting -user in when nobody is (GET), or to sign out (DELETE).""" +the gateway's virtual clock. Without data hooks of its own a site keeps its state in ``self.store``, or with +``@website(models=...)`` in an MCP server's database, which ``self.db`` opens sessions of (``website.database``). + +A store-backed site keeps a state per session, the CUA-Gym convention the website state verifier reads: a request's +session is its ``?sid=``, else the one its browser last named, which the site keeps in the ``agentenv_sid_`` +cookie, else the default one. ``self.store`` is the request's session's store, each session started as a copy of the +initial state; the default session, ``sid=default``, is the state the data plane loads, exports and resets, and the +one a browser that names no session uses, so a site browsed without ``sid`` works as it did before sessions. + +Two routes are built in, each replaced by a site's own ``@api`` route at its very path and answering before one whose +path parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the +acting user in when nobody is (GET), or to sign out (DELETE); and, for a store-backed site, ``/go?sid=``, that +session's ``{initial_state, current_state, state_diff}`` at the path ``website/v1``'s ``params.state.path`` names, +which a reader of a card without ``state`` falls back to, with the site's ``changed_sessions``, the named sessions whose +state changed, which data/reset would drop; a session no request has used yet answers the initial state, unchanged. A +store-backed site answers ``website/v1``'s ``state``, of the session its ``sid`` names, with the same ``WebsiteState``, +among agentenv's own routes, which is how the website state verifier reads it.""" from __future__ import annotations import inspect @@ -27,6 +38,7 @@ from starlette.routing import Route from ..agent_env_environment import ( + _CARD_CONFIG_ATTR, _OP_ATTR, AgentEnvEnvironment, AgentEnvStarletteApplication, @@ -37,17 +49,20 @@ get_data, reset_data, ) -from ..types import DataPart +from ..types import DataPart, EnvironmentExtension from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock, clock_patch_script, iso +from .database import ModelState, Sessions, Tables, is_hooks, metadata_of from .errors import NotFound from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness from .routes import (ApiContext, ApiRoute, endpoint, error_response, forwarded_prefix, openapi, openapi_path, request_role, reserved_path, routes_of) -from .session import SESSION_COOKIE, SessionSigner, expires_at -from .store import Collections +from .session import SessionSigner, expires_at, session_cookie_name, sid_cookie_name +from .state import WEBSITE_STATE_OP, WebsiteState +from .store import DEFAULT_SID, Collections, active_sid, session_id DEFAULT_HEALTH_PATH = "/api/health" SESSION_PATH = "/api/session" +STATE_PATH = "/go" _WEBSITE_ATTR = "_aee_website" _READY_ATTR = "_aee_website_ready" _DEFAULT_HOOK_ATTR = "_aee_default_store_hook" @@ -57,27 +72,36 @@ class WebsiteDeclaration(NamedTuple): """What ``@website`` declares: the site's domains, its static folder and seed data file (relative to the class's - file), its health path, and whether its HTML pages read the universe's time.""" + file), its health path, whether its HTML pages read the universe's time, and the models whose database keeps its + state instead of the default store.""" domains: tuple[str, ...] = () static: Optional[str] = None health: str = DEFAULT_HEALTH_PATH clock_patch: bool = False data: Optional[str] = None + models: Any = None def website(_cls: type | None = None, *, domains: list[str] | tuple[str, ...] = (), static: str | None = None, - health: str = DEFAULT_HEALTH_PATH, clock_patch: bool = False, data: str | None = None) -> Any: - """Declare a WebsiteEnvironment's site. ``data`` names a JSON file the default store starts with.""" + health: str = DEFAULT_HEALTH_PATH, clock_patch: bool = False, data: str | None = None, + models: Any = None) -> Any: + """Declare a WebsiteEnvironment's site. ``data`` names a JSON file the default store starts with; ``models`` + keeps the site's state in a database instead, an MCP server's (see ``website.database``), and the site's + data plane and ``/go`` are that database's.""" if _cls is not None: raise TypeError("use @website(...) with parentheses") if not health.startswith("/") or health == "/" or reserved_path(health): raise ValueError(f"health is a path of its own under '/', not {health!r}") + if models is not None and data is not None: + raise ValueError("data= seeds the default store; a site of models= is seeded through data/add") + if models is not None and not is_hooks(models): + metadata_of(models) def deco(cls: type) -> type: if not (isinstance(cls, type) and issubclass(cls, WebsiteEnvironment)): raise TypeError(f"@website declares a WebsiteEnvironment subclass, not {cls!r}") - setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data)) + setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data, models)) _check_health(cls) return cls @@ -98,6 +122,17 @@ class WebsiteEnvironment(AgentEnvEnvironment): app: ClassVar[Any] = None asgi: Any = None + db: Any = None + _store: Collections + + @property + def store(self) -> Collections: + """The state store of the request's session: the default session's outside a request.""" + return self._store.session(active_sid.get()) + + @store.setter + def store(self, value: Collections) -> None: + self._store = value def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) @@ -113,8 +148,14 @@ def __init__(self) -> None: self.store = Collections() if declaration.data is not None: self.store.load(json.loads(self._site_path(declaration.data).read_text(encoding="utf-8"))) + self.store.mark_initial() self.clock = UniverseClock() - self._signer = SessionSigner() + self._signer = SessionSigner(cookie_name=session_cookie_name(self.environment_name())) + models = declaration.models + self._tables = (None if models is None or is_hooks(models) + else Tables(metadata_of(models), lambda: self.database_url)) + self._models = None if models is None else ModelState(models if self._tables is None else self._tables) + self.db = Sessions(self._tables) if self._tables is not None else None @classmethod def declaration(cls) -> WebsiteDeclaration: @@ -124,6 +165,17 @@ def declaration(cls) -> WebsiteDeclaration: def routes(cls) -> list[ApiRoute]: return routes_of(cls) + def environment_name(self) -> str: + """The site's name, as its card names it: ``ENVIRONMENT_NAME``, else the card's declared name, else the + class's.""" + declared = (getattr(self, _CARD_CONFIG_ATTR, None) or {}).get("name") + return os.environ.get("ENVIRONMENT_NAME") or declared or type(self).__name__ + + @property + def session_cookie(self) -> str: + """The name of the site's session cookie, ``agentenv_session_``.""" + return self._session_signer().cookie_name + @property def database_url(self) -> str | None: """Where the site keeps its task state: its own schema of the env's database.""" @@ -145,12 +197,15 @@ def timezone(self) -> str | None: return self._clock().timezone def acting_user(self) -> str | None: - """Who a browser with no session is signed in as: the store's ``current_user`` unless a site says else.""" + """Who a browser with no session is signed in as: the store's ``current_user``, or the one a models site's data + named, unless a site says else.""" + if getattr(self, "_models", None) is not None: + return self._models.current_user return self.store.current_user if hasattr(self, "store") else None def context(self, request: Request) -> ApiContext: """The ``ApiContext`` of ``request``, for a mounted ``app``'s routes as for ``@api`` ones.""" - session = self._session_signer().decode(request.cookies.get(SESSION_COOKIE)) + session = self._session_signer().read(request.cookies) user = session["user"] if session is not None else self.acting_user() return ApiContext(request, user, request_role(request), session, self._session_signer()) @@ -162,11 +217,14 @@ def create_app(self) -> Starlette: api_routes = self.routes() taken = {openapi_path(r.path) for r in api_routes} root = self._site_path(declaration.static) if declaration.static is not None else None - asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self)]) + asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self), + Middleware(_Session, cookie=sid_cookie_name(self.environment_name()))]) AgentEnvStarletteApplication(self._build_card(), self).add_routes_to_app(asgi) asgi.router.routes.append(Route(declaration.health, self._health, methods=["GET"])) if SESSION_PATH not in taken: asgi.router.routes.append(Route(SESSION_PATH, self._session, methods=["GET", "DELETE"])) + if self._serves_state_path(): + asgi.router.routes.append(Route(STATE_PATH, self._state, methods=["GET"])) asgi.router.routes.extend(Route(r.path, _unreserved(endpoint(getattr(self, r.fn.__name__), r, self.context)), methods=[r.method]) for r in api_routes) asgi.router.default = _Fallback(root, type(self).app) @@ -190,10 +248,18 @@ def extension_params(self, uri: str) -> dict: if uri == WEBSITE_EXTENSION_URI: declaration = self.declaration() return {"domains": list(declaration.domains), "health": declaration.health, - "routes": [r.advertised() for r in self.routes()], "session_cookie": SESSION_COOKIE, - "session_path": SESSION_PATH} + "routes": [r.advertised() for r in self.routes()], "session_cookie": self.session_cookie, + "session_path": SESSION_PATH, **({"state": {"path": STATE_PATH}} if self._serves_state_path() else {})} return super().extension_params(uri) + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + """The extensions the base class serves, ``website/v1``'s ``state`` only for a store-backed site: one with data + hooks of its own keeps its state where ``self.store`` can't see it.""" + served = super().served_extensions(extensions) + if self.uses_store: + return served + return [_without_method(e, WEBSITE_STATE_OP) if e.uri == WEBSITE_EXTENSION_URI else e for e in served] + @extension(CLOCK_EXTENSION_URI, description="Sync this site to the gateway's virtual clock.") async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) -> dict: """Follow the gateway's virtual time at ``env_get_time_url`` from now on, in ``timezone`` when the task names @@ -204,8 +270,9 @@ async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) @extension(WEBSITE_EXTENSION_URI, op="openapi", method="GET", description=( "A site's server: domains, its health path and its @api routes by name; openapi returns the OpenAPI document " - "of its routes, ready whether the site is usable now and why not, and session the cookies a browser adds to " - "start signed in as user, or signed out without one.")) + "of its routes, ready whether the site is usable now and why not, state the task state it is in (its " + "initial and current state and their diff), and session the cookies a browser adds to start signed in as user, " + "or signed out without one.")) async def _website_openapi(self) -> dict: name = self._application.environment_card.name if self._application is not None else type(self).__name__ return openapi(name, self.routes()) @@ -217,6 +284,16 @@ async def _website_ready(self) -> dict: problems = await self._not_ready() return WebsiteReadiness(ready=not problems, problems=problems).model_dump() + @extension(WEBSITE_EXTENSION_URI, op=WEBSITE_STATE_OP, method="GET") + async def _website_state(self, sid: Optional[str] = None) -> dict: + """The site's task state, a ``WebsiteState``: its state when the task started, now, and what changed between + them, of session ``sid``'s store (the default session's without one), with the site's ``changed_sessions`` as + ``/go`` names them, or of a models site's database, which keeps one.""" + if self._models is not None: + return WebsiteState.model_validate(await self._models.state()).model_dump() + return {**WebsiteState.model_validate(self._store.session(sid).state()).model_dump(), + "changed_sessions": self._store.changed_sessions()} + @extension(WEBSITE_EXTENSION_URI, op="session") async def _website_session(self, user: Optional[str] = None) -> dict: cookie = self._session_signer().cookie(user) @@ -224,15 +301,20 @@ async def _website_session(self, user: Optional[str] = None) -> dict: @reset_data async def _store_reset(self) -> None: - self.store.reset() + if self._models is not None: + return await self._models.reset() + self._store.reset() @add_data async def _store_add(self, parts: list) -> None: - self.store.load_parts(parts) + if self._models is not None: + return await self._models.load_parts(parts) + self._store.load_parts(parts) + self._store.mark_initial() @get_data async def _store_get(self) -> list: - return [DataPart(data=self.store.export())] + return [DataPart(data=await self._models.export() if self._models is not None else self._store.export())] async def _not_ready(self) -> list[str]: return [check.__name__ for check, _marker in _marked_members(self, _READY_ATTR) if not await _ready(check)] @@ -252,6 +334,15 @@ async def _session(self, request: Request) -> Response: return context.apply(JSONResponse({"user": context.user, "role": context.role, "signed_in": context.user is not None})) + def _serves_state_path(self) -> bool: + """Whether the site serves its state at ``STATE_PATH``: it is store-backed and no ``@api`` route takes the path.""" + return self.uses_store and STATE_PATH not in {openapi_path(r.path) for r in self.routes()} + + async def _state(self, request: Request) -> Response: + if self._models is not None: + return JSONResponse(await self._models.state()) + return JSONResponse({**self.store.state(), "changed_sessions": self._store.changed_sessions()}) + def _clock(self) -> UniverseClock: if not hasattr(self, "clock"): raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") @@ -272,6 +363,12 @@ def _site_path(cls, path: str) -> Path: setattr(getattr(WebsiteEnvironment, _hook), _DEFAULT_HOOK_ATTR, True) +def _without_method(extension: EnvironmentExtension, method: str) -> EnvironmentExtension: + params = dict(extension.params or {}) + params["methods"] = {name: entry for name, entry in (params.get("methods") or {}).items() if name != method} + return extension.model_copy(update={"params": params}) + + async def _ready(check: Any) -> bool: try: return bool(await _maybe_await(check())) @@ -319,6 +416,34 @@ def _first(value: bytes | None) -> str: return value.decode("latin-1").split(",")[0].strip() if value else "" +class _Session: + """Runs the request in its session (``active_sid``): its ``sid`` query parameter's, else its ``cookie``'s; an + answer to a request naming a session other than its cookie's sets the cookie, so a page opened at ``?sid=`` keeps + its session for the requests it makes after.""" + + def __init__(self, app: Any, cookie: str) -> None: + self.app, self.cookie = app, cookie + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] != "http": + return await self.app(scope, receive, send) + request = Request(scope) + named, kept = request.query_params.get("sid"), request.cookies.get(self.cookie) + sid = session_id(named if named is not None else kept) + + async def remembered(message: dict) -> None: + if message["type"] == "http.response.start" and named is not None and session_id(kept) != sid: + MutableHeaders(scope=message).append( + "set-cookie", f"{self.cookie}={sid or DEFAULT_SID}; Path=/; HttpOnly; SameSite=Lax") + await send(message) + + token = active_sid.set(sid) + try: + await self.app(scope, receive, remembered) + finally: + active_sid.reset(token) + + class _Universe: """Stamps ``X-Universe-Now`` on every answer, and with ``clock_patch`` puts the clock script in its whole HTML (never a range of it), holding only the HTML up to its ```` (at most ``HEAD_SCAN_BYTES``), so a page streamed in chunks reaches the browser diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py index 939803342..15158ddc8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py @@ -26,7 +26,7 @@ from ..tool_context import DEFAULT_ROLE, caller_from_request from ..types import RPC_PATH, WELL_KNOWN_PATH from .errors import ApiError, BadRequest, Forbidden -from .session import SESSION_COOKIE, SessionSigner +from .session import SessionSigner logger = logging.getLogger(__name__) @@ -154,7 +154,7 @@ def _set_session(self, user: str | None) -> None: def apply(self, response: Response) -> Response: for value, expires in self._set_cookie: at = datetime.datetime.fromtimestamp(expires, datetime.timezone.utc) - response.set_cookie(SESSION_COOKIE, value, expires=at, path="/", httponly=True, samesite="lax") + response.set_cookie(self._signer.cookie_name, value, expires=at, path="/", httponly=True, samesite="lax") return response diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py index 20e4ad36f..b798afcd4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py @@ -1,6 +1,8 @@ """A site's signed session cookie: who the browser is signed in as, or that it signed out, signed with HMAC-SHA256 under ``WEBSITE_SESSION_SECRET``, or a secret of the process's own when that isn't set. The cookie names no domain, -so it holds under whatever host the site is reached at.""" +so it holds under whatever host the site is reached at, and its name is the site's own, ``agentenv_session_``, so +two sites a browser reaches under one gateway host (``/website/a/``, ``/website/b/``) keep a session each; a site still +reads the shared ``agentenv_session`` of older sites, for one release.""" from __future__ import annotations import base64 @@ -10,19 +12,46 @@ import hmac import json import os +import re import secrets import time SESSION_COOKIE = "agentenv_session" +SID_COOKIE = "agentenv_sid" SECRET_ENV = "WEBSITE_SESSION_SECRET" SESSION_TTL_S = 7 * 24 * 3600 SECRET_BYTES = 32 +_NOT_TOKEN = re.compile(r"[^A-Za-z0-9_.-]") + + +def _cookie_suffix(environment_name: str) -> str: + """The part of a site's cookie names that is its own: the name itself when a cookie name can hold each of its + characters, else the name with each such character made ``_`` and ``~`` and a hash of the whole name appended. No + name kept as itself holds a ``~``, so two distinct names never make one suffix.""" + if not _NOT_TOKEN.search(environment_name): + return environment_name + digest = hashlib.sha256(environment_name.encode("utf-8", "surrogatepass")).hexdigest()[:12] + return f"{_NOT_TOKEN.sub('_', environment_name)}~{digest}" + + +def session_cookie_name(environment_name: str | None) -> str: + """The session cookie of the site named ``environment_name``: ``agentenv_session_``, a name a cookie name + can't hold spelled as ``_cookie_suffix`` spells it; the shared ``agentenv_session`` for a site with no name.""" + return f"{SESSION_COOKIE}_{_cookie_suffix(environment_name)}" if environment_name else SESSION_COOKIE + + +def sid_cookie_name(environment_name: str | None) -> str: + """The cookie in which the site named ``environment_name`` keeps the store session its browser last named: + ``agentenv_sid_``, made as ``session_cookie_name`` makes its name, ``agentenv_sid`` for a site with no name. + Its prefix is no session cookie's, so no site's store-selection cookie is another site's session cookie.""" + return f"{SID_COOKIE}_{_cookie_suffix(environment_name)}" if environment_name else SID_COOKIE class SessionSigner: - def __init__(self, secret: str | bytes | None = None) -> None: + def __init__(self, secret: str | bytes | None = None, cookie_name: str = SESSION_COOKIE) -> None: secret = secret if secret is not None else os.environ.get(SECRET_ENV) self._key = (secret.encode() if isinstance(secret, str) else secret) or secrets.token_bytes(SECRET_BYTES) + self.cookie_name = cookie_name def encode(self, user: str | None, ttl_s: int = SESSION_TTL_S) -> tuple[str, int]: """The cookie value for ``user`` signed in, or signed out when None, and when it expires, in epoch @@ -45,11 +74,18 @@ def decode(self, value: str | None) -> dict | None: return None return {"user": data.get("u")} + def read(self, cookies: dict) -> dict | None: + """The session of a request's ``cookies``: its own cookie's, else the shared legacy cookie's.""" + session = self.decode(cookies.get(self.cookie_name)) + if session is None and self.cookie_name != SESSION_COOKIE: + session = self.decode(cookies.get(SESSION_COOKIE)) + return session + def cookie(self, user: str | None, ttl_s: int = SESSION_TTL_S) -> dict: """The cookie a browser sets to be signed in as ``user``, as a browser automation adds one; its domain is left to whoever adds it.""" value, expires = self.encode(user, ttl_s) - return {"name": SESSION_COOKIE, "value": value, "path": "/", "expires": expires, "httpOnly": True, + return {"name": self.cookie_name, "value": value, "path": "/", "expires": expires, "httpOnly": True, "sameSite": "Lax"} def _sign(self, payload: str) -> str: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py index 657ea78f0..63c316a75 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py @@ -1,11 +1,19 @@ -"""The default data store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON -object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as; -the data plane loads, exports and resets it.""" +"""The default state store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON +object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as. +It remembers the state it was last loaded with, so a verifier can read the initial state, the current one and the +difference between them. + +A store also keeps sessions, as CUA-Gym sites do, so browsers that each name a session id (``?sid=``) never see one +another's changes: ``session(sid)`` is a store of its own, started on first use as a copy of this store's initial +state, and the session ``default``, or none, is this store itself. ``reset`` drops every session; a session started +before a later ``load`` keeps the state it was started with.""" from __future__ import annotations import base64 +import contextvars import copy import json +import re from pathlib import Path from typing import Any, Iterable from urllib.parse import unquote, urlparse @@ -13,6 +21,17 @@ from ..types import DataPart, FilePart, Part, TextPart CURRENT_USER = "current_user" +DEFAULT_SID = "default" +SID_MAX = 128 +_NOT_SID = re.compile(r"[^A-Za-z0-9_-]") +active_sid: contextvars.ContextVar[str | None] = contextvars.ContextVar("agentenv_website_sid", default=None) + + +def session_id(value: str | None) -> str | None: + """The session id ``value`` names, as CUA-Gym sites read ``sid``: its letters, digits, ``_`` and ``-``, at most + ``SID_MAX`` of them; None, the default session, for one that is empty or ``default``.""" + sid = _NOT_SID.sub("", value or "")[:SID_MAX] + return None if sid in ("", DEFAULT_SID) else sid class Collections: @@ -21,12 +40,33 @@ class Collections: def __init__(self, data: dict | None = None) -> None: self._collections: dict[str, list[dict]] = {} self.current_user: str | None = None + self.initial: dict = {} + self._sessions: dict[str, Collections] = {} if data: self.load(data) + self.mark_initial() def __getitem__(self, name: str) -> list[dict]: return self._collections.setdefault(name, []) + def session(self, sid: str | None) -> Collections: + """The store of session ``sid``, started as a copy of this one's initial state on first use; this store + itself for the default session.""" + sid = session_id(sid) + if sid is None: + return self + if sid not in self._sessions: + self._sessions[sid] = Collections(self.initial) + return self._sessions[sid] + + def sessions(self) -> list[str]: + """The ids of the sessions started since the store was last reset, the default one aside.""" + return list(self._sessions) + + def changed_sessions(self) -> list[str]: + """The ids of the sessions, the default one aside, whose state differs from the one they started with.""" + return [sid for sid, store in self._sessions.items() if store.diff()] + def names(self) -> list[str]: return list(self._collections) @@ -70,11 +110,13 @@ def load(self, data: dict) -> None: def load_parts(self, parts: Iterable[Part]) -> None: for part in parts: - self.load(_part_data(part)) + self.load(part_data(part)) def reset(self) -> None: self._collections.clear() self.current_user = None + self.initial = {} + self._sessions.clear() def export(self) -> dict: data: dict = copy.deepcopy(self._collections) @@ -82,8 +124,49 @@ def export(self) -> dict: data[CURRENT_USER] = self.current_user return data + def mark_initial(self) -> None: + self.initial = self.export() + + def diff(self) -> dict: + return state_diff(self.initial, self.export()) + + def state(self) -> dict: + return verifier_state(self.initial, self.export()) + + +def state_diff(initial: dict, current: dict, key: Any = None) -> dict: + """Each collection that changed from ``initial`` to ``current``, as its ``added``, ``removed`` and ``changed`` + objects, matched by ``key(name, doc)``, their ``id`` by default; a changed one as its ``id``, ``before`` and + ``after``.""" + key = key or (lambda name, doc: _key(doc)) + changes: dict = {} + for name in dict.fromkeys([*initial, *current]): + if name == CURRENT_USER: + continue + before = {key(name, d): d for d in initial.get(name, [])} + after = {key(name, d): d for d in current.get(name, [])} + entry = { + "added": [d for k, d in after.items() if k not in before], + "removed": [d for k, d in before.items() if k not in after], + "changed": [{"id": d.get("id"), "before": before[k], "after": d} + for k, d in after.items() if k in before and before[k] != d], + } + if any(entry.values()): + changes[name] = entry + return changes + + +def verifier_state(initial: dict, current: dict, key: Any = None) -> dict: + """What ``/go`` answers: ``{initial_state, current_state, state_diff}``.""" + return {"initial_state": copy.deepcopy(initial), "current_state": current, + "state_diff": state_diff(initial, current, key)} + + +def _key(doc: dict) -> str: + return json.dumps(doc.get("id")) if doc.get("id") is not None else json.dumps(doc, sort_keys=True) + -def _part_data(part: Part) -> dict: +def part_data(part: Part) -> dict: """A part's JSON: a data part's data, a text part's text, or a file part's bytes or ``file://`` file.""" if isinstance(part, DataPart): return part.data diff --git a/packages/agentenv-protocol/tests/test_website_conformance.py b/packages/agentenv-protocol/tests/test_website_conformance.py index 281561bd0..af03260b3 100644 --- a/packages/agentenv-protocol/tests/test_website_conformance.py +++ b/packages/agentenv-protocol/tests/test_website_conformance.py @@ -6,6 +6,7 @@ import datetime from pathlib import Path +import httpx import pytest from starlette.applications import Starlette from starlette.requests import Request @@ -17,7 +18,7 @@ from agentenv_protocol.website import conformance from agentenv_protocol.website.clock import UNIVERSE_NOW_HEADER, iso -CHECKS = ["card", "any_host", "health", "universe_now", "data_plane", "clock", "forwarded", "cookies"] +CHECKS = ["card", "any_host", "health", "universe_now", "reserved_paths", "data_plane", "clock", "forwarded", "cookies"] def statuses(results: list[conformance.Result]) -> dict[str, str]: @@ -36,6 +37,7 @@ def notes(self) -> list: env = Notes() env.store.load({"current_user": "ann", "notes": [{"id": 1, "text": "hello"}]}) + env.store.mark_initial() return env @@ -56,10 +58,12 @@ def _handwritten(*, leak_host: bool = False, domain_cookie: bool = False, clock: clock_ops_first: tuple[str, ...] = (), leak_redirect: bool = False, stamp_rpc: bool = True, reset_keeps: bool = False, get_fails_after_reset: bool = False, rpc_reply: tuple[int, dict] | None = None, - ready_answer: object = None, ready_status: int = 200, rpc_path: str | None = None) -> Starlette: + ready_answer: object = None, ready_status: int = 200, rpc_path: str | None = None, + state_path: str | None = None, changed: bool = False) -> Starlette: """A server written without the SDK: the contract by hand, optionally broken. Its state is its ``state.data``; with ``ready_answer`` its card advertises ``website/v1``'s ``ready``, which answers it with ``ready_status``. With ``rpc_path`` its card's - ``url`` names its data plane there, not at ``/agentenv``.""" + ``url`` names its data plane there, not at ``/agentenv``, and with ``state_path`` its ``params.state.path`` names + where it serves its state, which shows changes since the initial state when ``changed``.""" state = {"items": ["a"], "url": None, "at": None, "reset": False} def now() -> datetime.datetime: @@ -94,6 +98,8 @@ async def card(request: Request) -> Response: "routes": [{"method": "GET", "path": "/items", "name": "items"}]}}] if ready_answer is not None: extensions[0]["params"]["methods"] = {"ready": {"method": "GET", "endpoint": "/ready"}} + if state_path is not None: + extensions[0]["params"]["state"] = {"path": state_path} if clock: extensions.append({"uri": "urn:agentenv:clock/v1", "params": { "endpoint": "/agentenv/ext/sync_time", "methods": { @@ -103,6 +109,9 @@ async def card(request: Request) -> Response: "capabilities": {"extensions": extensions, "operations": ["data/reset", "data/add", "data/get"]}})) + async def site_state(request: Request) -> Response: + return JSONResponse({"state_diff": {"items": {"added": ["b"]}} if changed else {}}) + async def rpc(request: Request) -> Response: body = await request.json() method = body["method"] @@ -132,10 +141,15 @@ async def sync_time(request: Request) -> Response: state["at"] = datetime.datetime.fromisoformat(virtual.replace("Z", "+00:00")) return stamp(JSONResponse({"gateway_virtual_time": virtual})) + async def unrouted(request: Request, exc: Exception) -> Response: + return stamp(JSONResponse({"error": "not found"}, status_code=getattr(exc, "status_code", 404))) + app = Starlette(routes=[Route("/", page), Route("/healthz", health), Route("/ready", ready), Route("/items", items), Route("/items/", items_slash), Route("/.well-known/agent-env.json", card), Route(rpc_path or "/agentenv", rpc, methods=["POST"]), - Route("/agentenv/ext/sync_time", sync_time, methods=["POST"])]) + Route("/agentenv/ext/sync_time", sync_time, methods=["POST"]), + *([Route(state_path, site_state)] if state_path else [])], + exception_handlers={404: unrouted, 405: unrouted}) app.state.data = state return app @@ -181,7 +195,7 @@ def test_an_answer_to_ready_that_isnt_a_website_readiness_fails_health_saying_wh def test_a_server_breaking_the_contract_fails_those_checks() -> None: results = statuses(conformance.check("http://testserver", client=TestClient(_handwritten(leak_host=True, domain_cookie=True, clock=False)))) - assert results["forwarded"] == results["cookies"] == results["clock"] == "failed" + assert results["forwarded"] == results["cookies"] == results["clock"] == "warning" assert results["card"] == results["health"] == results["data_plane"] == "passed" @@ -190,13 +204,13 @@ def test_the_forwarded_check_follows_no_redirect_and_reads_its_location(sdk_site assert forwarded.status == "passed" and "/api/notes/" in forwarded.detail leaked = {r.name: r for r in conformance.check("http://testserver", client=TestClient(_handwritten(leak_redirect=True)))}["forwarded"] - assert leaked.status == "failed" and "Location of /items/" in leaked.detail + assert leaked.status == "warning" and "Location of /items/" in leaked.detail def test_universe_now_is_checked_on_the_answers_of_every_check() -> None: result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(_handwritten(stamp_rpc=False)))}["universe_now"] - assert result.status == "failed" and "/agentenv" in result.detail + assert result.status == "warning" and "/agentenv" in result.detail def test_a_reset_that_clears_nothing_fails_the_data_plane_and_leaves_the_data() -> None: @@ -220,14 +234,28 @@ def test_a_data_request_without_a_2xx_json_rpc_result_fails_the_data_plane(reply assert result.status == "failed" and "no JSON-RPC result" in result.detail -def test_an_sdk_sites_data_round_trips_and_is_left_as_the_check_found_it_changed_or_not( +def test_an_sdk_site_keeps_its_initial_state_and_one_changed_since_skips_the_round_trip( + notes: WebsiteEnvironment, sdk_site: TestClient) -> None: + before = sdk_site.get("/go").json() + conformance.check("http://testserver", client=sdk_site) + assert sdk_site.get("/go").json() == before + notes.store.insert("notes", {"text": "new"}) + changed = sdk_site.get("/go").json() + assert changed["state_diff"] + result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] + assert result.status == "skipped" and "website/v1 state shows changes" in result.detail + assert sdk_site.get("/go").json() == changed + + +def test_a_named_session_with_changes_skips_the_round_trip_and_keeps_its_state( notes: WebsiteEnvironment, sdk_site: TestClient) -> None: - for _ in range(2): - before = notes.store.export() - result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] - assert result.status == "passed", result - assert notes.store.export() == before - notes.store.insert("notes", {"text": "new"}) + sdk_site.get("/api/notes", params={"sid": "task-1"}) + assert {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"].status == "passed" + notes.store.session("task-2")["notes"].append({"id": 2, "text": "draft"}) + result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] + assert result.status == "skipped" and "named sessions" in result.detail + assert sdk_site.get("/go", params={"sid": "task-2"}).json()["state_diff"]["notes"]["added"] == [ + {"id": 2, "text": "draft"}] def test_the_data_plane_is_reached_at_the_cards_url() -> None: @@ -243,22 +271,82 @@ def test_the_clock_check_calls_sync_time_whichever_operation_the_card_lists_firs assert results["clock"] == "passed", results -def test_a_server_with_no_card_health_or_header_fails_them() -> None: +@pytest.mark.parametrize(("changed", "status"), [(True, "skipped"), (False, "passed")]) +def test_a_card_without_state_is_read_at_the_state_path_it_names(changed: bool, status: str) -> None: + app = _handwritten(state_path="/grading/state", changed=changed) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == status, result + assert ("/grading/state shows changes" in result.detail) is changed + + +def test_a_server_with_no_card_health_or_header_fails_the_contract_and_warns_of_the_rest() -> None: bare = Starlette(routes=[Route("/", lambda request: JSONResponse({"not": "html"}))]) - results = statuses(conformance.check("http://testserver", client=TestClient(bare))) - assert results["card"] == results["any_host"] == results["health"] == results["universe_now"] == "failed" - assert results["data_plane"] == results["clock"] == "failed" - assert results["cookies"] == "skipped" + results = conformance.check("http://testserver", client=TestClient(bare)) + assert statuses(results) == {"card": "warning", "any_host": "failed", "health": "failed", "universe_now": "warning", + "reserved_paths": "passed", "data_plane": "skipped", "clock": "warning", + "forwarded": "passed", "cookies": "skipped"} + assert [r.name for r in results if r.required] == ["any_host", "health", "reserved_paths", "data_plane"] + + +def test_a_bare_server_meeting_only_the_contract_passes_with_warnings() -> None: + async def page(request: Request) -> Response: + return HTMLResponse("hi") + + bare = Starlette(routes=[Route("/", page), Route("/api/health", lambda request: JSONResponse({"ok": True}))]) + results = conformance.check("http://testserver", client=TestClient(bare)) + assert not any(r.fails() for r in results) and any(r.fails(strict=True) for r in results) + assert {r.name for r in results if r.status == "warning"} == {"card", "universe_now", "clock"} + + +def test_a_spa_fallback_that_serves_the_reserved_paths_fails() -> None: + async def page(request: Request) -> Response: + return HTMLResponse("app") + + spa = Starlette(routes=[Route("/{path:path}", page)]) + reserved = {r.name: r for r in conformance.check("http://testserver", client=TestClient(spa))}["reserved_paths"] + assert reserved.status == "failed" and reserved.detail.startswith("/agentenv answered a page") + + +@pytest.mark.parametrize("exact", ["/agentenv", "/.well-known/agent-env.json"]) +def test_a_page_at_a_reserved_path_itself_fails(exact: str) -> None: + async def page(request: Request) -> Response: + return HTMLResponse("app") + + server = Starlette(routes=[Route("/", page), Route(exact, page)]) + reserved = {r.name: r for r in conformance.check("http://testserver", client=TestClient(server))}["reserved_paths"] + assert reserved.status == "failed" and reserved.detail.startswith(f"{exact} answered a page") def test_main_prints_each_check_and_exits_by_outcome(monkeypatch, capsys) -> None: - monkeypatch.setattr(conformance, "check", lambda url, **options: [conformance.Result("card", "passed"), - conformance.Result("clock", "failed", "late")]) - assert conformance.main(["http://site"]) == 1 + results = [conformance.Result("health", "passed"), conformance.Result("clock", "warning", "late", required=False)] + monkeypatch.setattr(conformance, "check", lambda url, **options: results) + assert conformance.main(["http://site"]) == 0 printed = capsys.readouterr().out - assert "PASS card" in printed and "FAIL clock: late" in printed + assert "PASS health (required)" in printed and "WARN clock (recommended): late" in printed + assert "1 passed, 0 failed, 1 warning(s), 0 skipped" in printed + assert conformance.main(["http://site", "--strict"]) == 1 + results.append(conformance.Result("any_host", "failed", "json")) + assert conformance.main(["http://site"]) == 1 + assert "FAIL any_host (required): json" in capsys.readouterr().out def test_main_exits_2_when_the_server_cannot_be_reached(capsys) -> None: assert conformance.main(["http://127.0.0.1:9", "--timeout", "2"]) == 2 assert "couldn't reach" in capsys.readouterr().err + + +def test_the_cookie_check_holds_the_session_path_to_the_cookie_the_card_names(sdk_site: TestClient) -> None: + card = sdk_site.get("/.well-known/agent-env.json").json() + for extension in card["capabilities"]["extensions"]: + if extension["uri"] == "urn:agentenv:website/v1": + extension["params"]["session_cookie"] = "agentenv_session_other" + real = sdk_site.get + + def get(url, **kwargs): + if url.endswith("/.well-known/agent-env.json"): + return httpx.Response(200, json=card, request=httpx.Request("GET", url)) + return real(url, **kwargs) + + sdk_site.get = get + cookies = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["cookies"] + assert cookies.status == "warning" and "not the card's agentenv_session_other" in cookies.detail diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py new file mode 100644 index 000000000..1b12d96de --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -0,0 +1,213 @@ +"""A site of ``@website(models=...)``: its state is a database's, SQLAlchemy models' tables on DATABASE_URL (sqlite +here) or an object's own load/export/reset, which its data plane and /go read and write, and its routes reach +through ``self.db``.""" + +from __future__ import annotations + +import datetime +import sys +import types +from pathlib import Path + +import pytest +from starlette.testclient import TestClient + +from agentenv_protocol.website import ApiContext, WebsiteEnvironment, api, conformance, website + +sqlalchemy = pytest.importorskip("sqlalchemy") +from sqlalchemy import DateTime, ForeignKey, Integer, String, select # noqa: E402 +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column # noqa: E402 + +RPC = "/agentenv" + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = "users" + id: Mapped[str] = mapped_column(String, primary_key=True) + name: Mapped[str] = mapped_column(String) + + +class Ticket(Base): + __tablename__ = "tickets" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + owner: Mapped[str] = mapped_column(ForeignKey("users.id")) + title: Mapped[str] = mapped_column(String) + opened_at: Mapped[datetime.datetime] = mapped_column(DateTime) + + +SEED = {"current_user": "ann", "users": [{"id": "ann", "name": "Ann"}], + "tickets": [{"id": 1, "owner": "ann", "title": "printer", "opened_at": "2031-01-02T03:04:05"}]} + + +def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}).json() + assert "error" not in body, body + return body["result"] + + +@pytest.fixture +def dist(tmp_path: Path) -> Path: + (tmp_path / "index.html").write_text("desk") + return tmp_path + + +@pytest.fixture +def database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> str: + url = f"sqlite:///{tmp_path / 'service.db'}" + monkeypatch.setenv("DATABASE_URL", url) + return url + + +def helpdesk(dist: Path, models: object) -> type[WebsiteEnvironment]: + @website(domains=["desk.example"], static=str(dist), models=models) + class Helpdesk(WebsiteEnvironment): + @api.get("/api/tickets") + def tickets(self, ctx: ApiContext) -> list: + with self.db() as session: + return [{"id": t.id, "title": t.title} for t in session.scalars( + select(Ticket).where(Ticket.owner == ctx.user).order_by(Ticket.id))] + + @api.post("/api/tickets", status=201) + def open_ticket(self, body: dict, ctx: ApiContext) -> dict: + with self.db.begin() as session: + ticket = Ticket(owner=ctx.user, title=body["title"], opened_at=self.now().replace(tzinfo=None)) + session.add(ticket) + session.flush() + return {"id": ticket.id} + + return Helpdesk + + +def test_the_data_plane_loads_exports_and_resets_the_models_tables(dist: Path, database: str) -> None: + env = helpdesk(dist, Base)() + client = TestClient(env.create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + assert client.get("/api/session").json()["user"] == "ann" + assert client.get("/api/tickets").json() == [{"id": 1, "title": "printer"}] + rpc(client, "data/reset") + assert rpc(client, "data/get")["parts"][0]["data"] == {"users": [], "tickets": []} + + +def test_rows_that_leave_an_optional_column_out_load_as_given_in_either_order(dist: Path, database: str) -> None: + class Threads(DeclarativeBase): + pass + + class Note(Threads): + __tablename__ = "notes" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + parent_id: Mapped[int | None] = mapped_column(ForeignKey("notes.id"), nullable=True) + + client = TestClient(helpdesk(dist, Threads)().create_app()) + for seed in ([{"id": 1}, {"id": 2, "parent_id": 1}, {"id": 3, "parent_id": 1}, {"id": 4}], + [{"id": 1, "parent_id": None}, {"id": 2}, {"id": 3, "parent_id": 2}]): + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"notes": seed}}]}) + assert rpc(client, "data/get")["parts"][0]["data"]["notes"] == [{"parent_id": None, **row} for row in seed] + + +def test_a_row_the_gui_adds_is_the_mcp_servers_and_go_reports_it(dist: Path, database: str) -> None: + env = helpdesk(dist, Base)() + client = TestClient(env.create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) + assert client.post("/api/tickets", json={"title": "badge"}).json() == {"id": 2} + server = sqlalchemy.create_engine(database) + with server.connect() as connection: + assert connection.execute(select(Ticket.title).order_by(Ticket.id)).scalars().all() == ["printer", "badge"] + connection.execute(sqlalchemy.update(Ticket).where(Ticket.id == 1).values(title="printer jam")) + connection.commit() + state = client.get("/go", params={"sid": "default"}).json() + assert state["initial_state"] == SEED + diff = state["state_diff"]["tickets"] + assert [t["title"] for t in diff["added"]] == ["badge"] and diff["removed"] == [] + assert diff["changed"][0]["before"]["title"] == "printer" and diff["changed"][0]["after"]["title"] == "printer jam" + assert client.get("/go", params={"sid": "other"}).json() == state + + +def test_a_models_site_passes_the_conformance_check(dist: Path, database: str) -> None: + client = TestClient(helpdesk(dist, Base)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) + results = conformance.check("http://testserver", client=client) + assert all(r.status == "passed" for r in results), results + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + + +def test_models_may_be_a_metadata_or_a_module_and_refuse_anything_else(dist: Path, database: str) -> None: + module = types.ModuleType("helpdesk_models") + module.Base, module.Ticket = Base, Ticket + for models in (Base.metadata, module): + assert helpdesk(dist, models)()._tables.metadata is Base.metadata + with pytest.raises(TypeError, match="models="): + helpdesk(dist, object()) + with pytest.raises(ValueError, match="data/add"): + website(models=Base, data="seed.json") + + +def test_unknown_tables_and_a_missing_database_url_are_refused(dist: Path, monkeypatch: pytest.MonkeyPatch, + database: str) -> None: + client = TestClient(helpdesk(dist, Base)().create_app()) + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": "data/add", + "params": {"parts": [{"kind": "data", "data": {"orders": []}}]}}).json() + assert "error" in body + monkeypatch.delenv("DATABASE_URL") + with pytest.raises(RuntimeError, match="DATABASE_URL"): + helpdesk(dist, Base)().db() + + +def test_a_database_url_whose_driver_isnt_installed_names_it(dist: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "psycopg2", None) + monkeypatch.setitem(sys.modules, "psycopg", None) + monkeypatch.setenv("DATABASE_URL", "postgresql://desk@localhost/service") + with pytest.raises(ImportError, match=r"postgresql:// needs the database driver psycopg, .* carries psycopg"): + helpdesk(dist, Base)().db() + + +def test_an_object_with_its_own_hooks_is_the_sites_data(dist: Path) -> None: + class Rows: + def __init__(self) -> None: + self.rows: list = [] + + def load(self, data: dict) -> None: + self.rows += data["rows"] + + async def export(self) -> dict: + return {"rows": list(self.rows)} + + def reset(self) -> None: + self.rows.clear() + + rows = Rows() + env = helpdesk(dist, rows)() + client = TestClient(env.create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"current_user": "bo", "rows": [{"id": 1}]}}]}) + rows.rows.append({"id": 2}) + assert rpc(client, "data/get")["parts"][0]["data"] == {"current_user": "bo", "rows": [{"id": 1}, {"id": 2}]} + assert client.get("/go").json()["state_diff"] == {"rows": {"added": [{"id": 2}], "removed": [], "changed": []}} + assert env.acting_user() == "bo" + assert env.db is None + + +def test_editing_a_row_a_hook_still_owns_leaves_the_initial_state(dist: Path) -> None: + class Rows: + def __init__(self) -> None: + self.rows: list = [] + + def load(self, data: dict) -> None: + self.rows += data["rows"] + + def export(self) -> dict: + return {"rows": list(self.rows)} + + def reset(self) -> None: + self.rows.clear() + + rows = Rows() + client = TestClient(helpdesk(dist, rows)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"rows": [{"id": 1, "status": "open"}]}}]}) + rows.rows[0]["status"] = "closed" + assert client.get("/go").json()["state_diff"] == {"rows": {"added": [], "removed": [], "changed": [ + {"id": 1, "before": {"id": 1, "status": "open"}, "after": {"id": 1, "status": "closed"}}]}} diff --git a/packages/agentenv-protocol/tests/test_website_environment.py b/packages/agentenv-protocol/tests/test_website_environment.py index 05c118591..e2ccfccbb 100644 --- a/packages/agentenv-protocol/tests/test_website_environment.py +++ b/packages/agentenv-protocol/tests/test_website_environment.py @@ -37,6 +37,7 @@ from agentenv_protocol.website.clock import parse_time RPC = "/agentenv" +COOKIE = "agentenv_session_Shop" def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: @@ -238,6 +239,39 @@ def test_a_sites_data_is_read_through_the_data_plane_alone(client: TestClient) - assert "state" not in website_params and "state" not in website_params.get("methods", {}) +def test_each_sid_keeps_a_state_of_its_own_started_from_the_initial_state(client: TestClient, + shop: WebsiteEnvironment) -> None: + client.post("/api/orders", json={"sku": "mug"}) + assert TestClient(shop.asgi).get("/go", params={"sid": "task-7"}).json() == { + "initial_state": SEED, "current_state": SEED, "state_diff": {}, "changed_sessions": []} + page = client.get("/", params={"sid": "task-7"}) + assert page.headers["set-cookie"].startswith("agentenv_sid_Shop=task-7;") + assert "set-cookie" not in client.get("/", params={"sid": "task-7"}).headers + client.post("/api/orders", json={"sku": "pen"}) + client.post("/api/orders", params={"sid": "other"}, json={"sku": "cup"}) + added = {sid: [o.get("sku") for d in [client.get("/go", params={"sid": sid}).json()["state_diff"]] + for o in d.get("orders", {}).get("added", [])] for sid in ("default", "task-7", "other")} + assert added == {"default": ["mug"], "task-7": ["pen"], "other": ["cup"]} + assert client.get("/go").json()["state_diff"]["orders"]["added"][0]["sku"] == "cup" + client.get("/", params={"sid": "default"}) + assert client.get("/go").json()["state_diff"]["orders"]["added"][0]["sku"] == "mug" + assert rpc(client, "data/get")["parts"][0]["data"]["orders"][-1]["sku"] == "mug" + assert sorted(shop._store.sessions()) == ["other", "task-7"] + assert sorted(client.get("/go").json()["changed_sessions"]) == ["other", "task-7"] + rpc(client, "data/reset") + assert shop._store.sessions() == [] and client.get("/go", params={"sid": "task-7"}).json()["current_state"] == {} + + +def test_a_session_id_is_sanitised_as_cua_gym_sites_do() -> None: + from agentenv_protocol.website.store import session_id + + assert [session_id(v) for v in (None, "", "default", "../a b", "x" * 200)] == [None, None, None, "ab", "x" * 128] + store = Collections({"items": [{"id": 1}]}) + store.session("s")["items"].append({"id": 2}) + assert store.session("default") is store and store.export() == {"items": [{"id": 1}]} + assert store.session("s").diff() == {"items": {"added": [{"id": 2}], "removed": [], "changed": []}} + + def test_broad_routes_leave_the_built_in_ones_answering_their_paths(dist: Path) -> None: class Broad(site(dist)): @api.get("/{slug}") @@ -291,7 +325,7 @@ def test_the_acting_user_is_signed_in_until_a_session_says_else(client: TestClie first = client.get("/api/session") assert first.json() == {"user": "alice", "role": "default", "signed_in": True} cookie = first.headers["set-cookie"] - assert cookie.startswith(f"{SESSION_COOKIE}=") and "domain" not in cookie.lower() and "httponly" in cookie.lower() + assert cookie.startswith(f"{COOKIE}=") and "domain" not in cookie.lower() and "httponly" in cookie.lower() client.post("/api/sign-in", json={"user": "bob"}) assert client.get("/api/orders").json() == [{"id": 1, "user": "bob"}] assert client.get("/api/session").json()["user"] == "bob" @@ -299,7 +333,7 @@ def test_the_acting_user_is_signed_in_until_a_session_says_else(client: TestClie assert client.get("/api/session").json()["user"] is None # The jar holds the site's signed-out cookie under its own domain; a second cookie of the name would race it. client.cookies.clear() - client.cookies.set(SESSION_COOKIE, "forged.signature") + client.cookies.set(COOKIE, "forged.signature") assert client.get("/api/session").json()["user"] == "alice" @@ -317,11 +351,65 @@ def test_the_session_operation_mints_cookies_for_a_user_or_signed_out(shop: Webs minted = client.post("/agentenv/ext/website/session", json={"user": "bob"}).json() assert minted["user"] == "bob" and minted["expires_at"].endswith("Z") (cookie,) = minted["cookies"] - assert cookie["name"] == SESSION_COOKIE and "domain" not in cookie and cookie["path"] == "/" - browser = TestClient(shop.asgi, cookies={SESSION_COOKIE: cookie["value"]}) + assert cookie["name"] == COOKIE and "domain" not in cookie and cookie["path"] == "/" + browser = TestClient(shop.asgi, cookies={COOKIE: cookie["value"]}) assert browser.get("/api/session").json()["user"] == "bob" signed_out = client.post("/agentenv/ext/website/session", json={}).json()["cookies"][0]["value"] - assert TestClient(shop.asgi, cookies={SESSION_COOKIE: signed_out}).get("/api/session").json()["user"] is None + assert TestClient(shop.asgi, cookies={COOKIE: signed_out}).get("/api/session").json()["user"] is None + + +def test_each_site_keeps_its_own_session_cookie_and_reads_the_legacy_one(dist: Path, monkeypatch) -> None: + monkeypatch.setenv("ENVIRONMENT_NAME", "shop a/1") + first = site(dist)() + monkeypatch.setenv("ENVIRONMENT_NAME", "shop-b") + second = site(dist)() + for env in (first, second): + env.store.load(SEED) + named = "agentenv_session_shop_a_1~030f5785d67a" + assert (first.session_cookie, second.session_cookie) == (named, "agentenv_session_shop-b") + browser = TestClient(first.create_app()) + browser.post("/api/sign-in", json={"user": "bob"}) + other = TestClient(second.create_app(), cookies=browser.cookies) + other.post("/api/sign-in", json={"user": "alice"}) + assert set(other.cookies.keys()) == {named, "agentenv_session_shop-b"} + assert TestClient(first.asgi, cookies=other.cookies).get("/api/session").json()["user"] == "bob" + card = browser.get("/.well-known/agent-env.json").json() + assert _extension(card, WEBSITE_EXTENSION_URI)["params"]["session_cookie"] == named + legacy = first._session_signer().encode("bob")[0] + assert TestClient(first.asgi, cookies={SESSION_COOKIE: legacy}).get("/api/orders").json() == [{"id": 1, "user": "bob"}] + + +def test_no_site_s_session_choice_is_another_site_s_sign_in(dist: Path, monkeypatch) -> None: + monkeypatch.setenv("ENVIRONMENT_NAME", "notes") + notes = site(dist)() + monkeypatch.setenv("ENVIRONMENT_NAME", "notes_sid") + notes_sid = site(dist)() + browser = TestClient(notes.create_app()) + browser.post("/api/orders", params={"sid": "task-1"}, json={"sku": "mug"}) + other = TestClient(notes_sid.create_app(), cookies=browser.cookies) + other.post("/api/sign-in", json={"user": "bob"}) + back = TestClient(notes.asgi, cookies=other.cookies) + assert [o["sku"] for o in back.get("/go").json()["current_state"]["orders"]] == ["mug"] + assert TestClient(notes_sid.asgi, cookies=back.cookies).get("/api/session").json()["user"] == "bob" + + +def test_sites_whose_names_spell_alike_keep_their_own_cookies(dist: Path, monkeypatch) -> None: + apps = [] + for name in ("shop a/1", "shop_a_1"): + monkeypatch.setenv("ENVIRONMENT_NAME", name) + env = site(dist)() + env.store.load(SEED) + apps.append(env.create_app()) + browser = TestClient(apps[0]) + browser.post("/api/sign-in", json={"user": "bob"}) + browser.get("/", params={"sid": "task-1"}) + other = TestClient(apps[1], cookies=browser.cookies) + other.post("/api/sign-in", json={"user": "alice"}) + other.get("/", params={"sid": "task-2"}) + assert len(other.cookies.keys()) == 4 + back = TestClient(apps[0], cookies=other.cookies) + assert back.get("/api/session").json()["user"] == "bob" + assert back.cookies.get("agentenv_sid_shop_a_1~030f5785d67a") == "task-1" def test_every_answer_carries_the_universe_time(client: TestClient) -> None: @@ -335,13 +423,16 @@ def test_every_answer_carries_the_universe_time(client: TestClient) -> None: class _Clock: """A gateway-shaped /clock/time whose time a test sets.""" - def __init__(self, virtual: str) -> None: - self.virtual = virtual + def __init__(self, virtual: str, rate: object = None) -> None: + self.virtual, self.rate = virtual, rate outer = self class Handler(BaseHTTPRequestHandler): def do_GET(self) -> None: - body = json.dumps({"virtual_time": outer.virtual}).encode() + answer = {"virtual_time": outer.virtual} + if outer.rate is not None: + answer["virtual_seconds_per_real_second"] = outer.rate + body = json.dumps(answer).encode() self.send_response(200) self.send_header("Content-Length", str(len(body))) self.end_headers() @@ -544,8 +635,8 @@ async def _clock_rates(monkeypatch) -> None: real, virtual = [0.0], [datetime.datetime(2031, 1, 1, tzinfo=datetime.timezone.utc)] - async def read(url: str) -> datetime.datetime: - return virtual[0] + async def read(url: str) -> tuple[datetime.datetime, None]: + return virtual[0], None monkeypatch.setattr(clock_module.time, "monotonic", lambda: real[0]) monkeypatch.setattr(clock_module, "_read", read) @@ -573,6 +664,39 @@ def tick(seconds: float, virtual_seconds: float) -> None: assert clock._rate == 100.0 and clock.now() == virtual[0] +def test_the_clock_extrapolates_at_the_rate_the_clock_answers_with(monkeypatch) -> None: + asyncio.run(_answered_rates(monkeypatch)) + + +async def _answered_rates(monkeypatch) -> None: + from agentenv_protocol.website import UniverseClock + + gateway = _Clock("2031-01-01T00:00:00Z", rate=60) + try: + assert await clock_module._read(gateway.url) == (parse_time("2031-01-01T00:00:00Z"), 60.0) + for unusable in ("fast", True, -1, float("inf")): + gateway.rate = unusable + assert (await clock_module._read(gateway.url))[1] is None + finally: + gateway.close() + real, answers = [0.0], [(parse_time("2031-01-01T00:00:00Z"), 60.0)] + + async def read(url: str) -> tuple[datetime.datetime, float | None]: + return answers[0] + + monkeypatch.setattr(clock_module.time, "monotonic", lambda: real[0]) + monkeypatch.setattr(clock_module, "_read", read) + clock = UniverseClock() + await clock.sync("http://gateway/clock/time") + real[0] += 0.5 + assert clock._rate == 60.0 and clock.now() == answers[0][0] + datetime.timedelta(seconds=30) + real[0] += 0.5 + answers[0] = (parse_time("2031-02-01T00:00:00Z"), 0.0) + await clock.refresh() + real[0] += 10 + assert clock._rate == 0.0 and clock.now() == answers[0][0] + + def test_a_refresh_of_the_old_clock_never_overwrites_a_sync_to_a_new_one(monkeypatch) -> None: asyncio.run(_refresh_during_sync(monkeypatch)) @@ -583,12 +707,12 @@ async def _refresh_during_sync(monkeypatch) -> None: old, new = (datetime.datetime(year, 1, 1, tzinfo=datetime.timezone.utc) for year in (2031, 2040)) release, held = asyncio.Event(), [] - async def read(url: str) -> datetime.datetime: + async def read(url: str) -> tuple[datetime.datetime, None]: if url == "http://old/clock/time": if held: await release.wait() - return old - return new + return old, None + return new, None monkeypatch.setattr(clock_module, "_read", read) monkeypatch.setattr(clock_module, "REFRESH_S", 0.0) diff --git a/packages/agentenv-protocol/tests/test_website_notes_example.py b/packages/agentenv-protocol/tests/test_website_notes_example.py index 4d7110657..93eb51cc1 100644 --- a/packages/agentenv-protocol/tests/test_website_notes_example.py +++ b/packages/agentenv-protocol/tests/test_website_notes_example.py @@ -18,7 +18,7 @@ from starlette.responses import PlainTextResponse from starlette.testclient import TestClient -from agentenv_protocol.website import (ROLE_KEY_ENV, ROLE_KEY_HEADER, SESSION_COOKIE, UNIVERSE_NOW_HEADER, WebsiteEnvironment, +from agentenv_protocol.website import (ROLE_KEY_ENV, ROLE_KEY_HEADER, UNIVERSE_NOW_HEADER, WebsiteEnvironment, conformance, website) NOTES_DIR = Path(__file__).resolve().parents[1] / "examples" / "website" / "notes" @@ -104,7 +104,8 @@ def gateway_clock(): def test_the_example_passes_every_conformance_check(client: TestClient) -> None: results = conformance.check("http://testserver", client=client) assert {r.name: r.status for r in results} == dict.fromkeys( - ["card", "any_host", "health", "universe_now", "data_plane", "clock", "forwarded", "cookies"], "passed"), results + ["card", "any_host", "health", "universe_now", "reserved_paths", "data_plane", "clock", "forwarded", + "cookies"], "passed"), results def test_the_card_declares_the_site_its_health_path_and_its_routes_by_name(client: TestClient) -> None: @@ -121,7 +122,7 @@ def test_the_card_declares_the_site_its_health_path_and_its_routes_by_name(clien def test_a_browser_is_signed_in_as_the_acting_user_and_can_sign_in_as_another(client: TestClient) -> None: session = client.get("/api/session") assert session.json() == {"user": "ann", "role": "default", "signed_in": True} - assert SESSION_COOKIE in session.cookies + assert "agentenv_session_notes" in session.cookies assert [n["title"] for n in client.get("/api/notes").json()] == ["Buy milk", "Book dentist"] assert client.post("/api/sign-in", json={"user": "ben"}).json() == {"user": "ben"} assert client.get("/api/session").json()["user"] == "ben" @@ -195,6 +196,15 @@ def test_the_seed_loads_exports_and_resets_through_the_data_plane(client: TestCl assert next(note for note in toggled if note["id"] == 1)["done"] is True + +def test_a_page_opened_at_a_sid_keeps_that_sessions_notes_apart(client: TestClient, env: WebsiteEnvironment) -> None: + other = TestClient(env.asgi) + other.get("/", params={"sid": "task-1"}) + created = other.post("/api/notes", json={"title": "only in task-1"}).json() + assert client.get("/go", params={"sid": "task-1"}).json()["state_diff"]["notes"]["added"] == [created] + assert client.get("/go", params={"sid": "default"}).json()["state_diff"] == {} + assert all(note["title"] != "only in task-1" for note in client.get("/api/notes").json()) + GATEWAY_PREFIX = "/website/notes" diff --git a/packages/agentenv-protocol/tests/test_website_state.py b/packages/agentenv-protocol/tests/test_website_state.py new file mode 100644 index 000000000..b4e4ec882 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_state.py @@ -0,0 +1,132 @@ +"""What a website env's task state is: WebsiteState, the SDK's website/v1 state, and website_state and website_sites +reading it through cards.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from unittest.mock import patch + +import httpx +import pytest +from pydantic import ValidationError +from starlette.testclient import TestClient + +from agentenv_protocol import DataPart +from agentenv_protocol.agent_env_environment import get_data +from agentenv_protocol.client import find_extension_method +from agentenv_protocol.website import ( + STATE_PATH, + WEBSITE_EXTENSION_URI, + WEBSITE_STATE_OP, + WebsiteEnvironment, + WebsiteState, + api, + website, + website_sites, + website_state, +) + +_GW = "http://gw" + + +def _site_card(name: str, *, state: bool = True) -> dict: + methods = {"state": {"method": "GET", "endpoint": f"/svc/{name}/agentenv/ext/website/state"}} if state else {} + return {"name": name, "url": f"/svc/{name}/agentenv", "capabilities": {"extensions": [ + {"uri": WEBSITE_EXTENSION_URI, "params": {"methods": methods}}]}} + + +def _routed(handler): + real = httpx.AsyncClient + return patch.object(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handler))) + + +@pytest.fixture +def shop(tmp_path: Path) -> WebsiteEnvironment: + (tmp_path / "index.html").write_text("") + + @website(domains=["shop.test"], static=str(tmp_path)) + class Shop(WebsiteEnvironment): + @api.post("/api/orders") + def place(self, order: dict) -> dict: + return self.store.insert("orders", order) + + env = Shop() + env.store.load({"orders": [{"id": 1, "sku": "pen"}]}) + env.store.mark_initial() + return env + + +def test_a_website_state_ignores_fields_it_doesnt_know_and_refuses_a_state_that_isnt_an_object() -> None: + assert WebsiteState.model_validate({"current_state": {"a": 1}, "seen_at": "now"}).model_dump() == { + "initial_state": None, "current_state": {"a": 1}, "state_diff": None} + with pytest.raises(ValidationError): + WebsiteState.model_validate({"current_state": [1]}) + + +def test_an_sdk_site_advertises_state_and_answers_its_stores_state_there_and_at_the_path_it_names( + shop: WebsiteEnvironment) -> None: + client = TestClient(shop.create_app()) + card = client.get("/.well-known/agent-env.json").json() + op = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) + params = next(e for e in card["capabilities"]["extensions"] if e["uri"] == WEBSITE_EXTENSION_URI)["params"] + assert (op["method"], op["endpoint"], params["state"]) == ("GET", "/agentenv/ext/website/state", {"path": STATE_PATH}) + client.post("/api/orders", json={"sku": "mug"}) + answer = client.get(op["endpoint"], params={"sid": "default"}).json() + assert answer == client.get(STATE_PATH).json() == {**shop.store.state(), "changed_sessions": []} + assert answer["state_diff"]["orders"]["added"][0]["sku"] == "mug" + + +def test_an_sdk_sites_state_op_answers_the_session_its_sid_names(shop: WebsiteEnvironment) -> None: + client = TestClient(shop.create_app()) + client.post("/api/orders", params={"sid": "task-1"}, json={"sku": "mug"}) + named = client.get("/agentenv/ext/website/state", params={"sid": "task-1"}).json() + assert named == client.get(STATE_PATH, params={"sid": "task-1"}).json() + assert named["state_diff"]["orders"]["added"][0]["sku"] == "mug" and named["changed_sessions"] == ["task-1"] + default = client.get("/agentenv/ext/website/state").json() + assert WebsiteState.model_validate(default).state_diff == shop.store.state()["state_diff"] + assert default["current_state"]["orders"] == [{"id": 1, "sku": "pen"}] + + +def test_a_site_with_data_hooks_of_its_own_advertises_no_state(tmp_path: Path) -> None: + @website(domains=["own.test"]) + class Own(WebsiteEnvironment): + @get_data + def export(self) -> list: + return [DataPart(data={"mine": True})] + + client = TestClient(Own().create_app()) + card = client.get("/.well-known/agent-env.json").json() + params = next(e for e in card["capabilities"]["extensions"] if e["uri"] == WEBSITE_EXTENSION_URI)["params"] + assert find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) is None and "state" not in params + assert client.get("/agentenv/ext/website/state").status_code == 404 + + +def test_website_state_asks_the_cards_state_of_the_session_with_the_headers_given() -> None: + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append((request.url.path, dict(request.url.params), request.headers.get("authorization"))) + return httpx.Response(200, json={"initial_state": {}, "current_state": {"n": 1}, "state_diff": {"n": {}}}) + + with _routed(handler): + state = asyncio.run(website_state(_GW, _site_card("mail"), sid="s1", headers={"Authorization": "Bearer t"})) + assert asyncio.run(website_state(_GW, _site_card("mail", state=False))) is None + assert state == WebsiteState(initial_state={}, current_state={"n": 1}, state_diff={"n": {}}) + assert seen == [("/svc/mail/agentenv/ext/website/state", {"sid": "s1"}, "Bearer t")] + + +def test_an_answer_that_isnt_a_website_state_is_refused() -> None: + with _routed(lambda request: httpx.Response(200, json={"current_state": "full"})): + with pytest.raises(ValueError): + asyncio.run(website_state(_GW, _site_card("mail"))) + + +def test_website_sites_are_a_single_sites_own_card_or_a_composed_cards_website_children() -> None: + mail, calendar = _site_card("mail"), _site_card("calendar", state=False) + mcp = {"name": "mail", "url": "/svc/mcp-mail/agentenv", "capabilities": {"extensions": []}} + composed = {"name": "gw", "children_environments": [mail, mcp, calendar, "junk"]} + assert [(site.name, site.url) for site in website_sites(composed)] == [ + ("mail", "/svc/mail/agentenv"), ("calendar", "/svc/calendar/agentenv")] + assert [site.card for site in website_sites(mail)] == [mail] + assert website_sites({"name": "plain", "children_environments": [mcp]}) == [] diff --git a/src/agent_env/env/gateway/clock.py b/src/agent_env/env/gateway/clock.py index cceb14364..dda1dd7eb 100644 --- a/src/agent_env/env/gateway/clock.py +++ b/src/agent_env/env/gateway/clock.py @@ -217,11 +217,13 @@ def at_elapsed(self, elapsed: float) -> datetime | None: return _advance(self._t0, elapsed - self._lived + self._ahead) def read(self) -> dict: - """The frozen consumer contract: {"virtual_time": ""} — pure read, never advances.""" - current = self.now() - if current is None: - raise ClockError("clock is not armed") - return {"virtual_time": _iso(current)} + """The frozen consumer contract: {"virtual_time": ""} — pure read, never advances — plus the rate it + runs at, "virtual_seconds_per_real_second", so a reader extrapolates between reads without guessing it.""" + with self._lock: + if not self._armed or self._t0 is None: + raise ClockError("clock is not armed") + current = _advance(self._t0, (self._now() - self._anchor) * self._rate + self._ahead) + return {"virtual_time": _iso(current), "virtual_seconds_per_real_second": self._rate} def state(self) -> dict: with self._lock: diff --git a/tst/unit/env/gateway/clock_advance_test.py b/tst/unit/env/gateway/clock_advance_test.py index 0a76031f9..53bdbfbab 100644 --- a/tst/unit/env/gateway/clock_advance_test.py +++ b/tst/unit/env/gateway/clock_advance_test.py @@ -165,3 +165,22 @@ def test_route_refuses_a_backwards_advance(gateway): gateway._clock.set_time("2026-01-04T08:00:00Z", rate=0) resp = _put(gateway, "/clock/advance", {"virtual_time": T0}) assert resp.status_code == 400 and "only moves forward" in resp.json()["error"] + + +def _get(gw: Gateway, path: str) -> httpx.Response: + async def _do() -> httpx.Response: + transport = httpx.ASGITransport(app=gw._mcp.streamable_http_app()) + async with httpx.AsyncClient(transport=transport, base_url="http://gateway") as client: + return await client.get(path) + + return asyncio.run(_do()) + + +def test_clock_time_answers_the_rate_beside_the_time(gateway): + assert _get(gateway, "/clock/time").status_code == 404 + gateway._clock.set_time(T0, rate=60) + gateway._clock._now.tick(2) + assert _get(gateway, "/clock/time").json() == {"virtual_time": "2026-01-01T00:02:00Z", + "virtual_seconds_per_real_second": 60.0} + gateway._clock.advance("2026-01-02T00:00:00Z") + assert _get(gateway, "/clock/time").json()["virtual_seconds_per_real_second"] == 60.0 diff --git a/uv.lock b/uv.lock index f0342e1ea..241ad385f 100644 --- a/uv.lock +++ b/uv.lock @@ -298,6 +298,10 @@ computer = [ { name = "pillow" }, { name = "uvicorn" }, ] +db = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] dev = [ { name = "a2a-sdk", extra = ["http-server"] }, { name = "av", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, @@ -309,6 +313,7 @@ dev = [ { name = "pytest-asyncio" }, { name = "python-multipart" }, { name = "regex" }, + { name = "sqlalchemy" }, ] [package.metadata] @@ -325,6 +330,7 @@ requires-dist = [ { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.25,<2" }, { name = "pillow", marker = "extra == 'computer'", specifier = ">=10.4" }, { name = "pillow", marker = "extra == 'dev'", specifier = ">=10.4" }, + { name = "psycopg", extras = ["binary"], marker = "extra == 'db'", specifier = ">=3.1" }, { name = "pydantic", specifier = ">=2.11" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, @@ -332,12 +338,14 @@ requires-dist = [ { name = "python-multipart", marker = "extra == 'dev'", specifier = ">=0.0.13" }, { name = "regex", marker = "extra == 'agent'", specifier = ">=2023.12.25" }, { name = "regex", marker = "extra == 'dev'", specifier = ">=2023.12.25" }, + { name = "sqlalchemy", marker = "extra == 'db'", specifier = ">=2.1" }, + { name = "sqlalchemy", marker = "extra == 'dev'", specifier = ">=2.1" }, { name = "starlette", specifier = ">=0.37" }, { name = "uvicorn", marker = "extra == 'agent'", specifier = ">=0.30" }, { name = "uvicorn", marker = "extra == 'app'", specifier = ">=0.30" }, { name = "uvicorn", marker = "extra == 'computer'", specifier = ">=0.30" }, ] -provides-extras = ["agent", "app", "computer", "dev"] +provides-extras = ["agent", "app", "computer", "db", "dev"] [[package]] name = "aiofile" @@ -5004,6 +5012,86 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, ] +[[package]] +name = "psycopg" +version = "3.3.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/a3/55c9fb649391fddec8d7ba3005162cd76e8cb3b2eea3dc922684aa0013e4/psycopg-3.3.7.tar.gz", hash = "sha256:6574f23fd79bf129ee0616b49f6e358faf7ae1bf3a362d2ad599cf8f9c3d4935", size = 168912, upload-time = "2026-11-03T03:58:07.876Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/2a/2e05cc01b2817b8baff419823774bb928efd0a36de12da78f19482cfa329/psycopg-3.3.7-py3-none-any.whl", hash = "sha256:d592e88a077e01ff57f22d59f16e828b92f2b2d9f439b6679d582b48d74cf306", size = 216207, upload-time = "2026-11-03T03:51:21.09Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.7" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/19/07272d51290a502281fa3f54e76ef95fa72bd05abd913a57f033162d7768/psycopg_binary-3.3.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4f9ec0cc3b327f8ed1e2ba40441dd843be9c82b3ac5dfe75eb00ece02a454ec3", size = 4731229, upload-time = "2026-11-03T03:52:28.922Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a3/beb642b331f1611f7f87c75c5b0803bfa779f6ccd4e79e381961484a20b2/psycopg_binary-3.3.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0734da871182afdefcc9b3fbaf13c70408f47a2de64394208606d8f45c4027ec", size = 4779422, upload-time = "2026-11-03T03:52:34.545Z" }, + { url = "https://files.pythonhosted.org/packages/be/5d/41538a43e0364b8f8fc669056f8f2eeb884a7374c16f65695ee3868113af/psycopg_binary-3.3.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:05fa93c416af799f8d0db325e4a58ed0df68ef7ff85a48192b616b4e1be7be26", size = 5616589, upload-time = "2026-11-03T03:52:40.664Z" }, + { url = "https://files.pythonhosted.org/packages/5f/87/c290af551e9b71ef048601fe261e7f91212274fa1fb224619d35818aca33/psycopg_binary-3.3.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:652a6029e53635ef3106c726ef3b4c01429a586372dd516fc56e75ab8a2bd934", size = 5330007, upload-time = "2026-11-03T03:52:47.627Z" }, + { url = "https://files.pythonhosted.org/packages/af/e1/efa4ea43b53fcf0f33be16d968851b610fd02c45211306a97fc84446b877/psycopg_binary-3.3.7-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2fa90f30e4a5a1dff00301797ec66b5eba991566291dcc9df6dfb6fd68965e1", size = 6879399, upload-time = "2026-11-03T03:52:57.022Z" }, + { url = "https://files.pythonhosted.org/packages/f3/2e/7d6d734fc4be304b567c47e28c6eca98cdcb30eeb84d8cb352f4bdadb0c0/psycopg_binary-3.3.7-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:496da9a6f4f7fcdb8394b1273f9ffc10deb5a95dd2aa6e85212a732bfd9a2424", size = 5110921, upload-time = "2026-11-03T03:53:01.907Z" }, + { url = "https://files.pythonhosted.org/packages/d0/3c/d6e035a73062d3abbb8829bc294ae027eb7f99ca09bc0ddbea1d78627972/psycopg_binary-3.3.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d7434c3fc858d86eb09217bce226274245957299d22e2c7d8fc7b564409d242e", size = 4647372, upload-time = "2026-11-03T03:53:06.633Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f3/8078373f96894d3f4988a3ef924646bcecddce884dfc8f547a6d66cdb517/psycopg_binary-3.3.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a07ead8eacaa3b2e130ebf5357035c53fc9bbb623fe5d7c02d83944ffeae5d09", size = 4343403, upload-time = "2026-11-03T03:53:16.037Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4a/236c3be410e3c420249119c0ed7fc0372bd292f35f980efe1ca0825e42fc/psycopg_binary-3.3.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4e960429b237ccff892a489b9348ade8cf780a103fcea403ddf924e0a07ad671", size = 4058720, upload-time = "2026-11-03T03:53:22.364Z" }, + { url = "https://files.pythonhosted.org/packages/8a/78/50b5b85c16e3526c2e001d0faccf9241e2bf7407cd4ad2a16bd353e4a991/psycopg_binary-3.3.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34e2cff14aafa3d0649b48cd18c447c163e0a3ec7bbfd2e38314cd721f33add7", size = 4363326, upload-time = "2026-11-03T03:53:26.846Z" }, + { url = "https://files.pythonhosted.org/packages/80/95/559b7910fe35a5f63adc7425bcad46c85ee10880dae8096b7789ea6d75ac/psycopg_binary-3.3.7-cp311-cp311-win_amd64.whl", hash = "sha256:cb6fe750879c781d3d3667b6d0f66318071d0245e60b51ad9a00186708978089", size = 3636509, upload-time = "2026-11-03T03:53:31.315Z" }, + { url = "https://files.pythonhosted.org/packages/bb/da/3a41bfb4ca39f601abaa20f8170545b27247df4d4850ee3647a10091c219/psycopg_binary-3.3.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:edfd6abb5da972cae50984d41d44cc744a6842ee359b0df24b39fb1524b1e654", size = 4710682, upload-time = "2026-11-03T03:53:37.421Z" }, + { url = "https://files.pythonhosted.org/packages/df/d9/775f908d6af11f0d658526cdd564dc54f03463c53de245cd7e23544c133b/psycopg_binary-3.3.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d43d35af50c66fd5b0d2537ef20bddee61b68fbfbd24f8c44a0be6f65a68301", size = 4773036, upload-time = "2026-11-03T03:53:44.926Z" }, + { url = "https://files.pythonhosted.org/packages/8f/7f/40944fa341e752bb79410bd2f6542c71e3103a476ff4e88cc780fd53caac/psycopg_binary-3.3.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1a831476577db5e62f1aa3942a60c5088f459f2c14c0e2f4639a823d23b5ffb6", size = 5575526, upload-time = "2026-11-03T03:53:51.59Z" }, + { url = "https://files.pythonhosted.org/packages/df/70/0d68867bdb39834ae5a46d5afa1d12053dd12b061659652294fbaa092f71/psycopg_binary-3.3.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df30e4e2fe7b4f4ae401326f8f24fd572d1f08543b97c2daff9bcb623aebb41f", size = 5300375, upload-time = "2026-11-03T03:53:57.765Z" }, + { url = "https://files.pythonhosted.org/packages/bf/04/4ab0135ff483b6bdcdb7fa2b6c74efec93b0d5bb2e0ffc33b099d9cae98c/psycopg_binary-3.3.7-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71601439823d28250c28255b2204376a7d686c2ab1259e5b6e0fce70cd2309d5", size = 6852910, upload-time = "2026-11-03T03:54:03.57Z" }, + { url = "https://files.pythonhosted.org/packages/55/80/0d360077293a95f3c2539e2afe054fee657e1dd5278e00f2007b645f7841/psycopg_binary-3.3.7-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c9e740f8f9fda8ac878d11d590a8b802f9b0ebf7a2c4c1fbbd577b5ceaff6694", size = 5073829, upload-time = "2026-11-03T03:54:09.479Z" }, + { url = "https://files.pythonhosted.org/packages/f1/6d/8e1b4186702b55a54a2e8f8fb7b766484659c3298de58eec754b3c801691/psycopg_binary-3.3.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e23fb1b141e9f05724f5bc83177427b3c32ca7f814450721614d23c4b2828cf5", size = 4618943, upload-time = "2026-11-03T03:54:17.659Z" }, + { url = "https://files.pythonhosted.org/packages/8d/cc/34bf9da0ce2103c7bab4de0e2716646448355fceab847620b74d67839819/psycopg_binary-3.3.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d52a488169e52774a9e09d4d0d0082d6f19cb59c58f8308b53bf355ae160a1ac", size = 4302298, upload-time = "2026-11-03T03:54:25.646Z" }, + { url = "https://files.pythonhosted.org/packages/26/38/75b5dca8d42e983d900688dbc9ada57b55ebcc3406cbcea7f1b8c92eac32/psycopg_binary-3.3.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:b2cbb774a73d6f12fbc2ab4d2faddd05a043dd623dbd07df59ec42535b00fedf", size = 4020129, upload-time = "2026-11-03T03:54:30.715Z" }, + { url = "https://files.pythonhosted.org/packages/0f/6c/0dfac7abe24a08b98367960535d164544f89eb3eb317e16979db36365dc6/psycopg_binary-3.3.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:df59dd2a1aa038856952360bbc72599ed118fe14cef558a499476bf56b0ecb45", size = 4334125, upload-time = "2026-11-03T03:54:36.812Z" }, + { url = "https://files.pythonhosted.org/packages/36/60/f412c2accfb92f8382b812f45990965e4392674a1d0d1648fe743a8fe787/psycopg_binary-3.3.7-cp312-cp312-win_amd64.whl", hash = "sha256:7a1fd4e844630393800a86e55312c77dcdfa12dd7baf02da3230c7147a805793", size = 3619065, upload-time = "2026-11-03T03:54:41.265Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ff/936c13638c8b9bd4519d3ad7996e7600b066909156cb1cd6f55b73d2203e/psycopg_binary-3.3.7-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e0639d06c50c684bb9867fb4c0593b7327525275142c0b620053c55134572773", size = 4715405, upload-time = "2026-11-03T03:54:45.749Z" }, + { url = "https://files.pythonhosted.org/packages/7a/19/98a3d8dd5b61478ce242f017b70d68241d151826409ab3f88066ed2ec327/psycopg_binary-3.3.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:89b508e50ccfe93a440ce9450c33ff78a3e2462dab369e66ee63e725562421cb", size = 4775846, upload-time = "2026-11-03T03:54:51.138Z" }, + { url = "https://files.pythonhosted.org/packages/9d/74/c855489f8be5ee3d8bfd4cee523e036a49b3aa1bfdc6eb5b72c34828fdb2/psycopg_binary-3.3.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:a0aeb1b680b2ee414a0196a9dd94b17edaa75c1c04f7cddadc9aae95d0dd940c", size = 5575585, upload-time = "2026-11-03T03:54:57.816Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c9/860db102775b7fcf1f08fe06b0d845f926fe5ab89599d23007693354331d/psycopg_binary-3.3.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3cc435fd83ff3fe627370bb8208bbcfd79631be2ac3b49b7a5c7edb528011c9", size = 5301826, upload-time = "2026-11-03T03:55:05.309Z" }, + { url = "https://files.pythonhosted.org/packages/a7/37/5f09fb2d1ad05f439329350097c702c52c1355156c176e3515d12496f723/psycopg_binary-3.3.7-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b311c834ece61e9be0df48c6078ca8949ef50d43a884f4c05628f77a8b31442", size = 6852614, upload-time = "2026-11-03T03:55:14.88Z" }, + { url = "https://files.pythonhosted.org/packages/6e/62/befa167c2ea5bfdeb65ce0f799aa35d78eb693250b650963ec7cba1fa17d/psycopg_binary-3.3.7-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:15873c72c6db035b7bbe9d443dedc571753c713b7d10223aea6438310c0f468e", size = 5076255, upload-time = "2026-11-03T03:55:23.593Z" }, + { url = "https://files.pythonhosted.org/packages/eb/97/8e622891df044606e0819cd17b589d4b47834eddd3f941848edab207b7e8/psycopg_binary-3.3.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:191dc5f2056368e0556359be7782a3be201399c7f1dfc3fe0ebffde37446e1f2", size = 4619359, upload-time = "2026-11-03T03:55:31.516Z" }, + { url = "https://files.pythonhosted.org/packages/40/84/857ec7d5ab5e63ec05e98d22d0e8c3b85df5a49cdfb8b3fc84879424040a/psycopg_binary-3.3.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:21dccda9c04e52240b9d6d934c5b710c198fdd7d0c60a358d78927988bca154e", size = 4301370, upload-time = "2026-11-03T03:55:35.691Z" }, + { url = "https://files.pythonhosted.org/packages/d9/f3/9a21844b395d49dbe7e6fd53c7cddfc7e54766e7d9957c89a8fec6c75c2a/psycopg_binary-3.3.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9ae3e685302569c34bf159927e9667d3afd5dd34e68a92afabe05e2022dafc68", size = 4021521, upload-time = "2026-11-03T03:55:39.78Z" }, + { url = "https://files.pythonhosted.org/packages/08/d7/8eda1d662b7049a6729a6b5f30601561e5204bb7ff3ae15796640afdba53/psycopg_binary-3.3.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4c8ab53d4a7c9f68d4b2cf878d1e56880bb87a1008435a5c3770422ac1e1b4ef", size = 4334750, upload-time = "2026-11-03T03:55:45.384Z" }, + { url = "https://files.pythonhosted.org/packages/dd/92/2a1ad37faa52270594e92c483a28084d75a3c696b95ad8f6fc18faf39ead/psycopg_binary-3.3.7-cp313-cp313-win_amd64.whl", hash = "sha256:63fc0a5a226e63db37a3f038f2581ea8b8189112daa86bd8ae52c3897ca105da", size = 3618325, upload-time = "2026-11-03T03:55:49.757Z" }, + { url = "https://files.pythonhosted.org/packages/07/19/a0b79901d532257d28867c3b5345f37a30191670a8998b8fd779f935d053/psycopg_binary-3.3.7-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d72b6bd81535303b3f6726de330c257d8912948d9e0cc1761c41911132633112", size = 4723602, upload-time = "2026-11-03T03:55:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/55/43/a22a87c7a51fd59c90223bd4d4fc604444b8647890af2893c50e823c1bcf/psycopg_binary-3.3.7-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c3753802f809d8fcec4fbf633f3c3335909a2f1638a36f9b924edfa2f6e8796", size = 4786213, upload-time = "2026-11-03T03:56:03.022Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2f6fe94326baaedcea29fc06c13b1b8b5dfe81690c422eda09039c531dbb/psycopg_binary-3.3.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:590c075b8645bdddb332a61ca1be5be52e7ecb57d3a3be63eba5c6a38a60b789", size = 5588332, upload-time = "2026-11-03T03:56:08.561Z" }, + { url = "https://files.pythonhosted.org/packages/17/d4/4e60003f3094e71849d29ac174b355fc0b0b9b16eb5799591b05b62a3893/psycopg_binary-3.3.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:20dbb0f8e173a46feb803136446a89d4b04959342f5331e7f3f93b4fe7f2c32e", size = 5310722, upload-time = "2026-11-03T03:56:14.427Z" }, + { url = "https://files.pythonhosted.org/packages/66/76/08edccbdc1e25e99854297bdc93ad0de34b029ff707ff6567b54e0f4421f/psycopg_binary-3.3.7-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4171b7a2d0534baf7ac1792aade1f8aba74ca06357db79915eac265140fc12c8", size = 6866520, upload-time = "2026-11-03T03:56:21.962Z" }, + { url = "https://files.pythonhosted.org/packages/9f/59/b56fe4ed6747241fdd92c4859d17ea3a10bdbca586a1e1baff5bfba09eaf/psycopg_binary-3.3.7-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7965237729bfec12b3682dedf84538bc0936c2c442560dbca8cd66534d62747", size = 5085522, upload-time = "2026-11-03T03:56:30.456Z" }, + { url = "https://files.pythonhosted.org/packages/3c/07/4e7251607b1278aa47894fec895c6d961ff8b5d436be010557ba03647f69/psycopg_binary-3.3.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:100495111a2de4305c92996708469b5a461b24c28d152ba6bf2fbebe49fe02f2", size = 4633868, upload-time = "2026-11-03T03:56:36.786Z" }, + { url = "https://files.pythonhosted.org/packages/71/3d/b6b09ec08b9235eea343df05b9e5d978fb784c9adea01c4a1649c440436b/psycopg_binary-3.3.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f2018e449c7d60dd6e66261e7c4cd9a4274baf6d824994bbd55577435c081cea", size = 4319169, upload-time = "2026-11-03T03:56:41.292Z" }, + { url = "https://files.pythonhosted.org/packages/2c/93/ea77791ceaad20970a04d3fbc132138cfb2e6f845fd9604f5705da4b0dc9/psycopg_binary-3.3.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:8487a5e112b6a22508e603c78a0dc496372daefed22a9b9d4de4b89e2d4456bc", size = 4032876, upload-time = "2026-11-03T03:56:46.755Z" }, + { url = "https://files.pythonhosted.org/packages/8c/1a/56e6bb0fad9d2751cb120a833e6113b860131e26a127f8a0e4088f56994d/psycopg_binary-3.3.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a58b9a9e7b2d31367f605c53d87d501bee7c52f5c009dd8a2d341e5f2c2b6f8e", size = 4346131, upload-time = "2026-11-03T03:56:51.75Z" }, + { url = "https://files.pythonhosted.org/packages/8c/4d/efd82743bef4b5ff120f57f2c3ffb5b5e72f3f66dec1b17ad215c090c084/psycopg_binary-3.3.7-cp314-cp314-win_amd64.whl", hash = "sha256:40a4231674cd06465384ade0fa5886820c6f25b5a26d38b4ce946c391162cb0f", size = 3715677, upload-time = "2026-11-03T03:56:56.651Z" }, + { url = "https://files.pythonhosted.org/packages/5c/69/36a022cd3739bdb2d94003c9d3c220618f8c86b8596c29831d215b490a7b/psycopg_binary-3.3.7-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:d8acf1f90217e5a398daaa0370e078675c0b767a3d0422b74fd3ffd83bcd874d", size = 4722833, upload-time = "2026-11-03T03:57:03.208Z" }, + { url = "https://files.pythonhosted.org/packages/88/52/c97e936e4db0522c93ea39ba26b5cb21de178b69cd37f37ce5d77344bad3/psycopg_binary-3.3.7-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:3919dc8d37d1ed6a1247753111f3eee5a3b906846b27eb3f08af3c907bf15806", size = 4785089, upload-time = "2026-11-03T03:57:08.978Z" }, + { url = "https://files.pythonhosted.org/packages/be/85/9f78af7011735fc98a168a8a4b41abd07df53af0749eb5cf4daf076343a3/psycopg_binary-3.3.7-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:cdc2ce73b2409d2af6d2dbd24ced79cca8408fc57faef1bfc7a1fcf50391f1e1", size = 5586061, upload-time = "2026-11-03T03:57:14.849Z" }, + { url = "https://files.pythonhosted.org/packages/37/03/28a682eda331f59452b5745d776a72ad673f240a86f096d694344ad7e25f/psycopg_binary-3.3.7-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:022d7297c6ff13e3b9fe4772979b55dcea0f043b3a2d3211b92622403faffdd5", size = 5312779, upload-time = "2026-11-03T03:57:20.976Z" }, + { url = "https://files.pythonhosted.org/packages/8b/66/db3e08d00290ff08764506c643da0682565794d037e84787346da33a5e43/psycopg_binary-3.3.7-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b20a404dc921bde74c00c2ca082ba2864cf59ecc610675ac39b3a14a5aaacc", size = 6866400, upload-time = "2026-11-03T03:57:31.307Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2f/31b275285bbcce0df0c943908ee3543bdff6d46a657a3e7e101c4a39d5a3/psycopg_binary-3.3.7-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018ba4b5eeefd1400f7e2d042c853c14bc9d7a5e2b180b07e63a59e900a90e00", size = 5093067, upload-time = "2026-11-03T03:57:38.949Z" }, + { url = "https://files.pythonhosted.org/packages/6b/69/0868542c2cbfd622a35c39ada9b604e423610ac703dfcaa5f6b7908906a8/psycopg_binary-3.3.7-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:a09474c66e0068724ffd2ab7a6ae0314526d9156dc43688d242d6d00654e4a75", size = 4634472, upload-time = "2026-11-03T03:57:44.967Z" }, + { url = "https://files.pythonhosted.org/packages/42/60/7998247ac07e5094c4c552a852ee8b0a9207083692a140d79cf29a00f696/psycopg_binary-3.3.7-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:1da7f340bc7d3a7945422148001893ee003ab0e93722d6a996010cba96b71093", size = 4316481, upload-time = "2026-11-03T03:57:51.186Z" }, + { url = "https://files.pythonhosted.org/packages/57/b0/dae410a3c17f634f18f7f6bb1003a9986b2a52347593d87deaf509dd7056/psycopg_binary-3.3.7-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:f3e0ba732d49cebe08f62e8ba8995e7cf82c74f6b43b880d4f05bc592415787a", size = 4039869, upload-time = "2026-11-03T03:57:56.388Z" }, + { url = "https://files.pythonhosted.org/packages/0e/3c/daadb40466b98e0465c767006c72bd3eda3e2d581822cd4212debd5273f2/psycopg_binary-3.3.7-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:48528deb18273d53fc49382cf4eaf309c3013f1a2aa8e22b0fdec580ba235189", size = 4349380, upload-time = "2026-11-03T03:58:00.373Z" }, + { url = "https://files.pythonhosted.org/packages/1c/85/728f34d24afdb135a1cd7bfe79dd76da3ee7a6089a1ddd61ee737f158300/psycopg_binary-3.3.7-cp315-cp315-win_amd64.whl", hash = "sha256:5ca2e91d166c1cdd44073fac07c6dd1dd80011e474650d24778086bb57f59a22", size = 3716314, upload-time = "2026-11-03T03:58:06.014Z" }, +] + [[package]] name = "psycopg2-binary" version = "2.9.12" @@ -6519,7 +6607,7 @@ name = "sqlalchemy" version = "2.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version >= '3.13.14'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/36/dc/9addbf490110b5727380059ee542ad8353bdcee45a37ba6a5e0976966ae0/sqlalchemy-2.1.6.tar.gz", hash = "sha256:499eae5c9051efe96cb9a73ec2fbde8ffa4186f102b9094ebab8fcc9d3b99f52", size = 10589700, upload-time = "2026-10-21T16:25:06.497Z" } wheels = [ From ef1c24b3f9b21ac649c4a9403d880528442ac5b5 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 15:00:25 -0800 Subject: [PATCH 15/19] fix(website): a models= site's data load leaves out computed columns, so its own export loads back (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/database.py | 9 +++++++-- .../tests/test_website_database.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py index 34373ea47..97509fbcf 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -114,7 +114,8 @@ def export(self) -> dict: def load(self, data: dict) -> None: """Insert ``data``'s rows, each table's after the tables it refers to, in their order, each run of rows that name the same columns as one insert (one insert takes its columns from its first row, so a row that leaves an - optional column out goes in its own); refuses a name that is no table.""" + optional column out goes in its own), leaving out each computed column, which the database works out itself, so + what ``export`` gave loads back; refuses a name that is no table.""" tables = {table.name: table for table in self.metadata.sorted_tables} unknown = sorted(set(data) - set(tables)) if unknown: @@ -146,9 +147,13 @@ def key(self, name: str, row: dict) -> str: def _row(table: Any, row: dict) -> dict: - """``row`` with each JSON string a date, time, decimal or UUID column holds parsed to that type.""" + """``row`` without its computed columns' values, and with each JSON string a date, time, decimal or UUID column + holds parsed to that type.""" parsed = dict(row) for column in table.columns: + if column.computed is not None: + parsed.pop(column.name, None) + continue value = parsed.get(column.name) if isinstance(value, str): kind = _python_type(column) diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py index 1b12d96de..7eeb96b64 100644 --- a/packages/agentenv-protocol/tests/test_website_database.py +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -110,6 +110,25 @@ class Note(Threads): assert rpc(client, "data/get")["parts"][0]["data"]["notes"] == [{"parent_id": None, **row} for row in seed] +def test_an_export_with_a_computed_column_loads_back_as_the_database_computes_it(dist: Path, database: str) -> None: + class Priced(DeclarativeBase): + pass + + class Item(Priced): + __tablename__ = "items" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + price: Mapped[int] = mapped_column(Integer) + doubled: Mapped[int] = mapped_column(Integer, sqlalchemy.Computed("price * 2")) + + client = TestClient(helpdesk(dist, Priced)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"items": [{"id": 1, "price": 3}]}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert exported == {"items": [{"id": 1, "price": 3, "doubled": 6}]} + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + + def test_a_row_the_gui_adds_is_the_mcp_servers_and_go_reports_it(dist: Path, database: str) -> None: env = helpdesk(dist, Base)() client = TestClient(env.create_app()) From 9741dbf5d627371dfee1e02731158e4b43a0feff Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 16:00:45 -0800 Subject: [PATCH 16/19] fix(website): a models= site loads a self-referencing table's rows parents first, so its export loads back (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/database.py | 38 ++++++++++++++++--- .../tests/test_website_database.py | 27 +++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py index 97509fbcf..34181bf91 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -112,10 +112,11 @@ def export(self) -> dict: sqlalchemy.select(table).order_by(*table.primary_key.columns))] for table in self.metadata.sorted_tables} def load(self, data: dict) -> None: - """Insert ``data``'s rows, each table's after the tables it refers to, in their order, each run of rows that name - the same columns as one insert (one insert takes its columns from its first row, so a row that leaves an - optional column out goes in its own), leaving out each computed column, which the database works out itself, so - what ``export`` gave loads back; refuses a name that is no table.""" + """Insert ``data``'s rows, each table's after the tables it refers to, in their order but each row of a table that + refers to itself after the row it refers to (``_parents_first``), each run of rows that name the same columns as + one insert (one insert takes its columns from its first row, so a row that leaves an optional column out goes in + its own), leaving out each computed column, which the database works out itself, so what ``export`` gave loads + back; refuses a name that is no table.""" tables = {table.name: table for table in self.metadata.sorted_tables} unknown = sorted(set(data) - set(tables)) if unknown: @@ -125,7 +126,7 @@ def load(self, data: dict) -> None: rows = data.get(name) or [] if not (isinstance(rows, list) and all(isinstance(row, dict) for row in rows)): raise ValueError(f"table {name!r}'s data is not a list of rows") - for _, run in itertools.groupby(rows, key=lambda row: sorted(row)): + for _, run in itertools.groupby(_parents_first(table, rows), key=lambda row: sorted(row)): connection.execute(table.insert(), [_row(table, row) for row in run]) if connection.dialect.name == "postgresql": _advance_sequences(connection, tables.values()) @@ -142,6 +143,33 @@ def key(self, name: str, row: dict) -> str: return json.dumps([row.get(c) for c in columns] if columns else row, sort_keys=True, default=str) +def _parents_first(table: Any, rows: list[dict]) -> list[dict]: + """``rows`` in their order, but each row that refers to another row of ``table`` through a foreign key of the table's + own after that row, so a constraint that isn't deferred holds at each insert; rows in a cycle keep their order.""" + links = [(fk.parent.name, fk.column.name) for fk in table.foreign_keys if fk.column.table is table] + if not links: + return rows + by_value = {(referenced, row.get(referenced)): index for index, row in enumerate(rows) + for _, referenced in links if row.get(referenced) is not None} + ordered, placed, visiting = [], set(), set() + + def place(index: int) -> None: + if index in placed or index in visiting: + return + visiting.add(index) + for local, referenced in links: + parent = by_value.get((referenced, rows[index].get(local))) + if parent is not None and parent != index: + place(parent) + visiting.discard(index) + placed.add(index) + ordered.append(rows[index]) + + for index in range(len(rows)): + place(index) + return ordered + + _PARSERS: dict = {datetime.datetime: datetime.datetime.fromisoformat, datetime.date: datetime.date.fromisoformat, datetime.time: datetime.time.fromisoformat, decimal.Decimal: decimal.Decimal, uuid.UUID: uuid.UUID} diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py index 7eeb96b64..e2690f233 100644 --- a/packages/agentenv-protocol/tests/test_website_database.py +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -129,6 +129,33 @@ class Item(Priced): assert rpc(client, "data/get")["parts"][0]["data"] == exported +def test_an_export_whose_child_row_sorts_before_its_parent_loads_back_with_foreign_keys_enforced( + dist: Path, database: str) -> None: + class Threads(DeclarativeBase): + pass + + class Note(Threads): + __tablename__ = "notes" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + parent_id: Mapped[int | None] = mapped_column(ForeignKey("notes.id"), nullable=True) + + def enforce(connection, record) -> None: + connection.execute("PRAGMA foreign_keys=ON") + + sqlalchemy.event.listen(sqlalchemy.engine.Engine, "connect", enforce) + try: + client = TestClient(helpdesk(dist, Threads)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"notes": [ + {"id": 3, "parent_id": None}, {"id": 2, "parent_id": 3}, {"id": 1, "parent_id": 2}]}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert [row["id"] for row in exported["notes"]] == [1, 2, 3] + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + finally: + sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) + + def test_a_row_the_gui_adds_is_the_mcp_servers_and_go_reports_it(dist: Path, database: str) -> None: env = helpdesk(dist, Base)() client = TestClient(env.create_app()) From aabc036d8040bda1c26dcd50239336f7ac157ddb Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 18:19:47 -0800 Subject: [PATCH 17/19] refactor(website): no /go?sid= sessions and no site-side state for models= sites; data/get alone (FD-4683) - The store keeps no sessions: no `?sid=`, no `agentenv_sid_` cookie, no `changed_sessions`; `self.store` is the site's one store, which the data plane loads, exports and resets. - A models= site keeps no initial state and no diff; its data plane is its database's. - Conformance runs the round trip with no skip for changed state or sessions. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/notes/README.md | 5 +- .../agentenv_protocol/website/conformance.py | 39 +----- .../src/agentenv_protocol/website/database.py | 25 +--- .../agentenv_protocol/website/environment.py | 127 +++-------------- .../src/agentenv_protocol/website/session.py | 8 -- .../src/agentenv_protocol/website/store.py | 89 +----------- .../tests/test_website_conformance.py | 51 ++----- .../tests/test_website_database.py | 37 +---- .../tests/test_website_environment.py | 52 +------ .../tests/test_website_notes_example.py | 8 -- .../tests/test_website_state.py | 132 ------------------ 11 files changed, 50 insertions(+), 523 deletions(-) delete mode 100644 packages/agentenv-protocol/tests/test_website_state.py diff --git a/packages/agentenv-protocol/examples/website/notes/README.md b/packages/agentenv-protocol/examples/website/notes/README.md index fc5cba314..4f0c590d3 100644 --- a/packages/agentenv-protocol/examples/website/notes/README.md +++ b/packages/agentenv-protocol/examples/website/notes/README.md @@ -29,9 +29,8 @@ them. The page shows the universe's time. store's `current_user`), and `GET /api/session` signs that user in. `POST /api/sign-in` switches to another seeded user. A user's notes are invisible to every other user, and another user's note answers 404. - **State.** The site writes no data hooks, so it uses the default store. `data/add` loads the seed (as a data part, or - as a file part by bytes or `file://`), `data/get` exports it, `data/reset` empties it, and `/go` returns - `{initial_state, current_state, state_diff}`. A page opened at `?sid=` keeps a state of its own, started - from the seed, which `/go?sid=` reports; without `sid` it is the default session, the data plane's. + as a file part by bytes or `file://`), `data/get` exports it, and `data/reset` empties it. `data/get` is how + agentenv reads what a task left the site with. - **Time.** Notes are stamped with `self.now()`, which follows the gateway's clock once `clock/v1`'s `sync_time` names it. The frontend reads `X-Universe-Now` from every API response into `universeNow()` and shows it in the header. It uses this helper rather than `clock_patch=True`, so the page works the same against a server written without the diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py index 67a8a114a..d5bc89c26 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py @@ -14,14 +14,10 @@ redirect), and cookies tied to no domain, the session path's the one the card names. A failed one is a ``warning``. The round trip leaves the server's data as it found it, adding the export back when a step after the reset fails too. -data/add sets the state a task starts from, so a server whose state shows changes since its initial state, as an SDK -site's does, would have them folded into it, and data/reset drops a site's named sessions, which data/get doesn't -export: the round trip is skipped for a server whose state shows changes or ``changed_sessions``. Its state is its -card's ``website/v1`` ``state``, else, for a card without it, what the path its ``params.state.path`` names answers, -else ``/go``'s, deprecated. The clock check leaves the server following a clock that is gone once the check ends, so a -server it ran against keeps extrapolating from that clock's time until it's synced again. The checker's clock starts at -``CLOCK_START``, far enough from wall time that a synced server shows it, and a synced server's ``X-Universe-Now`` may -be ``CLOCK_SLACK_S`` seconds from it.""" +The clock check leaves the server following a clock that is gone once the check ends, so a server it ran against +keeps extrapolating from that clock's time until it's synced again. The checker's clock starts at ``CLOCK_START``, far +enough from wall time that a synced server shows it, and a synced server's ``X-Universe-Now`` may be ``CLOCK_SLACK_S`` +seconds from it.""" from __future__ import annotations import argparse @@ -42,9 +38,8 @@ from ..client import find_extension, find_extension_method from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, iso, parse_time -from .environment import DEFAULT_HEALTH_PATH, STATE_PATH +from .environment import DEFAULT_HEALTH_PATH from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness -from .state import WEBSITE_STATE_OP Status = Literal["passed", "failed", "warning", "skipped"] CLOCK_START = datetime.datetime(2031, 5, 6, 7, 8, 9, tzinfo=datetime.timezone.utc) @@ -204,10 +199,6 @@ def _data_plane(self, card: dict) -> tuple[Status, str]: if missing: return "failed", f"the card's operations lack {', '.join(missing)}" exported = self._rpc(METHOD_GET).get("parts") or [] - if where := self._changed_since_initial(card): - return "skipped", (f"data/get answered; {where} shows changes since the initial state, or named sessions " - "with changes, which data/reset and data/add would fold in or drop, so the round trip " - "was not run") self._rpc(METHOD_RESET) added_back = False try: @@ -229,26 +220,6 @@ def _data_plane(self, card: dict) -> tuple[Status, str]: return "failed", "data/get after data/reset and data/add of its own export returned other data" return "passed", f"{len(exported)} part(s) round-tripped through reset and add" - def _changed_since_initial(self, card: dict) -> str | None: - """Where the server's state was read when its ``state_diff`` names changes or it names ``changed_sessions``, - None when it names neither or can't be read: the card's ``website/v1`` ``state``, else the path its - ``params.state.path`` names, else ``/go``. Read aside, as no check's answer, since a server without one needn't - stamp its 404.""" - op = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) - if op is not None and op.get("endpoint"): - where, request = f"website/v1 {WEBSITE_STATE_OP}", (op["method"], self.base_url + op["endpoint"]) - else: - declared = ((find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {}).get("state") - path = declared.get("path") if isinstance(declared, dict) else None - where = path if isinstance(path, str) and path.startswith("/") else STATE_PATH - request = ("GET", self.base_url + where) - try: - state = self.client.request(*request).json() - except Exception: - return None - changed = isinstance(state, dict) and bool(state.get("state_diff") or state.get("changed_sessions")) - return where if changed else None - def _clock(self, card: dict, health: str) -> tuple[Status, str]: if find_extension(card, CLOCK_EXTENSION_URI) is None: return "failed", f"the card declares no {CLOCK_EXTENSION_URI}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py index 34181bf91..dbc260f02 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -12,15 +12,12 @@ naming it. Beside the tables, the data's string ``current_user`` names the user a browser starts signed in as, as the default -store's does, and the state data/add last loaded is the initial one ``/go`` reports. A models site keeps one state, -the database's, so ``/go`` answers it for every ``sid``.""" +store's does.""" from __future__ import annotations -import copy import datetime import decimal import itertools -import json import types import uuid from typing import Any, Callable, Iterable @@ -28,7 +25,7 @@ from pydantic_core import to_jsonable_python from ..agent_env_environment import _maybe_await -from .store import CURRENT_USER, part_data, verifier_state +from .store import CURRENT_USER, part_data HOOKS = ("load", "export", "reset") @@ -136,12 +133,6 @@ def reset(self) -> None: for table in reversed(self.metadata.sorted_tables): connection.execute(table.delete()) - def key(self, name: str, row: dict) -> str: - """A row's identity, its primary key's values, which ``/go``'s diff matches rows by.""" - table = next((t for t in self.metadata.sorted_tables if t.name == name), None) - columns = [c.name for c in table.primary_key.columns] if table is not None else [] - return json.dumps([row.get(c) for c in columns] if columns else row, sort_keys=True, default=str) - def _parents_first(table: Any, rows: list[dict]) -> list[dict]: """``rows`` in their order, but each row that refers to another row of ``table`` through a foreign key of the table's @@ -230,17 +221,16 @@ def engine(self) -> Any: class ModelState: - """A models site's data plane and ``/go``: ``hooks`` loads, exports and resets its data, the state data/add last - loaded is the initial one, and the data's ``current_user`` is kept beside it.""" + """A models site's data plane: ``hooks`` loads, exports and resets its data, and the data's ``current_user`` is + kept beside it.""" def __init__(self, hooks: Any) -> None: self.hooks = hooks - self.initial: dict = {} self.current_user: str | None = None async def reset(self) -> None: await _maybe_await(self.hooks.reset()) - self.initial, self.current_user = {}, None + self.current_user = None async def load_parts(self, parts: list) -> None: for part in parts: @@ -248,14 +238,9 @@ async def load_parts(self, parts: list) -> None: if CURRENT_USER in data: self.current_user = data.pop(CURRENT_USER) await _maybe_await(self.hooks.load(data)) - self.initial = copy.deepcopy(await self.export()) async def export(self) -> dict: data = dict(await _maybe_await(self.hooks.export())) if self.current_user is not None: data[CURRENT_USER] = self.current_user return data - - async def state(self) -> dict: - key = getattr(self.hooks, "key", None) - return verifier_state(copy.deepcopy(self.initial), await self.export(), key) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py index 9a7a27059..5c3b1eda8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -2,23 +2,13 @@ plain HTTP on ``PORT`` for any host name: its ``@api`` routes, its static folder at ``/`` with ``index.html`` for the pages a single-page app routes itself, its health path, and the data plane and card the base class serves. Every answer carries ``X-Universe-Now``, the universe's time, which ``self.now()`` reads and ``clock/v1``'s ``sync_time`` ties to -the gateway's virtual clock. Without data hooks of its own a site keeps its state in ``self.store``, or with -``@website(models=...)`` in an MCP server's database, which ``self.db`` opens sessions of (``website.database``). - -A store-backed site keeps a state per session, the CUA-Gym convention the website state verifier reads: a request's -session is its ``?sid=``, else the one its browser last named, which the site keeps in the ``agentenv_sid_`` -cookie, else the default one. ``self.store`` is the request's session's store, each session started as a copy of the -initial state; the default session, ``sid=default``, is the state the data plane loads, exports and resets, and the -one a browser that names no session uses, so a site browsed without ``sid`` works as it did before sessions. - -Two routes are built in, each replaced by a site's own ``@api`` route at its very path and answering before one whose -path parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the -acting user in when nobody is (GET), or to sign out (DELETE); and, for a store-backed site, ``/go?sid=``, that -session's ``{initial_state, current_state, state_diff}`` at the path ``website/v1``'s ``params.state.path`` names, -which a reader of a card without ``state`` falls back to, with the site's ``changed_sessions``, the named sessions whose -state changed, which data/reset would drop; a session no request has used yet answers the initial state, unchanged. A -store-backed site answers ``website/v1``'s ``state``, of the session its ``sid`` names, with the same ``WebsiteState``, -among agentenv's own routes, which is how the website state verifier reads it.""" +the gateway's virtual clock. Without data hooks of its own a site keeps its data in ``self.store``, or with +``@website(models=...)`` in an MCP server's database, which ``self.db`` opens sessions of (``website.database``); the +data plane loads, exports and resets it, and ``data/get`` is how agentenv reads what a task left a site with. + +One route is built in, replaced by a site's own ``@api`` route at its very path and answering before one whose path +parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the acting +user in when nobody is (GET), or to sign out (DELETE).""" from __future__ import annotations import inspect @@ -49,20 +39,18 @@ get_data, reset_data, ) -from ..types import DataPart, EnvironmentExtension +from ..types import DataPart from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock, clock_patch_script, iso from .database import ModelState, Sessions, Tables, is_hooks, metadata_of from .errors import NotFound from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness from .routes import (ApiContext, ApiRoute, endpoint, error_response, forwarded_prefix, openapi, openapi_path, request_role, reserved_path, routes_of) -from .session import SessionSigner, expires_at, session_cookie_name, sid_cookie_name -from .state import WEBSITE_STATE_OP, WebsiteState -from .store import DEFAULT_SID, Collections, active_sid, session_id +from .session import SessionSigner, expires_at, session_cookie_name +from .store import Collections DEFAULT_HEALTH_PATH = "/api/health" SESSION_PATH = "/api/session" -STATE_PATH = "/go" _WEBSITE_ATTR = "_aee_website" _READY_ATTR = "_aee_website_ready" _DEFAULT_HOOK_ATTR = "_aee_default_store_hook" @@ -87,8 +75,8 @@ def website(_cls: type | None = None, *, domains: list[str] | tuple[str, ...] = health: str = DEFAULT_HEALTH_PATH, clock_patch: bool = False, data: str | None = None, models: Any = None) -> Any: """Declare a WebsiteEnvironment's site. ``data`` names a JSON file the default store starts with; ``models`` - keeps the site's state in a database instead, an MCP server's (see ``website.database``), and the site's - data plane and ``/go`` are that database's.""" + keeps the site's data in a database instead, an MCP server's (see ``website.database``), and the site's + data plane is that database's.""" if _cls is not None: raise TypeError("use @website(...) with parentheses") if not health.startswith("/") or health == "/" or reserved_path(health): @@ -123,16 +111,6 @@ class WebsiteEnvironment(AgentEnvEnvironment): app: ClassVar[Any] = None asgi: Any = None db: Any = None - _store: Collections - - @property - def store(self) -> Collections: - """The state store of the request's session: the default session's outside a request.""" - return self._store.session(active_sid.get()) - - @store.setter - def store(self, value: Collections) -> None: - self._store = value def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) @@ -148,7 +126,6 @@ def __init__(self) -> None: self.store = Collections() if declaration.data is not None: self.store.load(json.loads(self._site_path(declaration.data).read_text(encoding="utf-8"))) - self.store.mark_initial() self.clock = UniverseClock() self._signer = SessionSigner(cookie_name=session_cookie_name(self.environment_name())) models = declaration.models @@ -217,14 +194,11 @@ def create_app(self) -> Starlette: api_routes = self.routes() taken = {openapi_path(r.path) for r in api_routes} root = self._site_path(declaration.static) if declaration.static is not None else None - asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self), - Middleware(_Session, cookie=sid_cookie_name(self.environment_name()))]) + asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self)]) AgentEnvStarletteApplication(self._build_card(), self).add_routes_to_app(asgi) asgi.router.routes.append(Route(declaration.health, self._health, methods=["GET"])) if SESSION_PATH not in taken: asgi.router.routes.append(Route(SESSION_PATH, self._session, methods=["GET", "DELETE"])) - if self._serves_state_path(): - asgi.router.routes.append(Route(STATE_PATH, self._state, methods=["GET"])) asgi.router.routes.extend(Route(r.path, _unreserved(endpoint(getattr(self, r.fn.__name__), r, self.context)), methods=[r.method]) for r in api_routes) asgi.router.default = _Fallback(root, type(self).app) @@ -249,17 +223,9 @@ def extension_params(self, uri: str) -> dict: declaration = self.declaration() return {"domains": list(declaration.domains), "health": declaration.health, "routes": [r.advertised() for r in self.routes()], "session_cookie": self.session_cookie, - "session_path": SESSION_PATH, **({"state": {"path": STATE_PATH}} if self._serves_state_path() else {})} + "session_path": SESSION_PATH} return super().extension_params(uri) - def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: - """The extensions the base class serves, ``website/v1``'s ``state`` only for a store-backed site: one with data - hooks of its own keeps its state where ``self.store`` can't see it.""" - served = super().served_extensions(extensions) - if self.uses_store: - return served - return [_without_method(e, WEBSITE_STATE_OP) if e.uri == WEBSITE_EXTENSION_URI else e for e in served] - @extension(CLOCK_EXTENSION_URI, description="Sync this site to the gateway's virtual clock.") async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) -> dict: """Follow the gateway's virtual time at ``env_get_time_url`` from now on, in ``timezone`` when the task names @@ -270,9 +236,8 @@ async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) @extension(WEBSITE_EXTENSION_URI, op="openapi", method="GET", description=( "A site's server: domains, its health path and its @api routes by name; openapi returns the OpenAPI document " - "of its routes, ready whether the site is usable now and why not, state the task state it is in (its " - "initial and current state and their diff), and session the cookies a browser adds to start signed in as user, " - "or signed out without one.")) + "of its routes, ready whether the site is usable now and why not, and session the cookies a browser adds to " + "start signed in as user, or signed out without one.")) async def _website_openapi(self) -> dict: name = self._application.environment_card.name if self._application is not None else type(self).__name__ return openapi(name, self.routes()) @@ -284,16 +249,6 @@ async def _website_ready(self) -> dict: problems = await self._not_ready() return WebsiteReadiness(ready=not problems, problems=problems).model_dump() - @extension(WEBSITE_EXTENSION_URI, op=WEBSITE_STATE_OP, method="GET") - async def _website_state(self, sid: Optional[str] = None) -> dict: - """The site's task state, a ``WebsiteState``: its state when the task started, now, and what changed between - them, of session ``sid``'s store (the default session's without one), with the site's ``changed_sessions`` as - ``/go`` names them, or of a models site's database, which keeps one.""" - if self._models is not None: - return WebsiteState.model_validate(await self._models.state()).model_dump() - return {**WebsiteState.model_validate(self._store.session(sid).state()).model_dump(), - "changed_sessions": self._store.changed_sessions()} - @extension(WEBSITE_EXTENSION_URI, op="session") async def _website_session(self, user: Optional[str] = None) -> dict: cookie = self._session_signer().cookie(user) @@ -303,18 +258,17 @@ async def _website_session(self, user: Optional[str] = None) -> dict: async def _store_reset(self) -> None: if self._models is not None: return await self._models.reset() - self._store.reset() + self.store.reset() @add_data async def _store_add(self, parts: list) -> None: if self._models is not None: return await self._models.load_parts(parts) - self._store.load_parts(parts) - self._store.mark_initial() + self.store.load_parts(parts) @get_data async def _store_get(self) -> list: - return [DataPart(data=await self._models.export() if self._models is not None else self._store.export())] + return [DataPart(data=await self._models.export() if self._models is not None else self.store.export())] async def _not_ready(self) -> list[str]: return [check.__name__ for check, _marker in _marked_members(self, _READY_ATTR) if not await _ready(check)] @@ -334,15 +288,6 @@ async def _session(self, request: Request) -> Response: return context.apply(JSONResponse({"user": context.user, "role": context.role, "signed_in": context.user is not None})) - def _serves_state_path(self) -> bool: - """Whether the site serves its state at ``STATE_PATH``: it is store-backed and no ``@api`` route takes the path.""" - return self.uses_store and STATE_PATH not in {openapi_path(r.path) for r in self.routes()} - - async def _state(self, request: Request) -> Response: - if self._models is not None: - return JSONResponse(await self._models.state()) - return JSONResponse({**self.store.state(), "changed_sessions": self._store.changed_sessions()}) - def _clock(self) -> UniverseClock: if not hasattr(self, "clock"): raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") @@ -363,12 +308,6 @@ def _site_path(cls, path: str) -> Path: setattr(getattr(WebsiteEnvironment, _hook), _DEFAULT_HOOK_ATTR, True) -def _without_method(extension: EnvironmentExtension, method: str) -> EnvironmentExtension: - params = dict(extension.params or {}) - params["methods"] = {name: entry for name, entry in (params.get("methods") or {}).items() if name != method} - return extension.model_copy(update={"params": params}) - - async def _ready(check: Any) -> bool: try: return bool(await _maybe_await(check())) @@ -416,34 +355,6 @@ def _first(value: bytes | None) -> str: return value.decode("latin-1").split(",")[0].strip() if value else "" -class _Session: - """Runs the request in its session (``active_sid``): its ``sid`` query parameter's, else its ``cookie``'s; an - answer to a request naming a session other than its cookie's sets the cookie, so a page opened at ``?sid=`` keeps - its session for the requests it makes after.""" - - def __init__(self, app: Any, cookie: str) -> None: - self.app, self.cookie = app, cookie - - async def __call__(self, scope: dict, receive: Any, send: Any) -> None: - if scope["type"] != "http": - return await self.app(scope, receive, send) - request = Request(scope) - named, kept = request.query_params.get("sid"), request.cookies.get(self.cookie) - sid = session_id(named if named is not None else kept) - - async def remembered(message: dict) -> None: - if message["type"] == "http.response.start" and named is not None and session_id(kept) != sid: - MutableHeaders(scope=message).append( - "set-cookie", f"{self.cookie}={sid or DEFAULT_SID}; Path=/; HttpOnly; SameSite=Lax") - await send(message) - - token = active_sid.set(sid) - try: - await self.app(scope, receive, remembered) - finally: - active_sid.reset(token) - - class _Universe: """Stamps ``X-Universe-Now`` on every answer, and with ``clock_patch`` puts the clock script in its whole HTML (never a range of it), holding only the HTML up to its ```` (at most ``HEAD_SCAN_BYTES``), so a page streamed in chunks reaches the browser diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py index b798afcd4..b84b47be5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py @@ -17,7 +17,6 @@ import time SESSION_COOKIE = "agentenv_session" -SID_COOKIE = "agentenv_sid" SECRET_ENV = "WEBSITE_SESSION_SECRET" SESSION_TTL_S = 7 * 24 * 3600 SECRET_BYTES = 32 @@ -40,13 +39,6 @@ def session_cookie_name(environment_name: str | None) -> str: return f"{SESSION_COOKIE}_{_cookie_suffix(environment_name)}" if environment_name else SESSION_COOKIE -def sid_cookie_name(environment_name: str | None) -> str: - """The cookie in which the site named ``environment_name`` keeps the store session its browser last named: - ``agentenv_sid_``, made as ``session_cookie_name`` makes its name, ``agentenv_sid`` for a site with no name. - Its prefix is no session cookie's, so no site's store-selection cookie is another site's session cookie.""" - return f"{SID_COOKIE}_{_cookie_suffix(environment_name)}" if environment_name else SID_COOKIE - - class SessionSigner: def __init__(self, secret: str | bytes | None = None, cookie_name: str = SESSION_COOKIE) -> None: secret = secret if secret is not None else os.environ.get(SECRET_ENV) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py index 63c316a75..bf767efc3 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py @@ -1,19 +1,11 @@ -"""The default state store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON -object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as. -It remembers the state it was last loaded with, so a verifier can read the initial state, the current one and the -difference between them. - -A store also keeps sessions, as CUA-Gym sites do, so browsers that each name a session id (``?sid=``) never see one -another's changes: ``session(sid)`` is a store of its own, started on first use as a copy of this store's initial -state, and the session ``default``, or none, is this store itself. ``reset`` drops every session; a session started -before a later ``load`` keeps the state it was started with.""" +"""The default data store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON +object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as; +the data plane loads, exports and resets it.""" from __future__ import annotations import base64 -import contextvars import copy import json -import re from pathlib import Path from typing import Any, Iterable from urllib.parse import unquote, urlparse @@ -21,17 +13,6 @@ from ..types import DataPart, FilePart, Part, TextPart CURRENT_USER = "current_user" -DEFAULT_SID = "default" -SID_MAX = 128 -_NOT_SID = re.compile(r"[^A-Za-z0-9_-]") -active_sid: contextvars.ContextVar[str | None] = contextvars.ContextVar("agentenv_website_sid", default=None) - - -def session_id(value: str | None) -> str | None: - """The session id ``value`` names, as CUA-Gym sites read ``sid``: its letters, digits, ``_`` and ``-``, at most - ``SID_MAX`` of them; None, the default session, for one that is empty or ``default``.""" - sid = _NOT_SID.sub("", value or "")[:SID_MAX] - return None if sid in ("", DEFAULT_SID) else sid class Collections: @@ -40,33 +21,12 @@ class Collections: def __init__(self, data: dict | None = None) -> None: self._collections: dict[str, list[dict]] = {} self.current_user: str | None = None - self.initial: dict = {} - self._sessions: dict[str, Collections] = {} if data: self.load(data) - self.mark_initial() def __getitem__(self, name: str) -> list[dict]: return self._collections.setdefault(name, []) - def session(self, sid: str | None) -> Collections: - """The store of session ``sid``, started as a copy of this one's initial state on first use; this store - itself for the default session.""" - sid = session_id(sid) - if sid is None: - return self - if sid not in self._sessions: - self._sessions[sid] = Collections(self.initial) - return self._sessions[sid] - - def sessions(self) -> list[str]: - """The ids of the sessions started since the store was last reset, the default one aside.""" - return list(self._sessions) - - def changed_sessions(self) -> list[str]: - """The ids of the sessions, the default one aside, whose state differs from the one they started with.""" - return [sid for sid, store in self._sessions.items() if store.diff()] - def names(self) -> list[str]: return list(self._collections) @@ -115,8 +75,6 @@ def load_parts(self, parts: Iterable[Part]) -> None: def reset(self) -> None: self._collections.clear() self.current_user = None - self.initial = {} - self._sessions.clear() def export(self) -> dict: data: dict = copy.deepcopy(self._collections) @@ -124,47 +82,6 @@ def export(self) -> dict: data[CURRENT_USER] = self.current_user return data - def mark_initial(self) -> None: - self.initial = self.export() - - def diff(self) -> dict: - return state_diff(self.initial, self.export()) - - def state(self) -> dict: - return verifier_state(self.initial, self.export()) - - -def state_diff(initial: dict, current: dict, key: Any = None) -> dict: - """Each collection that changed from ``initial`` to ``current``, as its ``added``, ``removed`` and ``changed`` - objects, matched by ``key(name, doc)``, their ``id`` by default; a changed one as its ``id``, ``before`` and - ``after``.""" - key = key or (lambda name, doc: _key(doc)) - changes: dict = {} - for name in dict.fromkeys([*initial, *current]): - if name == CURRENT_USER: - continue - before = {key(name, d): d for d in initial.get(name, [])} - after = {key(name, d): d for d in current.get(name, [])} - entry = { - "added": [d for k, d in after.items() if k not in before], - "removed": [d for k, d in before.items() if k not in after], - "changed": [{"id": d.get("id"), "before": before[k], "after": d} - for k, d in after.items() if k in before and before[k] != d], - } - if any(entry.values()): - changes[name] = entry - return changes - - -def verifier_state(initial: dict, current: dict, key: Any = None) -> dict: - """What ``/go`` answers: ``{initial_state, current_state, state_diff}``.""" - return {"initial_state": copy.deepcopy(initial), "current_state": current, - "state_diff": state_diff(initial, current, key)} - - -def _key(doc: dict) -> str: - return json.dumps(doc.get("id")) if doc.get("id") is not None else json.dumps(doc, sort_keys=True) - def part_data(part: Part) -> dict: """A part's JSON: a data part's data, a text part's text, or a file part's bytes or ``file://`` file.""" diff --git a/packages/agentenv-protocol/tests/test_website_conformance.py b/packages/agentenv-protocol/tests/test_website_conformance.py index af03260b3..9140ab39d 100644 --- a/packages/agentenv-protocol/tests/test_website_conformance.py +++ b/packages/agentenv-protocol/tests/test_website_conformance.py @@ -37,7 +37,6 @@ def notes(self) -> list: env = Notes() env.store.load({"current_user": "ann", "notes": [{"id": 1, "text": "hello"}]}) - env.store.mark_initial() return env @@ -58,12 +57,10 @@ def _handwritten(*, leak_host: bool = False, domain_cookie: bool = False, clock: clock_ops_first: tuple[str, ...] = (), leak_redirect: bool = False, stamp_rpc: bool = True, reset_keeps: bool = False, get_fails_after_reset: bool = False, rpc_reply: tuple[int, dict] | None = None, - ready_answer: object = None, ready_status: int = 200, rpc_path: str | None = None, - state_path: str | None = None, changed: bool = False) -> Starlette: + ready_answer: object = None, ready_status: int = 200, rpc_path: str | None = None) -> Starlette: """A server written without the SDK: the contract by hand, optionally broken. Its state is its ``state.data``; with ``ready_answer`` its card advertises ``website/v1``'s ``ready``, which answers it with ``ready_status``. With ``rpc_path`` its card's - ``url`` names its data plane there, not at ``/agentenv``, and with ``state_path`` its ``params.state.path`` names - where it serves its state, which shows changes since the initial state when ``changed``.""" + ``url`` names its data plane there, not at ``/agentenv``.""" state = {"items": ["a"], "url": None, "at": None, "reset": False} def now() -> datetime.datetime: @@ -98,8 +95,6 @@ async def card(request: Request) -> Response: "routes": [{"method": "GET", "path": "/items", "name": "items"}]}}] if ready_answer is not None: extensions[0]["params"]["methods"] = {"ready": {"method": "GET", "endpoint": "/ready"}} - if state_path is not None: - extensions[0]["params"]["state"] = {"path": state_path} if clock: extensions.append({"uri": "urn:agentenv:clock/v1", "params": { "endpoint": "/agentenv/ext/sync_time", "methods": { @@ -109,9 +104,6 @@ async def card(request: Request) -> Response: "capabilities": {"extensions": extensions, "operations": ["data/reset", "data/add", "data/get"]}})) - async def site_state(request: Request) -> Response: - return JSONResponse({"state_diff": {"items": {"added": ["b"]}} if changed else {}}) - async def rpc(request: Request) -> Response: body = await request.json() method = body["method"] @@ -147,8 +139,7 @@ async def unrouted(request: Request, exc: Exception) -> Response: app = Starlette(routes=[Route("/", page), Route("/healthz", health), Route("/ready", ready), Route("/items", items), Route("/items/", items_slash), Route("/.well-known/agent-env.json", card), Route(rpc_path or "/agentenv", rpc, methods=["POST"]), - Route("/agentenv/ext/sync_time", sync_time, methods=["POST"]), - *([Route(state_path, site_state)] if state_path else [])], + Route("/agentenv/ext/sync_time", sync_time, methods=["POST"])], exception_handlers={404: unrouted, 405: unrouted}) app.state.data = state return app @@ -234,28 +225,14 @@ def test_a_data_request_without_a_2xx_json_rpc_result_fails_the_data_plane(reply assert result.status == "failed" and "no JSON-RPC result" in result.detail -def test_an_sdk_site_keeps_its_initial_state_and_one_changed_since_skips_the_round_trip( - notes: WebsiteEnvironment, sdk_site: TestClient) -> None: - before = sdk_site.get("/go").json() - conformance.check("http://testserver", client=sdk_site) - assert sdk_site.get("/go").json() == before - notes.store.insert("notes", {"text": "new"}) - changed = sdk_site.get("/go").json() - assert changed["state_diff"] - result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] - assert result.status == "skipped" and "website/v1 state shows changes" in result.detail - assert sdk_site.get("/go").json() == changed - - -def test_a_named_session_with_changes_skips_the_round_trip_and_keeps_its_state( +def test_an_sdk_sites_data_round_trips_and_is_left_as_the_check_found_it_changed_or_not( notes: WebsiteEnvironment, sdk_site: TestClient) -> None: - sdk_site.get("/api/notes", params={"sid": "task-1"}) - assert {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"].status == "passed" - notes.store.session("task-2")["notes"].append({"id": 2, "text": "draft"}) - result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] - assert result.status == "skipped" and "named sessions" in result.detail - assert sdk_site.get("/go", params={"sid": "task-2"}).json()["state_diff"]["notes"]["added"] == [ - {"id": 2, "text": "draft"}] + for _ in range(2): + before = notes.store.export() + result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] + assert result.status == "passed", result + assert notes.store.export() == before + notes.store.insert("notes", {"text": "new"}) def test_the_data_plane_is_reached_at_the_cards_url() -> None: @@ -271,14 +248,6 @@ def test_the_clock_check_calls_sync_time_whichever_operation_the_card_lists_firs assert results["clock"] == "passed", results -@pytest.mark.parametrize(("changed", "status"), [(True, "skipped"), (False, "passed")]) -def test_a_card_without_state_is_read_at_the_state_path_it_names(changed: bool, status: str) -> None: - app = _handwritten(state_path="/grading/state", changed=changed) - result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] - assert result.status == status, result - assert ("/grading/state shows changes" in result.detail) is changed - - def test_a_server_with_no_card_health_or_header_fails_the_contract_and_warns_of_the_rest() -> None: bare = Starlette(routes=[Route("/", lambda request: JSONResponse({"not": "html"}))]) results = conformance.check("http://testserver", client=TestClient(bare)) diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py index e2690f233..2cc5c8b8e 100644 --- a/packages/agentenv-protocol/tests/test_website_database.py +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -1,6 +1,6 @@ """A site of ``@website(models=...)``: its state is a database's, SQLAlchemy models' tables on DATABASE_URL (sqlite -here) or an object's own load/export/reset, which its data plane and /go read and write, and its routes reach -through ``self.db``.""" +here) or an object's own load/export/reset, which its data plane reads and writes, and its routes reach through +``self.db``.""" from __future__ import annotations @@ -156,7 +156,7 @@ def enforce(connection, record) -> None: sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) -def test_a_row_the_gui_adds_is_the_mcp_servers_and_go_reports_it(dist: Path, database: str) -> None: +def test_a_row_the_gui_adds_is_the_mcp_servers_and_data_get_reports_it(dist: Path, database: str) -> None: env = helpdesk(dist, Base)() client = TestClient(env.create_app()) rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) @@ -166,12 +166,8 @@ def test_a_row_the_gui_adds_is_the_mcp_servers_and_go_reports_it(dist: Path, dat assert connection.execute(select(Ticket.title).order_by(Ticket.id)).scalars().all() == ["printer", "badge"] connection.execute(sqlalchemy.update(Ticket).where(Ticket.id == 1).values(title="printer jam")) connection.commit() - state = client.get("/go", params={"sid": "default"}).json() - assert state["initial_state"] == SEED - diff = state["state_diff"]["tickets"] - assert [t["title"] for t in diff["added"]] == ["badge"] and diff["removed"] == [] - assert diff["changed"][0]["before"]["title"] == "printer" and diff["changed"][0]["after"]["title"] == "printer jam" - assert client.get("/go", params={"sid": "other"}).json() == state + tickets = rpc(client, "data/get")["parts"][0]["data"]["tickets"] + assert [t["title"] for t in tickets] == ["printer jam", "badge"] def test_a_models_site_passes_the_conformance_check(dist: Path, database: str) -> None: @@ -232,28 +228,5 @@ def reset(self) -> None: rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"current_user": "bo", "rows": [{"id": 1}]}}]}) rows.rows.append({"id": 2}) assert rpc(client, "data/get")["parts"][0]["data"] == {"current_user": "bo", "rows": [{"id": 1}, {"id": 2}]} - assert client.get("/go").json()["state_diff"] == {"rows": {"added": [{"id": 2}], "removed": [], "changed": []}} assert env.acting_user() == "bo" assert env.db is None - - -def test_editing_a_row_a_hook_still_owns_leaves_the_initial_state(dist: Path) -> None: - class Rows: - def __init__(self) -> None: - self.rows: list = [] - - def load(self, data: dict) -> None: - self.rows += data["rows"] - - def export(self) -> dict: - return {"rows": list(self.rows)} - - def reset(self) -> None: - self.rows.clear() - - rows = Rows() - client = TestClient(helpdesk(dist, rows)().create_app()) - rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"rows": [{"id": 1, "status": "open"}]}}]}) - rows.rows[0]["status"] = "closed" - assert client.get("/go").json()["state_diff"] == {"rows": {"added": [], "removed": [], "changed": [ - {"id": 1, "before": {"id": 1, "status": "open"}, "after": {"id": 1, "status": "closed"}}]}} diff --git a/packages/agentenv-protocol/tests/test_website_environment.py b/packages/agentenv-protocol/tests/test_website_environment.py index e2ccfccbb..1cfb7c1d3 100644 --- a/packages/agentenv-protocol/tests/test_website_environment.py +++ b/packages/agentenv-protocol/tests/test_website_environment.py @@ -239,39 +239,6 @@ def test_a_sites_data_is_read_through_the_data_plane_alone(client: TestClient) - assert "state" not in website_params and "state" not in website_params.get("methods", {}) -def test_each_sid_keeps_a_state_of_its_own_started_from_the_initial_state(client: TestClient, - shop: WebsiteEnvironment) -> None: - client.post("/api/orders", json={"sku": "mug"}) - assert TestClient(shop.asgi).get("/go", params={"sid": "task-7"}).json() == { - "initial_state": SEED, "current_state": SEED, "state_diff": {}, "changed_sessions": []} - page = client.get("/", params={"sid": "task-7"}) - assert page.headers["set-cookie"].startswith("agentenv_sid_Shop=task-7;") - assert "set-cookie" not in client.get("/", params={"sid": "task-7"}).headers - client.post("/api/orders", json={"sku": "pen"}) - client.post("/api/orders", params={"sid": "other"}, json={"sku": "cup"}) - added = {sid: [o.get("sku") for d in [client.get("/go", params={"sid": sid}).json()["state_diff"]] - for o in d.get("orders", {}).get("added", [])] for sid in ("default", "task-7", "other")} - assert added == {"default": ["mug"], "task-7": ["pen"], "other": ["cup"]} - assert client.get("/go").json()["state_diff"]["orders"]["added"][0]["sku"] == "cup" - client.get("/", params={"sid": "default"}) - assert client.get("/go").json()["state_diff"]["orders"]["added"][0]["sku"] == "mug" - assert rpc(client, "data/get")["parts"][0]["data"]["orders"][-1]["sku"] == "mug" - assert sorted(shop._store.sessions()) == ["other", "task-7"] - assert sorted(client.get("/go").json()["changed_sessions"]) == ["other", "task-7"] - rpc(client, "data/reset") - assert shop._store.sessions() == [] and client.get("/go", params={"sid": "task-7"}).json()["current_state"] == {} - - -def test_a_session_id_is_sanitised_as_cua_gym_sites_do() -> None: - from agentenv_protocol.website.store import session_id - - assert [session_id(v) for v in (None, "", "default", "../a b", "x" * 200)] == [None, None, None, "ab", "x" * 128] - store = Collections({"items": [{"id": 1}]}) - store.session("s")["items"].append({"id": 2}) - assert store.session("default") is store and store.export() == {"items": [{"id": 1}]} - assert store.session("s").diff() == {"items": {"added": [{"id": 2}], "removed": [], "changed": []}} - - def test_broad_routes_leave_the_built_in_ones_answering_their_paths(dist: Path) -> None: class Broad(site(dist)): @api.get("/{slug}") @@ -379,20 +346,6 @@ def test_each_site_keeps_its_own_session_cookie_and_reads_the_legacy_one(dist: P assert TestClient(first.asgi, cookies={SESSION_COOKIE: legacy}).get("/api/orders").json() == [{"id": 1, "user": "bob"}] -def test_no_site_s_session_choice_is_another_site_s_sign_in(dist: Path, monkeypatch) -> None: - monkeypatch.setenv("ENVIRONMENT_NAME", "notes") - notes = site(dist)() - monkeypatch.setenv("ENVIRONMENT_NAME", "notes_sid") - notes_sid = site(dist)() - browser = TestClient(notes.create_app()) - browser.post("/api/orders", params={"sid": "task-1"}, json={"sku": "mug"}) - other = TestClient(notes_sid.create_app(), cookies=browser.cookies) - other.post("/api/sign-in", json={"user": "bob"}) - back = TestClient(notes.asgi, cookies=other.cookies) - assert [o["sku"] for o in back.get("/go").json()["current_state"]["orders"]] == ["mug"] - assert TestClient(notes_sid.asgi, cookies=back.cookies).get("/api/session").json()["user"] == "bob" - - def test_sites_whose_names_spell_alike_keep_their_own_cookies(dist: Path, monkeypatch) -> None: apps = [] for name in ("shop a/1", "shop_a_1"): @@ -402,14 +355,11 @@ def test_sites_whose_names_spell_alike_keep_their_own_cookies(dist: Path, monkey apps.append(env.create_app()) browser = TestClient(apps[0]) browser.post("/api/sign-in", json={"user": "bob"}) - browser.get("/", params={"sid": "task-1"}) other = TestClient(apps[1], cookies=browser.cookies) other.post("/api/sign-in", json={"user": "alice"}) - other.get("/", params={"sid": "task-2"}) - assert len(other.cookies.keys()) == 4 + assert len(other.cookies.keys()) == 2 back = TestClient(apps[0], cookies=other.cookies) assert back.get("/api/session").json()["user"] == "bob" - assert back.cookies.get("agentenv_sid_shop_a_1~030f5785d67a") == "task-1" def test_every_answer_carries_the_universe_time(client: TestClient) -> None: diff --git a/packages/agentenv-protocol/tests/test_website_notes_example.py b/packages/agentenv-protocol/tests/test_website_notes_example.py index 93eb51cc1..ad92a50df 100644 --- a/packages/agentenv-protocol/tests/test_website_notes_example.py +++ b/packages/agentenv-protocol/tests/test_website_notes_example.py @@ -197,14 +197,6 @@ def test_the_seed_loads_exports_and_resets_through_the_data_plane(client: TestCl -def test_a_page_opened_at_a_sid_keeps_that_sessions_notes_apart(client: TestClient, env: WebsiteEnvironment) -> None: - other = TestClient(env.asgi) - other.get("/", params={"sid": "task-1"}) - created = other.post("/api/notes", json={"title": "only in task-1"}).json() - assert client.get("/go", params={"sid": "task-1"}).json()["state_diff"]["notes"]["added"] == [created] - assert client.get("/go", params={"sid": "default"}).json()["state_diff"] == {} - assert all(note["title"] != "only in task-1" for note in client.get("/api/notes").json()) - GATEWAY_PREFIX = "/website/notes" diff --git a/packages/agentenv-protocol/tests/test_website_state.py b/packages/agentenv-protocol/tests/test_website_state.py deleted file mode 100644 index b4e4ec882..000000000 --- a/packages/agentenv-protocol/tests/test_website_state.py +++ /dev/null @@ -1,132 +0,0 @@ -"""What a website env's task state is: WebsiteState, the SDK's website/v1 state, and website_state and website_sites -reading it through cards.""" - -from __future__ import annotations - -import asyncio -from pathlib import Path -from unittest.mock import patch - -import httpx -import pytest -from pydantic import ValidationError -from starlette.testclient import TestClient - -from agentenv_protocol import DataPart -from agentenv_protocol.agent_env_environment import get_data -from agentenv_protocol.client import find_extension_method -from agentenv_protocol.website import ( - STATE_PATH, - WEBSITE_EXTENSION_URI, - WEBSITE_STATE_OP, - WebsiteEnvironment, - WebsiteState, - api, - website, - website_sites, - website_state, -) - -_GW = "http://gw" - - -def _site_card(name: str, *, state: bool = True) -> dict: - methods = {"state": {"method": "GET", "endpoint": f"/svc/{name}/agentenv/ext/website/state"}} if state else {} - return {"name": name, "url": f"/svc/{name}/agentenv", "capabilities": {"extensions": [ - {"uri": WEBSITE_EXTENSION_URI, "params": {"methods": methods}}]}} - - -def _routed(handler): - real = httpx.AsyncClient - return patch.object(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handler))) - - -@pytest.fixture -def shop(tmp_path: Path) -> WebsiteEnvironment: - (tmp_path / "index.html").write_text("") - - @website(domains=["shop.test"], static=str(tmp_path)) - class Shop(WebsiteEnvironment): - @api.post("/api/orders") - def place(self, order: dict) -> dict: - return self.store.insert("orders", order) - - env = Shop() - env.store.load({"orders": [{"id": 1, "sku": "pen"}]}) - env.store.mark_initial() - return env - - -def test_a_website_state_ignores_fields_it_doesnt_know_and_refuses_a_state_that_isnt_an_object() -> None: - assert WebsiteState.model_validate({"current_state": {"a": 1}, "seen_at": "now"}).model_dump() == { - "initial_state": None, "current_state": {"a": 1}, "state_diff": None} - with pytest.raises(ValidationError): - WebsiteState.model_validate({"current_state": [1]}) - - -def test_an_sdk_site_advertises_state_and_answers_its_stores_state_there_and_at_the_path_it_names( - shop: WebsiteEnvironment) -> None: - client = TestClient(shop.create_app()) - card = client.get("/.well-known/agent-env.json").json() - op = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) - params = next(e for e in card["capabilities"]["extensions"] if e["uri"] == WEBSITE_EXTENSION_URI)["params"] - assert (op["method"], op["endpoint"], params["state"]) == ("GET", "/agentenv/ext/website/state", {"path": STATE_PATH}) - client.post("/api/orders", json={"sku": "mug"}) - answer = client.get(op["endpoint"], params={"sid": "default"}).json() - assert answer == client.get(STATE_PATH).json() == {**shop.store.state(), "changed_sessions": []} - assert answer["state_diff"]["orders"]["added"][0]["sku"] == "mug" - - -def test_an_sdk_sites_state_op_answers_the_session_its_sid_names(shop: WebsiteEnvironment) -> None: - client = TestClient(shop.create_app()) - client.post("/api/orders", params={"sid": "task-1"}, json={"sku": "mug"}) - named = client.get("/agentenv/ext/website/state", params={"sid": "task-1"}).json() - assert named == client.get(STATE_PATH, params={"sid": "task-1"}).json() - assert named["state_diff"]["orders"]["added"][0]["sku"] == "mug" and named["changed_sessions"] == ["task-1"] - default = client.get("/agentenv/ext/website/state").json() - assert WebsiteState.model_validate(default).state_diff == shop.store.state()["state_diff"] - assert default["current_state"]["orders"] == [{"id": 1, "sku": "pen"}] - - -def test_a_site_with_data_hooks_of_its_own_advertises_no_state(tmp_path: Path) -> None: - @website(domains=["own.test"]) - class Own(WebsiteEnvironment): - @get_data - def export(self) -> list: - return [DataPart(data={"mine": True})] - - client = TestClient(Own().create_app()) - card = client.get("/.well-known/agent-env.json").json() - params = next(e for e in card["capabilities"]["extensions"] if e["uri"] == WEBSITE_EXTENSION_URI)["params"] - assert find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) is None and "state" not in params - assert client.get("/agentenv/ext/website/state").status_code == 404 - - -def test_website_state_asks_the_cards_state_of_the_session_with_the_headers_given() -> None: - seen = [] - - def handler(request: httpx.Request) -> httpx.Response: - seen.append((request.url.path, dict(request.url.params), request.headers.get("authorization"))) - return httpx.Response(200, json={"initial_state": {}, "current_state": {"n": 1}, "state_diff": {"n": {}}}) - - with _routed(handler): - state = asyncio.run(website_state(_GW, _site_card("mail"), sid="s1", headers={"Authorization": "Bearer t"})) - assert asyncio.run(website_state(_GW, _site_card("mail", state=False))) is None - assert state == WebsiteState(initial_state={}, current_state={"n": 1}, state_diff={"n": {}}) - assert seen == [("/svc/mail/agentenv/ext/website/state", {"sid": "s1"}, "Bearer t")] - - -def test_an_answer_that_isnt_a_website_state_is_refused() -> None: - with _routed(lambda request: httpx.Response(200, json={"current_state": "full"})): - with pytest.raises(ValueError): - asyncio.run(website_state(_GW, _site_card("mail"))) - - -def test_website_sites_are_a_single_sites_own_card_or_a_composed_cards_website_children() -> None: - mail, calendar = _site_card("mail"), _site_card("calendar", state=False) - mcp = {"name": "mail", "url": "/svc/mcp-mail/agentenv", "capabilities": {"extensions": []}} - composed = {"name": "gw", "children_environments": [mail, mcp, calendar, "junk"]} - assert [(site.name, site.url) for site in website_sites(composed)] == [ - ("mail", "/svc/mail/agentenv"), ("calendar", "/svc/calendar/agentenv")] - assert [site.card for site in website_sites(mail)] == [mail] - assert website_sites({"name": "plain", "children_environments": [mcp]}) == [] From 3184f7940028ae1722ec998a75b19b5e28cdd47a Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 20:02:32 -0800 Subject: [PATCH 18/19] fix(website): a models= site's load matches a composite foreign key whole and restores enum columns as their members (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/database.py | 36 +++++++++--- .../tests/test_website_database.py | 56 +++++++++++++++++++ 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py index dbc260f02..baed53719 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -17,6 +17,7 @@ import datetime import decimal +import enum import itertools import types import uuid @@ -136,20 +137,28 @@ def reset(self) -> None: def _parents_first(table: Any, rows: list[dict]) -> list[dict]: """``rows`` in their order, but each row that refers to another row of ``table`` through a foreign key of the table's - own after that row, so a constraint that isn't deferred holds at each insert; rows in a cycle keep their order.""" - links = [(fk.parent.name, fk.column.name) for fk in table.foreign_keys if fk.column.table is table] + own after that row, each key matched whole, every column of a composite one together, so a constraint that isn't + deferred holds at each insert; rows in a cycle keep their order.""" + links = [([element.parent.name for element in constraint.elements], + [element.column.name for element in constraint.elements]) + for constraint in table.foreign_key_constraints if constraint.referred_table is table] if not links: return rows - by_value = {(referenced, row.get(referenced)): index for index, row in enumerate(rows) - for _, referenced in links if row.get(referenced) is not None} + by_key: dict = {} + for index, row in enumerate(rows): + for n, (_, referenced) in enumerate(links): + key = tuple(row.get(column) for column in referenced) + if None not in key: + by_key[(n, key)] = index ordered, placed, visiting = [], set(), set() def place(index: int) -> None: if index in placed or index in visiting: return visiting.add(index) - for local, referenced in links: - parent = by_value.get((referenced, rows[index].get(local))) + for n, (local, _) in enumerate(links): + key = tuple(rows[index].get(column) for column in local) + parent = by_key.get((n, key)) if None not in key else None if parent is not None and parent != index: place(parent) visiting.discard(index) @@ -166,8 +175,9 @@ def place(index: int) -> None: def _row(table: Any, row: dict) -> dict: - """``row`` without its computed columns' values, and with each JSON string a date, time, decimal or UUID column - holds parsed to that type.""" + """``row`` without its computed columns' values, with each JSON string a date, time, decimal or UUID column holds + parsed to that type, and each an enum column holds made its member again, found by its value, as ``export`` writes + it, else by its name.""" parsed = dict(row) for column in table.columns: if column.computed is not None: @@ -178,9 +188,19 @@ def _row(table: Any, row: dict) -> dict: kind = _python_type(column) if kind in _PARSERS: parsed[column.name] = _PARSERS[kind](value.replace("Z", "+00:00") if kind is datetime.datetime else value) + elif isinstance(kind, type) and issubclass(kind, enum.Enum): + parsed[column.name] = _member(kind, value) return parsed +def _member(kind: type[enum.Enum], value: str) -> Any: + """The member of ``kind`` whose value is ``value``, else the one so named, else ``value`` as given.""" + try: + return kind(value) + except ValueError: + return kind.__members__.get(value, value) + + def _python_type(column: Any) -> type | None: try: return column.type.python_type diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py index 2cc5c8b8e..36fab7fd3 100644 --- a/packages/agentenv-protocol/tests/test_website_database.py +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -5,6 +5,7 @@ from __future__ import annotations import datetime +import enum import sys import types from pathlib import Path @@ -39,6 +40,11 @@ class Ticket(Base): opened_at: Mapped[datetime.datetime] = mapped_column(DateTime) +class State(enum.Enum): + OPEN = "open" + CLOSED = "closed" + + SEED = {"current_user": "ann", "users": [{"id": "ann", "name": "Ann"}], "tickets": [{"id": 1, "owner": "ann", "title": "printer", "opened_at": "2031-01-02T03:04:05"}]} @@ -156,6 +162,56 @@ def enforce(connection, record) -> None: sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) +def test_a_composite_self_reference_loads_back_parents_first_matching_the_whole_key(dist: Path, database: str) -> None: + class Tenants(DeclarativeBase): + pass + + class Node(Tenants): + __tablename__ = "nodes" + tenant: Mapped[str] = mapped_column(String, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + parent_tenant: Mapped[str | None] = mapped_column(String, nullable=True) + parent_id: Mapped[int | None] = mapped_column(Integer, nullable=True) + __table_args__ = (sqlalchemy.ForeignKeyConstraint(["parent_tenant", "parent_id"], + ["nodes.tenant", "nodes.id"]),) + + def enforce(connection, record) -> None: + connection.execute("PRAGMA foreign_keys=ON") + + sqlalchemy.event.listen(sqlalchemy.engine.Engine, "connect", enforce) + try: + client = TestClient(helpdesk(dist, Tenants)().create_app()) + rows = [{"tenant": t, "id": i, "parent_tenant": None, "parent_id": None} for t, i in (("b", 2), ("b", 5), ("c", 2))] + rows.append({"tenant": "a", "id": 1, "parent_tenant": "b", "parent_id": 2}) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"nodes": rows}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert [(row["tenant"], row["id"]) for row in exported["nodes"]] == [("a", 1), ("b", 2), ("b", 5), ("c", 2)] + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + finally: + sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) + + +def test_an_export_of_an_enum_column_loads_back_as_its_members(dist: Path, database: str) -> None: + class Desk(DeclarativeBase): + pass + + class Issue(Desk): + __tablename__ = "issues" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + state: Mapped[State] = mapped_column(sqlalchemy.Enum(State)) + + client = TestClient(helpdesk(dist, Desk)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"issues": [{"id": 1, "state": "open"}, + {"id": 2, "state": "CLOSED"}]}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert exported == {"issues": [{"id": 1, "state": "open"}, {"id": 2, "state": "closed"}]} + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + + def test_a_row_the_gui_adds_is_the_mcp_servers_and_data_get_reports_it(dist: Path, database: str) -> None: env = helpdesk(dist, Base)() client = TestClient(env.create_app()) From e7590246c2cf90d4667c892edebb95d3b9b46894 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 20:54:13 -0800 Subject: [PATCH 19/19] fix(website): a models= site orders a self-referencing chain of any length parents first, without recursing (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/database.py | 33 ++++++++++++------- .../tests/test_website_database.py | 16 +++++++++ 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py index baed53719..2e18d002b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -138,7 +138,8 @@ def reset(self) -> None: def _parents_first(table: Any, rows: list[dict]) -> list[dict]: """``rows`` in their order, but each row that refers to another row of ``table`` through a foreign key of the table's own after that row, each key matched whole, every column of a composite one together, so a constraint that isn't - deferred holds at each insert; rows in a cycle keep their order.""" + deferred holds at each insert; rows in a cycle keep their order. The walk keeps its own stack, so a chain of any + length is ordered.""" links = [([element.parent.name for element in constraint.elements], [element.column.name for element in constraint.elements]) for constraint in table.foreign_key_constraints if constraint.referred_table is table] @@ -152,21 +153,31 @@ def _parents_first(table: Any, rows: list[dict]) -> list[dict]: by_key[(n, key)] = index ordered, placed, visiting = [], set(), set() - def place(index: int) -> None: - if index in placed or index in visiting: - return - visiting.add(index) + def parents(index: int) -> list[int]: + found = [] for n, (local, _) in enumerate(links): key = tuple(rows[index].get(column) for column in local) parent = by_key.get((n, key)) if None not in key else None if parent is not None and parent != index: - place(parent) - visiting.discard(index) - placed.add(index) - ordered.append(rows[index]) + found.append(parent) + return found - for index in range(len(rows)): - place(index) + for start in range(len(rows)): + if start in placed: + continue + visiting.add(start) + stack = [(start, iter(parents(start)))] + while stack: + index, pending = stack[-1] + parent = next((p for p in pending if p not in placed and p not in visiting), None) + if parent is not None: + visiting.add(parent) + stack.append((parent, iter(parents(parent)))) + continue + stack.pop() + visiting.discard(index) + placed.add(index) + ordered.append(rows[index]) return ordered diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py index 36fab7fd3..1c623a482 100644 --- a/packages/agentenv-protocol/tests/test_website_database.py +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -162,6 +162,22 @@ def enforce(connection, record) -> None: sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) +def test_a_chain_longer_than_the_recursion_limit_is_ordered_parents_first() -> None: + import sys + + from agentenv_protocol.website.database import _parents_first + + metadata = sqlalchemy.MetaData() + table = sqlalchemy.Table("notes", metadata, sqlalchemy.Column("id", Integer, primary_key=True), + sqlalchemy.Column("parent_id", Integer, ForeignKey("notes.id"), nullable=True)) + count = sys.getrecursionlimit() * 2 + rows = [{"id": i, "parent_id": i + 1 if i + 1 < count else None} for i in range(count)] + ordered = _parents_first(table, rows) + assert [row["id"] for row in ordered] == list(range(count - 1, -1, -1)) + cycle = [{"id": 1, "parent_id": 2}, {"id": 2, "parent_id": 1}] + assert sorted(row["id"] for row in _parents_first(table, cycle)) == [1, 2] + + def test_a_composite_self_reference_loads_back_parents_first_matching_the_whole_key(dist: Path, database: str) -> None: class Tenants(DeclarativeBase): pass