From 8fe9e7631d90a5ba46385544a5ba0424c3e7c304 Mon Sep 17 00:00:00 2001 From: John Speaks Date: Mon, 5 Oct 2026 16:40:42 +0000 Subject: [PATCH 1/2] Cover managed two-agent CUJ on dedicated workspace --- pyproject.toml | 2 + tests/README.md | 1 + tests/e2e_cuj/AGENTS.md | 5 +- tests/e2e_cuj/conftest.py | 117 +++++ tests/e2e_cuj/test_ug_managed_sentinel.py | 501 ++++++++++++++++++++++ tests/integration/utils/terminal.py | 7 + uv.lock | 37 ++ 7 files changed, 668 insertions(+), 2 deletions(-) create mode 100644 tests/e2e_cuj/conftest.py create mode 100644 tests/e2e_cuj/test_ug_managed_sentinel.py diff --git a/pyproject.toml b/pyproject.toml index 39fbd9cf9..2feaf034b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,8 @@ fallback-version = "0.0.0" [dependency-groups] dev = [ "pytest>=9.0.3", + "pexpect>=4.9.0", + "pyte>=0.8.2", # Pinned so local and CI lint with the identical formatter/type-checker; # ruff/ty output shifts between releases, so bump these deliberately. "ruff==0.16.5", diff --git a/tests/README.md b/tests/README.md index 2245d68b4..a3741b337 100644 --- a/tests/README.md +++ b/tests/README.md @@ -9,6 +9,7 @@ mocks, monkeypatching, fake binaries/services, or fabricated ug state. | Unit/component | Existing `test_*.py` files | Individual behavior; dependencies may be mocked | | Existing e2e | `test_e2e*.py` | Real workspace behavior with some patched setup/internal calls | | Integration CUJs | `integration/test_*.py` | Public configure, TUI, script, command, protocol, and lifecycle journeys | +| Dedicated-workspace CUJs | `e2e_cuj/test_*.py` | Published workspace policy, real agent tasks, and observable agent traces | | Installation | `integration/test_installation.py` | Fresh installed package, CLI, and local helpers without credentials on Linux and advisory native Windows | `test_entry_points.py` also runs both installed console scripts (`ug` and `ucode`) diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index ca063c975..94a1656d2 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -5,6 +5,7 @@ Each CUJ owns a separate workspace. Subclass `BaseCujTest` from `base.py` and se `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` with OAuth M2M authentication. Do not share the workspace between concurrent runs. No stubbed configuration. -Run from the repository root: +Run from the repository root with Claude Code, Codex, and Databricks CLI installed: `uv run pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj`. -This keeps the parent suite's mocked fixtures out of CUJs. There are no tests yet. +This keeps the parent suite's mocked fixtures out of CUJs. Set +`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` for the dedicated workspace. diff --git a/tests/e2e_cuj/conftest.py b/tests/e2e_cuj/conftest.py new file mode 100644 index 000000000..e18f095ed --- /dev/null +++ b/tests/e2e_cuj/conftest.py @@ -0,0 +1,117 @@ +"""Fixtures for dedicated-workspace CUJs.""" + +from __future__ import annotations + +import os +import re +import shutil +import sys +import tempfile +from pathlib import Path + +import pytest + +_INTEGRATION = Path(__file__).parents[1] / "integration" +_ROOT = _INTEGRATION.parents[1] +_SUITE = Path(__file__).parent +for _path in (_ROOT, _INTEGRATION, _SUITE): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +from base import BaseCujTest # noqa: E402 +from utils.harness import UserSession # noqa: E402 + + +def _ug_binary() -> Path: + raw = os.environ.get("UG_INTEGRATION_BIN", "").strip() + if raw: + path = Path(raw).expanduser() + if not path.is_file(): + pytest.fail(f"UG_INTEGRATION_BIN is not a file: {path}", pytrace=False) + return path + + checkout_binary = _ROOT / ".venv" / ("Scripts/ug.exe" if os.name == "nt" else "bin/ug") + if checkout_binary.is_file(): + return checkout_binary + pytest.fail( + f"No checkout ug found at {checkout_binary}; set UG_INTEGRATION_BIN to an installed ug.", + pytrace=False, + ) + + +@pytest.fixture(scope="session") +def installed_binary(): + return _ug_binary() + + +@pytest.fixture +def session(request, installed_binary): + case = re.sub(r"[^a-zA-Z0-9_.-]", "_", request.node.name) + with tempfile.TemporaryDirectory(prefix="ug-cuj-run-", dir=_ROOT) as root_name: + root = Path(root_name) + project_parent = root if os.name == "nt" else None + with tempfile.TemporaryDirectory( + prefix="ug-cuj-project-", dir=project_parent + ) as project_name: + user = UserSession( + root, + Path(project_name), + installed_binary, + root / "artifacts" / case, + ) + if os.name == "nt": + app_data = user.home / "AppData/Roaming" + local_app_data = user.home / "AppData/Local" + case_temp = local_app_data / "Temp" + for path in (app_data, local_app_data, case_temp): + path.mkdir(parents=True, exist_ok=True) + user.env.update( + { + "APPDATA": str(app_data), + "LOCALAPPDATA": str(local_app_data), + "TEMP": str(case_temp), + "TMP": str(case_temp), + "TMPDIR": str(case_temp), + } + ) + try: + yield user + finally: + if any( + (user.home / ".ucode" / name).is_file() + for name in ("state.json", "managed-backups/manifest.json") + ): + if os.name == "posix": + from utils.terminal import TerminalProcess + + with TerminalProcess( + user, "ug", [str(user.binary), "revert"], "cleanup-revert" + ) as terminal: + terminal.finish() + else: + user.run("revert", timeout=120) + + +@pytest.fixture +def live_session(request, session): + test = request.instance + if not isinstance(test, BaseCujTest): + pytest.fail("live_session requires a BaseCujTest instance.", pytrace=False) + + for binary in ("databricks", "claude", "codex"): + if not shutil.which(binary, path=session.env["PATH"]): + pytest.fail(f"Required CUJ binary is missing: {binary}", pytrace=False) + + has_sp_credentials = all( + os.environ.get(name, "").strip() + for name in ("UG_CUJ_SP_CLIENT_ID", "UG_CUJ_SP_CLIENT_SECRET") + ) + if has_sp_credentials: + authorization = test.workspace.config.authenticate().get("Authorization", "") + bearer = authorization.removeprefix("Bearer ").strip() + else: + bearer = os.environ.get("DATABRICKS_BEARER", "").strip() + if not bearer: + pytest.fail("Workspace authentication returned no bearer token.", pytrace=False) + session.env["DATABRICKS_BEARER"] = bearer + return session diff --git a/tests/e2e_cuj/test_ug_managed_sentinel.py b/tests/e2e_cuj/test_ug_managed_sentinel.py new file mode 100644 index 000000000..7e5eeca92 --- /dev/null +++ b/tests/e2e_cuj/test_ug_managed_sentinel.py @@ -0,0 +1,501 @@ +"""End-to-end sentinel for the published managed Claude and Codex configuration. + +The selected managed workspace publishes one static configuration for both agents. This journey +fetches that configuration through one public ``ug configure`` command, checks the generated files, +then runs real tasks on the bare managed default and explicit Claude/Codex selections. +""" + +from __future__ import annotations + +import json +import os +import re +import time +import tomllib +import uuid + +import pytest +from base import BaseCujTest +from utils.evidence import FileTask +from utils.model_discovery import claude_model_in_picker +from utils.sql import query_count, resolve_trace_table, resolve_warehouse_id +from utils.terminal import AgentTerminal + +CLAUDE_MODELS = [ + "system.ai.claude-opus-4-8", + "system.ai.claude-sonnet-4-6", + "system.ai.claude-haiku-4-5", +] +CLAUDE_DEFAULTS = { + "default_model": "system.ai.claude-sonnet-4-6", + "default_opus_model": "system.ai.claude-opus-4-8", + "default_sonnet_model": "system.ai.claude-sonnet-4-6", + "default_haiku_model": "system.ai.claude-haiku-4-5", +} +CODEX_MODELS = ["system.ai.gpt-5-6-sol", "system.ai.gpt-5-6-luna"] +CODEX_DEFAULT = CODEX_MODELS[0] +CODEX_LUNA = CODEX_MODELS[1] +EXPECTED_HEADER_NAMES = {"x-ug-e2e-run", "x-ug-e2e-agent"} +CLAUDE_PICKER_LABELS = ("Claude Opus 4.8", "Claude Sonnet 4.6", "Claude Haiku 4.5") +TRACE_WAIT_SECONDS = 360 +TRACE_POLL_SECONDS = 10 + + +def _claude_models_visible(text: str) -> bool: + return all( + claude_model_in_picker(text, model, label) + for model, label in zip(CLAUDE_MODELS, CLAUDE_PICKER_LABELS, strict=True) + ) + + +def _claude_custom_model_ids(screen: str) -> list[str]: + return re.findall( + r"(?m)^\s*(?:[❯›>]\s*)?\d+\.\s+.*Custom model \(([^)]+)\)", + screen, + ) + + +def _managed_config(session) -> dict: + cache_path = session.home / ".ucode" / "managed-config.json" + cache = json.loads(cache_path.read_text()) + raw = cache.get("config") + assert isinstance(raw, dict), cache + return raw + + +def _managed_entries(raw: dict) -> dict[str, dict]: + entries = raw.get("enabled_agents") + assert isinstance(entries, list), raw + assert len(entries) == 2, raw + result = { + entry.get("agent"): entry.get("config") + for entry in entries + if isinstance(entry, dict) and isinstance(entry.get("config"), dict) + } + assert set(result) == {"CODING_AGENT_CLAUDE_CODE", "CODING_AGENT_CODEX"}, raw + return result + + +def _normalized_headers(headers: object) -> dict[str, str]: + assert isinstance(headers, dict), headers + assert all(isinstance(name, str) for name in headers), headers + result = {name.casefold(): value for name, value in headers.items()} + assert set(result) == EXPECTED_HEADER_NAMES, headers + assert all(isinstance(value, str) and value for value in result.values()), headers + return result + + +def _managed_header_subset(headers: object) -> dict[str, str]: + assert isinstance(headers, dict), headers + assert all(isinstance(name, str) for name in headers), headers + normalized = {name.casefold(): value for name, value in headers.items()} + assert EXPECTED_HEADER_NAMES <= set(normalized), headers + assert all( + isinstance(normalized[name], str) and normalized[name] for name in EXPECTED_HEADER_NAMES + ) + return {name: normalized[name] for name in EXPECTED_HEADER_NAMES} + + +def _claude_header_lines(value: object) -> dict[str, str]: + assert isinstance(value, str), value + headers = {} + for line in value.splitlines(): + name, separator, header_value = line.partition(":") + if separator: + headers[name.strip().casefold()] = header_value.strip() + return _managed_header_subset(headers) + + +def _assert_published_config(raw: dict, entries: dict[str, dict]) -> dict[str, str]: + assert raw["default_agent"] == "CODING_AGENT_CLAUDE_CODE", raw + claude = entries["CODING_AGENT_CLAUDE_CODE"] + codex = entries["CODING_AGENT_CODEX"] + + assert claude["smart_routing"] == {"enabled": False}, claude + assert claude["models"] == {"model_services": CLAUDE_MODELS}, claude + assert claude["default_models"] == CLAUDE_DEFAULTS, claude + assert claude["tracing"] == {"enabled": True}, claude + claude_headers = _normalized_headers(claude["http_headers"]) + assert claude_headers["x-ug-e2e-agent"] == "claude", claude_headers + + assert codex["smart_routing"] == {"enabled": False}, codex + assert codex["models"] == {"model_services": CODEX_MODELS}, codex + assert codex["default_models"] == {"default_model": CODEX_DEFAULT}, codex + assert codex["tracing"] == {"enabled": True}, codex + codex_headers = _normalized_headers(codex["http_headers"]) + assert codex_headers["x-ug-e2e-agent"] == "codex", codex_headers + assert codex_headers["x-ug-e2e-run"] == claude_headers["x-ug-e2e-run"], ( + claude_headers, + codex_headers, + ) + return claude_headers + + +def _assert_generated_configs(session, claude_headers: dict[str, str]) -> None: + claude_settings_path = session.home / ".claude" / "ucode-settings.json" + claude_settings = json.loads(claude_settings_path.read_text()) + assert claude_settings["availableModels"] == CLAUDE_MODELS, claude_settings + claude_picker = claude_settings["modelPicker"] + assert claude_picker["replaceBuiltInOptions"] is True, claude_settings + assert claude_picker["options"] == [ + {"model": model, "label": label} + for model, label in zip( + CLAUDE_MODELS, + ("Claude Opus 4.8", "Claude Sonnet 4.6", "Claude Haiku 4.5"), + strict=True, + ) + ], claude_settings + claude_env = claude_settings["env"] + assert claude_env["ANTHROPIC_DEFAULT_OPUS_MODEL"] == CLAUDE_MODELS[0] + "[1m]", claude_settings + assert claude_env["ANTHROPIC_DEFAULT_SONNET_MODEL"] == (CLAUDE_MODELS[1] + "[1m]"), ( + claude_settings + ) + assert claude_env["ANTHROPIC_DEFAULT_HAIKU_MODEL"] == CLAUDE_MODELS[2], claude_settings + assert _claude_header_lines(claude_env["ANTHROPIC_CUSTOM_HEADERS"]) == claude_headers, ( + claude_settings + ) + assert claude_env["CLAUDE_CODE_ENABLE_TELEMETRY"] == "1", claude_settings + assert claude_env["OTEL_TRACES_EXPORTER"] == "otlp", claude_settings + assert claude_env["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"].endswith( + "/ai-gateway/otel/v1/traces" + ), claude_settings + + codex_profile_path = session.home / ".codex" / "ucode.config.toml" + codex_profile = tomllib.loads(codex_profile_path.read_text()) + assert codex_profile["model"] == CODEX_DEFAULT, codex_profile + codex_provider = codex_profile["model_providers"]["Databricks"] + assert _managed_header_subset(codex_provider["http_headers"]) == { + "x-ug-e2e-run": claude_headers["x-ug-e2e-run"], + "x-ug-e2e-agent": "codex", + }, codex_profile + codex_catalog_path = session.home / ".ucode" / "codex-model-catalog.json" + codex_catalog = json.loads(codex_catalog_path.read_text()) + assert [ + model["slug"] for model in codex_catalog["models"] if model.get("visibility") == "list" + ] == CODEX_MODELS, codex_catalog + codex_app_config = tomllib.loads((session.home / ".codex" / "config.toml").read_text()) + assert codex_app_config["model_catalog_json"] == str(codex_catalog_path), codex_app_config + + +def _trace_pair_count( + workspace: str, + bearer: str, + warehouse_id: str, + table: str, + pairs: list[tuple[str, str, str]], +) -> int: + """Count distinct marker/model pairs in both agents' trace attribute layouts.""" + clauses = [] + parameters = [] + for index, (marker, model, _marker_source) in enumerate(pairs): + clauses.append( + "(marker_source = :marker_source_" + f"{index} AND marker = :marker_{index} AND model = :model_{index})" + ) + parameters.extend( + [ + { + "name": f"marker_source_{index}", + "value": _marker_source, + "type": "STRING", + }, + {"name": f"marker_{index}", "value": marker, "type": "STRING"}, + {"name": f"model_{index}", "value": model, "type": "STRING"}, + ] + ) + + marker_expressions = { + "resource": "variant_get(resource.attributes, '$[\"ug_integration_marker\"]', 'STRING')", + "span": "variant_get(attributes, '$[\"ug_integration_marker\"]', 'STRING')", + } + span_names = { + "resource": "claude_code.llm_request", + "span": "model_client.stream_responses_api", + } + unions = [] + for source in ("resource", "span"): + marker_expression = marker_expressions[source] + model_key = "gen_ai.request.model" if source == "resource" else "model" + unions.append( + "SELECT " + f"'{source}' AS marker_source, " + f"{marker_expression} AS marker, " + f"variant_get(attributes, '$[\"{model_key}\"]', 'STRING') AS model " + f"FROM {table} WHERE kind = 'SPAN_KIND_INTERNAL' " + f"AND name = '{span_names[source]}' " + "AND time > current_timestamp() - INTERVAL 90 MINUTES" + ) + statement = ( + "SELECT COUNT(*) FROM (SELECT marker_source, marker, model FROM (" + + " UNION ALL ".join(unions) + + ") AS candidate_rows WHERE " + + " OR ".join(clauses) + + " GROUP BY marker_source, marker, model) AS observed" + ) + return query_count(workspace, bearer, warehouse_id, statement, parameters) + + +def _native_gateway_model(model: str) -> str: + """Return the model name emitted by the gateway's native client span.""" + return re.sub(r"\[(?:1m|200k)\]$", "", model).removeprefix("system.ai.") + + +def _trace_client_pair_count( + workspace: str, + bearer: str, + warehouse_id: str, + table: str, + pairs: list[tuple[str, str, str]], +) -> int: + """Count expected native gateway client models for the task markers.""" + input_messages = "CAST(variant_get(attributes, '$[\"gen_ai.input.messages\"]') AS STRING)" + model = "variant_get(attributes, '$[\"gen_ai.request.model\"]', 'STRING')" + expected_cases = [] + parameters = [] + for index, (marker, expected_model, _marker_source) in enumerate(pairs): + expected_cases.append( + f"WHEN input_messages LIKE concat('%', :client_marker_{index}, '%') " + f"AND model = :client_model_{index} THEN :client_marker_{index}" + ) + parameters.extend( + [ + { + "name": f"client_marker_{index}", + "value": marker, + "type": "STRING", + }, + { + "name": f"client_model_{index}", + "value": _native_gateway_model(expected_model), + "type": "STRING", + }, + ] + ) + statement = ( + "WITH candidates AS (" + "SELECT " + f"{input_messages} AS input_messages, " + f"{model} AS model " + f"FROM {table} " + "WHERE kind = 'SPAN_KIND_CLIENT' " + "AND time > current_timestamp() - INTERVAL 90 MINUTES) " + "SELECT COUNT(DISTINCT CASE " + " ".join(expected_cases) + " END) FROM candidates" + ) + return query_count(workspace, bearer, warehouse_id, statement, parameters) + + +def _wait_for_trace_pairs( + workspace: str, + bearer: str, + warehouse_id: str, + table: str, + pairs: list[tuple[str, str, str]], + session, +) -> None: + deadline = time.monotonic() + TRACE_WAIT_SECONDS + observed = 0 + observed_client = 0 + while True: + observed = _trace_pair_count(workspace, bearer, warehouse_id, table, pairs) + observed_client = _trace_client_pair_count(workspace, bearer, warehouse_id, table, pairs) + if observed >= len(pairs) and observed_client >= len(pairs): + session.record( + "trace-query.json", + { + "table": table, + "expected_pairs": [ + {"marker": marker, "model": model, "marker_source": source} + for marker, model, source in pairs + ], + "observed_pairs": observed, + "observed_native_client_pairs": observed_client, + "native_client_model_scope": "expected model per marker; auxiliary models allowed", + }, + ) + return + remaining = deadline - time.monotonic() + if remaining <= 0: + pytest.fail( + "Managed sentinel trace evidence timed out: " + f"observed {observed}/{len(pairs)} internal and " + f"{observed_client}/{len(pairs)} native client marker/model pairs in {table}. " + "The generated headers below are local writer evidence; the public trace table " + "does not expose arbitrary HTTP request headers, so this failure does not claim " + "that X-UG-E2E-* headers reached the gateway. Native client spans may also " + "contain auxiliary model calls for the same task." + ) + time.sleep(min(TRACE_POLL_SECONDS, remaining)) + + +class TestCujManagedConfig(BaseCujTest): + WORKSPACE_URL = "https://dbc-135c115c-c255.cloud.databricks.com" + + def test_cuj_managed_config_sentinel(self, live_session): + """Scenario: configure one published two-agent policy and run both agents. + + Expected: no agent selector; configured models, defaults, tracing and headers in generated + files; six completed tasks and matching agent inference plus native gateway client spans. + Auxiliary client calls for a task may use another model. Gateway receipt of headers is not + asserted here. + """ + session = live_session + workspace = self.WORKSPACE_URL + configured_workspace = os.environ.get("UG_CUJ1_WORKSPACE", "").strip().rstrip("/") + if configured_workspace: + assert configured_workspace == workspace, configured_workspace + configured = session.run( + "configure", + "--workspace", + workspace, + "--skip-upgrade", + timeout=300, + ) + assert "Select coding agents to configure:" not in configured.stdout, configured.stdout + + raw = _managed_config(session) + entries = _managed_entries(raw) + claude_headers = _assert_published_config(raw, entries) + _assert_generated_configs(session, claude_headers) + codex_models = session.codex_model_ids( + ["app-server", "--listen", "stdio://"], name="managed-sentinel-codex-models" + ) + assert codex_models == CODEX_MODELS, codex_models + session.record( + "managed-sentinel-wire.json", + { + "default_agent": raw["default_agent"], + "agents": { + agent: { + "models": config["models"], + "default_models": config["default_models"], + "smart_routing": config["smart_routing"], + "tracing": config["tracing"], + "http_headers": sorted(config["http_headers"]), + } + for agent, config in entries.items() + }, + }, + ) + + bearer = session.env["DATABRICKS_BEARER"] + trace_table = resolve_trace_table(workspace, bearer) + warehouse_id = os.environ.get("UG_INTEGRATION_WAREHOUSE_ID", "").strip() + warehouse_id = warehouse_id or resolve_warehouse_id(workspace, bearer) + run_id = uuid.uuid4().hex + trace_pairs: list[tuple[str, str, str]] = [] + + root_task = FileTask(session) + root_marker = f"ug-managed-sentinel-{run_id}-root" + trace_pairs.append((root_marker, CLAUDE_DEFAULTS["default_model"], "resource")) + session.env["OTEL_RESOURCE_ATTRIBUTES"] = f"ug_integration_marker={root_marker}" + with AgentTerminal( + session, "claude", [str(session.binary)], "managed-sentinel-root" + ) as tui: + tui.boot() + tui.submit(f"{root_task.prompt} Trace marker: {root_marker}") + tui.wait_for_task(root_task) + picker_screen = tui.open_model_picker(model_visible=_claude_models_visible) + assert _claude_custom_model_ids(picker_screen) == CLAUDE_MODELS, picker_screen + tui.exit_normally() + root_task.assert_completed(session, "claude") + + claude_opus_task = FileTask(session) + claude_opus_marker = f"ug-managed-sentinel-{run_id}-claude-opus" + trace_pairs.append( + (claude_opus_marker, CLAUDE_DEFAULTS["default_opus_model"] + "[1m]", "resource") + ) + session.env["OTEL_RESOURCE_ATTRIBUTES"] = f"ug_integration_marker={claude_opus_marker}" + with AgentTerminal( + session, + "claude", + [str(session.binary), "claude", "--model", "opus"], + "managed-sentinel-claude-opus", + ) as tui: + tui.boot() + tui.submit(f"{claude_opus_task.prompt} Trace marker: {claude_opus_marker}") + tui.wait_for_task(claude_opus_task) + tui.exit_normally() + claude_opus_task.assert_completed(session, "claude") + + claude_sonnet_task = FileTask(session) + claude_sonnet_marker = f"ug-managed-sentinel-{run_id}-claude-sonnet" + trace_pairs.append( + (claude_sonnet_marker, CLAUDE_DEFAULTS["default_sonnet_model"] + "[1m]", "resource") + ) + session.env["OTEL_RESOURCE_ATTRIBUTES"] = f"ug_integration_marker={claude_sonnet_marker}" + with AgentTerminal( + session, + "claude", + [str(session.binary), "claude", "--model", "sonnet"], + "managed-sentinel-claude-sonnet", + ) as tui: + tui.boot() + tui.submit(f"{claude_sonnet_task.prompt} Trace marker: {claude_sonnet_marker}") + tui.wait_for_task(claude_sonnet_task) + tui.exit_normally() + claude_sonnet_task.assert_completed(session, "claude") + + claude_haiku_task = FileTask(session) + claude_haiku_marker = f"ug-managed-sentinel-{run_id}-claude-haiku" + trace_pairs.append( + (claude_haiku_marker, CLAUDE_DEFAULTS["default_haiku_model"], "resource") + ) + session.env["OTEL_RESOURCE_ATTRIBUTES"] = f"ug_integration_marker={claude_haiku_marker}" + with AgentTerminal( + session, + "claude", + [str(session.binary), "claude", "--model", "haiku"], + "managed-sentinel-claude-haiku", + ) as tui: + tui.boot() + tui.submit(f"{claude_haiku_task.prompt} Trace marker: {claude_haiku_marker}") + tui.wait_for_task(claude_haiku_task) + tui.exit_normally() + claude_haiku_task.assert_completed(session, "claude") + + codex_default_task = FileTask(session) + codex_default_marker = f"ug-managed-sentinel-{run_id}-codex-default" + trace_pairs.append((codex_default_marker, CODEX_DEFAULT, "span")) + session.env.pop("OTEL_RESOURCE_ATTRIBUTES", None) + with AgentTerminal( + session, + "codex", + [ + str(session.binary), + "codex", + "--", + "--config", + f'otel.span_attributes.ug_integration_marker="{codex_default_marker}"', + ], + "managed-sentinel-codex-default", + ) as tui: + tui.boot() + tui.submit(f"{codex_default_task.prompt} Trace marker: {codex_default_marker}") + tui.wait_for_task(codex_default_task) + tui.exit_normally() + codex_default_task.assert_completed(session, "codex") + + codex_luna_task = FileTask(session) + codex_luna_marker = f"ug-managed-sentinel-{run_id}-codex-luna" + trace_pairs.append((codex_luna_marker, CODEX_LUNA, "span")) + with AgentTerminal( + session, + "codex", + [ + str(session.binary), + "codex", + "--", + "--model", + CODEX_LUNA, + "--config", + f'otel.span_attributes.ug_integration_marker="{codex_luna_marker}"', + ], + "managed-sentinel-codex-luna", + ) as tui: + tui.boot() + tui.submit(f"{codex_luna_task.prompt} Trace marker: {codex_luna_marker}") + tui.wait_for_task(codex_luna_task) + tui.exit_normally() + codex_luna_task.assert_completed(session, "codex") + + _wait_for_trace_pairs(workspace, bearer, warehouse_id, trace_table, trace_pairs, session) diff --git a/tests/integration/utils/terminal.py b/tests/integration/utils/terminal.py index 9f4c4fda7..e6a544474 100644 --- a/tests/integration/utils/terminal.py +++ b/tests/integration/utils/terminal.py @@ -232,6 +232,13 @@ def boot(self, timeout=120): # These are interactions with ordinary UI choices, not pre-written # onboarding state. Only the test's disposable project is trusted. dialogs = [ + ( + "external-imports", + "Allow external CLAUDE.md file imports?" in text + and "No, disable external imports" in text + and "Yes, allow external imports" in text, + "\r", + ), ( "theme", "Choose the text style" in text and "Dark mode" in text, diff --git a/uv.lock b/uv.lock index b63869f81..a2e745a30 100644 --- a/uv.lock +++ b/uv.lock @@ -569,6 +569,18 @@ wheels = [ { url = "https://pypi-proxy.cloud.databricks.com/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "pexpect" +version = "4.9.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } +dependencies = [ + { name = "ptyprocess" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -605,6 +617,15 @@ wheels = [ { url = "https://pypi-proxy.cloud.databricks.com/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "ptyprocess" +version = "0.7.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, +] + [[package]] name = "pyasn1" version = "0.6.4" @@ -748,6 +769,18 @@ crypto = [ { name = "cryptography" }, ] +[[package]] +name = "pyte" +version = "0.8.2" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/ab/ab/b599762933eba04de7dc5b31ae083112a6c9a9db15b01d3109ad797559d9/pyte-0.8.2.tar.gz", hash = "sha256:5af970e843fa96a97149d64e170c984721f20e52227a2f57f0a54207f08f083f", size = 92301, upload-time = "2023-11-12T09:33:43.217Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/59/d0/bb522283b90853afbf506cd5b71c650cf708829914efd0003d615cf426cd/pyte-0.8.2-py3-none-any.whl", hash = "sha256:85db42a35798a5aafa96ac4d8da78b090b2c933248819157fc0e6f78876a0135", size = 31627, upload-time = "2023-11-12T09:33:41.096Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -1096,6 +1129,8 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "pexpect" }, + { name = "pyte" }, { name = "pytest" }, { name = "ruff" }, { name = "ty" }, @@ -1114,6 +1149,8 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "pexpect", specifier = ">=4.9.0" }, + { name = "pyte", specifier = ">=0.8.2" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "ruff", specifier = "==0.16.5" }, { name = "ty", specifier = "==0.0.75" }, From bc493ea1489ec2db0b39669f62adb9e8335bb05d Mon Sep 17 00:00:00 2001 From: John Speaks Date: Mon, 5 Oct 2026 16:40:47 +0000 Subject: [PATCH 2/2] Gate full integration on CUJ1 managed sentinel --- .github/workflows/integration.yml | 49 ++++++++++++++++++++++++++++-- tests/test_integration_contract.py | 2 +- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7d4365166..2a1682437 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -428,11 +428,56 @@ jobs: ${{ runner.temp }}/ug-integration/artifacts/ ${{ runner.temp }}/ug-integration/wheels/ + cuj1: + name: CUJ1 managed config sentinel + if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && inputs.suite != 'tui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + runs-on: ubuntu-22.04 + timeout-minutes: 70 + concurrency: + group: e2e-cuj1 + cancel-in-progress: false + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22.19.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: 0.9.8 + - uses: databricks/setup-cli@bdb89f81c11a5bd647fd55b585b7c396ec68a25a # v1.0.0 + with: + version: 1.17.0 + - name: Run CUJ1 against its dedicated workspace + shell: bash + env: + UV_DEFAULT_INDEX: ${{ env.PACKAGE_INDEX }} + UG_CUJ1_WORKSPACE: ${{ secrets.UG_CUJ1_WORKSPACE }} + UG_CUJ_SP_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} + UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} + run: | + test -n "$UG_CUJ1_WORKSPACE" + npm install --global --no-audit --no-fund \ + "@anthropic-ai/claude-code@$CLAUDE_VERSION" "@openai/codex@$CODEX_VERSION" + mkdir -p "$RUNNER_TEMP/ug-cuj1" + uv lock + uv run --frozen --python 3.12 pytest --confcutdir=tests/e2e_cuj \ + tests/e2e_cuj --junitxml="$RUNNER_TEMP/ug-cuj1/junit.xml" + - name: Upload CUJ1 evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: integration-cuj1 + include-hidden-files: true + path: | + ${{ runner.temp }}/ug-cuj1/*.xml + cujs: name: All integration tests if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} # installation-windows remains advisory until its initial platform issues are fixed. - needs: [installation, workspace, smoke, full, managed] + needs: [installation, workspace, smoke, full, managed, cuj1] runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -454,7 +499,7 @@ jobs: if suite != "smoke": required.append("full") if suite in {"full", "live"}: - required.append("managed") + required.extend(["managed", "cuj1"]) failed = [job for job in required if results[job]["result"] != "success"] if failed: raise SystemExit("Integration jobs did not pass: " + ", ".join(failed)) diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 1e37cf160..69e060fd3 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -71,7 +71,7 @@ def test_integration_ci_gate_requires_selected_managed_jobs(suite, managed_resul assert script is not None results = { job: {"result": "success"} - for job in ("installation", "workspace", "smoke", "full", "managed") + for job in ("installation", "workspace", "smoke", "full", "managed", "cuj1") } results["managed"]["result"] = managed_result for job in {