ci: parallelize CI, cargo-nextest, gate timing tests, convert to zx

.config/nextest.toml

@@ -0,0 +1,41 @@
+# cargo-nextest configuration.
+#
+# Tests run one process per test, which isolates the process-global V8/Pyodide
+# platform, warm-isolate pool, module caches, and env that previously forced
+# `cargo test -- --test-threads=1`. See CLAUDE.md > Testing. Doctests are not run
+# by nextest — ci.sh runs `cargo test --doc` separately.
+
+[profile.default]
+# Backstop for a genuinely hung test (the runtime's own watchdogs fire well
+# before this); terminate rather than hang the run.
+slow-timeout = { period = "120s", terminate-after = 1 }
+# Timing-sensitive cases are gated behind SECURE_EXEC_RUN_TIMING_TESTS at their
+# call sites (and #[ignore] for any standalone ones); both are skipped by default.
+
+# The Pyodide "*_suite" tests are single collapsed #[test]s that run every Python
+# case sequentially in one process; Pyodide is CPU-heavy, so under the CPU
+# contention of a parallel run they legitimately need minutes. Give them room —
+# their own internal watchdogs still catch real hangs. (Overrides are per-profile
+# in nextest, so this is repeated under [profile.ci].)
+[[profile.default.overrides]]
+filter = 'test(/python.*_suite/)'
+slow-timeout = { period = "600s", terminate-after = 1 }
+
+[profile.ci]
+# Absorb rare load-induced flakes without masking real failures — a test that
+# only passes on retry is still surfaced in the report.
+retries = 2
+fail-fast = false
+slow-timeout = { period = "180s", terminate-after = 1 }
+
+[[profile.ci.overrides]]
+filter = 'test(/python.*_suite/)'
+slow-timeout = { period = "600s", terminate-after = 1 }
+
+[profile.timing]
+# Nightly serial lane for the timing-sensitive tests: single-threaded and
+# un-retried so wall-clock assertions are measured on a quiet box. Invoked as
+# `SECURE_EXEC_RUN_TIMING_TESTS=1 cargo nextest run --profile timing --run-ignored all`.
+test-threads = 1
+retries = 0
+slow-timeout = { period = "300s", terminate-after = 1 }

.github/workflows/bench.yml

@@ -13,7 +13,7 @@ jobs:
   gate:
     name: Bench gate
     if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, agentos-builder]
     timeout-minutes: 40
     steps:
       - uses: actions/checkout@v4
@@ -57,7 +57,7 @@ jobs:
   nightly:
     name: Nightly benchmark matrix
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, agentos-builder]
     timeout-minutes: 90
     steps:
       - uses: actions/checkout@v4

.github/workflows/ci-nightly.yml

@@ -0,0 +1,31 @@
+name: CI Nightly
+
+# The PR/push CI (ci.yml) is hermetic and fast: it skips network e2e and the
+# #[ignore]d resource-saturation tests. This job runs that expensive tail once
+# a day (and on demand) so the coverage still exists without taxing every PR.
+on:
+  schedule:
+    - cron: "23 8 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ci-nightly-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  extras:
+    runs-on: [self-hosted, agentos-builder]
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: wasm32-wasip1
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: nextest
+      - run: npx --yes zx@8 scripts/ci.mjs nightly-extras

.github/workflows/ci.yml

@@ -1,3 +1,14 @@
+# INTENTIONAL: the cache-heavy rust/registry jobs here (wasm-commands, rust-lint,
+# rust-test, js-test) run on the agent-os self-hosted builders
+# (runs-on: [self-hosted, agentos-builder]), NOT GitHub-hosted runners. The cheap
+# job (js-checks — typecheck/boundary/publish tests) stays on GitHub Actions,
+# which spins up small/dynamically and doesn't need the warm cache.
+# agent-os compiles the secure-exec crates (via the prepare-build clone), so both
+# repos share ONE warm secure-exec build cache on those boxes — the cargo
+# registry, the rusty_v8 V8 prebuilt, and sccache'd crate objects (shared
+# CARGO_HOME + SCCACHE_DIR). secure-exec CI warms the cache agent-os reuses, and
+# vice versa. On the self-hosted jobs the persistent on-disk target/ + CARGO_HOME
+# + sccache replace the GHA cache steps, which is why they're absent there.
 name: CI
 
 on:
@@ -6,9 +17,74 @@ on:
   push:
     branches: [main]
 
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
-  ci:
+  # Guest WASM command binaries are needed by both the cargo service tests and
+  # the JS turbo tests. Build them once here and hand them to both via an
+  # artifact so neither pays the cold build twice.
+  wasm-commands:
+    runs-on: [self-hosted, agentos-builder]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: wasm32-wasip1
+      - run: pnpm install --frozen-lockfile --filter "@secure-exec/core..."
+      # Cache the built guest WASM commands. `make -C registry/native wasm` is a
+      # ~4.5min `cargo build -Z build-std` against a PATCHED std + vendored crates,
+      # so it recompiles std-from-source every run and the persistent target/
+      # can't warm it — but its OUTPUT is deterministic from the registry command
+      # sources. Key on those sources: a command/patch/toolchain change rebuilds,
+      # an unchanged tree restores in seconds. This is the one spot a GHA cache
+      # beats the on-disk target (the build is uncacheable; the output isn't).
+      - uses: actions/cache@v4
+        id: wasm-cache
+        with:
+          path: |
+            packages/core/commands
+            registry/native/target/wasm32-wasip1/release/commands
+          key: wasm-commands-v1-${{ hashFiles('registry/native/crates/**', 'registry/native/patches/**', 'registry/native/scripts/**', 'registry/native/Makefile', 'registry/native/Cargo.toml', 'registry/native/Cargo.lock', 'rust-toolchain.toml') }}
+      - if: steps.wasm-cache.outputs.cache-hit != 'true'
+        run: make -C registry/native wasm
+      - if: steps.wasm-cache.outputs.cache-hit != 'true'
+        run: node packages/core/scripts/copy-wasm-commands.mjs
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wasm-commands
+          path: |
+            packages/core/commands
+            registry/native/target/wasm32-wasip1/release/commands
+          retention-days: 1
+          if-no-files-found: error
+
+  # Cheap job (typecheck / boundary checks / publish tests, no heavy cargo
+  # compile) — runs on GitHub-hosted runners, not the self-hosted pool, which is
+  # reserved for the cache-heavy rust/registry jobs.
+  js-checks:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: . -> target
+          key: js-checks
+      - run: npx --yes zx@8 scripts/ci.mjs js-checks
+
+  rust-lint:
+    runs-on: [self-hosted, agentos-builder]
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
@@ -18,17 +94,41 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt, clippy
-      - uses: Swatinem/rust-cache@v2
+      - run: pnpm install --frozen-lockfile --filter "@secure-exec/build-tools..."
+      - run: npx --yes zx@8 scripts/ci.mjs rust-lint
+
+  rust-test:
+    runs-on: [self-hosted, agentos-builder]
+    needs: wasm-commands
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
         with:
-          workspaces: |
-            . -> target
-      - uses: actions/cache@v4
+          node-version: 22
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: taiki-e/install-action@v2
         with:
-          path: ~/.cargo/.rusty_v8
-          key: ${{ runner.os }}-rusty-v8-${{ hashFiles('Cargo.lock', 'crates/v8-runtime/Cargo.toml', 'crates/v8-runtime/build.rs', 'crates/build-support/v8_bridge_build.rs', 'packages/build-tools/scripts/build-v8-bridge.mjs') }}
-          restore-keys: |
-            ${{ runner.os }}-rusty-v8-
-      - run: find . -name node_modules -prune -exec rm -rf {} +
-      - run: bash scripts/ci.sh
+          tool: nextest
+      - run: pnpm install --frozen-lockfile --filter "@secure-exec/core..." --filter "@secure-exec/build-tools..."
+      - uses: actions/download-artifact@v4
+        with:
+          name: wasm-commands
+      - run: npx --yes zx@8 scripts/ci.mjs rust-test
         env:
-          CI_FORK_PULL_REQUEST: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && '1' || '0' }}
+          NEXTEST_PROFILE: ci
+
+  js-test:
+    runs-on: [self-hosted, agentos-builder]
+    needs: wasm-commands
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/download-artifact@v4
+        with:
+          name: wasm-commands
+      - run: npx --yes zx@8 scripts/ci.mjs js-test

.github/workflows/publish.yaml

@@ -61,7 +61,7 @@ jobs:
       matrix:
         include:
           - platform: linux-x64-gnu
-            runner: ubuntu-22.04
+            runner: [self-hosted, agentos-builder]
             target: x86_64-unknown-linux-gnu
             binary: secure-exec-sidecar
           - platform: linux-arm64-gnu

CLAUDE.md

@@ -67,6 +67,19 @@ Every bound that protects a shared resource — memory/heap, CPU/wall-clock, fd/
 - **Keep them current:** the benches are a maintained surface, not a one-off audit. New runtime surface (a syscall, a polyfill module, an executor capability, a registry command tier) needs a matrix op or focused lane in the same change or a filed follow-up; perf-relevant changes to existing surface must re-run the affected rows and update coverage when the shape changes (new lanes, renamed ops). If a bench stops compiling or a lane can no longer run, fix or explicitly skip it with a reason — never delete coverage silently.
 - **Agent OS boundary:** agent-os keeps product-surface benches only (session tax, ACP) and consumes this framework.
 
+## Testing
+
+- **Tests must run in parallel. Never fall back to `--test-threads=1`.** Serializing the whole suite to dodge a race is banned — it hides real shared-state bugs and roughly triples wall-clock. If tests interfere, fix the interference (see the categories below) or isolate the offender; do not pin the runner to one thread. Per-test process isolation via `cargo nextest` is the sanctioned way to neutralize process-global state while keeping parallelism (doctests run separately via `cargo test --doc`).
+- **No process-global state coupling between tests.** A test must not depend on or mutate shared process state that a concurrent sibling sees: `std::env::set_var`/`remove_var` (an `EnvVarGuard` that mutates real env is a smell — thread config through explicit args/builders instead), `static`/`OnceLock`/`lazy_static` caches, warm-isolate/worker pools, or global counters. If a test genuinely needs a process-global (env, a shared cache's warm state), it belongs in a dedicated serial lane, not the default parallel suite.
+- **Timing-sensitive tests are disabled by default.** Any test whose pass/fail depends on wall-clock — `sleep`-then-assert-elapsed, `Duration`/`Instant` comparisons in assertions, "watchdog/timeout fired within N ms" — flakes under the CPU contention of a parallel run and MUST be gated off by default (`#[ignore]` or an env/feature gate — in this repo the env gate is `SECURE_EXEC_RUN_TIMING_TESTS`, exercised only in the quiet serial nightly lane) and run only there. A test that merely *measures* timing never stays in the default suite.
+- **A safeguard-firing test stays in the default suite ONLY IF the safeguard fires fast — the deciding factor is duration, not what the test proves.** A configured limit actually firing (CPU-time → runaway killed, fuel → exit 124, fd/socket/heap cap → denied) is a keeper **only when the test configures a SHORT bound** (a small CPU-time/fuel/timeout — sub-second where the mechanism allows) so the runaway dies almost immediately. If the test's runtime is instead dominated by *waiting out* a multi-second window it does NOT keep — most notably the **default ~30s V8 CPU-time watchdog** driving an infinite-loop guest module, but equally a network-denied op that hangs until a `from_secs(N)` collect/poll timeout, or a wasm fuel/timeout enforced only by a slow poll path. Those are timeout-dependent: gate them off by default (early-return on `SECURE_EXEC_RUN_TIMING_TESTS` at the call site, or `#[ignore]`) and run them only in the nightly lane. **Prefer** shrinking the configured window so the safeguard fires fast AND the coverage stays on every PR; gate only when a short window genuinely isn't achievable. This stacks with the resource-saturation `#[ignore]` rule below.
+- **Auto-skip expensive resource-saturation tests.** A test that proves the *absence* of a bound by actually saturating a resource — a JS/WASM infinite loop pinning a core for the watchdog window, a heap/alloc bomb, a fork bomb, anything that aborts the process — must be `#[ignore = "expensive: <resource> saturation; run with --ignored"]` (vitest: `it.skip`/env gate). These pin cores or crash the runner. Still test the *safeguards* (a configured limit/watchdog/quota actually firing) — those are bounded, fast, and are the regression guard that the protection works; keep them in the default suite.
+- **Unique ports and temp dirs per test.** Bind ephemeral ports (`:0`), never a fixed port; use a unique temp dir per test, never a fixed shared path. Fixed ports/paths collide the moment two tests run concurrently.
+- **Tests never build their dependencies — they error if a prerequisite is missing.** A test harness must not shell out to `cargo build` (or any build) on demand: that build runs inside the test's own timeout and, under parallelism, spawns N concurrent builds — a deterministic-timeout footgun. If the sidecar binary (or any built artifact) is absent or stale, the harness fails fast with a clear message ("build `X` first: `cargo build -p ...`"), it does not build it. CI builds artifacts in an explicit earlier step and pins them (e.g. `SECURE_EXEC_SIDECAR_BIN`); local dev builds first, then runs tests.
+- **Never delete coverage silently.** A skip/ignore/quarantine carries a one-line reason and, ideally, a follow-up. Rule of thumb: a test that ends only when a watchdog whose *absence* it documents fires → `#[ignore]`; a test that ends because a *safeguard* fires → keep it running.
+- **CI test-phase wall = the single slowest test, then the compile.** nextest runs every `#[test]` in its own process in parallel, so the wall ≈ the *longest individual test*, NOT the sum. Optimize the tallest pole only — split it, gate it if it's timeout/watchdog-bound, or shrink its configured limit; splitting anything *below* the current critical path is wasted churn that just lights idle cores. Once the slowest surviving test drops under the crate's compile time, the **build is the floor** and more test surgery buys nothing. (This repo: warm rust-test is ~2m build + the slowest test; measure per-test durations before optimizing.)
+- **Split a collapsed `*_suite` test for nextest ONLY when its cases are *independently* slow — the collapse guards two things, and splitting fixes one but breaks the other.** Integration suites batch sub-cases into one `#[test]` (`python_suite`, `wasm_suite`, `kill_cleanup_suite`) to dodge a shared-process V8/Pyodide teardown/init crash. nextest's process-per-test removes that crash, so cases *can* be split to parallelize — pattern: a `mod <name>_split` whose `macro_rules!` emits one gated `#[test]` per case (`if std::env::var_os("NEXTEST").is_none() { return; } super::<case>();`), plus an early `return` under `NEXTEST` in the collapsed suite so `cargo test` still owns it locally; cases with an in-process ordering/warm-state dependency stay grouped in one sequential `#[test]` named `*_suite` (to inherit the `nextest.toml` slow-timeout override). **But the collapse ALSO amortizes warm process-global state, which splitting defeats:** a case that's cheap only because prior cases warmed the shared Pyodide/V8 (cold Pyodide init, a cold `import micropip`, an infinite-loop module's prewarm running into the ~30s CPU watchdog) becomes *slower* split — each fresh process pays the cold cost — and grouping the bare offenders can reintroduce the SIGSEGV the collapse existed to prevent. So split only independently-expensive real-work cases; leave warm-amortized suites collapsed with a comment saying why, and always confirm the split binary's measured wall actually dropped before keeping the change.
+
 ## Project Boundaries
 
 - Keep the secure-exec runtime Agent OS-agnostic: no ACP, sessions, `agentos-protocol`, `agentos-client`, or `agentos-sidecar` dependencies in runtime code.

crates/execution/tests/javascript_v8.rs

@@ -38,6 +38,13 @@ Deleted coverage:
   stable artifact and markdown benchmark tests remain.
 */
 
+// Timing-sensitive assertions flake under the CPU contention of a parallel test
+// run (see CLAUDE.md > Testing). Gated off by default; the nightly timing lane
+// sets SECURE_EXEC_RUN_TIMING_TESTS=1 to enforce them.
+fn run_timing_sensitive_tests() -> bool {
+    std::env::var_os("SECURE_EXEC_RUN_TIMING_TESTS").is_some()
+}
+
 fn write_fixture(path: &Path, contents: &str) {
     if let Some(parent) = path.parent() {
         fs::create_dir_all(parent).expect("create fixture parent dirs");
@@ -2680,9 +2687,9 @@ fn javascript_execution_v8_timer_callbacks_fire_and_clear_correctly() {
     }
     setImmediate(tick);
   });
-  if (immediateChainMs >= 500) {
-    throw new Error(`setImmediate chain too slow: ${immediateChainMs}ms`);
-  }
+  // The upper-bound check on the chain latency lives host-side in Rust so it can
+  // be gated to the nightly timing lane (see run_timing_sensitive_tests); the
+  // guest only reports the measurement.
   console.log(`setImmediate-chain-ms=${immediateChainMs}`);
 })().catch((error) => {
   process.exitCode = 1;
@@ -2707,10 +2714,12 @@ fn javascript_execution_v8_timer_callbacks_fire_and_clear_correctly() {
         .parse()
         .expect("parse setImmediate timing");
     println!("setImmediate 1000-chain elapsed ms: {chain_ms}");
-    assert!(
-        chain_ms < 500,
-        "setImmediate 1000-chain elapsed too high: {chain_ms}ms"
-    );
+    if run_timing_sensitive_tests() {
+        assert!(
+            chain_ms < 500,
+            "setImmediate 1000-chain elapsed too high: {chain_ms}ms"
+        );
+    }
 
     let only_immediate_execution = engine
         .start_execution(StartJavascriptExecutionRequest {