From 30bc1aa152b34a63e1d9f96970c090ec49d7455c Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Fri, 3 Jul 2026 12:50:08 -0700 Subject: [PATCH] ci: parallelize CI, cargo-nextest, gate timing tests, zx, run heavy workflows on self-hosted agentos-builder pool - Split the serial CI into parallel jobs (js-checks, rust-lint, rust-test, js-test) + a wasm-commands artifact job. - Switch the Rust suite to cargo-nextest (process-per-test); flock no-op + per-process compile cache; timing tests gated behind SECURE_EXEC_RUN_TIMING_TESTS. - Convert scripts/ci.sh to scripts/ci.mjs (zx). - Run all heavyweight workflows (ci, bench, ci-nightly, and the x64 publish build) on the self-hosted agentos-builder pool instead of GitHub-hosted runners; the persistent on-disk target/ + shared CARGO_HOME + sccache replace the GHA cache steps. arm64 publish + the lightweight context/publish jobs stay on GitHub. - Document the test-parallelism policy in CLAUDE.md. --- .config/nextest.toml | 41 +++++ .github/workflows/bench.yml | 4 +- .github/workflows/ci-nightly.yml | 31 ++++ .github/workflows/ci.yml | 124 +++++++++++-- .github/workflows/publish.yaml | 2 +- CLAUDE.md | 13 ++ crates/execution/tests/javascript_v8.rs | 23 ++- crates/execution/tests/python_prewarm.rs | 40 +++++ crates/execution/tests/wasm.rs | 11 ++ crates/sidecar/tests/acp_session.rs | 15 +- crates/sidecar/tests/builtin_conformance.rs | 27 ++- crates/sidecar/tests/kill_cleanup.rs | 51 ++++++ crates/sidecar/tests/python.rs | 109 +++++++++++- crates/sidecar/tests/service.rs | 84 +++++---- crates/sidecar/tests/support/mod.rs | 23 +-- .../tests/embedded_runtime_session.rs | 41 +++-- crates/v8-runtime/tests/event_loop.rs | 19 +- scripts/ci.mjs | 168 ++++++++++++++++++ scripts/ci.sh | 55 ------ 19 files changed, 719 insertions(+), 162 deletions(-) create mode 100644 .config/nextest.toml create mode 100644 .github/workflows/ci-nightly.yml create mode 100644 scripts/ci.mjs delete mode 100755 scripts/ci.sh diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 000000000..b5dec7922 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,41 @@ +# cargo-nextest configuration. +# +# Tests run one process per test, which isolates the process-global V8/Pyodide +# platform, warm-isolate pool, module caches, and env that previously forced +# `cargo test -- --test-threads=1`. See CLAUDE.md > Testing. Doctests are not run +# by nextest — ci.sh runs `cargo test --doc` separately. + +[profile.default] +# Backstop for a genuinely hung test (the runtime's own watchdogs fire well +# before this); terminate rather than hang the run. +slow-timeout = { period = "120s", terminate-after = 1 } +# Timing-sensitive cases are gated behind SECURE_EXEC_RUN_TIMING_TESTS at their +# call sites (and #[ignore] for any standalone ones); both are skipped by default. + +# The Pyodide "*_suite" tests are single collapsed #[test]s that run every Python +# case sequentially in one process; Pyodide is CPU-heavy, so under the CPU +# contention of a parallel run they legitimately need minutes. Give them room — +# their own internal watchdogs still catch real hangs. (Overrides are per-profile +# in nextest, so this is repeated under [profile.ci].) +[[profile.default.overrides]] +filter = 'test(/python.*_suite/)' +slow-timeout = { period = "600s", terminate-after = 1 } + +[profile.ci] +# Absorb rare load-induced flakes without masking real failures — a test that +# only passes on retry is still surfaced in the report. +retries = 2 +fail-fast = false +slow-timeout = { period = "180s", terminate-after = 1 } + +[[profile.ci.overrides]] +filter = 'test(/python.*_suite/)' +slow-timeout = { period = "600s", terminate-after = 1 } + +[profile.timing] +# Nightly serial lane for the timing-sensitive tests: single-threaded and +# un-retried so wall-clock assertions are measured on a quiet box. Invoked as +# `SECURE_EXEC_RUN_TIMING_TESTS=1 cargo nextest run --profile timing --run-ignored all`. +test-threads = 1 +retries = 0 +slow-timeout = { period = "300s", terminate-after = 1 } diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 35f4debbb..1e6751b43 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -13,7 +13,7 @@ jobs: gate: name: Bench gate if: github.event_name == 'pull_request' - runs-on: ubuntu-latest + runs-on: [self-hosted, agentos-builder] timeout-minutes: 40 steps: - uses: actions/checkout@v4 @@ -57,7 +57,7 @@ jobs: nightly: name: Nightly benchmark matrix if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest + runs-on: [self-hosted, agentos-builder] timeout-minutes: 90 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml new file mode 100644 index 000000000..61d8bd780 --- /dev/null +++ b/.github/workflows/ci-nightly.yml @@ -0,0 +1,31 @@ +name: CI Nightly + +# The PR/push CI (ci.yml) is hermetic and fast: it skips network e2e and the +# #[ignore]d resource-saturation tests. This job runs that expensive tail once +# a day (and on demand) so the coverage still exists without taxing every PR. +on: + schedule: + - cron: "23 8 * * *" + workflow_dispatch: + +concurrency: + group: ci-nightly-${{ github.ref }} + cancel-in-progress: false + +jobs: + extras: + runs-on: [self-hosted, agentos-builder] + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-wasip1 + - uses: taiki-e/install-action@v2 + with: + tool: nextest + - run: npx --yes zx@8 scripts/ci.mjs nightly-extras diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8fcd0e23..e424e982e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,3 +1,14 @@ +# INTENTIONAL: the cache-heavy rust/registry jobs here (wasm-commands, rust-lint, +# rust-test, js-test) run on the agent-os self-hosted builders +# (runs-on: [self-hosted, agentos-builder]), NOT GitHub-hosted runners. The cheap +# job (js-checks — typecheck/boundary/publish tests) stays on GitHub Actions, +# which spins up small/dynamically and doesn't need the warm cache. +# agent-os compiles the secure-exec crates (via the prepare-build clone), so both +# repos share ONE warm secure-exec build cache on those boxes — the cargo +# registry, the rusty_v8 V8 prebuilt, and sccache'd crate objects (shared +# CARGO_HOME + SCCACHE_DIR). secure-exec CI warms the cache agent-os reuses, and +# vice versa. On the self-hosted jobs the persistent on-disk target/ + CARGO_HOME +# + sccache replace the GHA cache steps, which is why they're absent there. name: CI on: @@ -6,9 +17,74 @@ on: push: branches: [main] +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: - ci: + # Guest WASM command binaries are needed by both the cargo service tests and + # the JS turbo tests. Build them once here and hand them to both via an + # artifact so neither pays the cold build twice. + wasm-commands: + runs-on: [self-hosted, agentos-builder] + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-wasip1 + - run: pnpm install --frozen-lockfile --filter "@secure-exec/core..." + # Cache the built guest WASM commands. `make -C registry/native wasm` is a + # ~4.5min `cargo build -Z build-std` against a PATCHED std + vendored crates, + # so it recompiles std-from-source every run and the persistent target/ + # can't warm it — but its OUTPUT is deterministic from the registry command + # sources. Key on those sources: a command/patch/toolchain change rebuilds, + # an unchanged tree restores in seconds. This is the one spot a GHA cache + # beats the on-disk target (the build is uncacheable; the output isn't). + - uses: actions/cache@v4 + id: wasm-cache + with: + path: | + packages/core/commands + registry/native/target/wasm32-wasip1/release/commands + key: wasm-commands-v1-${{ hashFiles('registry/native/crates/**', 'registry/native/patches/**', 'registry/native/scripts/**', 'registry/native/Makefile', 'registry/native/Cargo.toml', 'registry/native/Cargo.lock', 'rust-toolchain.toml') }} + - if: steps.wasm-cache.outputs.cache-hit != 'true' + run: make -C registry/native wasm + - if: steps.wasm-cache.outputs.cache-hit != 'true' + run: node packages/core/scripts/copy-wasm-commands.mjs + - uses: actions/upload-artifact@v4 + with: + name: wasm-commands + path: | + packages/core/commands + registry/native/target/wasm32-wasip1/release/commands + retention-days: 1 + if-no-files-found: error + + # Cheap job (typecheck / boundary checks / publish tests, no heavy cargo + # compile) — runs on GitHub-hosted runners, not the self-hosted pool, which is + # reserved for the cache-heavy rust/registry jobs. + js-checks: runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: . -> target + key: js-checks + - run: npx --yes zx@8 scripts/ci.mjs js-checks + + rust-lint: + runs-on: [self-hosted, agentos-builder] steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -18,17 +94,41 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy - - uses: Swatinem/rust-cache@v2 + - run: pnpm install --frozen-lockfile --filter "@secure-exec/build-tools..." + - run: npx --yes zx@8 scripts/ci.mjs rust-lint + + rust-test: + runs-on: [self-hosted, agentos-builder] + needs: wasm-commands + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 with: - workspaces: | - . -> target - - uses: actions/cache@v4 + node-version: 22 + - uses: dtolnay/rust-toolchain@stable + - uses: taiki-e/install-action@v2 with: - path: ~/.cargo/.rusty_v8 - key: ${{ runner.os }}-rusty-v8-${{ hashFiles('Cargo.lock', 'crates/v8-runtime/Cargo.toml', 'crates/v8-runtime/build.rs', 'crates/build-support/v8_bridge_build.rs', 'packages/build-tools/scripts/build-v8-bridge.mjs') }} - restore-keys: | - ${{ runner.os }}-rusty-v8- - - run: find . -name node_modules -prune -exec rm -rf {} + - - run: bash scripts/ci.sh + tool: nextest + - run: pnpm install --frozen-lockfile --filter "@secure-exec/core..." --filter "@secure-exec/build-tools..." + - uses: actions/download-artifact@v4 + with: + name: wasm-commands + - run: npx --yes zx@8 scripts/ci.mjs rust-test env: - CI_FORK_PULL_REQUEST: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && '1' || '0' }} + NEXTEST_PROFILE: ci + + js-test: + runs-on: [self-hosted, agentos-builder] + needs: wasm-commands + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - uses: dtolnay/rust-toolchain@stable + - uses: actions/download-artifact@v4 + with: + name: wasm-commands + - run: npx --yes zx@8 scripts/ci.mjs js-test diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index f4e4eb071..e2c04359d 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -61,7 +61,7 @@ jobs: matrix: include: - platform: linux-x64-gnu - runner: ubuntu-22.04 + runner: [self-hosted, agentos-builder] target: x86_64-unknown-linux-gnu binary: secure-exec-sidecar - platform: linux-arm64-gnu diff --git a/CLAUDE.md b/CLAUDE.md index 5d71e891b..41f540918 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -67,6 +67,19 @@ Every bound that protects a shared resource — memory/heap, CPU/wall-clock, fd/ - **Keep them current:** the benches are a maintained surface, not a one-off audit. New runtime surface (a syscall, a polyfill module, an executor capability, a registry command tier) needs a matrix op or focused lane in the same change or a filed follow-up; perf-relevant changes to existing surface must re-run the affected rows and update coverage when the shape changes (new lanes, renamed ops). If a bench stops compiling or a lane can no longer run, fix or explicitly skip it with a reason — never delete coverage silently. - **Agent OS boundary:** agent-os keeps product-surface benches only (session tax, ACP) and consumes this framework. +## Testing + +- **Tests must run in parallel. Never fall back to `--test-threads=1`.** Serializing the whole suite to dodge a race is banned — it hides real shared-state bugs and roughly triples wall-clock. If tests interfere, fix the interference (see the categories below) or isolate the offender; do not pin the runner to one thread. Per-test process isolation via `cargo nextest` is the sanctioned way to neutralize process-global state while keeping parallelism (doctests run separately via `cargo test --doc`). +- **No process-global state coupling between tests.** A test must not depend on or mutate shared process state that a concurrent sibling sees: `std::env::set_var`/`remove_var` (an `EnvVarGuard` that mutates real env is a smell — thread config through explicit args/builders instead), `static`/`OnceLock`/`lazy_static` caches, warm-isolate/worker pools, or global counters. If a test genuinely needs a process-global (env, a shared cache's warm state), it belongs in a dedicated serial lane, not the default parallel suite. +- **Timing-sensitive tests are disabled by default.** Any test whose pass/fail depends on wall-clock — `sleep`-then-assert-elapsed, `Duration`/`Instant` comparisons in assertions, "watchdog/timeout fired within N ms" — flakes under the CPU contention of a parallel run and MUST be gated off by default (`#[ignore]` or an env/feature gate — in this repo the env gate is `SECURE_EXEC_RUN_TIMING_TESTS`, exercised only in the quiet serial nightly lane) and run only there. A test that merely *measures* timing never stays in the default suite. +- **A safeguard-firing test stays in the default suite ONLY IF the safeguard fires fast — the deciding factor is duration, not what the test proves.** A configured limit actually firing (CPU-time → runaway killed, fuel → exit 124, fd/socket/heap cap → denied) is a keeper **only when the test configures a SHORT bound** (a small CPU-time/fuel/timeout — sub-second where the mechanism allows) so the runaway dies almost immediately. If the test's runtime is instead dominated by *waiting out* a multi-second window it does NOT keep — most notably the **default ~30s V8 CPU-time watchdog** driving an infinite-loop guest module, but equally a network-denied op that hangs until a `from_secs(N)` collect/poll timeout, or a wasm fuel/timeout enforced only by a slow poll path. Those are timeout-dependent: gate them off by default (early-return on `SECURE_EXEC_RUN_TIMING_TESTS` at the call site, or `#[ignore]`) and run them only in the nightly lane. **Prefer** shrinking the configured window so the safeguard fires fast AND the coverage stays on every PR; gate only when a short window genuinely isn't achievable. This stacks with the resource-saturation `#[ignore]` rule below. +- **Auto-skip expensive resource-saturation tests.** A test that proves the *absence* of a bound by actually saturating a resource — a JS/WASM infinite loop pinning a core for the watchdog window, a heap/alloc bomb, a fork bomb, anything that aborts the process — must be `#[ignore = "expensive: saturation; run with --ignored"]` (vitest: `it.skip`/env gate). These pin cores or crash the runner. Still test the *safeguards* (a configured limit/watchdog/quota actually firing) — those are bounded, fast, and are the regression guard that the protection works; keep them in the default suite. +- **Unique ports and temp dirs per test.** Bind ephemeral ports (`:0`), never a fixed port; use a unique temp dir per test, never a fixed shared path. Fixed ports/paths collide the moment two tests run concurrently. +- **Tests never build their dependencies — they error if a prerequisite is missing.** A test harness must not shell out to `cargo build` (or any build) on demand: that build runs inside the test's own timeout and, under parallelism, spawns N concurrent builds — a deterministic-timeout footgun. If the sidecar binary (or any built artifact) is absent or stale, the harness fails fast with a clear message ("build `X` first: `cargo build -p ...`"), it does not build it. CI builds artifacts in an explicit earlier step and pins them (e.g. `SECURE_EXEC_SIDECAR_BIN`); local dev builds first, then runs tests. +- **Never delete coverage silently.** A skip/ignore/quarantine carries a one-line reason and, ideally, a follow-up. Rule of thumb: a test that ends only when a watchdog whose *absence* it documents fires → `#[ignore]`; a test that ends because a *safeguard* fires → keep it running. +- **CI test-phase wall = the single slowest test, then the compile.** nextest runs every `#[test]` in its own process in parallel, so the wall ≈ the *longest individual test*, NOT the sum. Optimize the tallest pole only — split it, gate it if it's timeout/watchdog-bound, or shrink its configured limit; splitting anything *below* the current critical path is wasted churn that just lights idle cores. Once the slowest surviving test drops under the crate's compile time, the **build is the floor** and more test surgery buys nothing. (This repo: warm rust-test is ~2m build + the slowest test; measure per-test durations before optimizing.) +- **Split a collapsed `*_suite` test for nextest ONLY when its cases are *independently* slow — the collapse guards two things, and splitting fixes one but breaks the other.** Integration suites batch sub-cases into one `#[test]` (`python_suite`, `wasm_suite`, `kill_cleanup_suite`) to dodge a shared-process V8/Pyodide teardown/init crash. nextest's process-per-test removes that crash, so cases *can* be split to parallelize — pattern: a `mod _split` whose `macro_rules!` emits one gated `#[test]` per case (`if std::env::var_os("NEXTEST").is_none() { return; } super::();`), plus an early `return` under `NEXTEST` in the collapsed suite so `cargo test` still owns it locally; cases with an in-process ordering/warm-state dependency stay grouped in one sequential `#[test]` named `*_suite` (to inherit the `nextest.toml` slow-timeout override). **But the collapse ALSO amortizes warm process-global state, which splitting defeats:** a case that's cheap only because prior cases warmed the shared Pyodide/V8 (cold Pyodide init, a cold `import micropip`, an infinite-loop module's prewarm running into the ~30s CPU watchdog) becomes *slower* split — each fresh process pays the cold cost — and grouping the bare offenders can reintroduce the SIGSEGV the collapse existed to prevent. So split only independently-expensive real-work cases; leave warm-amortized suites collapsed with a comment saying why, and always confirm the split binary's measured wall actually dropped before keeping the change. + ## Project Boundaries - Keep the secure-exec runtime Agent OS-agnostic: no ACP, sessions, `agentos-protocol`, `agentos-client`, or `agentos-sidecar` dependencies in runtime code. diff --git a/crates/execution/tests/javascript_v8.rs b/crates/execution/tests/javascript_v8.rs index 66ffe6619..8aa630f17 100644 --- a/crates/execution/tests/javascript_v8.rs +++ b/crates/execution/tests/javascript_v8.rs @@ -38,6 +38,13 @@ Deleted coverage: stable artifact and markdown benchmark tests remain. */ +// Timing-sensitive assertions flake under the CPU contention of a parallel test +// run (see CLAUDE.md > Testing). Gated off by default; the nightly timing lane +// sets SECURE_EXEC_RUN_TIMING_TESTS=1 to enforce them. +fn run_timing_sensitive_tests() -> bool { + std::env::var_os("SECURE_EXEC_RUN_TIMING_TESTS").is_some() +} + fn write_fixture(path: &Path, contents: &str) { if let Some(parent) = path.parent() { fs::create_dir_all(parent).expect("create fixture parent dirs"); @@ -2680,9 +2687,9 @@ fn javascript_execution_v8_timer_callbacks_fire_and_clear_correctly() { } setImmediate(tick); }); - if (immediateChainMs >= 500) { - throw new Error(`setImmediate chain too slow: ${immediateChainMs}ms`); - } + // The upper-bound check on the chain latency lives host-side in Rust so it can + // be gated to the nightly timing lane (see run_timing_sensitive_tests); the + // guest only reports the measurement. console.log(`setImmediate-chain-ms=${immediateChainMs}`); })().catch((error) => { process.exitCode = 1; @@ -2707,10 +2714,12 @@ fn javascript_execution_v8_timer_callbacks_fire_and_clear_correctly() { .parse() .expect("parse setImmediate timing"); println!("setImmediate 1000-chain elapsed ms: {chain_ms}"); - assert!( - chain_ms < 500, - "setImmediate 1000-chain elapsed too high: {chain_ms}ms" - ); + if run_timing_sensitive_tests() { + assert!( + chain_ms < 500, + "setImmediate 1000-chain elapsed too high: {chain_ms}ms" + ); + } let only_immediate_execution = engine .start_execution(StartJavascriptExecutionRequest { diff --git a/crates/execution/tests/python_prewarm.rs b/crates/execution/tests/python_prewarm.rs index 7d70143e2..103351998 100644 --- a/crates/execution/tests/python_prewarm.rs +++ b/crates/execution/tests/python_prewarm.rs @@ -145,8 +145,48 @@ fn python_execution_invalidates_prewarm_stamp_when_pyodide_bundle_changes() { // Separate libtest cases in this binary still trip a V8 teardown/init crash, so // keep the prewarm coverage in one top-level suite until that boundary is fixed. +// +// cargo-nextest runs every `#[test]` in its OWN process, so that crash cannot +// occur there — `mod python_prewarm_split` below exposes each case as a separate +// test that nextest runs in parallel across cores. Skip the collapsed run under +// nextest so the work isn't done twice; the split cases skip themselves under +// `cargo test`. nextest sets `NEXTEST=1` in each test process; libtest/`cargo +// test` does not. #[test] fn python_prewarm_suite() { + if std::env::var_os("NEXTEST").is_some() { + return; + } python_execution_prewarms_once_when_compile_cache_is_ready(); python_execution_invalidates_prewarm_stamp_when_pyodide_bundle_changes(); } + +/// Per-case split of `python_prewarm_suite` for cargo-nextest (process-per-test). +/// +/// Each `#[test]` here runs exactly one Pyodide prewarm case in its own process, +/// so the shared-process V8/Pyodide teardown/init crash that forces the collapsed +/// `python_prewarm_suite` above does not apply, and the cases run in parallel +/// across cores instead of serially. Each case skips itself under plain `cargo +/// test` (where `NEXTEST` is unset) so the collapsed suite owns the run there; the +/// collapsed suite likewise skips under nextest so the work isn't duplicated. +mod python_prewarm_split { + macro_rules! nextest_cases { + ($($case:ident),+ $(,)?) => { + $( + #[test] + fn $case() { + // Covered by the collapsed `super::python_prewarm_suite` under `cargo test`. + if std::env::var_os("NEXTEST").is_none() { + return; + } + super::$case(); + } + )+ + }; + } + + nextest_cases!( + python_execution_prewarms_once_when_compile_cache_is_ready, + python_execution_invalidates_prewarm_stamp_when_pyodide_bundle_changes, + ); +} diff --git a/crates/execution/tests/wasm.rs b/crates/execution/tests/wasm.rs index f8850c8c5..d23f31357 100644 --- a/crates/execution/tests/wasm.rs +++ b/crates/execution/tests/wasm.rs @@ -2744,6 +2744,17 @@ fn wasm_deep_recursion_respects_configured_stack_byte_limit() { // Separate libtest cases in this binary still trip a V8 teardown/init crash, so // keep the WASM runtime coverage in one top-level suite until that boundary is fixed. +// +// NOT split for cargo-nextest (unlike `python_suite`/`kill_cleanup_suite`): three +// cases here run an infinite-loop guest module (`wasm_execution_times_out_when_ +// fuel_budget_is_exhausted`, `..._poll_path_times_out_...`, `..._allows_prewarm_ +// timeout_to_differ_...`). In this collapsed run they are cheap because earlier +// cases warmed the process-global V8 state, but in a COLD nextest process the +// infinite loop is bounded only by the ~30s V8 CPU-time watchdog, so each costs +// ~30s in isolation and grouping only those three SIGSEGVs (the very shared- +// process teardown crash this suite guards against). Splitting therefore makes +// the binary's wall WORSE (33-92s vs this ~20s collapsed run) and is unsafe, so +// the coverage stays collapsed here. #[test] fn wasm_suite() { wasm_contexts_preserve_vm_and_module_configuration(); diff --git a/crates/sidecar/tests/acp_session.rs b/crates/sidecar/tests/acp_session.rs index 0aa803dc4..320ad540c 100644 --- a/crates/sidecar/tests/acp_session.rs +++ b/crates/sidecar/tests/acp_session.rs @@ -29,6 +29,13 @@ use std::task::{Context, Poll}; use std::time::{Duration, Instant}; use tokio::io::{split, AsyncBufReadExt, AsyncWrite, AsyncWriteExt, BufReader, DuplexStream}; +// Timing-sensitive assertions flake under the CPU contention of a parallel test +// run (see CLAUDE.md > Testing). Gated off by default; the nightly timing lane +// sets SECURE_EXEC_RUN_TIMING_TESTS=1 to enforce them. +fn run_timing_sensitive_tests() -> bool { + std::env::var_os("SECURE_EXEC_RUN_TIMING_TESTS").is_some() +} + fn sample_init_result() -> Map { Map::from_iter([ ( @@ -937,7 +944,9 @@ async fn acp_response_write_failures_put_the_client_into_a_failed_state() { ) .await .expect_err("subsequent request should fail fast"); - assert!(started_at.elapsed() < Duration::from_millis(50)); + if run_timing_sensitive_tests() { + assert!(started_at.elapsed() < Duration::from_millis(50)); + } assert!( matches!(subsequent_error, AcpClientError::Io(_)), "unexpected subsequent error: {subsequent_error:?}" @@ -981,7 +990,9 @@ async fn acp_request_method_timeout_overrides_apply_to_initialize_and_prompt() { .expect("initialize join") .expect_err("initialize should time out"); assert!(matches!(initialize_error, AcpClientError::Timeout(_))); - assert!(initialize_started.elapsed() < Duration::from_millis(20)); + if run_timing_sensitive_tests() { + assert!(initialize_started.elapsed() < Duration::from_millis(20)); + } assert!(initialize_error .to_string() .contains("ACP request initialize (id=1) timed out after 5ms")); diff --git a/crates/sidecar/tests/builtin_conformance.rs b/crates/sidecar/tests/builtin_conformance.rs index 8a9054d95..92dd2692a 100644 --- a/crates/sidecar/tests/builtin_conformance.rs +++ b/crates/sidecar/tests/builtin_conformance.rs @@ -28,6 +28,13 @@ use support::{ wire_session, wire_vm, write_fixture, }; +// Timing-sensitive assertions flake under the CPU contention of a parallel test +// run (see CLAUDE.md > Testing). Gated off by default; the nightly timing lane +// sets SECURE_EXEC_RUN_TIMING_TESTS=1 to enforce them. +fn run_timing_sensitive_tests() -> bool { + std::env::var_os("SECURE_EXEC_RUN_TIMING_TESTS").is_some() +} + const ALLOWED_NODE_BUILTINS: &[&str] = &[ "assert", "buffer", @@ -1596,10 +1603,12 @@ console.log(JSON.stringify({ let elapsed_ms = result["elapsedMs"] .as_u64() .expect("vm timeout elapsed milliseconds"); - assert!( - elapsed_ms <= 200, - "vm timeout exceeded 200ms: {elapsed_ms}ms ({result})" - ); + if run_timing_sensitive_tests() { + assert!( + elapsed_ms <= 200, + "vm timeout exceeded 200ms: {elapsed_ms}ms ({result})" + ); + } assert_eq!( result["timeoutCode"], Value::String(String::from("ERR_SCRIPT_EXECUTION_TIMEOUT")) @@ -4181,10 +4190,12 @@ console.log(JSON.stringify({ hasRefAfterUnref: timer.hasRef() })); stderr.trim().is_empty(), "guest process should not wait long enough to fire the timer:\n{stderr}" ); - assert!( - elapsed < Duration::from_millis(1_500), - "guest process waited too long for an unref'd timer: {elapsed:?}" - ); + if run_timing_sensitive_tests() { + assert!( + elapsed < Duration::from_millis(1_500), + "guest process waited too long for an unref'd timer: {elapsed:?}" + ); + } let payload: Value = serde_json::from_str(stdout.trim()).expect("parse timer stdout JSON"); assert_eq!(payload["hasRefAfterUnref"], Value::Bool(false)); diff --git a/crates/sidecar/tests/kill_cleanup.rs b/crates/sidecar/tests/kill_cleanup.rs index f7c2ea99f..321ac93f2 100644 --- a/crates/sidecar/tests/kill_cleanup.rs +++ b/crates/sidecar/tests/kill_cleanup.rs @@ -260,6 +260,13 @@ fn append_process_output(buffer: &mut String, chunk: &[u8], process_id: &str, ch } fn kill_process_terminates_running_wasm_execution() { + // Timeout-dependent: the infinite-loop wasm module's prewarm runs into the + // ~30s V8 CPU-time watchdog before the kill lands, so this case takes ~30s + // regardless. Gate it to the nightly timing lane (SECURE_EXEC_RUN_TIMING_TESTS=1) + // rather than pay 30s on every PR. See CLAUDE.md > Testing. + if std::env::var_os("SECURE_EXEC_RUN_TIMING_TESTS").is_none() { + return; + } assert_node_available(); let mut sidecar = new_sidecar("kill-process-wasm"); @@ -552,6 +559,16 @@ fn remove_connection_disposes_owned_sessions_and_vms() { fn kill_cleanup_suite() { // Multiple libtest cases in this V8-backed integration binary still trip // teardown/init crashes, so keep the coverage in one top-level suite. + // + // cargo-nextest runs every `#[test]` in its OWN process, so that crash cannot + // occur there — `mod kill_cleanup_split` below exposes each case as a separate + // test that nextest runs in parallel across cores. Skip the collapsed run under + // nextest so the work isn't done twice; the split cases skip themselves under + // `cargo test`. nextest sets `NEXTEST=1` in each test process; libtest/`cargo + // test` does not. + if std::env::var_os("NEXTEST").is_some() { + return; + } close_session_removes_the_session_and_disposes_owned_vms(); dispose_vm_succeeds_even_when_a_guest_process_is_running(); kill_process_terminates_running_guest_execution(); @@ -559,3 +576,37 @@ fn kill_cleanup_suite() { kill_process_terminates_running_wasm_execution(); remove_connection_disposes_owned_sessions_and_vms(); } + +/// Per-case split of `kill_cleanup_suite` for cargo-nextest (process-per-test). +/// +/// Each `#[test]` here runs exactly one case in its own process, so the +/// shared-process V8 teardown/init crash that forces the collapsed +/// `kill_cleanup_suite` above does not apply, and the cases run in parallel across +/// cores instead of serially. Each case skips itself under plain `cargo test` +/// (where `NEXTEST` is unset) so the collapsed suite owns the run there; the +/// collapsed suite likewise skips under nextest so the work isn't duplicated. +mod kill_cleanup_split { + macro_rules! nextest_cases { + ($($case:ident),+ $(,)?) => { + $( + #[test] + fn $case() { + // Covered by the collapsed `super::kill_cleanup_suite` under `cargo test`. + if std::env::var_os("NEXTEST").is_none() { + return; + } + super::$case(); + } + )+ + }; + } + + nextest_cases!( + close_session_removes_the_session_and_disposes_owned_vms, + dispose_vm_succeeds_even_when_a_guest_process_is_running, + kill_process_terminates_running_guest_execution, + sigkill_synthesizes_exit_for_shared_v8_guest_execution, + kill_process_terminates_running_wasm_execution, + remove_connection_disposes_owned_sessions_and_vms, + ); +} diff --git a/crates/sidecar/tests/python.rs b/crates/sidecar/tests/python.rs index d30918383..8d4853baf 100644 --- a/crates/sidecar/tests/python.rs +++ b/crates/sidecar/tests/python.rs @@ -3878,8 +3878,20 @@ fn python_cli_suite() { #[test] fn python_suite() { - // Multiple libtest cases in this V8/Pyodide-backed integration binary - // still trip teardown/init crashes, so keep the coverage in one suite. + // Multiple libtest cases in this V8/Pyodide-backed integration binary still + // trip shared-process teardown/init crashes, so under plain `cargo test` + // (one process for the whole binary) keep the coverage in one suite. + // + // cargo-nextest runs every `#[test]` in its OWN process, so that crash + // cannot occur there — `mod python_split` below exposes each case as a + // separate test that nextest runs in parallel across cores (this suite was + // the single longest test in CI at ~400s serial). Skip the collapsed run + // under nextest so the work isn't done twice; the split cases skip + // themselves under `cargo test`. nextest sets `NEXTEST=1` in each test + // process; libtest/`cargo test` does not. + if std::env::var_os("NEXTEST").is_some() { + return; + } static_file_server_rejects_traversal_paths(); python_runtime_executes_code_end_to_end(); python_runtime_executes_workspace_py_file_by_path(); @@ -3910,3 +3922,96 @@ fn python_suite() { python_cli_suite(); python_rootfs_suite(); } + +/// Per-case split of `python_suite` for cargo-nextest (process-per-test). +/// +/// Each `#[test]` here runs exactly one Pyodide case in its own process, so the +/// shared-process V8/Pyodide teardown/init crash that forces the collapsed +/// `python_suite` above does not apply, and the ~36 cases run in parallel across +/// cores instead of serially. Each case skips itself under plain `cargo test` +/// (where `NEXTEST` is unset) so the collapsed suite owns the run there; the +/// collapsed suite likewise skips under nextest so the work isn't duplicated. +mod python_split { + macro_rules! nextest_cases { + ($($case:ident),+ $(,)?) => { + $( + #[test] + fn $case() { + // Covered by the collapsed `super::python_suite` under `cargo test`. + if std::env::var_os("NEXTEST").is_none() { + return; + } + super::$case(); + } + )+ + }; + } + + nextest_cases!( + static_file_server_rejects_traversal_paths, + python_runtime_executes_code_end_to_end, + python_runtime_executes_workspace_py_file_by_path, + python_runtime_reports_syntax_errors_over_stderr, + python_runtime_blocks_pyodide_js_escape_hatches, + concurrent_python_processes_stay_isolated_across_vms, + python_runtime_mounts_workspace_over_the_kernel_vfs, + python_runtime_supports_file_delete_and_rename, + python_runtime_supports_raw_tcp_and_udp_sockets, + python_runtime_supports_symlink_readlink_and_metadata, + workspace_files_are_shared_between_javascript_and_python_runtimes, + python_workspace_mount_respects_read_only_root_permissions, + python_runtime_blocks_mapped_pyodide_cache_symlink_metadata_escape, + python_runtime_blocks_mapped_pyodide_cache_symlink_swap_toctou_escape, + python_runtime_routes_stdin_writes_and_close_to_pyodide, + python_runtime_supports_interactive_input_prompts_and_multiple_streaming_writes, + python_runtime_close_stdin_triggers_input_eof_and_empty_read, + python_runtime_kill_process_terminates_blocked_stdin_reads, + python_runtime_imports_bundled_numpy_without_network, + python_runtime_imports_bundled_pandas_without_network, + python_runtime_routes_dns_and_http_through_sidecar_bridge, + python_runtime_routes_requests_through_sidecar_bridge, + python_runtime_surfaces_network_permission_errors, + python_runtime_runs_node_subprocesses_through_sidecar_bridge, + python_runtime_surfaces_subprocess_permission_errors, + python_command_runs_inline_code, + python_command_runs_script_with_argv, + python_command_runs_module_with_dash_m, + python_command_reads_program_from_stdin, + python_command_runs_interactive_repl, + python_command_runs_as_nested_child_process, + python_command_pip_installs_via_micropip, + python_reads_and_writes_arbitrary_vm_paths, + python_pip_installs_persist_across_invocations, + ); + + // The network-DENIED micropip case can't load the micropip package on its + // own (network is denied, so the fetch hangs); it relies on micropip already + // being loaded in the same *process* by a prior network-ALLOWED install + // (Pyodide's module state is process-global). Under nextest's + // process-per-test model that shared state is gone, so pair it in one + // sequential case with a network-allowed install that loads micropip first. + // The other micropip/pip installs are self-contained (they load micropip + // themselves) and stay split for parallelism. Skips under `cargo test`, + // where the collapsed `super::python_suite` owns the run. + // Named `*_suite` so it inherits nextest.toml's 600s slow-timeout override + // for `test(/python.*_suite/)` when the denied case runs (nightly). + #[test] + fn python_micropip_suite() { + if std::env::var_os("NEXTEST").is_none() { + return; + } + // Both micropip installs are heavy COLD work under process-per-test (no + // warm micropip load to reuse): the network-allowed install is ~24s, and + // the network-denied variant is timeout-bound (~2min on the denied-op + // timeout). Neither is on the critical path, so run the whole group only in + // the nightly timing lane (SECURE_EXEC_RUN_TIMING_TESTS=1); `cargo test`'s + // collapsed `super::python_suite` still covers both locally. The denied + // case also needs the allowed install loaded in the SAME process, so they + // stay grouped. See CLAUDE.md > Testing. + if std::env::var_os("SECURE_EXEC_RUN_TIMING_TESTS").is_none() { + return; + } + super::python_runtime_supports_micropip_package_installation(); + super::python_runtime_micropip_install_respects_network_permissions(); + } +} diff --git a/crates/sidecar/tests/service.rs b/crates/sidecar/tests/service.rs index 1e05ba6d2..4057a3306 100644 --- a/crates/sidecar/tests/service.rs +++ b/crates/sidecar/tests/service.rs @@ -132,7 +132,6 @@ mod service { A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SOA, SRV, TXT, }; use hickory_resolver::proto::rr::{RData, Record, RecordType}; - use nix::fcntl::{Flock, FlockArg}; use nix::libc; use rustls::client::danger::{ HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier, @@ -166,7 +165,6 @@ mod service { use serde_json::{json, Value}; use std::collections::BTreeMap; use std::fs; - use std::fs::OpenOptions; use std::io::{BufReader, Read, Write}; use std::net::{SocketAddr, TcpListener, UdpSocket}; use std::os::unix::fs::PermissionsExt; @@ -238,37 +236,43 @@ ykAheWCsAteSEWVc0w==\n\ RequestFrame::new(request_id, ownership, payload) } + // Timing-sensitive assertions flake under the CPU contention of a parallel + // test run (see CLAUDE.md > Testing). Gated off by default; the nightly + // timing lane sets SECURE_EXEC_RUN_TIMING_TESTS=1 to enforce them. + fn run_timing_sensitive_tests() -> bool { + std::env::var_os("SECURE_EXEC_RUN_TIMING_TESTS").is_some() + } + fn acquire_sidecar_runtime_test_lock() { - static LOCK_FILE: OnceLock> = OnceLock::new(); - let _ = LOCK_FILE.get_or_init(|| { - let path = std::env::temp_dir().join("secure-exec-sidecar-runtime-tests.lock"); - let file = OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(&path) - .unwrap_or_else(|error| { - panic!("open sidecar test runtime lock {}: {error}", path.display()) - }); - Flock::lock(file, FlockArg::LockExclusive).unwrap_or_else(|(_, error)| { - panic!("lock sidecar test runtime {}: {error}", path.display()) - }) - }); + // No-op under cargo-nextest: each test runs in its own process, so the + // process-global V8 platform, env, and (now per-process) compile cache + // are already isolated. Previously an exclusive flock serialized + // runtime-touching tests across binaries to guard the shared fixed + // compile-cache path; that path is now unique per process. See + // CLAUDE.md > Testing. } fn create_test_sidecar_with_config( config: NativeSidecarConfig, ) -> NativeSidecar { - let isolated_cache_suffix = std::env::var(ISOLATED_SERVICE_CACHE_SUFFIX_ENV).ok(); - if isolated_cache_suffix.is_none() { - acquire_sidecar_runtime_test_lock(); - } - let compile_cache_root = isolated_cache_suffix - .map(|suffix| { - std::env::temp_dir().join(format!("secure-exec-sidecar-test-cache-{suffix}")) - }) - .unwrap_or_else(|| std::env::temp_dir().join("secure-exec-sidecar-test-cache")); + // Unique compile-cache dir per test process (a re-exec child supplies an + // explicit suffix; otherwise derive one from PID + a sequence counter). + // Under cargo-nextest each test is its own process, so this gives every + // test an isolated cache instead of the old fixed shared path — no flock + // needed. See CLAUDE.md > Testing. + let cache_suffix = std::env::var(ISOLATED_SERVICE_CACHE_SUFFIX_ENV) + .ok() + .unwrap_or_else(|| { + static CACHE_SEQ: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + format!( + "{}-{}", + std::process::id(), + CACHE_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + ) + }); + let compile_cache_root = + std::env::temp_dir().join(format!("secure-exec-sidecar-test-cache-{cache_suffix}")); NativeSidecar::with_config( RecordingBridge::default(), NativeSidecarConfig { @@ -9174,6 +9178,12 @@ ykAheWCsAteSEWVc0w==\n\ } fn wasm_command_timeout_is_enforced_by_sidecar_poll_path() { + // Timeout-dependent: an infinite-loop wasm module whose termination is + // enforced by the sidecar poll path only after ~30s. Gate it to the + // nightly timing lane rather than pay ~30s per PR. See CLAUDE.md > Testing. + if !run_timing_sensitive_tests() { + return; + } let command_root = temp_dir("secure-exec-sidecar-command-resolution-wasm-timeout"); write_fixture( &command_root.join("spin"), @@ -19011,20 +19021,24 @@ console.log(`BODY:${{body}}`); dispose_result.expect("join dispose task"); let (poll_response, poll_elapsed) = poll_result.expect("join poll task"); assert_eq!(poll_response, Value::Null); - assert!( - poll_elapsed <= Duration::from_millis(200), - "net.poll stayed blocked too long: {poll_elapsed:?}" - ); + if run_timing_sensitive_tests() { + assert!( + poll_elapsed <= Duration::from_millis(200), + "net.poll stayed blocked too long: {poll_elapsed:?}" + ); + } let sidecar = std::rc::Rc::try_unwrap(sidecar) .expect("recover sidecar after local tasks") .into_inner(); (sidecar, started.elapsed()) })); let (mut sidecar, dispose_elapsed) = concurrency_elapsed; - assert!( - dispose_elapsed <= Duration::from_millis(200), - "dispose should not wait behind guest net.poll: {dispose_elapsed:?}" - ); + if run_timing_sensitive_tests() { + assert!( + dispose_elapsed <= Duration::from_millis(200), + "dispose should not wait behind guest net.poll: {dispose_elapsed:?}" + ); + } call_javascript_sync_rpc( &mut sidecar, diff --git a/crates/sidecar/tests/support/mod.rs b/crates/sidecar/tests/support/mod.rs index 6ea2039f5..a329769fb 100644 --- a/crates/sidecar/tests/support/mod.rs +++ b/crates/sidecar/tests/support/mod.rs @@ -4,7 +4,6 @@ mod bridge_support; pub use bridge_support::RecordingBridge; -use nix::fcntl::{Flock, FlockArg}; use secure_exec_sidecar::protocol::{ DisposeReason, EventFrame, GuestRuntimeKind, OwnershipScope, RequestFrame, RequestId, RequestPayload, ResponseFrame, @@ -12,32 +11,18 @@ use secure_exec_sidecar::protocol::{ use secure_exec_sidecar::{DispatchResult, NativeSidecar, NativeSidecarConfig}; use std::collections::{BTreeMap, HashMap}; use std::fs; -use std::fs::OpenOptions; use std::path::{Path, PathBuf}; use std::process::Command; -use std::sync::OnceLock; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; pub const TEST_AUTH_TOKEN: &str = "sidecar-test-token"; const MAX_COLLECTED_PROCESS_STREAM_BYTES: usize = 1024 * 1024; pub fn acquire_sidecar_runtime_test_lock() { - static LOCK_FILE: OnceLock> = OnceLock::new(); - let _ = LOCK_FILE.get_or_init(|| { - let path = std::env::temp_dir().join("secure-exec-sidecar-runtime-tests.lock"); - let file = OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(&path) - .unwrap_or_else(|error| { - panic!("open sidecar test runtime lock {}: {error}", path.display()) - }); - Flock::lock(file, FlockArg::LockExclusive).unwrap_or_else(|(_, error)| { - panic!("lock sidecar test runtime {}: {error}", path.display()) - }) - }); + // No-op under cargo-nextest: each test runs in its own process, so the + // process-global V8 platform, env, and compile cache are already isolated. + // Previously an exclusive flock serialized runtime tests across binaries. See + // CLAUDE.md > Testing. } pub fn assert_node_available() { diff --git a/crates/v8-runtime/tests/embedded_runtime_session.rs b/crates/v8-runtime/tests/embedded_runtime_session.rs index cb23fe0c2..a39b55296 100644 --- a/crates/v8-runtime/tests/embedded_runtime_session.rs +++ b/crates/v8-runtime/tests/embedded_runtime_session.rs @@ -7,6 +7,13 @@ use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; +// Timing-sensitive assertions flake under the CPU contention of a parallel test +// run (see CLAUDE.md > Testing). Gated off by default; the nightly timing lane +// sets SECURE_EXEC_RUN_TIMING_TESTS=1 to enforce them. +fn run_timing_sensitive_tests() -> bool { + std::env::var_os("SECURE_EXEC_RUN_TIMING_TESTS").is_some() +} + static NEXT_TEST_SESSION_ID: AtomicU64 = AtomicU64::new(1); fn next_session_id() -> String { @@ -340,10 +347,12 @@ fn assert_queued_work_waits_for_slot_release() -> io::Result<()> { "expected one active slot with the second session still queued", || runtime.active_slot_count() == 1 && runtime.session_count() == 2, ); - assert!( - receiver_b.recv_timeout(Duration::from_millis(150)).is_err(), - "queued session should not emit an execution result before the first slot is released" - ); + if run_timing_sensitive_tests() { + assert!( + receiver_b.recv_timeout(Duration::from_millis(150)).is_err(), + "queued session should not emit an execution result before the first slot is released" + ); + } runtime.dispatch(RuntimeCommand::DestroySession { session_id: session_a.clone(), @@ -416,12 +425,14 @@ fn assert_shared_runtime_handles_share_concurrency_quota() -> io::Result<()> { "expected one runtime-wide slot budget shared across all embedded runtime handles", || runtime.active_slot_count() == 3 && runtime.session_count() == 4, ); - assert!( - receivers[3] - .recv_timeout(Duration::from_millis(150)) - .is_err(), - "the fourth client should stay queued while the first three handles occupy the shared slots" - ); + if run_timing_sensitive_tests() { + assert!( + receivers[3] + .recv_timeout(Duration::from_millis(150)) + .is_err(), + "the fourth client should stay queued while the first three handles occupy the shared slots" + ); + } runtime.dispatch(RuntimeCommand::DestroySession { session_id: session_ids[0].clone(), @@ -482,10 +493,12 @@ fn assert_terminate_interrupts_sync_bridge_wait() -> io::Result<()> { runtime.session_handle(session_id.clone()).terminate()?; let terminated = wait_for_execution_result(&receiver, &session_id); - assert!( - terminate_started.elapsed() < Duration::from_secs(1), - "terminate() should return promptly while the sync bridge call is blocked" - ); + if run_timing_sensitive_tests() { + assert!( + terminate_started.elapsed() < Duration::from_secs(1), + "terminate() should return promptly while the sync bridge call is blocked" + ); + } assert!( matches!( terminated, diff --git a/crates/v8-runtime/tests/event_loop.rs b/crates/v8-runtime/tests/event_loop.rs index 950f66158..8ceab0886 100644 --- a/crates/v8-runtime/tests/event_loop.rs +++ b/crates/v8-runtime/tests/event_loop.rs @@ -11,6 +11,13 @@ use std::time::{Duration, Instant}; const WASM_FORTY_TWO_BYTES: &str = "0,97,115,109,1,0,0,0,1,5,1,96,0,1,127,3,2,1,0,7,12,1,8,102,111,114,116,121,84,119,111,0,0,10,6,1,4,0,65,42,11"; const EVENT_LOOP_WATCHDOG_TIMEOUT: Duration = Duration::from_secs(6); +// Timing-sensitive assertions flake under the CPU contention of a parallel test +// run (see CLAUDE.md > Testing). Gated off by default; the nightly timing lane +// sets SECURE_EXEC_RUN_TIMING_TESTS=1 to enforce them. +fn run_timing_sensitive_tests() -> bool { + std::env::var_os("SECURE_EXEC_RUN_TIMING_TESTS").is_some() +} + struct EventLoopWatchdog { cancel_tx: Option>, join_handle: Option>, @@ -331,11 +338,13 @@ fn event_loop_waits_for_refed_guest_timers_between_interval_ticks() { "event loop exited before four 500ms timer ticks elapsed: {:?}", elapsed ); - assert!( - elapsed < Duration::from_secs(5), - "event loop did not exit promptly after timers drained: {:?}", - elapsed - ); + if run_timing_sensitive_tests() { + assert!( + elapsed < Duration::from_secs(5), + "event loop did not exit promptly after timers drained: {:?}", + elapsed + ); + } let source = v8::String::new( scope, diff --git a/scripts/ci.mjs b/scripts/ci.mjs new file mode 100644 index 000000000..8665db180 --- /dev/null +++ b/scripts/ci.mjs @@ -0,0 +1,168 @@ +#!/usr/bin/env zx +// CI entry point (zx). Run with: `npx zx scripts/ci.mjs ` (or `pnpm dlx zx`). +// +// Phases (disjoint so .github/workflows/ci.yml can run them on parallel runners): +// all (default) full serial run — local use +// js-checks install, build, generated/type/boundary checks, publish tests +// rust-lint cargo fmt + clippy +// rust-test guest WASM commands + full cargo workspace tests (nextest) +// js-test turbo test suites against a pinned sidecar (no network e2e) +// nightly-extras network-e2e turbo tests + timing + #[ignore]d expensive tests +// +// zx injects the $, cd, echo, fs, path, argv, os globals — no imports needed. + +$.verbose = true; // stream each command's output live + +const ROOT = path.resolve(__dirname, ".."); +cd(ROOT); + +// Self-hosted-runner layout (a pinned toolchain under /workspace) — mirror the +// old bash bootstrap so those runners resolve cargo/rustc correctly. +const selfHostedToolchain = + "/workspace/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin"; +if (fs.existsSync("/workspace/.cargo") && fs.existsSync(selfHostedToolchain)) { + process.env.CARGO_HOME = "/workspace/.cargo"; + process.env.RUSTUP_HOME = "/workspace/.rustup"; + process.env.PATH = `/workspace/.cargo/bin:${process.env.PATH}`; + process.env.RUSTC = `${selfHostedToolchain}/rustc`; + process.env.RUSTDOC = `${selfHostedToolchain}/rustdoc`; +} +process.env.CARGO_HTTP_TIMEOUT ??= "120"; +process.env.CARGO_NET_RETRY ??= "10"; + +// Run a labeled step inside a foldable GitHub Actions log group. +async function step(label, cmd) { + echo(`::group::==> ${label}`); + try { + await cmd(); + } finally { + echo("::endgroup::"); + } +} + +// $ with extra environment for a single command (like bash `env VAR=val cmd`). +function withEnv(extra) { + return $({ env: { ...process.env, ...extra } }); +} + +// The service fs/shell regression tests stage guest WASM command binaries and +// fail hard when missing. In CI these arrive prebuilt via the wasm-commands +// artifact; skip the ~12-minute build when they are already present. +async function buildWasmCommands() { + const dir = path.join(ROOT, "packages/core/commands"); + if (fs.existsSync(dir) && fs.readdirSync(dir).length > 0) { + echo("==> guest WASM commands already present; skipping build"); + return; + } + await step("make -C registry/native wasm", () => $`make -C registry/native wasm`); + await step("copy-wasm-commands", () => $`node packages/core/scripts/copy-wasm-commands.mjs`); +} + +// Pin the sidecar binary for the JS test suites. Without this the first vitest +// file that needs a sidecar sees the v8-bridge asset regenerated by the turbo +// ^build tasks (mtime newer than the binary) and runs `cargo build` INSIDE its +// own 120s test timeout — the deterministic-timeout footgun we chased earlier. +async function pinSidecar() { + await step("cargo build -p secure-exec-sidecar", () => $`cargo build -p secure-exec-sidecar`); + process.env.SECURE_EXEC_SIDECAR_BIN = path.join( + ROOT, + "target/debug/secure-exec-sidecar", + ); +} + +async function jsChecks() { + await step("pnpm install", () => $`pnpm install --frozen-lockfile`); + await step("pnpm build", () => $`pnpm build`); + await step("check-generated", () => $`pnpm run check-generated`); + await step("check-types", () => $`pnpm check-types`); + await step("boundary check test", () => $`node --test scripts/check-secure-exec-boundary.test.mjs`); + await step("boundary check", () => $`node scripts/check-secure-exec-boundary.mjs`); + await step("no-escaping-deps test", () => $`node --test scripts/check-no-escaping-local-deps.test.mjs`); + await step("no-escaping-deps check", () => $`node scripts/check-no-escaping-local-deps.mjs`); + await step("publish check-types", () => $`pnpm --dir scripts/publish run check-types`); + await step("publish tests", () => $`pnpm --dir scripts/publish test`); +} + +async function rustLint() { + await step("cargo fmt --check", () => $`cargo fmt --check`); + await step("cargo clippy", () => $`cargo clippy --workspace --all-targets -- -D warnings`); +} + +async function rustTest() { + await buildWasmCommands(); + // cargo-nextest runs one process per test, isolating the process-global + // V8/Pyodide platform + caches that previously forced --test-threads=1. + // Timing-sensitive tests are skipped by default. Doctests are not run by + // nextest, so run them separately. + const profile = process.env.NEXTEST_PROFILE || "default"; + await step(`cargo nextest run (${profile})`, () => + withEnv({ CARGO_INCREMENTAL: "0" })`cargo nextest run --workspace --profile ${profile}`, + ); + await step("cargo test --doc", () => + withEnv({ CARGO_INCREMENTAL: "0" })`cargo test --workspace --doc`, + ); +} + +async function jsTest() { + await step("pnpm install", () => $`pnpm install --frozen-lockfile`); + await step("pnpm build", () => $`pnpm build`); + await buildWasmCommands(); + await pinSidecar(); + // Network e2e coverage runs in the nightly job, keeping the PR path hermetic. + await step("turbo test", () => $`pnpm exec turbo run test --concurrency=1`); +} + +async function nightlyExtras() { + await step("pnpm install", () => $`pnpm install --frozen-lockfile`); + await step("pnpm build", () => $`pnpm build`); + await buildWasmCommands(); + await pinSidecar(); + await step("turbo test (network e2e)", () => + withEnv({ SECURE_EXEC_E2E_NETWORK: "1" })`pnpm exec turbo run test --concurrency=1 --force`, + ); + // Timing-sensitive tests are gated off the default suite (they flake under + // parallel CPU contention); run them here on a quiet, serial lane. + await step("cargo nextest (timing lane)", () => + withEnv({ CARGO_INCREMENTAL: "0", SECURE_EXEC_RUN_TIMING_TESTS: "1" })`cargo nextest run --workspace --profile timing --run-ignored all`, + ); + // Expensive resource-saturation tests are #[ignore]d in the default suite; + // the nightly is where they actually run. + await step("cargo test --ignored", () => + withEnv({ CARGO_INCREMENTAL: "0" })`cargo test --workspace -- --ignored --test-threads=1`, + ); +} + +const phase = argv._[0] ?? "all"; +switch (phase) { + case "js-checks": + await jsChecks(); + break; + case "rust-lint": + await rustLint(); + break; + case "rust-test": + await rustTest(); + break; + case "js-test": + await jsTest(); + break; + case "nightly-extras": + await nightlyExtras(); + break; + case "all": + await jsChecks(); + await rustLint(); + await rustTest(); + await pinSidecar(); + if (process.env.CI_FORK_PULL_REQUEST === "1") { + await step("turbo test", () => $`pnpm exec turbo run test --concurrency=1`); + } else { + await step("turbo test (network e2e)", () => + withEnv({ SECURE_EXEC_E2E_NETWORK: "1" })`pnpm exec turbo run test --concurrency=1`, + ); + } + break; + default: + echo(`unknown ci phase: ${phase}`); + process.exit(1); +} diff --git a/scripts/ci.sh b/scripts/ci.sh deleted file mode 100755 index 3d49cfdc1..000000000 --- a/scripts/ci.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT_DIR" - -if [[ -d /workspace/.cargo && -d /workspace/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin ]]; then - export CARGO_HOME=/workspace/.cargo - export RUSTUP_HOME=/workspace/.rustup - export PATH="/workspace/.cargo/bin:${PATH}" - export RUSTC=/workspace/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin/rustc - export RUSTDOC=/workspace/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin/rustdoc -fi - -export CARGO_HTTP_TIMEOUT="${CARGO_HTTP_TIMEOUT:-120}" -export CARGO_NET_RETRY="${CARGO_NET_RETRY:-10}" - -run_step() { - echo "" - echo "==> $*" - "$@" -} - -run_step pnpm install --frozen-lockfile -run_step pnpm build -run_step pnpm run check-generated -run_step pnpm check-types -run_step node --test scripts/check-secure-exec-boundary.test.mjs -run_step node scripts/check-secure-exec-boundary.mjs -run_step node --test scripts/check-no-escaping-local-deps.test.mjs -run_step node scripts/check-no-escaping-local-deps.mjs -run_step pnpm --dir scripts/publish run check-types -run_step pnpm --dir scripts/publish test -run_step cargo fmt --check -run_step cargo clippy --workspace --all-targets -- -D warnings -# Service fs/shell regression tests stage guest WASM command binaries -# (registry/native or packages/core/commands) and fail hard when missing. -run_step make -C registry/native wasm -run_step node packages/core/scripts/copy-wasm-commands.mjs -run_step env CARGO_INCREMENTAL=0 cargo test --workspace -- --test-threads=1 - -# Pin the sidecar binary for the JS test suites. Without this, the first -# vitest file that needs a sidecar sees the v8-bridge asset regenerated by -# the turbo ^build tasks (an mtime newer than the binary) and runs -# `cargo build -p secure-exec-sidecar` INSIDE its own 120s test timeout — -# a multi-minute rebuild on CI runners (and N concurrent cargo builds under -# file parallelism), which presented as deterministic test timeouts. -run_step cargo build -p secure-exec-sidecar -export SECURE_EXEC_SIDECAR_BIN="$ROOT_DIR/target/debug/secure-exec-sidecar" - -if [[ "${CI_FORK_PULL_REQUEST:-0}" == "1" ]]; then - run_step pnpm exec turbo run test --concurrency=1 -else - run_step env SECURE_EXEC_E2E_NETWORK=1 pnpm exec turbo run test --concurrency=1 -fi