From d22d2b509b4199911d138f7dff791a0ba75b7e09 Mon Sep 17 00:00:00 2001 From: squid-protocol Date: Sun, 25 Oct 2026 03:02:16 -0400 Subject: [PATCH] java-compile: keep the engine scan across runs through the Actions cache (Part of #5142) java-base-generation.yml seeds a content-keyed scan cache per corpus, java-compile.yml restores it (restore-only) and passes --scan-cache to the unchanged check and the engine-scan build. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/java-base-generation.yml | 31 ++++++++++++++---- .github/workflows/java-compile.yml | 15 +++++++++ tests/tools/test_cache_keys.py | 37 ++++++++++++++++++++++ 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/.github/workflows/java-base-generation.yml b/.github/workflows/java-base-generation.yml index a100918c59..35bb076ff1 100644 --- a/.github/workflows/java-base-generation.yml +++ b/.github/workflows/java-base-generation.yml @@ -74,8 +74,19 @@ jobs: lookup-only: true key: java-same-base-${{ matrix.corpus }}-${{ hashFiles('gitgalaxy/gitgalaxy/cobol_to_java_controller.py', 'gitgalaxy/gitgalaxy/cobol_refractor_controller.py', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_cobol/**', 'gitgalaxy/gitgalaxy/standards/yaml_reader.py', 'gitgalaxy/gitgalaxy/standards/fact_channel_confidence.json', 'gitgalaxy/tests/tools/java_target_matrix.py', 'gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py') }} + # #5142: the engine scan (dsf: ~90 s per side) is kept under its own content key (the scan-deciding code, not the + # translator), so the pull-request check restores it instead of scanning. java_target_matrix.scan_key is the exact + # key inside; this one only has to change whenever that one can. + - name: Is the engine scan already kept? + id: scan-kept + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/java_scan_cache + lookup-only: true + key: java-scan-${{ matrix.corpus }}-${{ hashFiles('gitgalaxy/gitgalaxy/**', '!gitgalaxy/gitgalaxy/tools/cobol_to_java/**', '!gitgalaxy/gitgalaxy/cobol_to_java_controller.py', '!gitgalaxy/gitgalaxy/cobol_refractor_controller.py') }}-${{ hashFiles('gitgalaxy/gitgalaxy/tools/cobol_to_java/det/copy/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/cics_task_containers.py') }}-${{ hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py', 'gitgalaxy/tests/tools/java_target_matrix.py') }} + - name: Setup Python - if: steps.kept.outputs.cache-hit != 'true' + if: steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true' uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" @@ -85,7 +96,7 @@ jobs: gitgalaxy/ci/requirements/*.txt - name: Install GitGalaxy - if: steps.kept.outputs.cache-hit != 'true' + if: steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true' run: | cd gitgalaxy tests/tools/ci_retry.sh 4 10 -- python -m pip install --require-hashes -r ci/requirements/bootstrap.txt @@ -93,21 +104,21 @@ jobs: - name: Cache the pinned corpora id: corpora - if: steps.kept.outputs.cache-hit != 'true' + if: steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .mainframe_corpora key: mainframe-corpora-${{ hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json') }} - name: Fetch the pinned corpora - if: steps.kept.outputs.cache-hit != 'true' + if: steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true' run: | cd gitgalaxy rm -rf "$GITGALAXY_MAINFRAME_CORPORA/_scans" PYTHONPATH="$PWD" python tests/tools/mainframe_corpus.py fetch - name: Save the pinned corpora (main) - if: steps.kept.outputs.cache-hit != 'true' && steps.corpora.outputs.cache-hit != 'true' && github.event_name != 'pull_request' + if: (steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true') && steps.corpora.outputs.cache-hit != 'true' && github.event_name != 'pull_request' continue-on-error: true # best effort: a miss only costs the next run a clone uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -116,7 +127,7 @@ jobs: - name: Generate this commit's Java (both paths) id: keep-gen - if: steps.kept.outputs.cache-hit != 'true' + if: steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true' env: CORPUS: ${{ matrix.corpus }} CONFIGS: ${{ matrix.configs }} @@ -129,6 +140,7 @@ jobs: "$GITGALAXY_MAINFRAME_CORPORA/$CORPUS" \ --same-as "$GITHUB_WORKSPACE/gitgalaxy" \ --base-cache "$RUNNER_TEMP/java_base_cache" \ + --scan-cache "$RUNNER_TEMP/java_scan_cache" \ --work "$RUNNER_TEMP/java_keep" \ "${ONLY[@]}" @@ -138,3 +150,10 @@ jobs: with: path: ${{ runner.temp }}/java_base_cache key: java-same-base-${{ matrix.corpus }}-${{ hashFiles('gitgalaxy/gitgalaxy/cobol_to_java_controller.py', 'gitgalaxy/gitgalaxy/cobol_refractor_controller.py', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_cobol/**', 'gitgalaxy/gitgalaxy/standards/yaml_reader.py', 'gitgalaxy/gitgalaxy/standards/fact_channel_confidence.json', 'gitgalaxy/tests/tools/java_target_matrix.py', 'gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py') }} + + - name: Save the engine scan under this content (main) + if: steps.scan-kept.outputs.cache-hit != 'true' && steps.keep-gen.outcome == 'success' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/java_scan_cache + key: java-scan-${{ matrix.corpus }}-${{ hashFiles('gitgalaxy/gitgalaxy/**', '!gitgalaxy/gitgalaxy/tools/cobol_to_java/**', '!gitgalaxy/gitgalaxy/cobol_to_java_controller.py', '!gitgalaxy/gitgalaxy/cobol_refractor_controller.py') }}-${{ hashFiles('gitgalaxy/gitgalaxy/tools/cobol_to_java/det/copy/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/cics_task_containers.py') }}-${{ hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py', 'gitgalaxy/tests/tools/java_target_matrix.py') }} diff --git a/.github/workflows/java-compile.yml b/.github/workflows/java-compile.yml index 706ce95d75..a92d482f9c 100644 --- a/.github/workflows/java-compile.yml +++ b/.github/workflows/java-compile.yml @@ -186,6 +186,19 @@ jobs: # main commits that touch those paths (the push run is path-filtered), so most PRs missed and regenerated both sides. key: java-same-base-${{ matrix.corpus }}-${{ github.event_name == 'pull_request' && hashFiles('base/gitgalaxy/cobol_to_java_controller.py', 'base/gitgalaxy/cobol_refractor_controller.py', 'base/gitgalaxy/tools/cobol_to_java/**', 'base/gitgalaxy/tools/cobol_to_cobol/**', 'base/gitgalaxy/standards/yaml_reader.py', 'base/gitgalaxy/standards/fact_channel_confidence.json', 'base/tests/tools/java_target_matrix.py', 'base/tests/cobol_mainframe/corpora.json', 'base/tests/tools/mainframe_corpus.py') || hashFiles('gitgalaxy/gitgalaxy/cobol_to_java_controller.py', 'gitgalaxy/gitgalaxy/cobol_refractor_controller.py', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_cobol/**', 'gitgalaxy/gitgalaxy/standards/yaml_reader.py', 'gitgalaxy/gitgalaxy/standards/fact_channel_confidence.json', 'gitgalaxy/tests/tools/java_target_matrix.py', 'gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py') }} + # #5142: the engine scan is kept across runs (java-base-generation.yml seeds it; restore-only here, #4968). The key only + # has to change whenever java_target_matrix.scan_key can; the exact key decides inside the tool. Same inputs as the seeding. + - name: Key the engine scan cache + id: scankey + run: echo "key=java-scan-${{ matrix.corpus }}-${{ github.event_name == 'pull_request' && hashFiles('base/gitgalaxy/**', '!base/gitgalaxy/tools/cobol_to_java/**', '!base/gitgalaxy/cobol_to_java_controller.py', '!base/gitgalaxy/cobol_refractor_controller.py') || hashFiles('gitgalaxy/gitgalaxy/**', '!gitgalaxy/gitgalaxy/tools/cobol_to_java/**', '!gitgalaxy/gitgalaxy/cobol_to_java_controller.py', '!gitgalaxy/gitgalaxy/cobol_refractor_controller.py') }}-${{ github.event_name == 'pull_request' && hashFiles('base/gitgalaxy/tools/cobol_to_java/det/copy/**', 'base/gitgalaxy/tools/cobol_to_java/cics_task_containers.py') || hashFiles('gitgalaxy/gitgalaxy/tools/cobol_to_java/det/copy/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/cics_task_containers.py') }}-${{ github.event_name == 'pull_request' && hashFiles('base/tests/cobol_mainframe/corpora.json', 'base/tests/tools/mainframe_corpus.py', 'base/tests/tools/java_target_matrix.py') || hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py', 'gitgalaxy/tests/tools/java_target_matrix.py') }}" >> "$GITHUB_OUTPUT" + + - name: Restore the engine scan + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/java_scan_cache + key: ${{ steps.scankey.outputs.key }} + restore-keys: java-scan-${{ matrix.corpus }}- + - name: Is the generated Java unchanged? (pull requests) id: same if: github.event_name == 'pull_request' @@ -201,6 +214,7 @@ jobs: "$GITGALAXY_MAINFRAME_CORPORA/$CORPUS" \ --same-as "$GITHUB_WORKSPACE/base" \ --base-cache "$RUNNER_TEMP/java_base_cache" \ + --scan-cache "$RUNNER_TEMP/java_scan_cache" \ --work "$RUNNER_TEMP/java_same" \ --summary "$GITHUB_STEP_SUMMARY" \ "${ONLY[@]}" @@ -272,6 +286,7 @@ jobs: "$GITGALAXY_MAINFRAME_CORPORA/$CORPUS" \ "${REUSE[@]}" \ --scan \ + --scan-cache "$RUNNER_TEMP/java_scan_cache" \ --work "$RUNNER_TEMP/java_matrix_scan" \ --m2 "$HOME/.m2/repository" \ --summary "$GITHUB_STEP_SUMMARY" \ diff --git a/tests/tools/test_cache_keys.py b/tests/tools/test_cache_keys.py index 95448bd5e3..c96a473071 100644 --- a/tests/tools/test_cache_keys.py +++ b/tests/tools/test_cache_keys.py @@ -158,6 +158,43 @@ def test_same_base_cache_is_restored_by_java_compile_and_seeded_by_its_own_workf assert "lookup-only: true" in lookup, "a push whose content key exists must not regenerate" +def _scan_inputs(expr): + """The hashFiles patterns of an expression, `!` kept and the checkout prefix (`base/` or `gitgalaxy/`) stripped.""" + return [re.sub(r"^(!?)(base|gitgalaxy)/", r"\1", a) for a in re.findall(r"'([^']+)'", expr)] + + +def test_engine_scan_cache_is_restored_by_java_compile_and_seeded_by_its_own_workflow(): + """#5142: the dsf engine scan (~90 s a side) is kept across runs. Same split as the same-base generation: java-compile + only restores (#4968), java-base-generation.yml saves. The key is a coarse content hash; the tool's scan_key is exact.""" + compile_text = (WORKFLOWS / "java-compile.yml").read_text(encoding="utf-8") + seed_text = (WORKFLOWS / "java-base-generation.yml").read_text(encoding="utf-8") + steps = [(wf, k, s) for wf, k, s in _cache_steps() if "java-scan-" in _key(s) or "java_scan_cache" in s] + assert sorted((wf, k) for wf, k, _ in steps) == [ + ("java-base-generation.yml", "restore"), + ("java-base-generation.yml", "save"), + ("java-compile.yml", "restore"), + ], steps + lookup = next(s for wf, k, s in steps if wf == "java-base-generation.yml" and k == "restore") + save = next(s for wf, k, s in steps if wf == "java-base-generation.yml" and k == "save") + assert "lookup-only: true" in lookup and _key(lookup) == _key(save) + seeded = _scan_inputs(_key(save)) + # the compile side computes the same key from the base checkout on a PR and from its own on a push + (keyed,) = re.findall(r'echo "key=(java-scan-[^"]*)"', compile_text) + parts = re.findall(r"pull_request' && hashFiles\(([^)]*)\) \|\| hashFiles\(([^)]*)\)", keyed) + assert len(parts) == 3, keyed + for side in (0, 1): + got = [a for p in parts for a in _scan_inputs(p[side])] + assert got == seeded, "the compile-side scan key hashes different inputs than the seeding" + assert any(a.startswith("!") for a in seeded) and "!gitgalaxy/cobol_to_java_controller.py" in seeded + assert seeded[0] == "gitgalaxy/**" and "tests/tools/java_target_matrix.py" in seeded + # the key must change whenever the tool's exact key can: the files the scan reads from the otherwise-left-out translator + assert "gitgalaxy/tools/cobol_to_java/det/copy/**" in seeded + assert "gitgalaxy/tools/cobol_to_java/cics_task_containers.py" in seeded + for text in (compile_text, seed_text): + assert '--scan-cache "$RUNNER_TEMP/java_scan_cache"' in text + assert compile_text.count('--scan-cache "$RUNNER_TEMP/java_scan_cache"') == 2, "the check and the scan build" + + def test_the_seeding_workflow_is_never_cancelled_or_gated_and_never_runs_on_a_pull_request(): text = (WORKFLOWS / "java-base-generation.yml").read_text(encoding="utf-8") assert "cancel-in-progress: false" in text and "github.sha" in text.split("concurrency:")[1].split("jobs:")[0]