java-compile: keep the engine scan across runs (Part of #5142)

.github/workflows/java-base-generation.yml

@@ -74,8 +74,19 @@ jobs:
           lookup-only: true
           key: java-same-base-${{ matrix.corpus }}-${{ hashFiles('gitgalaxy/gitgalaxy/cobol_to_java_controller.py', 'gitgalaxy/gitgalaxy/cobol_refractor_controller.py', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_cobol/**', 'gitgalaxy/gitgalaxy/standards/yaml_reader.py', 'gitgalaxy/gitgalaxy/standards/fact_channel_confidence.json', 'gitgalaxy/tests/tools/java_target_matrix.py', 'gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py') }}
 
+      # #5142: the engine scan (dsf: ~90 s per side) is kept under its own content key (the scan-deciding code, not the
+      # translator), so the pull-request check restores it instead of scanning. java_target_matrix.scan_key is the exact
+      # key inside; this one only has to change whenever that one can.
+      - name: Is the engine scan already kept?
+        id: scan-kept
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ runner.temp }}/java_scan_cache
+          lookup-only: true
+          key: java-scan-${{ matrix.corpus }}-${{ hashFiles('gitgalaxy/gitgalaxy/**', '!gitgalaxy/gitgalaxy/tools/cobol_to_java/**', '!gitgalaxy/gitgalaxy/cobol_to_java_controller.py', '!gitgalaxy/gitgalaxy/cobol_refractor_controller.py') }}-${{ hashFiles('gitgalaxy/gitgalaxy/tools/cobol_to_java/det/copy/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/cics_task_containers.py') }}-${{ hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py', 'gitgalaxy/tests/tools/java_target_matrix.py') }}
+
       - name: Setup Python
-        if: steps.kept.outputs.cache-hit != 'true'
+        if: steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true'
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
@@ -85,29 +96,29 @@ jobs:
             gitgalaxy/ci/requirements/*.txt
 
       - name: Install GitGalaxy
-        if: steps.kept.outputs.cache-hit != 'true'
+        if: steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true'
         run: |
           cd gitgalaxy
           tests/tools/ci_retry.sh 4 10 -- python -m pip install --require-hashes -r ci/requirements/bootstrap.txt
           pip install --no-build-isolation -e .
 
       - name: Cache the pinned corpora
         id: corpora
-        if: steps.kept.outputs.cache-hit != 'true'
+        if: steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true'
         uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: .mainframe_corpora
           key: mainframe-corpora-${{ hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json') }}
 
       - name: Fetch the pinned corpora
-        if: steps.kept.outputs.cache-hit != 'true'
+        if: steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true'
         run: |
           cd gitgalaxy
           rm -rf "$GITGALAXY_MAINFRAME_CORPORA/_scans"
           PYTHONPATH="$PWD" python tests/tools/mainframe_corpus.py fetch
 
       - name: Save the pinned corpora (main)
-        if: steps.kept.outputs.cache-hit != 'true' && steps.corpora.outputs.cache-hit != 'true' && github.event_name != 'pull_request'
+        if: (steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true') && steps.corpora.outputs.cache-hit != 'true' && github.event_name != 'pull_request'
         continue-on-error: true  # best effort: a miss only costs the next run a clone
         uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
@@ -116,7 +127,7 @@ jobs:
 
       - name: Generate this commit's Java (both paths)
         id: keep-gen
-        if: steps.kept.outputs.cache-hit != 'true'
+        if: steps.kept.outputs.cache-hit != 'true' || steps.scan-kept.outputs.cache-hit != 'true'
         env:
           CORPUS: ${{ matrix.corpus }}
           CONFIGS: ${{ matrix.configs }}
@@ -129,6 +140,7 @@ jobs:
             "$GITGALAXY_MAINFRAME_CORPORA/$CORPUS" \
             --same-as "$GITHUB_WORKSPACE/gitgalaxy" \
             --base-cache "$RUNNER_TEMP/java_base_cache" \
+            --scan-cache "$RUNNER_TEMP/java_scan_cache" \
             --work "$RUNNER_TEMP/java_keep" \
             "${ONLY[@]}"
 
@@ -138,3 +150,10 @@ jobs:
         with:
           path: ${{ runner.temp }}/java_base_cache
           key: java-same-base-${{ matrix.corpus }}-${{ hashFiles('gitgalaxy/gitgalaxy/cobol_to_java_controller.py', 'gitgalaxy/gitgalaxy/cobol_refractor_controller.py', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_cobol/**', 'gitgalaxy/gitgalaxy/standards/yaml_reader.py', 'gitgalaxy/gitgalaxy/standards/fact_channel_confidence.json', 'gitgalaxy/tests/tools/java_target_matrix.py', 'gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py') }}
+
+      - name: Save the engine scan under this content (main)
+        if: steps.scan-kept.outputs.cache-hit != 'true' && steps.keep-gen.outcome == 'success'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ runner.temp }}/java_scan_cache
+          key: java-scan-${{ matrix.corpus }}-${{ hashFiles('gitgalaxy/gitgalaxy/**', '!gitgalaxy/gitgalaxy/tools/cobol_to_java/**', '!gitgalaxy/gitgalaxy/cobol_to_java_controller.py', '!gitgalaxy/gitgalaxy/cobol_refractor_controller.py') }}-${{ hashFiles('gitgalaxy/gitgalaxy/tools/cobol_to_java/det/copy/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/cics_task_containers.py') }}-${{ hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py', 'gitgalaxy/tests/tools/java_target_matrix.py') }}

.github/workflows/java-compile.yml

@@ -186,6 +186,19 @@ jobs:
           # main commits that touch those paths (the push run is path-filtered), so most PRs missed and regenerated both sides.
           key: java-same-base-${{ matrix.corpus }}-${{ github.event_name == 'pull_request' && hashFiles('base/gitgalaxy/cobol_to_java_controller.py', 'base/gitgalaxy/cobol_refractor_controller.py', 'base/gitgalaxy/tools/cobol_to_java/**', 'base/gitgalaxy/tools/cobol_to_cobol/**', 'base/gitgalaxy/standards/yaml_reader.py', 'base/gitgalaxy/standards/fact_channel_confidence.json', 'base/tests/tools/java_target_matrix.py', 'base/tests/cobol_mainframe/corpora.json', 'base/tests/tools/mainframe_corpus.py') || hashFiles('gitgalaxy/gitgalaxy/cobol_to_java_controller.py', 'gitgalaxy/gitgalaxy/cobol_refractor_controller.py', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_cobol/**', 'gitgalaxy/gitgalaxy/standards/yaml_reader.py', 'gitgalaxy/gitgalaxy/standards/fact_channel_confidence.json', 'gitgalaxy/tests/tools/java_target_matrix.py', 'gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py') }}
 
+      # #5142: the engine scan is kept across runs (java-base-generation.yml seeds it; restore-only here, #4968). The key only
+      # has to change whenever java_target_matrix.scan_key can; the exact key decides inside the tool. Same inputs as the seeding.
+      - name: Key the engine scan cache
+        id: scankey
+        run: echo "key=java-scan-${{ matrix.corpus }}-${{ github.event_name == 'pull_request' && hashFiles('base/gitgalaxy/**', '!base/gitgalaxy/tools/cobol_to_java/**', '!base/gitgalaxy/cobol_to_java_controller.py', '!base/gitgalaxy/cobol_refractor_controller.py') || hashFiles('gitgalaxy/gitgalaxy/**', '!gitgalaxy/gitgalaxy/tools/cobol_to_java/**', '!gitgalaxy/gitgalaxy/cobol_to_java_controller.py', '!gitgalaxy/gitgalaxy/cobol_refractor_controller.py') }}-${{ github.event_name == 'pull_request' && hashFiles('base/gitgalaxy/tools/cobol_to_java/det/copy/**', 'base/gitgalaxy/tools/cobol_to_java/cics_task_containers.py') || hashFiles('gitgalaxy/gitgalaxy/tools/cobol_to_java/det/copy/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/cics_task_containers.py') }}-${{ github.event_name == 'pull_request' && hashFiles('base/tests/cobol_mainframe/corpora.json', 'base/tests/tools/mainframe_corpus.py', 'base/tests/tools/java_target_matrix.py') || hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py', 'gitgalaxy/tests/tools/java_target_matrix.py') }}" >> "$GITHUB_OUTPUT"
+
+      - name: Restore the engine scan
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ runner.temp }}/java_scan_cache
+          key: ${{ steps.scankey.outputs.key }}
+          restore-keys: java-scan-${{ matrix.corpus }}-
+
       - name: Is the generated Java unchanged? (pull requests)
         id: same
         if: github.event_name == 'pull_request'
@@ -201,6 +214,7 @@ jobs:
             "$GITGALAXY_MAINFRAME_CORPORA/$CORPUS" \
             --same-as "$GITHUB_WORKSPACE/base" \
             --base-cache "$RUNNER_TEMP/java_base_cache" \
+            --scan-cache "$RUNNER_TEMP/java_scan_cache" \
             --work "$RUNNER_TEMP/java_same" \
             --summary "$GITHUB_STEP_SUMMARY" \
             "${ONLY[@]}"
@@ -272,6 +286,7 @@ jobs:
             "$GITGALAXY_MAINFRAME_CORPORA/$CORPUS" \
             "${REUSE[@]}" \
             --scan \
+            --scan-cache "$RUNNER_TEMP/java_scan_cache" \
             --work "$RUNNER_TEMP/java_matrix_scan" \
             --m2 "$HOME/.m2/repository" \
             --summary "$GITHUB_STEP_SUMMARY" \

tests/tools/test_cache_keys.py

@@ -158,6 +158,43 @@ def test_same_base_cache_is_restored_by_java_compile_and_seeded_by_its_own_workf
     assert "lookup-only: true" in lookup, "a push whose content key exists must not regenerate"
 
 
+def _scan_inputs(expr):
+    """The hashFiles patterns of an expression, `!` kept and the checkout prefix (`base/` or `gitgalaxy/`) stripped."""
+    return [re.sub(r"^(!?)(base|gitgalaxy)/", r"\1", a) for a in re.findall(r"'([^']+)'", expr)]
+
+
+def test_engine_scan_cache_is_restored_by_java_compile_and_seeded_by_its_own_workflow():
+    """#5142: the dsf engine scan (~90 s a side) is kept across runs. Same split as the same-base generation: java-compile
+    only restores (#4968), java-base-generation.yml saves. The key is a coarse content hash; the tool's scan_key is exact."""
+    compile_text = (WORKFLOWS / "java-compile.yml").read_text(encoding="utf-8")
+    seed_text = (WORKFLOWS / "java-base-generation.yml").read_text(encoding="utf-8")
+    steps = [(wf, k, s) for wf, k, s in _cache_steps() if "java-scan-" in _key(s) or "java_scan_cache" in s]
+    assert sorted((wf, k) for wf, k, _ in steps) == [
+        ("java-base-generation.yml", "restore"),
+        ("java-base-generation.yml", "save"),
+        ("java-compile.yml", "restore"),
+    ], steps
+    lookup = next(s for wf, k, s in steps if wf == "java-base-generation.yml" and k == "restore")
+    save = next(s for wf, k, s in steps if wf == "java-base-generation.yml" and k == "save")
+    assert "lookup-only: true" in lookup and _key(lookup) == _key(save)
+    seeded = _scan_inputs(_key(save))
+    # the compile side computes the same key from the base checkout on a PR and from its own on a push
+    (keyed,) = re.findall(r'echo "key=(java-scan-[^"]*)"', compile_text)
+    parts = re.findall(r"pull_request' && hashFiles\(([^)]*)\) \|\| hashFiles\(([^)]*)\)", keyed)
+    assert len(parts) == 3, keyed
+    for side in (0, 1):
+        got = [a for p in parts for a in _scan_inputs(p[side])]
+        assert got == seeded, "the compile-side scan key hashes different inputs than the seeding"
+    assert any(a.startswith("!") for a in seeded) and "!gitgalaxy/cobol_to_java_controller.py" in seeded
+    assert seeded[0] == "gitgalaxy/**" and "tests/tools/java_target_matrix.py" in seeded
+    # the key must change whenever the tool's exact key can: the files the scan reads from the otherwise-left-out translator
+    assert "gitgalaxy/tools/cobol_to_java/det/copy/**" in seeded
+    assert "gitgalaxy/tools/cobol_to_java/cics_task_containers.py" in seeded
+    for text in (compile_text, seed_text):
+        assert '--scan-cache "$RUNNER_TEMP/java_scan_cache"' in text
+    assert compile_text.count('--scan-cache "$RUNNER_TEMP/java_scan_cache"') == 2, "the check and the scan build"
+
+
 def test_the_seeding_workflow_is_never_cancelled_or_gated_and_never_runs_on_a_pull_request():
     text = (WORKFLOWS / "java-base-generation.yml").read_text(encoding="utf-8")
     assert "cancel-in-progress: false" in text and "github.sha" in text.split("concurrency:")[1].split("jobs:")[0]