java-compile.yml

name: Generated Java Compiles

# The COBOL -> Java generator (#3121): the Spring Boot projects `cobol-to-java`
# writes must COMPILE, under every supported target config (#3613). Each job
# refactors one pinned mainframe corpus (tests/cobol_mainframe/corpora.json),
# generates the Java under a set of configs, and runs the real build (Maven, or
# Gradle for the gradle rows) with JDK 17 and 21: tests/tools/java_target_matrix.py.
# CardDemo runs every config; the other corpora run default / plain-records /
# gradle, so the generator meets each corpus's shapes. Each set runs twice: from
# the refactoring stage alone, and against an engine scan (--scan), where the
# generator builds from the verified skeleton (#3614) -- CICS transaction
# endpoints and COMMAREA / channel DTOs (#3615).
#
# Runs when the generator or what feeds it changes, on push to main, weekly
# (dependency drift on Maven Central), and on demand.
#
# On a pull request, each job first asks whether the PR changes the generated Java
# at all (java_target_matrix.py --same-as): it generates both paths with the PR's
# code and with the base branch's code and compares every file. Identical output
# needs no compile -- the base already compiles (push to main builds it) -- so the
# JDK / Gradle setup and both builds are skipped and the summary says why. Only what
# the build reads is compared (src/, pom.xml, the Gradle files): a change confined to
# porting tickets, worklists or audits needs no compile. Any build-input difference,
# or a side that fails to generate, builds -- from the projects the check already
# generated, so the corpus is not refactored (or scanned) a third time. The four
# generations run side by side, and the base's are cached per content hash of the generation inputs, so a
# second push to the same PR generates only the head. Pushes, the weekly run and
# manual runs always build from scratch.

on:
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]  # #5104: a draft starts only the cheap audits
    branches: [main]
    paths:
      - "gitgalaxy/cobol_to_java_controller.py"
      - "gitgalaxy/cobol_refractor_controller.py"
      - "gitgalaxy/tools/cobol_to_java/**"
      - "gitgalaxy/tools/cobol_to_cobol/**"
      - "gitgalaxy/standards/yaml_reader.py"
      - "gitgalaxy/standards/fact_channel_confidence.json"
      - "tests/tools/java_target_matrix.py"
      - "tests/cobol_mainframe/corpora.json"
      - ".github/workflows/java-compile.yml"
      - "tests/tools/mainframe_corpus.py"  # #4825: read by this workflow (workflow_inputs.py)
  push:
    branches: [main]
    paths:
      - "gitgalaxy/cobol_to_java_controller.py"
      - "gitgalaxy/cobol_refractor_controller.py"
      - "gitgalaxy/tools/cobol_to_java/**"
      - "gitgalaxy/tools/cobol_to_cobol/**"
      - "gitgalaxy/standards/yaml_reader.py"
      - "gitgalaxy/standards/fact_channel_confidence.json"
      - "tests/tools/java_target_matrix.py"
      - "tests/cobol_mainframe/corpora.json"
      - ".github/workflows/java-compile.yml"
  schedule:
    - cron: "17 6 * * 1"
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true  # #4825: read-only, so a superseded main run is cancelled too (frees runners for PRs)

jobs:
  # #4964 (sibling of #4935): a push to main whose tree already passed THIS workflow on its PR head (a squash of an
  # up-to-date PR) is not run again; any doubt runs it (tests/tools/skip_tested_tree.py has the rule, and tests).
  # No PR run on the head (the PR's paths filter skipped it) is "not proven", so the push runs. Always concludes success.
  # #5129: only a main push run can be skipped, so the gate runs on push only: a PR run (and a schedule / dispatch run,
  # which never skips) starts its heavy jobs without waiting on it. A skipped gate leaves `skip` empty: they run.
  tested-tree:
    if: ${{ github.event_name == 'push' }}
    runs-on: ubuntu-latest
    timeout-minutes: 5
    permissions:
      contents: read
      actions: read
      pull-requests: read
    outputs:
      skip: ${{ steps.decide.outputs.skip }}
    steps:
      - name: Checkout the gate script
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          sparse-checkout: tests/tools/skip_tested_tree.py
          sparse-checkout-cone-mode: false
          persist-credentials: false

      - name: Decide
        id: decide
        env:
          GH_TOKEN: ${{ github.token }}
        run: python tests/tools/skip_tested_tree.py java-compile.yml

  compile:
    needs: tested-tree
    if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.draft == false) && (!cancelled() && needs.tested-tree.outputs.skip != 'true') }}
    name: compile (${{ matrix.corpus }})
    runs-on: ubuntu-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        include:
          - corpus: aws-mainframe-modernization-carddemo
            configs: ""  # every config in java_target_matrix.MATRIX
          - corpus: cics-banking-sample-application-cbsa
            configs: "default plain-records gradle"
          - corpus: cics-genapp
            configs: "default plain-records gradle"
          - corpus: zopeneditor-sample
            configs: "default plain-records gradle"
          - corpus: zecs
            configs: "default plain-records gradle"
          - corpus: dsf
            configs: "default plain-records gradle"
    env:
      GITGALAXY_LICENSE_KEY: COMMUNITY_FREE_TIER
      GITGALAXY_MAINFRAME_CORPORA: ${{ github.workspace }}/.mainframe_corpora
      PYTHONUTF8: "1"
    steps:
      - name: Checkout GitGalaxy
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          path: gitgalaxy
          persist-credentials: false

      - name: Setup Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            gitgalaxy/pyproject.toml
            gitgalaxy/ci/requirements/*.txt

      - name: Install GitGalaxy
        run: |
          cd gitgalaxy
          python -m pip install --require-hashes -r ci/requirements/bootstrap.txt
          pip install --no-build-isolation -e .

      - name: Cache the pinned corpora
        id: corpora
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .mainframe_corpora
          key: mainframe-corpora-${{ hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json') }}

      - name: Fetch the pinned corpora
        run: |
          cd gitgalaxy
          rm -rf "$GITGALAXY_MAINFRAME_CORPORA/_scans"
          PYTHONPATH="$PWD" python tests/tools/mainframe_corpus.py fetch

      # #5024: restore-only on pull requests. A PR's cache scope is its own ref, so a PR that saved here left
      # a copy per PR (a PR that edits corpora.json) that nothing else could read. Main saves it, once per key.
      - name: Save the pinned corpora (main)
        if: github.event_name != 'pull_request' && steps.corpora.outputs.cache-hit != 'true'
        continue-on-error: true  # best effort: a miss only costs the next run a clone
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .mainframe_corpora
          key: mainframe-corpora-${{ hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json') }}

      - name: Checkout the base branch (pull requests)
        if: github.event_name == 'pull_request'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          path: base
          persist-credentials: false

      # #4968: restore-only on every ref. A PR's cache scope is its own ref, so a PR that saved here left one copy per PR
      # (13-14 per corpus) that nothing else could read. java-base-generation.yml saves it (#5142), keyed on the content a PR's base has.
      - name: Restore the base branch's generation
        id: base-gen
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ runner.temp }}/java_base_cache
          # #5142: seeded by java-base-generation.yml (a push run here is cancelled or tested-tree-skipped before it could save). Keyed on the CONTENT of the files that decide the generation (the paths: list above), of the base checkout on a PR
          # and of main's own checkout on a push. The old key was the base COMMIT, which has an entry only for the ~43 pct of
          # main commits that touch those paths (the push run is path-filtered), so most PRs missed and regenerated both sides.
          key: java-same-base-${{ matrix.corpus }}-${{ github.event_name == 'pull_request' && hashFiles('base/gitgalaxy/cobol_to_java_controller.py', 'base/gitgalaxy/cobol_refractor_controller.py', 'base/gitgalaxy/tools/cobol_to_java/**', 'base/gitgalaxy/tools/cobol_to_cobol/**', 'base/gitgalaxy/standards/yaml_reader.py', 'base/gitgalaxy/standards/fact_channel_confidence.json', 'base/tests/tools/java_target_matrix.py', 'base/tests/cobol_mainframe/corpora.json', 'base/tests/tools/mainframe_corpus.py') || hashFiles('gitgalaxy/gitgalaxy/cobol_to_java_controller.py', 'gitgalaxy/gitgalaxy/cobol_refractor_controller.py', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_cobol/**', 'gitgalaxy/gitgalaxy/standards/yaml_reader.py', 'gitgalaxy/gitgalaxy/standards/fact_channel_confidence.json', 'gitgalaxy/tests/tools/java_target_matrix.py', 'gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py') }}

      # #5142: the engine scan is kept across runs (java-base-generation.yml seeds it; restore-only here, #4968). The key only
      # has to change whenever java_target_matrix.scan_key can; the exact key decides inside the tool. Same inputs as the seeding.
      - name: Key the engine scan cache
        id: scankey
        run: echo "key=java-scan-${{ matrix.corpus }}-${{ github.event_name == 'pull_request' && hashFiles('base/gitgalaxy/**', '!base/gitgalaxy/tools/cobol_to_java/**', '!base/gitgalaxy/cobol_to_java_controller.py', '!base/gitgalaxy/cobol_refractor_controller.py') || hashFiles('gitgalaxy/gitgalaxy/**', '!gitgalaxy/gitgalaxy/tools/cobol_to_java/**', '!gitgalaxy/gitgalaxy/cobol_to_java_controller.py', '!gitgalaxy/gitgalaxy/cobol_refractor_controller.py') }}-${{ github.event_name == 'pull_request' && hashFiles('base/gitgalaxy/tools/cobol_to_java/det/copy/**', 'base/gitgalaxy/tools/cobol_to_java/cics_task_containers.py') || hashFiles('gitgalaxy/gitgalaxy/tools/cobol_to_java/det/copy/**', 'gitgalaxy/gitgalaxy/tools/cobol_to_java/cics_task_containers.py') }}-${{ github.event_name == 'pull_request' && hashFiles('base/tests/cobol_mainframe/corpora.json', 'base/tests/tools/mainframe_corpus.py', 'base/tests/tools/java_target_matrix.py') || hashFiles('gitgalaxy/tests/cobol_mainframe/corpora.json', 'gitgalaxy/tests/tools/mainframe_corpus.py', 'gitgalaxy/tests/tools/java_target_matrix.py') }}" >> "$GITHUB_OUTPUT"

      - name: Restore the engine scan
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ runner.temp }}/java_scan_cache
          key: ${{ steps.scankey.outputs.key }}
          restore-keys: java-scan-${{ matrix.corpus }}-

      - name: Is the generated Java unchanged? (pull requests)
        id: same
        if: github.event_name == 'pull_request'
        continue-on-error: true  # "changed" (exit 1) just means: build
        env:
          CORPUS: ${{ matrix.corpus }}
          CONFIGS: ${{ matrix.configs }}
        run: |
          cd gitgalaxy
          ONLY=()
          if [ -n "$CONFIGS" ]; then ONLY=(--only $CONFIGS); fi
          python tests/tools/java_target_matrix.py \
            "$GITGALAXY_MAINFRAME_CORPORA/$CORPUS" \
            --same-as "$GITHUB_WORKSPACE/base" \
            --base-cache "$RUNNER_TEMP/java_base_cache" \
            --scan-cache "$RUNNER_TEMP/java_scan_cache" \
            --work "$RUNNER_TEMP/java_same" \
            --summary "$GITHUB_STEP_SUMMARY" \
            "${ONLY[@]}"

      - name: Setup JDK 17 and 21
        if: steps.same.outputs.identical != 'true'
        uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
        with:
          distribution: temurin
          java-version: |
            21
            17

      - name: Setup Gradle
        if: steps.same.outputs.identical != 'true'
        uses: gradle/actions/setup-gradle@3f5f9adaf7d9fecd50b5935e54106014257a94e6 # v6.4.0
        with:
          gradle-version: "8.10.2"
          cache-disabled: true  # #4968: its own key ends in the run's sha (64 gradle-home-v2 entries); keyed on hashes below

      # #4968: the dependency caches are keyed on the files that decide what the builds download, and restore-only on a
      # PR ref (one copy per PR otherwise); main saves them.
      - name: Restore the Gradle and Maven dependency caches
        id: deps
        if: steps.same.outputs.identical != 'true'
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
            ~/.m2/repository
          key: java-compile-deps-${{ hashFiles('gitgalaxy/gitgalaxy/tools/cobol_to_java/cobol_to_java_build_forge.py', 'gitgalaxy/tests/tools/java_target_matrix.py') }}
          restore-keys: java-compile-deps-

      - name: Generate and compile
        if: steps.same.outputs.identical != 'true'
        env:
          CORPUS: ${{ matrix.corpus }}
          CONFIGS: ${{ matrix.configs }}
        run: |
          cd gitgalaxy
          export JDK_17="$JAVA_HOME_17_X64" JDK_21="$JAVA_HOME_21_X64" JAVA_HOME="$JAVA_HOME_17_X64"
          ONLY=()
          if [ -n "$CONFIGS" ]; then ONLY=(--only $CONFIGS); fi
          REUSE=()  # a pull request's check already generated the head's projects
          if [ -f "$RUNNER_TEMP/java_same/head/.complete" ]; then REUSE=(--generated "$RUNNER_TEMP/java_same/head"); fi
          PYTHONPATH="$PWD" python tests/tools/java_target_matrix.py \
            "$GITGALAXY_MAINFRAME_CORPORA/$CORPUS" \
            "${REUSE[@]}" \
            --work "$RUNNER_TEMP/java_matrix" \
            --m2 "$HOME/.m2/repository" \
            --summary "$GITHUB_STEP_SUMMARY" \
            "${ONLY[@]}"

      - name: Generate and compile (engine scan, verified skeleton)
        # report both paths even when the first fails; skipped when the Java is unchanged
        if: ${{ !cancelled() && steps.same.outputs.identical != 'true' }}
        env:
          CORPUS: ${{ matrix.corpus }}
          CONFIGS: ${{ matrix.configs }}
        run: |
          cd gitgalaxy
          export JDK_17="$JAVA_HOME_17_X64" JDK_21="$JAVA_HOME_21_X64" JAVA_HOME="$JAVA_HOME_17_X64"
          ONLY=()
          if [ -n "$CONFIGS" ]; then ONLY=(--only $CONFIGS); fi
          REUSE=()
          if [ -f "$RUNNER_TEMP/java_same/head_scan/.complete" ]; then REUSE=(--generated "$RUNNER_TEMP/java_same/head_scan"); fi
          PYTHONPATH="$PWD" python tests/tools/java_target_matrix.py \
            "$GITGALAXY_MAINFRAME_CORPORA/$CORPUS" \
            "${REUSE[@]}" \
            --scan \
            --scan-cache "$RUNNER_TEMP/java_scan_cache" \
            --work "$RUNNER_TEMP/java_matrix_scan" \
            --m2 "$HOME/.m2/repository" \
            --summary "$GITHUB_STEP_SUMMARY" \
            "${ONLY[@]}"

      - name: Save the Gradle and Maven dependency caches (not on a pull request)
        if: ${{ !cancelled() && github.event_name != 'pull_request' && steps.same.outputs.identical != 'true' && steps.deps.outputs.cache-hit != 'true' }}
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
            ~/.m2/repository
          key: java-compile-deps-${{ hashFiles('gitgalaxy/gitgalaxy/tools/cobol_to_java/cobol_to_java_build_forge.py', 'gitgalaxy/tests/tools/java_target_matrix.py') }}