From 27f0aa0dfdc206307813f968c4270ea6bc448c9b Mon Sep 17 00:00:00 2001 From: Dan Calavrezo <195309321+dcalavrezo-qorix@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:33:46 +0300 Subject: [PATCH] coverage: adoption fixes from first consumers Three issues surfaced by early adopters of the coverage module (persistency #380, inc_someip_gateway #256): - generate_coverage_html: --yaml is now OPTIONAL. Without it, justification processing is skipped and the COVERAGE_THRESHOLD gate applies to the raw line coverage computed from the LCOV data (NOT llvm-cov's text TOTAL, which omits baseline-only files and would let untested files escape the gate). Previously the script hard-errored before extracting any HTML, which read as 'no report is generated' for repos without a justification YAML. - generate_coverage_html: new --archive-dir assembles the artifacts tree without zipping it. Preferred for CI uploads: actions/upload-artifact zips its input itself, so uploading the --archive zip nested it in a second zip. - README: document both changes, plus the requirement that the coverage bazelrc import must precede a user.bazelrc try-import (bazelrc is last-wins; the local-override file must stay last). Integration test extended: no-yaml mode (HTML produced, raw gate enforced at high/low thresholds) and --archive-dir content checks. --- coverage/README.md | 20 +- coverage/generate_coverage_html.sh | 171 ++++++++++++------ .../integration_tests/run_integration_test.sh | 25 +++ 3 files changed, 152 insertions(+), 64 deletions(-) diff --git a/coverage/README.md b/coverage/README.md index 0de94968..dc9368d5 100644 --- a/coverage/README.md +++ b/coverage/README.md @@ -137,7 +137,11 @@ Everything in scope but untested shows up at 0%; everything outside the scope Copy the `coverage:llvm_cov` block from [integration_tests/.bazelrc](integration_tests/.bazelrc) into your -repository's bazelrc (directly or via `import`). The two labels to adapt: +repository's bazelrc (directly or via `import`). If your `.bazelrc` ends with +a `try-import %workspace%/user.bazelrc` (or similar local-override file), +place the coverage import BEFORE it — bazelrc conflicts resolve last-wins, +and the local override file must stay last to keep working. The two labels +to adapt: ``` coverage:llvm_cov --coverage_output_generator=@score_tooling//coverage:merger @@ -189,12 +193,20 @@ bazel coverage --config=llvm_cov //... --build_tests_only bazel run @score_tooling//coverage:generate_coverage_html -- \ --yaml tools/coverage/coverage_justifications.yaml -# CI variant: archive HTML + LCOV + JUnit XMLs, gate at 95%: +# CI variant: assemble HTML + LCOV + JUnit XMLs for artifact upload, gate at 95%: COVERAGE_THRESHOLD=95 bazel run @score_tooling//coverage:generate_coverage_html -- \ --yaml tools/coverage/coverage_justifications.yaml \ - --archive coverage_artifacts + --archive-dir coverage_artifacts +# then: actions/upload-artifact with path: coverage_artifacts +# (upload-artifact zips its input itself — use --archive only when you +# want a local .zip; uploading that zip would nest it in a second zip) ``` +`--yaml` is optional: without it, justification processing is skipped and the +`COVERAGE_THRESHOLD` gate applies to the **raw** line coverage. Start without +a YAML; add one (`version: 1` + `justifications: []`) when you introduce your +first `COV_JUSTIFIED` marker. + `--build_tests_only` matters: without it, coverage builds (not runs) every target matched by the pattern, including e.g. `manual`-tagged or platform-incompatible test binaries. @@ -203,7 +215,7 @@ platform-incompatible test binaries. | Need | Knob | |---|---| -| Different gate | `COVERAGE_THRESHOLD=` env var (default 100; exit 1 below) | +| Different gate | `COVERAGE_THRESHOLD=` env var (default 100; exit 1 below; gates effective coverage with `--yaml`, raw coverage without) | | Output directory | positional `output-dir` argument (default `coverage_`) | | Platform-specific justifications | `--platform linux\|qnx` (default linux) | | JUnit XMLs subtree in the archive | `--testlogs-subdir ` (default: whole `bazel-testlogs`) | diff --git a/coverage/generate_coverage_html.sh b/coverage/generate_coverage_html.sh index 7597637a..7c77d366 100755 --- a/coverage/generate_coverage_html.sh +++ b/coverage/generate_coverage_html.sh @@ -18,15 +18,21 @@ # Usage (from the CONSUMER workspace): # bazel coverage --config=llvm_cov //... --build_tests_only # bazel run @score_tooling//coverage:generate_coverage_html -- \ -# --yaml \ -# [--archive ] [--platform ] \ -# [--testlogs-subdir ] [output-dir] +# [--yaml ] \ +# [--archive ] [--archive-dir ] \ +# [--platform ] [--testlogs-subdir ] [output-dir] # # Arguments: # --yaml Justification YAML, relative to the workspace -# root (required). -# --archive Also create a zip archive named .zip +# root. When omitted, justification processing and +# the effective-coverage metric are skipped and the +# threshold gate applies to the RAW line coverage. +# --archive Create a zip archive named .zip # containing the HTML report, raw LCOV data and JUnit XMLs. +# --archive-dir Assemble the same content as --archive into +# WITHOUT zipping — preferred for CI artifact +# uploads (actions/upload-artifact zips its input +# itself; a pre-zipped file would be zipped twice). # --platform Target platform for justification filtering # (default: linux). Also affects the default output # directory (coverage_). @@ -37,13 +43,15 @@ # (default: coverage_) # # Environment: -# COVERAGE_THRESHOLD Minimum effective line coverage percentage -# (default: 100). The script exits non-zero when -# effective coverage is below this threshold. +# COVERAGE_THRESHOLD Minimum line coverage percentage (default: 100). +# The script exits non-zero when the gated metric +# (effective coverage with --yaml, raw coverage +# without) is below this threshold. set -euo pipefail ARCHIVE_NAME="" +ARCHIVE_DIR="" PLATFORM="linux" OUTPUT_DIR="" JUSTIFICATION_YAML_REL="" @@ -59,6 +67,10 @@ while [[ $# -gt 0 ]]; do ARCHIVE_NAME="${2:?--archive requires a name argument}" shift 2 ;; + --archive-dir) + ARCHIVE_DIR="${2:?--archive-dir requires a directory argument}" + shift 2 + ;; --platform) PLATFORM="${2:?--platform requires a platform argument (e.g. linux or qnx)}" shift 2 @@ -74,10 +86,9 @@ while [[ $# -gt 0 ]]; do esac done -if [[ -z "${JUSTIFICATION_YAML_REL}" ]]; then - echo "ERROR: --yaml is required." >&2 - exit 1 -fi +# --yaml is optional: without it, justification processing and the effective +# coverage metric are skipped and the threshold gate applies to the RAW line +# coverage from llvm-cov's summary instead. # Set default output directory based on platform if not explicitly provided. if [[ -z "${OUTPUT_DIR}" ]]; then @@ -126,80 +137,120 @@ fi echo "Coverage report written to: ${OUTPUT_DIR}" # --------------------------------------------------------------------------- -# Run coverage justification processing. +# Run coverage justification processing (only when --yaml was given) and +# enforce the coverage threshold. # --------------------------------------------------------------------------- -JUSTIFICATION_YAML="${BUILD_WORKSPACE_DIRECTORY}/${JUSTIFICATION_YAML_REL}" +THRESHOLD="${COVERAGE_THRESHOLD:-100}" +JUSTIFICATION_DIR="" -if [[ ! -f "${JUSTIFICATION_YAML}" ]]; then - echo "ERROR: ${JUSTIFICATION_YAML} not found." >&2 - exit 1 -fi +if [[ -n "${JUSTIFICATION_YAML_REL}" ]]; then + JUSTIFICATION_YAML="${BUILD_WORKSPACE_DIRECTORY}/${JUSTIFICATION_YAML_REL}" -echo "" -echo "Running coverage justification processing..." - -JUSTIFICATION_DIR="${TMPDIR_EXTRACT}/justification_report" -mkdir -p "${JUSTIFICATION_DIR}" - -# Run justify.py / effective_coverage.py via nested bazel invocations from the -# consumer workspace. This deliberately avoids runfiles resolution across -# module boundaries (canonical repo names vary between Bazel versions). -bazel run @score_tooling//coverage:justify -- \ - --yaml "${JUSTIFICATION_YAML}" \ - --source-root "${BUILD_WORKSPACE_DIRECTORY}" \ - --platform "${PLATFORM}" \ - --output "${JUSTIFICATION_DIR}/manifest.json" - -bazel run @score_tooling//coverage:effective_coverage -- \ - --html-dir "${OUTPUT_DIR}" \ - --manifest "${JUSTIFICATION_DIR}/manifest.json" \ - --output "${JUSTIFICATION_DIR}/report.json" - -# Display effective coverage summary and enforce the threshold. -if [[ ! -f "${JUSTIFICATION_DIR}/summary.txt" ]]; then - echo "ERROR: Effective coverage summary was not produced." >&2 - exit 1 -fi + if [[ ! -f "${JUSTIFICATION_YAML}" ]]; then + echo "ERROR: ${JUSTIFICATION_YAML} not found." >&2 + exit 1 + fi -echo "" -cat "${JUSTIFICATION_DIR}/summary.txt" + echo "" + echo "Running coverage justification processing..." + + JUSTIFICATION_DIR="${TMPDIR_EXTRACT}/justification_report" + mkdir -p "${JUSTIFICATION_DIR}" + + # Run justify.py / effective_coverage.py via nested bazel invocations from the + # consumer workspace. This deliberately avoids runfiles resolution across + # module boundaries (canonical repo names vary between Bazel versions). + bazel run @score_tooling//coverage:justify -- \ + --yaml "${JUSTIFICATION_YAML}" \ + --source-root "${BUILD_WORKSPACE_DIRECTORY}" \ + --platform "${PLATFORM}" \ + --output "${JUSTIFICATION_DIR}/manifest.json" + + bazel run @score_tooling//coverage:effective_coverage -- \ + --html-dir "${OUTPUT_DIR}" \ + --manifest "${JUSTIFICATION_DIR}/manifest.json" \ + --output "${JUSTIFICATION_DIR}/report.json" + + # Display effective coverage summary and enforce the threshold. + if [[ ! -f "${JUSTIFICATION_DIR}/summary.txt" ]]; then + echo "ERROR: Effective coverage summary was not produced." >&2 + exit 1 + fi -# Extract effective coverage percentage for threshold check. -EFFECTIVE_PCT=$(grep -oP 'Effective line coverage:\s+\K[0-9.]+' \ - "${JUSTIFICATION_DIR}/summary.txt" 2>/dev/null || echo "0") + echo "" + cat "${JUSTIFICATION_DIR}/summary.txt" + + # Extract effective coverage percentage for threshold check. + GATE_PCT=$(grep -oP 'Effective line coverage:\s+\K[0-9.]+' \ + "${JUSTIFICATION_DIR}/summary.txt" 2>/dev/null || echo "0") + GATE_KIND="Effective" +else + # No justification YAML: gate on the raw line coverage computed from the + # LCOV data. Deliberately NOT llvm-cov's text summary TOTAL — that summary + # omits baseline-only files (in-scope files no test links against), which + # would let untested files escape the gate. The LCOV includes them. + echo "" + echo "INFO: no --yaml given; justification processing skipped, gating on raw line coverage." + if [[ ! -f "${TMPDIR_EXTRACT}/lcov_report/lcov.dat" ]]; then + echo "ERROR: lcov_report/lcov.dat not found in ${COVERAGE_REPORT}" >&2 + exit 1 + fi + GATE_PCT=$(awk -F: '/^LF:/ {lf += $2} /^LH:/ {lh += $2} + END { if (lf > 0) printf "%.2f", lh * 100 / lf; }' \ + "${TMPDIR_EXTRACT}/lcov_report/lcov.dat") + if [[ -z "${GATE_PCT}" ]]; then + echo "ERROR: could not compute raw line coverage from lcov_report/lcov.dat" >&2 + exit 1 + fi + echo "Raw line coverage: ${GATE_PCT}%" + GATE_KIND="Raw" +fi # Threshold check (default: 100%). Fails the run when below. -THRESHOLD="${COVERAGE_THRESHOLD:-100}" -if ! awk "BEGIN {exit (${EFFECTIVE_PCT} >= ${THRESHOLD}) ? 0 : 1}"; then - echo "ERROR: Effective coverage ${EFFECTIVE_PCT}% is below threshold ${THRESHOLD}%" >&2 +if ! awk "BEGIN {exit (${GATE_PCT} >= ${THRESHOLD}) ? 0 : 1}"; then + echo "ERROR: ${GATE_KIND} coverage ${GATE_PCT}% is below threshold ${THRESHOLD}%" >&2 RC=1 else RC=0 fi # --------------------------------------------------------------------------- -# Optional: create a zip archive with the HTML report, raw LCOV data and -# JUnit XML test results. +# Optional: assemble the HTML report, raw LCOV data, justification report and +# JUnit XML test results into an artifacts tree. +# --archive zip the tree into .zip (and remove the tree) +# --archive-dir keep the tree at — preferred for CI artifact +# uploads, since actions/upload-artifact zips its input +# anyway (a pre-zipped file would be zipped twice) # --------------------------------------------------------------------------- -if [[ -n "${ARCHIVE_NAME}" ]]; then - mkdir -p artifacts +assemble_artifacts() { + local dest="$1" + mkdir -p "${dest}" # Copy JUnit XML test results preserving directory structure. - find "bazel-testlogs/${TESTLOGS_SUBDIR}" -name 'test.xml' -exec cp --parents {} artifacts/ \; + find "bazel-testlogs/${TESTLOGS_SUBDIR}" -name 'test.xml' -exec cp --parents {} "${dest}/" \; # Copy the HTML coverage report - cp -r "${OUTPUT_DIR}" artifacts/ + cp -r "${OUTPUT_DIR}" "${dest}/" # Include the LCOV .dat file from the reporter zip. if [[ -f "${TMPDIR_EXTRACT}/lcov_report/lcov.dat" ]]; then - cp "${TMPDIR_EXTRACT}/lcov_report/lcov.dat" artifacts/coverage_report.dat + cp "${TMPDIR_EXTRACT}/lcov_report/lcov.dat" "${dest}/coverage_report.dat" fi # Include the justification report (manifest + effective coverage json). - if [[ -d "${JUSTIFICATION_DIR}" ]]; then - cp -r "${JUSTIFICATION_DIR}" artifacts/ + if [[ -n "${JUSTIFICATION_DIR}" && -d "${JUSTIFICATION_DIR}" ]]; then + cp -r "${JUSTIFICATION_DIR}" "${dest}/" fi +} +if [[ -n "${ARCHIVE_DIR}" ]]; then + rm -rf "${ARCHIVE_DIR}" + assemble_artifacts "${ARCHIVE_DIR}" + echo "Coverage artifacts written to: ${ARCHIVE_DIR}/" +fi + +if [[ -n "${ARCHIVE_NAME}" ]]; then + assemble_artifacts artifacts zip -r "${ARCHIVE_NAME}.zip" artifacts/ rm -rf artifacts/ echo "Coverage archive written to: ${ARCHIVE_NAME}.zip" diff --git a/coverage/integration_tests/run_integration_test.sh b/coverage/integration_tests/run_integration_test.sh index 946e1642..927bf428 100755 --- a/coverage/integration_tests/run_integration_test.sh +++ b/coverage/integration_tests/run_integration_test.sh @@ -40,6 +40,31 @@ COVERAGE_THRESHOLD=10 bazel run @score_tooling//coverage:generate_coverage_html --yaml "${YAML}" echo "OK: gate passed as expected" +echo "=== Without --yaml: HTML still produced, gate applies to RAW coverage ===" +if COVERAGE_THRESHOLD=100 bazel run @score_tooling//coverage:generate_coverage_html; then + echo "ERROR: raw-coverage gate passed at threshold 100" >&2 + exit 1 +fi +COVERAGE_THRESHOLD=10 bazel run @score_tooling//coverage:generate_coverage_html +if [[ ! -f coverage_linux/index.html ]]; then + echo "ERROR: HTML report missing after no-yaml run" >&2 + exit 1 +fi +echo "OK: no-yaml mode works (HTML produced, raw gate enforced)" + +echo "=== --archive-dir must produce an unzipped artifacts tree ===" +COVERAGE_THRESHOLD=10 bazel run @score_tooling//coverage:generate_coverage_html -- \ + --yaml "${YAML}" --archive-dir artifacts_dir +for f in artifacts_dir/coverage_linux/index.html artifacts_dir/coverage_report.dat \ + artifacts_dir/justification_report/summary.txt; do + if [[ ! -f "$f" ]]; then + echo "ERROR: ${f} missing from --archive-dir output" >&2 + exit 1 + fi +done +rm -rf artifacts_dir +echo "OK: --archive-dir works" + echo "=== Untested files must appear at exact 0% in the LCOV ===" unzip -p coverage_artifacts.zip artifacts/coverage_report.dat > lcov.dat