From 982c30e5208390057fd8cedc88a77931bf9e8326 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Mon, 10 Aug 2026 16:44:13 +0200 Subject: [PATCH 01/11] [UR][CI] Add unified test result reporting for UR tests Introduce common tooling for collecting and reporting results from UR conformance and adapter-specific test executions. The implementation: - provides a common execution and reporting path for UR test suites, - collects detailed test statuses and execution statistics, - parses LIT output and JUnit XML to obtain test-level information, - generates human-readable console output and GitHub Step Summary, - preserves full execution logs and XML results as CI artifacts, - separates test execution, result parsing, and summary generation to support future consumers of test result data. This is the first implementation step towards a broader test management and test health monitoring system. Database integration, historical analysis, and automated PR triage are outside the scope of this change. --- .ci/requirements.txt | 1 + .github/workflows/ur-build-hw.yml | 41 +++- devops/actions/run-tests/ur/action.yml | 127 ++++++++++ devops/scripts/ur-test | 8 + devops/scripts/ur_test_tools/__init__.py | 36 +++ devops/scripts/ur_test_tools/cli.py | 149 ++++++++++++ devops/scripts/ur_test_tools/constants.py | 36 +++ .../scripts/ur_test_tools/models/__init__.py | 25 ++ devops/scripts/ur_test_tools/models/config.py | 48 ++++ .../scripts/ur_test_tools/models/test_data.py | 50 ++++ .../scripts/ur_test_tools/outputs/__init__.py | 9 + .../scripts/ur_test_tools/outputs/console.py | 119 ++++++++++ .../ur_test_tools/outputs/github_actions.py | 20 ++ .../scripts/ur_test_tools/parsers/__init__.py | 19 ++ .../ur_test_tools/parsers/log_parser.py | 151 ++++++++++++ .../ur_test_tools/parsers/stats_parser.py | 16 ++ .../ur_test_tools/parsers/xml_parser.py | 90 +++++++ .../ur_test_tools/summary_generator.py | 224 ++++++++++++++++++ devops/scripts/ur_test_tools/test_runner.py | 160 +++++++++++++ .../ur_test_tools/validation/__init__.py | 9 + .../validation/data_validator.py | 36 +++ .../validation/path_validator.py | 96 ++++++++ unified-runtime/test/CMakeLists.txt | 3 +- .../test/conformance/CMakeLists.txt | 6 +- .../third_party/requirements_testing.txt | 3 + 25 files changed, 1469 insertions(+), 13 deletions(-) create mode 100644 devops/actions/run-tests/ur/action.yml create mode 100755 devops/scripts/ur-test create mode 100644 devops/scripts/ur_test_tools/__init__.py create mode 100644 devops/scripts/ur_test_tools/cli.py create mode 100644 devops/scripts/ur_test_tools/constants.py create mode 100644 devops/scripts/ur_test_tools/models/__init__.py create mode 100644 devops/scripts/ur_test_tools/models/config.py create mode 100644 devops/scripts/ur_test_tools/models/test_data.py create mode 100644 devops/scripts/ur_test_tools/outputs/__init__.py create mode 100644 devops/scripts/ur_test_tools/outputs/console.py create mode 100644 devops/scripts/ur_test_tools/outputs/github_actions.py create mode 100644 devops/scripts/ur_test_tools/parsers/__init__.py create mode 100644 devops/scripts/ur_test_tools/parsers/log_parser.py create mode 100644 devops/scripts/ur_test_tools/parsers/stats_parser.py create mode 100644 devops/scripts/ur_test_tools/parsers/xml_parser.py create mode 100644 devops/scripts/ur_test_tools/summary_generator.py create mode 100644 devops/scripts/ur_test_tools/test_runner.py create mode 100644 devops/scripts/ur_test_tools/validation/__init__.py create mode 100644 devops/scripts/ur_test_tools/validation/data_validator.py create mode 100644 devops/scripts/ur_test_tools/validation/path_validator.py diff --git a/.ci/requirements.txt b/.ci/requirements.txt index 45eb253548496..6c6abcada3eaf 100644 --- a/.ci/requirements.txt +++ b/.ci/requirements.txt @@ -1,3 +1,4 @@ junitparser==3.2.0 google-cloud-storage==3.3.0 PyGithub==2.8.1 +defusedxml==0.7.1 diff --git a/.github/workflows/ur-build-hw.yml b/.github/workflows/ur-build-hw.yml index ef5ce3bae5712..d10d715fde9db 100644 --- a/.github/workflows/ur-build-hw.yml +++ b/.github/workflows/ur-build-hw.yml @@ -157,21 +157,42 @@ jobs: # This is to check that install command does not fail run: cmake --install build - - name: Test adapter specific + - name: Build artifact name suffix + id: artifact_suffix + shell: bash env: - ZE_ENABLE_LOADER_DEBUG_TRACE: 1 - LIT_OPTS: "--timeout 120 -j 50" - # These tests cause timeouts on CI - LIT_FILTER_OUT: "(adapters/level_zero/memcheck.test|adapters/level_zero/v2/deferred_kernel_memcheck.test)" - run: cmake --build build -j $(($(nproc)/3)) -- check-unified-runtime-adapter + RUNNER_NAME: ${{ inputs.runner_name }} + STATIC_LOADER: ${{ matrix.adapter.static_Loader }} + OTHER_NAME: ${{ matrix.adapter.other_name }} + run: | + RUNNER_SUFFIX=$(echo "${RUNNER_NAME}" | sed 's/^UR_//') + SUFFIX="${RUNNER_SUFFIX}" + + if [ "${STATIC_LOADER}" = "ON" ]; then + SUFFIX="${SUFFIX}-static" + fi + + if [ -n "${OTHER_NAME}" ]; then + SUFFIX="${SUFFIX}-${OTHER_NAME}" + fi + + echo "suffix=${SUFFIX}" >> $GITHUB_OUTPUT + + - name: Test adapter specific + uses: ./devops/actions/run-tests/ur + with: + test_type: 'adapter-specific' + build_dir: build + artifact_name: 'ur-${{matrix.adapter.name}}-${{steps.artifact_suffix.outputs.suffix}}-adapter-specific' # Don't run adapter specific tests when building multiple adapters if: ${{ matrix.adapter.other_name == '' }} - name: Test adapters - env: - ZE_ENABLE_LOADER_DEBUG_TRACE: 1 - LIT_OPTS: "--timeout 120 -j 50" - run: cmake --build build -j $(($(nproc)/3)) -- check-unified-runtime-conformance + uses: ./devops/actions/run-tests/ur + with: + test_type: 'conformance' + build_dir: build + artifact_name: 'ur-${{matrix.adapter.name}}-${{steps.artifact_suffix.outputs.suffix}}-conformance' - name: Debug CI platform information if: ${{ always() }} diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml new file mode 100644 index 0000000000000..e6f99ea1455e8 --- /dev/null +++ b/devops/actions/run-tests/ur/action.yml @@ -0,0 +1,127 @@ +name: 'Run UR Tests' + +inputs: + test_type: + description: 'Type of tests to run (adapter-specific or conformance)' + required: true + build_dir: + required: false + default: 'build' + artifact_name: + description: 'Name for uploaded test artifacts (logs, XML)' + required: false + default: '' + +runs: + using: "composite" + steps: + - name: Run ${{ inputs.test_type }} tests + id: run_tests + continue-on-error: true + shell: bash + env: + BUILD_DIR: ${{ inputs.build_dir }} + TEST_TYPE: ${{ inputs.test_type }} + run: | + # Run tests via unified CLI (handles validation, config, cmake execution) + # Use temp file instead of command substitution to preserve outputs even on error + OUTPUTS_FILE=$(mktemp) + set +e # Don't exit on error + devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE" > "$OUTPUTS_FILE" 2>&1 + EXIT_CODE=$? + set -e # Re-enable exit on error + + # Parse Python output and set GitHub Actions outputs + while IFS='=' read -r key value; do + if [ -n "$key" ] && [ -n "$value" ]; then + echo "$key=$value" >> $GITHUB_OUTPUT + echo " Set output: $key=$value" + fi + done < <(grep -E "^(log-file|xml-file|skip-artifacts)=" "$OUTPUTS_FILE") + + rm -f "$OUTPUTS_FILE" + exit $EXIT_CODE + + - name: Filter test log + if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }} + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + run: | + if [ ! -f "$LOG_FILE" ]; then + echo "::warning::Test log not found" + exit 0 + fi + + echo "::group::Test Log" + devops/scripts/ur-test filter-log "$LOG_FILE" || { echo "::error::Filter failed"; head -1000 "$LOG_FILE"; } + echo "::endgroup::" + + - name: Report test failures + if: steps.run_tests.outcome != 'success' + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + run: | + devops/scripts/ur-test extract-errors "$LOG_FILE" + exit 1 + + - name: Test summary + if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }} + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + XML_FILE: ${{ steps.run_tests.outputs['xml-file'] || 'adapter_tests_results.xml' }} + ARTIFACT_NAME: ${{ inputs.artifact_name }} + WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [ ! -f "$LOG_FILE" ]; then + echo "::warning::Test log not found" + exit 0 + fi + + if [ ! -s "$LOG_FILE" ]; then + echo "::warning::Test log is empty" + exit 0 + fi + + devops/scripts/ur-test summary "$LOG_FILE" "${XML_FILE:-}" + + # Add link to artifacts if they will be uploaded + if [ -n "$ARTIFACT_NAME" ]; then + echo "" + echo "---" + echo "" + echo "Full logs and XML are uploaded as workflow artifacts:" + echo "$WORKFLOW_URL" + fi + + - name: Copy artifacts + if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + XML_FILE: ${{ steps.run_tests.outputs['xml-file'] || 'adapter_tests_results.xml' }} + run: | + set -f # Disable glob expansion + + # Clean artifacts directory to avoid mixing files from different test runs + rm -rf test_artifacts + mkdir -p test_artifacts + + if [ -f "$LOG_FILE" ]; then + cp "$LOG_FILE" test_artifacts/ + fi + + if [ -n "$XML_FILE" ] && [ -f "$XML_FILE" ]; then + cp "$XML_FILE" test_artifacts/ + fi + + - name: Upload test artifacts + if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.artifact_name }} + path: test_artifacts/ + retention-days: 7 + if-no-files-found: warn diff --git a/devops/scripts/ur-test b/devops/scripts/ur-test new file mode 100755 index 0000000000000..fcb4aaf4af43a --- /dev/null +++ b/devops/scripts/ur-test @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +"""Unified Runtime test management CLI.""" + +import sys +from ur_test_tools.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/devops/scripts/ur_test_tools/__init__.py b/devops/scripts/ur_test_tools/__init__.py new file mode 100644 index 0000000000000..c8d7cdb6a98ac --- /dev/null +++ b/devops/scripts/ur_test_tools/__init__.py @@ -0,0 +1,36 @@ +"""UR Test Tools - Modular test orchestration and summary generation.""" + +__version__ = "1.0.0" +__author__ = "Unified Runtime Team" + +from .models import ( + TestLists, + TestCounts, + TimingSummary, + TestConfig, + SummaryConfigFromLines, +) +from .test_runner import TestRunner +from .summary_generator import SummaryReporter +from .validation import ( + PathValidator, +) +from .outputs import ( + ConsoleOutput, + GitHubActionsOutput, +) + +__all__ = [ + "__version__", + "__author__", + "TestLists", + "TestCounts", + "TimingSummary", + "TestConfig", + "SummaryConfigFromLines", + "TestRunner", + "SummaryReporter", + "PathValidator", + "ConsoleOutput", + "GitHubActionsOutput", +] diff --git a/devops/scripts/ur_test_tools/cli.py b/devops/scripts/ur_test_tools/cli.py new file mode 100644 index 0000000000000..76db046db4f70 --- /dev/null +++ b/devops/scripts/ur_test_tools/cli.py @@ -0,0 +1,149 @@ +"""CLI entry points for UR test tools.""" + +import sys +import os +from pathlib import Path + +from .models.config import SummaryConfigFromLines, TestConfig, TestExecutionContext +from .validation.path_validator import PathValidator +from .parsers.log_parser import ( + LITLogParser, + read_log_file, +) +from .outputs.console import ConsoleOutput +from .summary_generator import SummaryReporter +from .test_runner import ( + TestRunner, + get_test_config, +) +from .outputs.github_actions import GitHubActionsOutput + + +def main() -> int: + """Unified CLI entry point.""" + if len(sys.argv) < 2: + print("Error: Missing command", file=sys.stderr) + return 1 + + command = sys.argv[1] + + if command == "run": + return main_ci_utils("run-tests") + + elif command in ("summary", "extract-errors", "filter-log"): + internal_cmd = "show-summary" if command == "summary" else command + return main_test_summary(internal_cmd) + + else: + print(f"Error: Unknown command '{command}'", file=sys.stderr) + return 1 + + +def main_test_summary(command: str) -> int: + """Entry point for ur_test_summary CLI.""" + try: + if len(sys.argv) < 3: + print( + f"Error: {sys.argv[0]} [xml_file]", + file=sys.stderr, + ) + return 1 + + log_file = sys.argv[2] + PathValidator.validate_log_path(log_file) + lines = read_log_file(log_file) + parser = LITLogParser(lines) + + if command == "extract-errors": + for line in parser.extract_error_details(): + print(line, end="") + + elif command == "filter-log": + for line in ConsoleOutput.filter_log_for_display(lines): + print(line, end="") + + elif command == "show-summary": + xml_file = PathValidator.validate_optional_path( + sys.argv[3] if len(sys.argv) > 3 else "", "XML", allow_absolute=True + ) + config = SummaryConfigFromLines(log_lines=lines, xml_file=xml_file or None) + SummaryReporter(config).generate() + + else: + print(f"Error: Unknown command '{command}'", file=sys.stderr) + return 1 + + return 0 + + except (OSError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +def main_ci_utils(command: str) -> int: + """Entry point for ur_ci_utils CLI.""" + if command == "run-tests": + return _run_tests_command() + + else: + print(f"Error: Unknown command '{command}'", file=sys.stderr) + return 1 + + +def _run_tests_command() -> int: + """Execute run-tests command.""" + if len(sys.argv) < 5: + print( + f"Error: run-tests ", + file=sys.stderr, + ) + return 1 + + test_type = sys.argv[2] + build_dir = sys.argv[3] + workspace = sys.argv[4] + + # Validate inputs + gha = GitHubActionsOutput() + if not PathValidator.validate_build_dir(build_dir, workspace): + gha.print_error("Invalid build_dir") + return 1 + + try: + config = get_test_config(test_type) + except ValueError as e: + gha.print_error(str(e)) + return 1 + + # Convert to paths and create context + workspace_path = Path(workspace).resolve() + build_dir_path = workspace_path / build_dir + + xml_output_name = f"{test_type.replace('-', '_')}_results.xml" + xml_output_path = (build_dir_path / xml_output_name).absolute() + xml_output_path.parent.mkdir(parents=True, exist_ok=True) + + log_file_path = workspace_path / config.log_file + + env = os.environ.copy() + + context = TestExecutionContext( + test_type=test_type, + build_dir=build_dir_path, + workspace=workspace_path, + xml_output_path=xml_output_path, + log_file_path=log_file_path, + config=config, + env=env, + ) + + # Validate context + try: + context.validate() + except ValueError as e: + gha.print_error(str(e)) + return 1 + + # Run tests + runner = TestRunner(context) + return runner.run() diff --git a/devops/scripts/ur_test_tools/constants.py b/devops/scripts/ur_test_tools/constants.py new file mode 100644 index 0000000000000..11610e6d5e894 --- /dev/null +++ b/devops/scripts/ur_test_tools/constants.py @@ -0,0 +1,36 @@ +"""Constants for UR test tools.""" + +import re + +# File I/O +MAX_LINES_TO_SCAN = 1000 + +SEPARATOR_WIDTH = 70 + +# Job Calculation +MAX_JOBS = 16 + +# LIT Configuration +DEFAULT_LIT_TIMEOUT = 120 +DEFAULT_LIT_JOBS = 50 + +# Test Type Identifiers +TEST_TYPE_ADAPTER_SPECIFIC = "adapter-specific" + +# Constants +TEST_NOT_SELECTED_MSG = "Test not selected" +SLOWEST_TESTS_HEADER = "Slowest Tests:" +TEST_TIMES_HEADERS = ("Tests Times:", "Test Times:") + +# LIT Output Patterns +FAIL_TIMEOUT_PATTERN = re.compile(r"^(FAIL|TIMEOUT):") +TEST_LIST_HEADER_PATTERN = re.compile( + r"^(Passed|Unsupported|Failed|Expectedly Failed|" + r"Timed Out|Unexpectedly Passed|Unresolved) Tests \(" +) +STATS_PATTERN = re.compile( + r"^\s*(Total Discovered|Expected Passes|Expectedly Failed|" + r"Excluded|Unsupported|Skipped|Passed|Passed With Retry|" + r"Failed|Timed Out|Unexpectedly Passed|Unresolved)(\s+Tests)?\s*:" +) +TEST_CATEGORY_PATTERN = re.compile(r"^([A-Za-z]+(?: [A-Za-z]+)*) Tests \((\d+)\):") diff --git a/devops/scripts/ur_test_tools/models/__init__.py b/devops/scripts/ur_test_tools/models/__init__.py new file mode 100644 index 0000000000000..f242b3bc63400 --- /dev/null +++ b/devops/scripts/ur_test_tools/models/__init__.py @@ -0,0 +1,25 @@ +"""Models package - Data structures for UR test tools.""" + +from .test_data import ( + TestLists, + TestCounts, + TimingSummary, + SkippedTestsResult, + ExcludedTestsResult, +) +from .config import ( + TestConfig, + TestExecutionContext, + SummaryConfigFromLines, +) + +__all__ = [ + "TestLists", + "TestCounts", + "TimingSummary", + "SkippedTestsResult", + "ExcludedTestsResult", + "TestConfig", + "TestExecutionContext", + "SummaryConfigFromLines", +] diff --git a/devops/scripts/ur_test_tools/models/config.py b/devops/scripts/ur_test_tools/models/config.py new file mode 100644 index 0000000000000..9094e4dadee91 --- /dev/null +++ b/devops/scripts/ur_test_tools/models/config.py @@ -0,0 +1,48 @@ +"""Configuration dataclasses for UR test tools.""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional, Dict, List + + +@dataclass +class TestConfig: + """Test execution configuration.""" + + target: str + log_file: str + lit_filter_out: Optional[str] = None + + def __post_init__(self): + """Validate configuration on creation.""" + if not all([self.target, self.log_file]): + raise ValueError("target and log_file are required") + + +@dataclass +class TestExecutionContext: + """Context for test execution.""" + + test_type: str + build_dir: Path + workspace: Path + xml_output_path: Path + log_file_path: Path + config: TestConfig + env: Dict[str, str] = field(default_factory=dict) + + def validate(self) -> None: + try: + workspace_resolved = self.workspace.resolve() + for path in [self.log_file_path, self.xml_output_path, self.build_dir]: + path.resolve().relative_to(workspace_resolved) + except ValueError as e: + raise ValueError(f"Path outside workspace: {e}") from e + + +@dataclass +class SummaryConfigFromLines: + """Configuration for summary generation from parsed log lines.""" + + log_lines: List[str] + xml_file: Optional[str] = None diff --git a/devops/scripts/ur_test_tools/models/test_data.py b/devops/scripts/ur_test_tools/models/test_data.py new file mode 100644 index 0000000000000..782c475609dc8 --- /dev/null +++ b/devops/scripts/ur_test_tools/models/test_data.py @@ -0,0 +1,50 @@ +"""Type definitions for test data structures.""" + +from typing import TypedDict, List + + +class TestLists(TypedDict, total=False): + """Type definition for test list dictionary.""" + + Passed: List[str] + Failed: List[str] + Skipped: List[str] + Unsupported: List[str] + Excluded: List[str] + Unresolved: List[str] + + +class TestCounts(TypedDict, total=False): + """Type definition for test count dictionary.""" + + Passed: int + Failed: int + Skipped: int + Unsupported: int + Excluded: int + Unresolved: int + + +class TimingSummary(TypedDict): + """Type definition for test timing summary.""" + + slowest: List[str] + histogram: List[str] + + +class SkippedTestsResult(TypedDict): + """Result of skipped tests analysis.""" + + tests: List[str] + count: int + source: str + note: str + + +class ExcludedTestsResult(TypedDict): + """Result of excluded tests analysis.""" + + tests: List[str] + count: int + source: str + note: str diff --git a/devops/scripts/ur_test_tools/outputs/__init__.py b/devops/scripts/ur_test_tools/outputs/__init__.py new file mode 100644 index 0000000000000..9273918957af5 --- /dev/null +++ b/devops/scripts/ur_test_tools/outputs/__init__.py @@ -0,0 +1,9 @@ +"""Outputs package - Console and GitHub Actions output generation.""" + +from .console import ConsoleOutput +from .github_actions import GitHubActionsOutput + +__all__ = [ + "ConsoleOutput", + "GitHubActionsOutput", +] diff --git a/devops/scripts/ur_test_tools/outputs/console.py b/devops/scripts/ur_test_tools/outputs/console.py new file mode 100644 index 0000000000000..a0b5f98fb2d86 --- /dev/null +++ b/devops/scripts/ur_test_tools/outputs/console.py @@ -0,0 +1,119 @@ +"""Console output formatting for test results.""" + +from typing import List, Optional + +from ..constants import ( + SEPARATOR_WIDTH, + SLOWEST_TESTS_HEADER, + STATS_PATTERN, + TEST_CATEGORY_PATTERN, + TEST_TIMES_HEADERS, +) +from ..models.test_data import TimingSummary +from ..parsers.log_parser import LITLogParser + + +class ConsoleOutput: + """Format test results for console output.""" + + @staticmethod + def print_test_group( + title: str, tests: List[str], note: str = "", count: Optional[int] = None + ) -> None: + """Print GitHub Actions collapsible group with test list.""" + test_count = count if count is not None else len(tests) + print(f"::group::{title} ({test_count})") + if note: + print(note) + print() + for test in tests: + print(test) + print("::endgroup::") + + @staticmethod + def print_statistics(stats: List[str]) -> None: + """Print statistics section.""" + if stats: + print("=== Test Statistics ===") + for stat in stats: + print(stat.rstrip()) + print() + + @staticmethod + def print_timing_summary(lines: List[str]) -> None: + """Print timing information section.""" + parser = LITLogParser(lines) + time_info = parser.extract_time_summary() + + testing_time = None + for line in lines: + if line.strip().startswith("Testing Time:"): + testing_time = line.strip() + break + + if not (time_info["slowest"] or time_info["histogram"] or testing_time): + return + + print("::group::Test Timing Summary") + + if testing_time: + print(testing_time) + print() + + if time_info["slowest"]: + print(SLOWEST_TESTS_HEADER) + print("-" * SEPARATOR_WIDTH) + for line in time_info["slowest"]: + print(line) + print() + + if time_info["histogram"]: + print("Test Times Distribution:") + print("-" * SEPARATOR_WIDTH) + for line in time_info["histogram"]: + print(line) + + print("::endgroup::") + + @staticmethod + def filter_log_for_display(lines: List[str]) -> List[str]: + """Remove statistics, test lists, and timing from log.""" + result = [] + skip_until_empty = False + in_timing = False + + for line in lines: + stripped = line.strip() + + # Skip statistics lines + if STATS_PATTERN.match(line): + continue + + # Skip test category sections + if TEST_CATEGORY_PATTERN.match(line): + skip_until_empty = True + continue + + # Skip timing sections + if stripped == SLOWEST_TESTS_HEADER or stripped in TEST_TIMES_HEADERS: + in_timing = True + continue + + if in_timing and stripped.replace("*", "") == "": + in_timing = False + continue + + if in_timing: + continue + + if stripped.startswith("Testing Time:"): + continue + + if skip_until_empty: + if not stripped: + skip_until_empty = False + continue + + result.append(line) + + return result diff --git a/devops/scripts/ur_test_tools/outputs/github_actions.py b/devops/scripts/ur_test_tools/outputs/github_actions.py new file mode 100644 index 0000000000000..e892ad3f2f036 --- /dev/null +++ b/devops/scripts/ur_test_tools/outputs/github_actions.py @@ -0,0 +1,20 @@ +"""GitHub Actions-specific output formatting.""" + +import sys + + +class GitHubActionsOutput: + """Format output for GitHub Actions.""" + + @staticmethod + def print_error(message: str) -> None: + print(f"::error::{message}", file=sys.stderr) + + @staticmethod + def print_warning(message: str) -> None: + print(f"::warning::{message}", file=sys.stderr) + + @staticmethod + def set_output(name: str, value: str) -> None: + print(f"{name}={value}", flush=True) + sys.stdout.flush() diff --git a/devops/scripts/ur_test_tools/parsers/__init__.py b/devops/scripts/ur_test_tools/parsers/__init__.py new file mode 100644 index 0000000000000..55721e06f10c6 --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/__init__.py @@ -0,0 +1,19 @@ +"""Parsers package - Data extraction from logs and XML.""" + +from .log_parser import ( + LITLogParser, + read_log_file, +) +from .xml_parser import ( + JUnitXMLParser, + ParsedXMLTests, +) +from .stats_parser import get_count_from_stats + +__all__ = [ + "LITLogParser", + "read_log_file", + "JUnitXMLParser", + "ParsedXMLTests", + "get_count_from_stats", +] diff --git a/devops/scripts/ur_test_tools/parsers/log_parser.py b/devops/scripts/ur_test_tools/parsers/log_parser.py new file mode 100644 index 0000000000000..2291973b0ebea --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/log_parser.py @@ -0,0 +1,151 @@ +"""Parse LIT text output for test information.""" + +import sys +from pathlib import Path +from typing import Iterator, List, Tuple + +from ..constants import ( + FAIL_TIMEOUT_PATTERN, + TEST_LIST_HEADER_PATTERN, + STATS_PATTERN, + TEST_CATEGORY_PATTERN, + SLOWEST_TESTS_HEADER, + TEST_TIMES_HEADERS, +) +from ..models.test_data import TestLists, TestCounts, TimingSummary + + +def _read_with_utf8_fallback(path: str, read_func): + try: + with open(path, "r", encoding="utf-8", errors="strict") as f: + return read_func(f) + except UnicodeDecodeError: + print( + f"Warning: File contains non-UTF-8 characters, replacing with U+FFFD", + file=sys.stderr, + ) + with open(path, "r", encoding="utf-8", errors="replace") as f: + return read_func(f) + + +def read_log_file(log_path: str) -> List[str]: + path = Path(log_path) + file_size = path.stat().st_size + + if file_size > 10 * 1024 * 1024: # 10 MB + print( + f"Large log file: {file_size / (1024 * 1024):.1f} MB. " + f"This may indicate a test problem.", + file=sys.stderr, + ) + + try: + return _read_with_utf8_fallback(log_path, lambda f: f.readlines()) + except OSError as e: + raise OSError(f"Cannot read log file: {e}") from e + + +class LITLogParser: + """Parse LIT (llvm-lit) text output for test information.""" + + def __init__(self, lines: List[str]): + self.lines = lines + + def extract_error_details(self) -> List[str]: + result = [] + in_error = False + + for line in self.lines: + if FAIL_TIMEOUT_PATTERN.match(line): + in_error = True + + # Stop at test list headers or timing summaries + if in_error and ( + TEST_LIST_HEADER_PATTERN.match(line) + or line.strip() == SLOWEST_TESTS_HEADER + or line.strip() in TEST_TIMES_HEADERS + ): + break + + if in_error: + result.append(line) + + return result + + def extract_statistics(self) -> List[str]: + return [line for line in self.lines if STATS_PATTERN.match(line)] + + def extract_time_summary(self) -> TimingSummary: + result: TimingSummary = {"slowest": [], "histogram": []} + current_section = None + skip_next_hr = False + + for line in self.lines: + stripped = line.strip() + + if stripped == SLOWEST_TESTS_HEADER: + current_section = "slowest" + skip_next_hr = True + continue + elif stripped in TEST_TIMES_HEADERS: + current_section = "histogram" + skip_next_hr = True + continue + + if skip_next_hr and stripped.startswith("---"): + skip_next_hr = False + continue + + if current_section == "slowest": + if not stripped: + current_section = None + elif not stripped.startswith("---"): + result["slowest"].append(line.rstrip()) + elif current_section == "histogram": + if not stripped: + current_section = None + elif stripped.replace("*", "") == "": + current_section = None + elif stripped.startswith("[") or stripped.replace("-", "") == "": + result["histogram"].append(line.rstrip()) + else: + current_section = None + + return result + + def extract_test_lists(self) -> Tuple[TestLists, TestCounts]: + categories: TestLists = {} + declared_counts: TestCounts = {} + current_category = None + current_tests = [] + current_declared_count = 0 + + for line in self.lines: + match = TEST_CATEGORY_PATTERN.match(line) + if match: + if current_category: + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + + current_category = match.group(1) + current_declared_count = int(match.group(2)) + current_tests = [] + continue + + if current_category: + if not line.strip(): + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + current_category = None + current_tests = [] + current_declared_count = 0 + else: + test_name = line.strip() + if test_name: + current_tests.append(test_name) + + if current_category: + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + + return categories, declared_counts diff --git a/devops/scripts/ur_test_tools/parsers/stats_parser.py b/devops/scripts/ur_test_tools/parsers/stats_parser.py new file mode 100644 index 0000000000000..5424e6c084405 --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/stats_parser.py @@ -0,0 +1,16 @@ +"""Parse statistics from LIT output.""" + +import re +from typing import List + + +def get_count_from_stats(stats: List[str], keywords: List[str]) -> int: + # Build regex pattern from keywords (compile once per call) + pattern = re.compile("|".join(re.escape(kw) for kw in keywords)) + + for stat in stats: + if pattern.search(stat): + match = re.search(r"(\d+)", stat) + if match: + return int(match.group(1)) + return 0 diff --git a/devops/scripts/ur_test_tools/parsers/xml_parser.py b/devops/scripts/ur_test_tools/parsers/xml_parser.py new file mode 100644 index 0000000000000..ac2eb41e83700 --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/xml_parser.py @@ -0,0 +1,90 @@ +"""Parse JUnit XML test results.""" + +import sys +from pathlib import Path +from typing import List, NamedTuple, Optional + +import defusedxml.ElementTree as ET + +from ..constants import TEST_NOT_SELECTED_MSG + + +class ParsedXMLTests(NamedTuple): + """Skipped and excluded tests from XML parsing.""" + + skipped: List[str] + excluded: List[str] + + +def _format_test_name(classname: str, name: str) -> str: + if classname and name: + return f"{classname}.{name}" + return name + + +class JUnitXMLParser: + """Parse JUnit XML from LIT --xunit-xml-output.""" + + def __init__(self, xml_path: Optional[str]): + self.xml_path = xml_path + self._root = None + + def parse(self) -> bool: + if not self.xml_path: + return False + + xml_file = Path(self.xml_path) + if not xml_file.exists(): + return False + + try: + tree = ET.parse(self.xml_path) + self._root = tree.getroot() + return True + except ET.ParseError as e: + print( + f"Warning: Failed to parse XML file {self.xml_path}: {e}", + file=sys.stderr, + ) + return False + except (OSError, ValueError) as e: + print( + f"Warning: Error reading XML file {self.xml_path}: {e}", file=sys.stderr + ) + return False + + def extract_tests_from_xml(self) -> ParsedXMLTests: + if not self.parse(): + return ParsedXMLTests([], []) + + skipped = [] + excluded = [] + + for testcase in self._root.findall(".//testcase"): + skipped_elem = testcase.find("skipped") + if skipped_elem is None: + continue + + message = skipped_elem.get("message", "") + test_name = _format_test_name( + testcase.get("classname", ""), testcase.get("name", "") + ) + + if not test_name: + continue + + # Separate by message type + if TEST_NOT_SELECTED_MSG in message: + excluded.append(test_name) + else: + skipped.append(test_name) + + return ParsedXMLTests(skipped=skipped, excluded=excluded) + + def extract_skipped_tests(self) -> List[str]: + skipped, _ = self.extract_tests_from_xml() + return skipped + + def extract_excluded_tests(self) -> List[str]: + _, excluded = self.extract_tests_from_xml() + return excluded diff --git a/devops/scripts/ur_test_tools/summary_generator.py b/devops/scripts/ur_test_tools/summary_generator.py new file mode 100644 index 0000000000000..e4ccac6c8f8fa --- /dev/null +++ b/devops/scripts/ur_test_tools/summary_generator.py @@ -0,0 +1,224 @@ +"""Generate test summary reports.""" + +import sys +from typing import List + +from .models.config import SummaryConfigFromLines +from .models.test_data import ( + TestLists, + TestCounts, + SkippedTestsResult, + ExcludedTestsResult, +) +from .parsers.log_parser import LITLogParser +from .parsers.xml_parser import JUnitXMLParser +from .parsers.stats_parser import get_count_from_stats +from .outputs.console import ConsoleOutput +from .validation.data_validator import validate_test_counts + + +class SummaryReporter: + """Generate comprehensive test summary.""" + + def __init__(self, config: SummaryConfigFromLines): + self.config = config + + def generate(self) -> None: + parser = LITLogParser(self.config.log_lines) + stats = parser.extract_statistics() + test_lists, declared_counts = parser.extract_test_lists() + total_discovered = get_count_from_stats(stats, ["Total Discovered"]) + + xml_parser = JUnitXMLParser(self.config.xml_file) + parsed_xml = xml_parser.extract_tests_from_xml() + skipped_xml = parsed_xml.skipped + excluded_xml = parsed_xml.excluded + + ConsoleOutput.print_statistics(stats) + + skipped_result = self._analyze_skipped_tests(test_lists, stats, skipped_xml) + self._validate_skipped_counts(skipped_result, declared_counts, stats) + self._display_skipped_tests(skipped_result) + if skipped_result["count"] > 0: + self._cleanup_skipped_from_test_lists(test_lists) + + excluded_result = self._analyze_excluded_tests(test_lists, stats, excluded_xml) + self._validate_excluded_counts(excluded_result, declared_counts, stats) + self._display_excluded_tests(excluded_result) + if excluded_result["count"] > 0: + self._cleanup_excluded_from_test_lists(test_lists) + + self._display_remaining_categories(test_lists) + + validate_test_counts( + total_discovered, + test_lists, + skipped_result["count"], + excluded_result["count"], + ) + + ConsoleOutput.print_timing_summary(self.config.log_lines) + + def _analyze_skipped_tests( + self, test_lists: TestLists, stats: List[str], skipped_xml: List[str] + ) -> SkippedTestsResult: + """Analyze skipped tests (priority: XML > Log > Stats).""" + skipped_from_log = test_lists.get("Skipped", test_lists.get("Unsupported", [])) + stats_count = get_count_from_stats(stats, ["Skipped", "Unsupported"]) + + # Priority 1: XML data (most reliable - structured output) + if skipped_xml: + return SkippedTestsResult( + tests=skipped_xml, + count=len(skipped_xml), + source="xml", + note="", + ) + + # Priority 2: Log data + if skipped_from_log: + return SkippedTestsResult( + tests=skipped_from_log, + count=len(skipped_from_log), + source="log", + note="", + ) + + # Priority 3: Stats only (no individual test names) + if stats_count: + return SkippedTestsResult( + tests=[], + count=stats_count, + source="stats", + note="Warning: Test names not available", + ) + + # No data available + return SkippedTestsResult(tests=[], count=0, source="none", note="") + + def _validate_skipped_counts( + self, result: SkippedTestsResult, declared_counts: TestCounts, stats: List[str] + ) -> None: + """Validate skipped counts (warns on mismatch).""" + actual_count = result["count"] + if actual_count == 0: + return # Nothing to validate + + declared_count = declared_counts.get( + "Skipped", declared_counts.get("Unsupported", 0) + ) + stats_count = get_count_from_stats(stats, ["Skipped", "Unsupported"]) + + # Build list of mismatches + mismatches = [] + + if declared_count and declared_count != actual_count: + mismatches.append(f"log header: {declared_count}") + + if stats_count and stats_count != actual_count: + mismatches.append(f"statistics: {stats_count}") + + # Display warning only if mismatches found + if mismatches: + sources_str = ", ".join(mismatches) + print( + f"Warning: Skipped test count mismatch. " + f"Using {actual_count} from {result['source']}, " + f"but found {sources_str}", + file=sys.stderr, + ) + + def _display_skipped_tests(self, result: SkippedTestsResult) -> None: + if result["count"] > 0: + ConsoleOutput.print_test_group( + "Skipped Tests", + result["tests"], + note=result["note"], + count=result["count"] if not result["tests"] else None, + ) + + def _cleanup_skipped_from_test_lists(self, test_lists: TestLists) -> None: + test_lists.pop("Skipped", None) + test_lists.pop("Unsupported", None) + + def _analyze_excluded_tests( + self, test_lists: TestLists, stats: List[str], excluded_xml: List[str] + ) -> ExcludedTestsResult: + """Analyze excluded tests (priority: Log > XML > Stats).""" + excluded_from_log = test_lists.get("Excluded", []) + stats_count = get_count_from_stats(stats, ["Excluded"]) + + # Priority 1: Log data + if excluded_from_log: + return ExcludedTestsResult( + tests=excluded_from_log, + count=len(excluded_from_log), + source="log", + note="", + ) + + # Priority 2: XML data + if excluded_xml: + return ExcludedTestsResult( + tests=excluded_xml, count=len(excluded_xml), source="xml", note="" + ) + + # Priority 3: Stats only (no individual test names) + if stats_count: + return ExcludedTestsResult( + tests=[], + count=stats_count, + source="stats", + note="Warning: Test names not available", + ) + + # No data available + return ExcludedTestsResult(tests=[], count=0, source="none", note="") + + def _validate_excluded_counts( + self, result: ExcludedTestsResult, declared_counts: TestCounts, stats: List[str] + ) -> None: + """Validate excluded counts (warns on mismatch).""" + actual_count = result["count"] + if actual_count == 0: + return # Nothing to validate + + declared_count = declared_counts.get("Excluded", 0) + stats_count = get_count_from_stats(stats, ["Excluded"]) + + # Build list of mismatches + mismatches = [] + + if declared_count and declared_count != actual_count: + mismatches.append(f"log header: {declared_count}") + + if stats_count and stats_count != actual_count: + mismatches.append(f"statistics: {stats_count}") + + # Display warning only if mismatches found + if mismatches: + sources_str = ", ".join(mismatches) + print( + f"Warning: Excluded test count mismatch. " + f"Using {actual_count} from {result['source']}, " + f"but found {sources_str}", + file=sys.stderr, + ) + + def _display_excluded_tests(self, result: ExcludedTestsResult) -> None: + if result["count"] > 0: + ConsoleOutput.print_test_group( + "Excluded Tests", + result["tests"], + note=result["note"], + count=result["count"] if not result["tests"] else None, + ) + + def _cleanup_excluded_from_test_lists(self, test_lists: TestLists) -> None: + test_lists.pop("Excluded", None) + + def _display_remaining_categories(self, test_lists: TestLists) -> None: + for category, tests in test_lists.items(): + if tests: + ConsoleOutput.print_test_group(f"{category} Tests", tests) + diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py new file mode 100644 index 0000000000000..136a43800412f --- /dev/null +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -0,0 +1,160 @@ +"""Test execution.""" + +import os +import sys +import subprocess # nosec B404 - Used safely with list args, no shell=True +from pathlib import Path +from typing import List, Optional + +from .constants import ( + DEFAULT_LIT_TIMEOUT, + DEFAULT_LIT_JOBS, + TEST_TYPE_ADAPTER_SPECIFIC, + MAX_LINES_TO_SCAN, + MAX_JOBS, +) +from .models.config import TestConfig, TestExecutionContext +from .outputs.github_actions import GitHubActionsOutput +from .parsers.log_parser import _read_with_utf8_fallback + + +def get_test_config(test_type: str) -> TestConfig: + """Get test configuration for test type.""" + if test_type == "adapter-specific": + return TestConfig( + target="check-unified-runtime-adapter", + log_file="adapter_tests.log", + lit_filter_out=( + "(adapters/level_zero/memcheck.test|" + "adapters/level_zero/v2/deferred_kernel_memcheck.test)" + ), + ) + elif test_type == "conformance": + return TestConfig( + target="check-unified-runtime-conformance", + log_file="conformance_tests.log", + ) + else: + raise ValueError(f"Invalid test_type: {test_type}") + + +def calculate_jobs() -> int: + """Calculate parallel jobs (nproc/3 capped at MAX_JOBS).""" + try: + nproc = os.cpu_count() or 4 + return min(nproc // 3, MAX_JOBS) + except (OSError, AttributeError): + return 4 + + +def check_log_has_tests(log_file: str) -> bool: + """Check if log contains test results.""" + + def _scan_for_testing(f): + for _ in range(MAX_LINES_TO_SCAN): + line = f.readline() + if not line: + break + if "Testing:" in line: + return True + return False + + try: + return _read_with_utf8_fallback(log_file, _scan_for_testing) + except OSError: + return False + + +class TestRunner: + """Execute UR tests.""" + + def __init__(self, context: TestExecutionContext): + self.context = context + self.github_output = GitHubActionsOutput() + self.jobs = calculate_jobs() + + def run(self) -> int: + """Run tests and return exit code.""" + self._setup_environment() + + result = self._execute_tests() + if result is None: + return 1 + + if not self._validate_output(): + return 1 + + self._publish_outputs(result) + return result.returncode + + def _setup_environment(self) -> None: + lit_opts = ( + f"--show-unsupported --show-pass --show-xfail --no-progress-bar " + f"-v --timeout {DEFAULT_LIT_TIMEOUT} -j {DEFAULT_LIT_JOBS} " + f"--time-tests --show-flakypass --show-skipped " + f"--xunit-xml-output {self.context.xml_output_path}" + ) + self.context.env["LIT_OPTS"] = lit_opts + + if self.context.config.lit_filter_out: + self.context.env["LIT_FILTER_OUT"] = self.context.config.lit_filter_out + + self.context.env["ZE_ENABLE_LOADER_DEBUG_TRACE"] = "1" + + def _build_cmake_command(self) -> List[str]: + return [ + "cmake", + "--build", + str(self.context.build_dir), + "-j", + str(self.jobs), + "--", + self.context.config.target, + ] + + def _execute_tests(self) -> Optional[subprocess.CompletedProcess]: + cmd = self._build_cmake_command() + + print(f"Running: {' '.join(cmd)}", file=sys.stderr) + print(f"Log: {self.context.log_file_path}, Jobs: {self.jobs}", file=sys.stderr) + print(f"Expected XML: {self.context.xml_output_path}", file=sys.stderr) + + try: + with open(self.context.log_file_path, "w", encoding="utf-8") as log: + return subprocess.run( # nosec B603 B607 + cmd, + stdout=log, + stderr=subprocess.STDOUT, + env=self.context.env, + cwd=self.context.workspace, + ) + except (OSError, PermissionError) as e: + self.github_output.print_error(f"Test execution failed: {e}") + return None + + def _validate_output(self) -> bool: + log_path = self.context.log_file_path + + if not log_path.exists() or log_path.stat().st_size == 0: + self.github_output.print_error("No log generated") + return False + + return True + + def _publish_outputs(self, result: subprocess.CompletedProcess) -> None: + self.github_output.set_output("log-file", str(self.context.log_file_path)) + + if ( + self.context.test_type == TEST_TYPE_ADAPTER_SPECIFIC + and not check_log_has_tests(str(self.context.log_file_path)) + ): + print("No adapter-specific tests found", file=sys.stderr) + self.github_output.set_output("skip-artifacts", "1") + return + + if self.context.xml_output_path.exists(): + self.github_output.set_output("xml-file", str(self.context.xml_output_path)) + else: + self.github_output.print_warning( + f"Expected XML file not found at {self.context.xml_output_path}" + ) diff --git a/devops/scripts/ur_test_tools/validation/__init__.py b/devops/scripts/ur_test_tools/validation/__init__.py new file mode 100644 index 0000000000000..081e0c05ae8f7 --- /dev/null +++ b/devops/scripts/ur_test_tools/validation/__init__.py @@ -0,0 +1,9 @@ +"""Validation package - Security and data validation.""" + +from .path_validator import PathValidator +from .data_validator import validate_test_counts + +__all__ = [ + "PathValidator", + "validate_test_counts", +] diff --git a/devops/scripts/ur_test_tools/validation/data_validator.py b/devops/scripts/ur_test_tools/validation/data_validator.py new file mode 100644 index 0000000000000..b63cc715f685c --- /dev/null +++ b/devops/scripts/ur_test_tools/validation/data_validator.py @@ -0,0 +1,36 @@ +"""Test data validation.""" + +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..models.test_data import TestLists + + +def validate_test_counts( + total_discovered: int, + test_lists: "TestLists", + displayed_skipped: int, + displayed_excluded: int, +) -> None: + """Validate test counts match (prints warnings on mismatch).""" + if not total_discovered: + return + + sum_categories = sum(len(tests) for tests in test_lists.values()) + + # Add skipped tests if they were displayed but not in test_lists + if displayed_skipped > 0 and "Skipped" not in test_lists: + sum_categories += displayed_skipped + + # Add excluded tests if they were displayed but not in test_lists + if displayed_excluded > 0 and "Excluded" not in test_lists: + sum_categories += displayed_excluded + + if total_discovered != sum_categories: + print() + print( + f"::warning::Test count mismatch: Total Discovered = " + f"{total_discovered}, but sum of all categories = {sum_categories}" + ) + print() diff --git a/devops/scripts/ur_test_tools/validation/path_validator.py b/devops/scripts/ur_test_tools/validation/path_validator.py new file mode 100644 index 0000000000000..c2094e142ca62 --- /dev/null +++ b/devops/scripts/ur_test_tools/validation/path_validator.py @@ -0,0 +1,96 @@ +"""Path validation for security.""" + +from pathlib import Path +from typing import Optional + + +class PathValidator: + """Validate paths for security and correctness.""" + + @staticmethod + def validate_build_dir(build_dir: str, workspace: Optional[str] = None) -> bool: + """Validate build directory is safe and within workspace.""" + if not build_dir or ".." in build_dir or build_dir.startswith("/"): + return False + + # Block shell metacharacters, quotes, and control characters + # to prevent injection in f-strings, env vars, and logs + dangerous_chars = {";", "&", "#", "$", "|", "`", "\\", "'", '"', "\n", "\r"} + if any(c in build_dir for c in dangerous_chars): + return False + + if workspace: + try: + build_path = Path(build_dir).resolve(strict=False) + workspace_path = Path(workspace).resolve(strict=False) + build_path.relative_to(workspace_path) + return True + except (ValueError, OSError): + return False + return True + + @staticmethod + def validate_log_path(path: str) -> None: + """Validate log file path (detects path traversal).""" + try: + # Resolve path to detect encoded forms of path traversal (e.g., %2e%2e) + resolved = Path(path).resolve(strict=False) + + # Check for path traversal in original string (simple check) + if ".." in path: + raise ValueError( + f"Invalid log file path (path traversal not allowed): {path}" + ) + + # Verify file exists + if not resolved.exists(): + raise ValueError(f"Log file not found: {path}") + except (OSError, ValueError) as e: + if isinstance(e, ValueError): + raise + raise OSError(f"Invalid log file path: {path} ({e})") from e + + @staticmethod + def validate_optional_path( + path: str, path_type: str, allow_absolute: bool = False + ) -> str: + """Validate optional file path.""" + if not path: + return "" + + try: + # Resolve path to detect encoded forms of path traversal + Path(path).resolve(strict=False) + + # Check for path traversal in original string + if ".." in path: + raise ValueError( + f"Invalid {path_type} file path (path traversal): {path}" + ) + + # Check absolute path restriction + if not allow_absolute and path.startswith("/"): + raise ValueError( + f"Invalid {path_type} file path " + f"(absolute paths not allowed): {path}" + ) + except (OSError, ValueError) as e: + if isinstance(e, ValueError): + raise + raise OSError(f"Invalid {path_type} file path: {path} ({e})") from e + + return path + + @staticmethod + def ensure_within_workspace(path: Path, workspace: Path) -> Path: + """Ensure path is within workspace.""" + resolved = path.resolve() + workspace_resolved = workspace.resolve() + + try: + resolved.relative_to(workspace_resolved) + return resolved + except ValueError as e: + raise ValueError( + f"Path outside workspace: {path} not in {workspace}" + ) from e diff --git a/unified-runtime/test/CMakeLists.txt b/unified-runtime/test/CMakeLists.txt index 2d3e36efbc12f..30f700ca4355f 100644 --- a/unified-runtime/test/CMakeLists.txt +++ b/unified-runtime/test/CMakeLists.txt @@ -86,7 +86,8 @@ function(add_ur_lit_testsuite suite) if(UR_STANDALONE_BUILD) add_custom_target(${TARGET} - COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv + COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" + --show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped USES_TERMINAL ) else() diff --git a/unified-runtime/test/conformance/CMakeLists.txt b/unified-runtime/test/conformance/CMakeLists.txt index 57b39ddbe7c4e..444805b22045a 100644 --- a/unified-runtime/test/conformance/CMakeLists.txt +++ b/unified-runtime/test/conformance/CMakeLists.txt @@ -41,8 +41,10 @@ foreach(adapter ${UR_ADAPTERS_LIST}) if(NOT "${adapter}" STREQUAL "mock") if(UR_STANDALONE_BUILD) add_custom_target(check-unified-runtime-conformance-${adapter} - COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" - -v -Dselector=${adapter}:* + COMMAND ${CMAKE_COMMAND} -E env "LIT_OPTS=$ENV{LIT_OPTS}" + "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" + --show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped + -Dselector=${adapter}:* DEPENDS deps_check-unified-runtime-conformance ) else() diff --git a/unified-runtime/third_party/requirements_testing.txt b/unified-runtime/third_party/requirements_testing.txt index c41dbabde3fe3..2ac825e67cfbd 100644 --- a/unified-runtime/third_party/requirements_testing.txt +++ b/unified-runtime/third_party/requirements_testing.txt @@ -4,3 +4,6 @@ lit==18.1.8 # For timeouts in lit psutil==7.0.0 + +# For secure XML parsing in test result processing +defusedxml==0.7.1 From 1f301e2547bed943cd95d570fddeab04287321c5 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Tue, 25 Aug 2026 12:01:15 +0200 Subject: [PATCH 02/11] Address code review feedback --- .ci/requirements.txt | 1 - .github/workflows/ur-build-hw.yml | 4 ++-- devops/scripts/ur_test_tools/__init__.py | 3 --- devops/scripts/ur_test_tools/cli.py | 4 ---- devops/scripts/ur_test_tools/constants.py | 5 ++--- devops/scripts/ur_test_tools/parsers/log_parser.py | 2 +- devops/scripts/ur_test_tools/summary_generator.py | 8 -------- devops/scripts/ur_test_tools/test_runner.py | 5 ++--- 8 files changed, 7 insertions(+), 25 deletions(-) diff --git a/.ci/requirements.txt b/.ci/requirements.txt index 6c6abcada3eaf..45eb253548496 100644 --- a/.ci/requirements.txt +++ b/.ci/requirements.txt @@ -1,4 +1,3 @@ junitparser==3.2.0 google-cloud-storage==3.3.0 PyGithub==2.8.1 -defusedxml==0.7.1 diff --git a/.github/workflows/ur-build-hw.yml b/.github/workflows/ur-build-hw.yml index d10d715fde9db..9a9d77183f740 100644 --- a/.github/workflows/ur-build-hw.yml +++ b/.github/workflows/ur-build-hw.yml @@ -179,13 +179,13 @@ jobs: echo "suffix=${SUFFIX}" >> $GITHUB_OUTPUT - name: Test adapter specific + # Don't run adapter specific tests when building multiple adapters + if: ${{ matrix.adapter.other_name == '' }} uses: ./devops/actions/run-tests/ur with: test_type: 'adapter-specific' build_dir: build artifact_name: 'ur-${{matrix.adapter.name}}-${{steps.artifact_suffix.outputs.suffix}}-adapter-specific' - # Don't run adapter specific tests when building multiple adapters - if: ${{ matrix.adapter.other_name == '' }} - name: Test adapters uses: ./devops/actions/run-tests/ur diff --git a/devops/scripts/ur_test_tools/__init__.py b/devops/scripts/ur_test_tools/__init__.py index c8d7cdb6a98ac..2707bdbdf15db 100644 --- a/devops/scripts/ur_test_tools/__init__.py +++ b/devops/scripts/ur_test_tools/__init__.py @@ -1,8 +1,5 @@ """UR Test Tools - Modular test orchestration and summary generation.""" -__version__ = "1.0.0" -__author__ = "Unified Runtime Team" - from .models import ( TestLists, TestCounts, diff --git a/devops/scripts/ur_test_tools/cli.py b/devops/scripts/ur_test_tools/cli.py index 76db046db4f70..9d8931b09e46e 100644 --- a/devops/scripts/ur_test_tools/cli.py +++ b/devops/scripts/ur_test_tools/cli.py @@ -20,7 +20,6 @@ def main() -> int: - """Unified CLI entry point.""" if len(sys.argv) < 2: print("Error: Missing command", file=sys.stderr) return 1 @@ -40,7 +39,6 @@ def main() -> int: def main_test_summary(command: str) -> int: - """Entry point for ur_test_summary CLI.""" try: if len(sys.argv) < 3: print( @@ -81,7 +79,6 @@ def main_test_summary(command: str) -> int: def main_ci_utils(command: str) -> int: - """Entry point for ur_ci_utils CLI.""" if command == "run-tests": return _run_tests_command() @@ -91,7 +88,6 @@ def main_ci_utils(command: str) -> int: def _run_tests_command() -> int: - """Execute run-tests command.""" if len(sys.argv) < 5: print( f"Error: run-tests ", diff --git a/devops/scripts/ur_test_tools/constants.py b/devops/scripts/ur_test_tools/constants.py index 11610e6d5e894..1500db2770dbc 100644 --- a/devops/scripts/ur_test_tools/constants.py +++ b/devops/scripts/ur_test_tools/constants.py @@ -3,13 +3,12 @@ import re # File I/O +# Conservative upper bound for locating the LIT "Testing:" marker. +# Non-verbose CI runs normally emit it much earlier. MAX_LINES_TO_SCAN = 1000 SEPARATOR_WIDTH = 70 -# Job Calculation -MAX_JOBS = 16 - # LIT Configuration DEFAULT_LIT_TIMEOUT = 120 DEFAULT_LIT_JOBS = 50 diff --git a/devops/scripts/ur_test_tools/parsers/log_parser.py b/devops/scripts/ur_test_tools/parsers/log_parser.py index 2291973b0ebea..51566013a0280 100644 --- a/devops/scripts/ur_test_tools/parsers/log_parser.py +++ b/devops/scripts/ur_test_tools/parsers/log_parser.py @@ -2,7 +2,7 @@ import sys from pathlib import Path -from typing import Iterator, List, Tuple +from typing import List, Tuple from ..constants import ( FAIL_TIMEOUT_PATTERN, diff --git a/devops/scripts/ur_test_tools/summary_generator.py b/devops/scripts/ur_test_tools/summary_generator.py index e4ccac6c8f8fa..97246cf7344de 100644 --- a/devops/scripts/ur_test_tools/summary_generator.py +++ b/devops/scripts/ur_test_tools/summary_generator.py @@ -66,7 +66,6 @@ def _analyze_skipped_tests( skipped_from_log = test_lists.get("Skipped", test_lists.get("Unsupported", [])) stats_count = get_count_from_stats(stats, ["Skipped", "Unsupported"]) - # Priority 1: XML data (most reliable - structured output) if skipped_xml: return SkippedTestsResult( tests=skipped_xml, @@ -75,7 +74,6 @@ def _analyze_skipped_tests( note="", ) - # Priority 2: Log data if skipped_from_log: return SkippedTestsResult( tests=skipped_from_log, @@ -84,7 +82,6 @@ def _analyze_skipped_tests( note="", ) - # Priority 3: Stats only (no individual test names) if stats_count: return SkippedTestsResult( tests=[], @@ -93,7 +90,6 @@ def _analyze_skipped_tests( note="Warning: Test names not available", ) - # No data available return SkippedTestsResult(tests=[], count=0, source="none", note="") def _validate_skipped_counts( @@ -148,7 +144,6 @@ def _analyze_excluded_tests( excluded_from_log = test_lists.get("Excluded", []) stats_count = get_count_from_stats(stats, ["Excluded"]) - # Priority 1: Log data if excluded_from_log: return ExcludedTestsResult( tests=excluded_from_log, @@ -157,13 +152,11 @@ def _analyze_excluded_tests( note="", ) - # Priority 2: XML data if excluded_xml: return ExcludedTestsResult( tests=excluded_xml, count=len(excluded_xml), source="xml", note="" ) - # Priority 3: Stats only (no individual test names) if stats_count: return ExcludedTestsResult( tests=[], @@ -172,7 +165,6 @@ def _analyze_excluded_tests( note="Warning: Test names not available", ) - # No data available return ExcludedTestsResult(tests=[], count=0, source="none", note="") def _validate_excluded_counts( diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py index 136a43800412f..3284791270af8 100644 --- a/devops/scripts/ur_test_tools/test_runner.py +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -11,7 +11,6 @@ DEFAULT_LIT_JOBS, TEST_TYPE_ADAPTER_SPECIFIC, MAX_LINES_TO_SCAN, - MAX_JOBS, ) from .models.config import TestConfig, TestExecutionContext from .outputs.github_actions import GitHubActionsOutput @@ -39,10 +38,10 @@ def get_test_config(test_type: str) -> TestConfig: def calculate_jobs() -> int: - """Calculate parallel jobs (nproc/3 capped at MAX_JOBS).""" + """Calculate parallel jobs (nproc/3, min 1).""" try: nproc = os.cpu_count() or 4 - return min(nproc // 3, MAX_JOBS) + return max(1, nproc // 3) except (OSError, AttributeError): return 4 From 1c120a3cd97dd3b6e7c06b65682167222d43d548 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Thu, 27 Aug 2026 16:14:38 +0200 Subject: [PATCH 03/11] Move ZE_ENABLE_LOADER_DEBUG_TRACE to workflow for L0-only and consolidate test type and filter constants --- .github/workflows/ur-build-hw.yml | 6 ++++++ devops/scripts/ur_test_tools/constants.py | 8 ++++++++ devops/scripts/ur_test_tools/test_runner.py | 13 +++++-------- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ur-build-hw.yml b/.github/workflows/ur-build-hw.yml index 9a9d77183f740..7870cb85f1b34 100644 --- a/.github/workflows/ur-build-hw.yml +++ b/.github/workflows/ur-build-hw.yml @@ -181,6 +181,9 @@ jobs: - name: Test adapter specific # Don't run adapter specific tests when building multiple adapters if: ${{ matrix.adapter.other_name == '' }} + env: + # Enable Level Zero loader debug trace for L0 adapters only + ZE_ENABLE_LOADER_DEBUG_TRACE: ${{ startsWith(matrix.adapter.name, 'L0') && '1' || '' }} uses: ./devops/actions/run-tests/ur with: test_type: 'adapter-specific' @@ -188,6 +191,9 @@ jobs: artifact_name: 'ur-${{matrix.adapter.name}}-${{steps.artifact_suffix.outputs.suffix}}-adapter-specific' - name: Test adapters + env: + # Enable Level Zero loader debug trace for L0 adapters only + ZE_ENABLE_LOADER_DEBUG_TRACE: ${{ startsWith(matrix.adapter.name, 'L0') && '1' || '' }} uses: ./devops/actions/run-tests/ur with: test_type: 'conformance' diff --git a/devops/scripts/ur_test_tools/constants.py b/devops/scripts/ur_test_tools/constants.py index 1500db2770dbc..7a7106a0ce1d4 100644 --- a/devops/scripts/ur_test_tools/constants.py +++ b/devops/scripts/ur_test_tools/constants.py @@ -15,6 +15,14 @@ # Test Type Identifiers TEST_TYPE_ADAPTER_SPECIFIC = "adapter-specific" +TEST_TYPE_CONFORMANCE = "conformance" + +# LIT Test Filters +# These tests cause timeouts on CI and are excluded from adapter-specific runs +LIT_FILTER_OUT_ADAPTER_SPECIFIC = ( + "(adapters/level_zero/memcheck.test|" + "adapters/level_zero/v2/deferred_kernel_memcheck.test)" +) # Constants TEST_NOT_SELECTED_MSG = "Test not selected" diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py index 3284791270af8..bb9e0ddea2f38 100644 --- a/devops/scripts/ur_test_tools/test_runner.py +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -10,6 +10,8 @@ DEFAULT_LIT_TIMEOUT, DEFAULT_LIT_JOBS, TEST_TYPE_ADAPTER_SPECIFIC, + TEST_TYPE_CONFORMANCE, + LIT_FILTER_OUT_ADAPTER_SPECIFIC, MAX_LINES_TO_SCAN, ) from .models.config import TestConfig, TestExecutionContext @@ -19,16 +21,13 @@ def get_test_config(test_type: str) -> TestConfig: """Get test configuration for test type.""" - if test_type == "adapter-specific": + if test_type == TEST_TYPE_ADAPTER_SPECIFIC: return TestConfig( target="check-unified-runtime-adapter", log_file="adapter_tests.log", - lit_filter_out=( - "(adapters/level_zero/memcheck.test|" - "adapters/level_zero/v2/deferred_kernel_memcheck.test)" - ), + lit_filter_out=LIT_FILTER_OUT_ADAPTER_SPECIFIC, ) - elif test_type == "conformance": + elif test_type == TEST_TYPE_CONFORMANCE: return TestConfig( target="check-unified-runtime-conformance", log_file="conformance_tests.log", @@ -98,8 +97,6 @@ def _setup_environment(self) -> None: if self.context.config.lit_filter_out: self.context.env["LIT_FILTER_OUT"] = self.context.config.lit_filter_out - self.context.env["ZE_ENABLE_LOADER_DEBUG_TRACE"] = "1" - def _build_cmake_command(self) -> List[str]: return [ "cmake", From f9f3d9798356e30a3582e5ab36c82266fc09911c Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Thu, 27 Aug 2026 16:23:06 +0200 Subject: [PATCH 04/11] Use GitHubActionsOutput static methods directly --- devops/scripts/ur_test_tools/cli.py | 7 +++---- devops/scripts/ur_test_tools/test_runner.py | 13 ++++++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/devops/scripts/ur_test_tools/cli.py b/devops/scripts/ur_test_tools/cli.py index 9d8931b09e46e..0811eaffe920b 100644 --- a/devops/scripts/ur_test_tools/cli.py +++ b/devops/scripts/ur_test_tools/cli.py @@ -100,15 +100,14 @@ def _run_tests_command() -> int: workspace = sys.argv[4] # Validate inputs - gha = GitHubActionsOutput() if not PathValidator.validate_build_dir(build_dir, workspace): - gha.print_error("Invalid build_dir") + GitHubActionsOutput.print_error("Invalid build_dir") return 1 try: config = get_test_config(test_type) except ValueError as e: - gha.print_error(str(e)) + GitHubActionsOutput.print_error(str(e)) return 1 # Convert to paths and create context @@ -137,7 +136,7 @@ def _run_tests_command() -> int: try: context.validate() except ValueError as e: - gha.print_error(str(e)) + GitHubActionsOutput.print_error(str(e)) return 1 # Run tests diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py index bb9e0ddea2f38..895954c2d2562 100644 --- a/devops/scripts/ur_test_tools/test_runner.py +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -68,7 +68,6 @@ class TestRunner: def __init__(self, context: TestExecutionContext): self.context = context - self.github_output = GitHubActionsOutput() self.jobs = calculate_jobs() def run(self) -> int: @@ -125,32 +124,32 @@ def _execute_tests(self) -> Optional[subprocess.CompletedProcess]: cwd=self.context.workspace, ) except (OSError, PermissionError) as e: - self.github_output.print_error(f"Test execution failed: {e}") + GitHubActionsOutput.print_error(f"Test execution failed: {e}") return None def _validate_output(self) -> bool: log_path = self.context.log_file_path if not log_path.exists() or log_path.stat().st_size == 0: - self.github_output.print_error("No log generated") + GitHubActionsOutput.print_error("No log generated") return False return True def _publish_outputs(self, result: subprocess.CompletedProcess) -> None: - self.github_output.set_output("log-file", str(self.context.log_file_path)) + GitHubActionsOutput.set_output("log-file", str(self.context.log_file_path)) if ( self.context.test_type == TEST_TYPE_ADAPTER_SPECIFIC and not check_log_has_tests(str(self.context.log_file_path)) ): print("No adapter-specific tests found", file=sys.stderr) - self.github_output.set_output("skip-artifacts", "1") + GitHubActionsOutput.set_output("skip-artifacts", "1") return if self.context.xml_output_path.exists(): - self.github_output.set_output("xml-file", str(self.context.xml_output_path)) + GitHubActionsOutput.set_output("xml-file", str(self.context.xml_output_path)) else: - self.github_output.print_warning( + GitHubActionsOutput.print_warning( f"Expected XML file not found at {self.context.xml_output_path}" ) From 03d33f3277e2e3fa2008ade39aba2327786de834 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Fri, 28 Aug 2026 08:31:02 +0200 Subject: [PATCH 05/11] Refactor test_runner: cleanup and explicit output contract --- devops/scripts/ur_test_tools/test_runner.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py index 895954c2d2562..78a49c9a39681 100644 --- a/devops/scripts/ur_test_tools/test_runner.py +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -81,7 +81,7 @@ def run(self) -> int: if not self._validate_output(): return 1 - self._publish_outputs(result) + self._publish_outputs() return result.returncode def _setup_environment(self) -> None: @@ -123,7 +123,7 @@ def _execute_tests(self) -> Optional[subprocess.CompletedProcess]: env=self.context.env, cwd=self.context.workspace, ) - except (OSError, PermissionError) as e: + except OSError as e: GitHubActionsOutput.print_error(f"Test execution failed: {e}") return None @@ -136,7 +136,7 @@ def _validate_output(self) -> bool: return True - def _publish_outputs(self, result: subprocess.CompletedProcess) -> None: + def _publish_outputs(self) -> None: GitHubActionsOutput.set_output("log-file", str(self.context.log_file_path)) if ( @@ -147,6 +147,8 @@ def _publish_outputs(self, result: subprocess.CompletedProcess) -> None: GitHubActionsOutput.set_output("skip-artifacts", "1") return + GitHubActionsOutput.set_output("skip-artifacts", "0") + if self.context.xml_output_path.exists(): GitHubActionsOutput.set_output("xml-file", str(self.context.xml_output_path)) else: From a71a7171ae5065a257be321e2caf1f59800cc70d Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Fri, 28 Aug 2026 09:37:05 +0200 Subject: [PATCH 06/11] Separate LIT reporting semantics from execution policy Clarify the distinction between common reporting options (what test categories to show) and execution-specific options (how to format output for CI vs standalone builds). --- devops/scripts/ur_test_tools/constants.py | 22 +++++++++++++++++++++ devops/scripts/ur_test_tools/test_runner.py | 22 +++++++++++++-------- unified-runtime/test/CMakeLists.txt | 22 ++++++++++++++++++++- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/devops/scripts/ur_test_tools/constants.py b/devops/scripts/ur_test_tools/constants.py index 7a7106a0ce1d4..cb03253e130a1 100644 --- a/devops/scripts/ur_test_tools/constants.py +++ b/devops/scripts/ur_test_tools/constants.py @@ -10,6 +10,28 @@ SEPARATOR_WIDTH = 70 # LIT Configuration +# Following LLVM's model: define common defaults + allow invocation-specific overrides. +# +# LIT_COMMON_REPORTING_OPTIONS describes WHAT test information to show (semantics). +# These align with standalone builds (unified-runtime/test/CMakeLists.txt). +# +# LIT_CI_OPTIONS describes HOW to format output for CI tooling (execution policy). +# Standalone uses --succinct instead for developer-friendly interactive feedback. +LIT_COMMON_REPORTING_OPTIONS = [ + "--show-unsupported", + "--show-pass", + "--show-xfail", + "--time-tests", + "--show-flakypass", + "--show-skipped", +] + +# CI-specific options: verbose output for log parsing and reporting tools +LIT_CI_OPTIONS = [ + "-v", + "--no-progress-bar", +] + DEFAULT_LIT_TIMEOUT = 120 DEFAULT_LIT_JOBS = 50 diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py index 78a49c9a39681..a5ecb66867113 100644 --- a/devops/scripts/ur_test_tools/test_runner.py +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -9,6 +9,8 @@ from .constants import ( DEFAULT_LIT_TIMEOUT, DEFAULT_LIT_JOBS, + LIT_COMMON_REPORTING_OPTIONS, + LIT_CI_OPTIONS, TEST_TYPE_ADAPTER_SPECIFIC, TEST_TYPE_CONFORMANCE, LIT_FILTER_OUT_ADAPTER_SPECIFIC, @@ -85,13 +87,15 @@ def run(self) -> int: return result.returncode def _setup_environment(self) -> None: - lit_opts = ( - f"--show-unsupported --show-pass --show-xfail --no-progress-bar " - f"-v --timeout {DEFAULT_LIT_TIMEOUT} -j {DEFAULT_LIT_JOBS} " - f"--time-tests --show-flakypass --show-skipped " - f"--xunit-xml-output {self.context.xml_output_path}" - ) - self.context.env["LIT_OPTS"] = lit_opts + # Build LIT options: common reporting + CI-specific + dynamic config + lit_opts_parts = [ + *LIT_COMMON_REPORTING_OPTIONS, + *LIT_CI_OPTIONS, + "--timeout", str(DEFAULT_LIT_TIMEOUT), + "-j", str(DEFAULT_LIT_JOBS), + "--xunit-xml-output", str(self.context.xml_output_path), + ] + self.context.env["LIT_OPTS"] = " ".join(lit_opts_parts) if self.context.config.lit_filter_out: self.context.env["LIT_FILTER_OUT"] = self.context.config.lit_filter_out @@ -150,7 +154,9 @@ def _publish_outputs(self) -> None: GitHubActionsOutput.set_output("skip-artifacts", "0") if self.context.xml_output_path.exists(): - GitHubActionsOutput.set_output("xml-file", str(self.context.xml_output_path)) + GitHubActionsOutput.set_output( + "xml-file", str(self.context.xml_output_path) + ) else: GitHubActionsOutput.print_warning( f"Expected XML file not found at {self.context.xml_output_path}" diff --git a/unified-runtime/test/CMakeLists.txt b/unified-runtime/test/CMakeLists.txt index 30f700ca4355f..3fc44c1a9ef50 100644 --- a/unified-runtime/test/CMakeLists.txt +++ b/unified-runtime/test/CMakeLists.txt @@ -86,8 +86,28 @@ function(add_ur_lit_testsuite suite) if(UR_STANDALONE_BUILD) add_custom_target(${TARGET} + # LIT invocation for standalone builds + # + # Keep reporting-related options aligned with LIT_COMMON_REPORTING_OPTIONS + # in devops/scripts/ur_test_tools/constants.py. + # + # The invocation is intentionally not identical to CI: + # - Standalone tests use --succinct for developer-friendly interactive output + # - CI uses -v because detailed output is consumed by reporting tools COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" - --show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped + # Reporting options (keep aligned with LIT_COMMON_REPORTING_OPTIONS) + --show-unsupported + --show-pass + --show-xfail + --time-tests + --show-flakypass + --show-skipped + # Standalone-specific: interactive developer feedback + --no-progress-bar + --succinct + # Execution policy + --timeout 120 + -j 50 USES_TERMINAL ) else() From 47366c053e02c73207b069f122d4b799e5c570d1 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Fri, 28 Aug 2026 09:55:11 +0200 Subject: [PATCH 07/11] Fix formatting: split arguments and remove trailing blank line --- devops/scripts/ur_test_tools/summary_generator.py | 1 - devops/scripts/ur_test_tools/test_runner.py | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/devops/scripts/ur_test_tools/summary_generator.py b/devops/scripts/ur_test_tools/summary_generator.py index 97246cf7344de..20bbf83cf1f74 100644 --- a/devops/scripts/ur_test_tools/summary_generator.py +++ b/devops/scripts/ur_test_tools/summary_generator.py @@ -213,4 +213,3 @@ def _display_remaining_categories(self, test_lists: TestLists) -> None: for category, tests in test_lists.items(): if tests: ConsoleOutput.print_test_group(f"{category} Tests", tests) - diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py index a5ecb66867113..ef4118daaf3f7 100644 --- a/devops/scripts/ur_test_tools/test_runner.py +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -91,9 +91,12 @@ def _setup_environment(self) -> None: lit_opts_parts = [ *LIT_COMMON_REPORTING_OPTIONS, *LIT_CI_OPTIONS, - "--timeout", str(DEFAULT_LIT_TIMEOUT), - "-j", str(DEFAULT_LIT_JOBS), - "--xunit-xml-output", str(self.context.xml_output_path), + "--timeout", + str(DEFAULT_LIT_TIMEOUT), + "-j", + str(DEFAULT_LIT_JOBS), + "--xunit-xml-output", + str(self.context.xml_output_path), ] self.context.env["LIT_OPTS"] = " ".join(lit_opts_parts) From 1fde48dad41c6287cb853954da242c3506b2ef48 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Fri, 28 Aug 2026 10:27:14 +0200 Subject: [PATCH 08/11] Remove fallback file names - runner is single source of truth --- devops/actions/run-tests/ur/action.yml | 33 +++++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index e6f99ea1455e8..70e7442302cf4 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -46,10 +46,15 @@ runs: if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }} shell: bash env: - LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} run: | + if [ -z "$LOG_FILE" ]; then + echo "::warning::Test log path not provided by runner; skipping log filtering." + exit 0 + fi + if [ ! -f "$LOG_FILE" ]; then - echo "::warning::Test log not found" + echo "::warning::Test log '$LOG_FILE' does not exist; skipping log filtering." exit 0 fi @@ -61,8 +66,13 @@ runs: if: steps.run_tests.outcome != 'success' shell: bash env: - LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} run: | + if [ -z "$LOG_FILE" ] || [ ! -f "$LOG_FILE" ]; then + echo "::warning::Test execution failed, but no test log is available." + exit 1 + fi + devops/scripts/ur-test extract-errors "$LOG_FILE" exit 1 @@ -70,18 +80,23 @@ runs: if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }} shell: bash env: - LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} - XML_FILE: ${{ steps.run_tests.outputs['xml-file'] || 'adapter_tests_results.xml' }} + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} + XML_FILE: ${{ steps.run_tests.outputs['xml-file'] }} ARTIFACT_NAME: ${{ inputs.artifact_name }} WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | + if [ -z "$LOG_FILE" ]; then + echo "::warning::Test log path not provided by runner; skipping summary." + exit 0 + fi + if [ ! -f "$LOG_FILE" ]; then - echo "::warning::Test log not found" + echo "::warning::Test log '$LOG_FILE' does not exist; skipping summary." exit 0 fi if [ ! -s "$LOG_FILE" ]; then - echo "::warning::Test log is empty" + echo "::warning::Test log is empty; skipping summary." exit 0 fi @@ -100,8 +115,8 @@ runs: if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} shell: bash env: - LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} - XML_FILE: ${{ steps.run_tests.outputs['xml-file'] || 'adapter_tests_results.xml' }} + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} + XML_FILE: ${{ steps.run_tests.outputs['xml-file'] }} run: | set -f # Disable glob expansion From 9fd79d84a2fbeac3298fbdf037d8e068939df018 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Fri, 28 Aug 2026 11:35:46 +0200 Subject: [PATCH 09/11] Refactor test result model to address review comments New architecture: - Parser layer: produces ParsedLogData and ParsedXMLData observations - Reconciliation layer: combines observations into TestRunResult - Summary generator: consumes TestRunResult with group_by_status() --- devops/scripts/ur_test_tools/__init__.py | 17 +- .../scripts/ur_test_tools/models/__init__.py | 11 + .../ur_test_tools/models/test_results.py | 110 +++++++ .../scripts/ur_test_tools/parsers/__init__.py | 12 + .../ur_test_tools/parsers/log_parser.py | 70 ++++ .../ur_test_tools/parsers/parser_models.py | 73 +++++ .../ur_test_tools/parsers/xml_parser.py | 86 +++++ .../scripts/ur_test_tools/reconciliation.py | 114 +++++++ .../ur_test_tools/summary_generator.py | 310 +++++++----------- 9 files changed, 597 insertions(+), 206 deletions(-) create mode 100644 devops/scripts/ur_test_tools/models/test_results.py create mode 100644 devops/scripts/ur_test_tools/parsers/parser_models.py create mode 100644 devops/scripts/ur_test_tools/reconciliation.py diff --git a/devops/scripts/ur_test_tools/__init__.py b/devops/scripts/ur_test_tools/__init__.py index 2707bdbdf15db..45e58ada0bb6a 100644 --- a/devops/scripts/ur_test_tools/__init__.py +++ b/devops/scripts/ur_test_tools/__init__.py @@ -1,11 +1,15 @@ """UR Test Tools - Modular test orchestration and summary generation.""" from .models import ( - TestLists, - TestCounts, - TimingSummary, + # Configuration TestConfig, SummaryConfigFromLines, + # Normalized result model + TestStatus, + TestResult, + TestRunResult, + # Legacy (to be removed) + TimingSummary, ) from .test_runner import TestRunner from .summary_generator import SummaryReporter @@ -20,11 +24,12 @@ __all__ = [ "__version__", "__author__", - "TestLists", - "TestCounts", - "TimingSummary", "TestConfig", "SummaryConfigFromLines", + "TestStatus", + "TestResult", + "TestRunResult", + "TimingSummary", # Legacy "TestRunner", "SummaryReporter", "PathValidator", diff --git a/devops/scripts/ur_test_tools/models/__init__.py b/devops/scripts/ur_test_tools/models/__init__.py index f242b3bc63400..f4b1b46261e40 100644 --- a/devops/scripts/ur_test_tools/models/__init__.py +++ b/devops/scripts/ur_test_tools/models/__init__.py @@ -12,14 +12,25 @@ TestExecutionContext, SummaryConfigFromLines, ) +from .test_results import ( + TestStatus, + TestResult, + TestRunResult, +) __all__ = [ + # Legacy models (to be deprecated) "TestLists", "TestCounts", "TimingSummary", "SkippedTestsResult", "ExcludedTestsResult", + # Configuration "TestConfig", "TestExecutionContext", "SummaryConfigFromLines", + # Normalized result model + "TestStatus", + "TestResult", + "TestRunResult", ] diff --git a/devops/scripts/ur_test_tools/models/test_results.py b/devops/scripts/ur_test_tools/models/test_results.py new file mode 100644 index 0000000000000..098305f159efa --- /dev/null +++ b/devops/scripts/ur_test_tools/models/test_results.py @@ -0,0 +1,110 @@ +"""Canonical test result model - independent of data sources. + +This module defines the normalized representation of test results that is used +throughout the system. Parsers translate source-specific formats (LIT log, +JUnit XML) into this model, and consumers (summary, validation, future DB) +operate only on these normalized types. +""" + +from dataclasses import dataclass +from enum import Enum +from typing import Dict, List, Optional + + +class TestStatus(str, Enum): + """Canonical test result status. + + Maps to LIT result codes. The string values match LIT's internal codes, + not the human-readable output labels. + + LIT Output Label -> TestStatus: + "Passed" -> PASS + "Passed With Retry" -> FLAKYPASS + "Expectedly Failed" -> XFAIL + "Unexpectedly Passed" -> XPASS + "Failed" -> FAIL + "Unresolved" -> UNRESOLVED + "Unsupported" -> UNSUPPORTED + "Timed Out" -> TIMEOUT + "Skipped" -> SKIPPED + "Excluded" -> EXCLUDED + """ + + PASS = "PASS" + FLAKYPASS = "FLAKYPASS" + XFAIL = "XFAIL" + XPASS = "XPASS" + FAIL = "FAIL" + UNRESOLVED = "UNRESOLVED" + UNSUPPORTED = "UNSUPPORTED" + TIMEOUT = "TIMEOUT" + SKIPPED = "SKIPPED" + EXCLUDED = "EXCLUDED" + + @property + def is_failure(self) -> bool: + """Check if this status represents a test failure. + + LIT treats FAIL, XPASS, UNRESOLVED, and TIMEOUT as failures. + """ + return self in ( + TestStatus.FAIL, + TestStatus.XPASS, + TestStatus.UNRESOLVED, + TestStatus.TIMEOUT, + ) + + @property + def display_label(self) -> str: + """Get human-readable label for display.""" + return { + TestStatus.PASS: "Passed", + TestStatus.FLAKYPASS: "Passed With Retry", + TestStatus.XFAIL: "Expectedly Failed", + TestStatus.XPASS: "Unexpectedly Passed", + TestStatus.FAIL: "Failed", + TestStatus.UNRESOLVED: "Unresolved", + TestStatus.UNSUPPORTED: "Unsupported", + TestStatus.TIMEOUT: "Timed Out", + TestStatus.SKIPPED: "Skipped", + TestStatus.EXCLUDED: "Excluded", + }[self] + + +@dataclass +class TestResult: + """Normalized result for a single test. + + Represents test outcome independent of where the information was obtained. + """ + + name: str + status: TestStatus + duration_ms: Optional[float] = None + + +@dataclass +class TestRunResult: + """Complete normalized result of a test run. + + Contains all test results and run-level metadata. + """ + + tests: List[TestResult] + total_discovered: Optional[int] = None + testing_time_ms: Optional[float] = None + + def count_by_status(self, status: TestStatus) -> int: + """Count tests with given status.""" + return sum(1 for test in self.tests if test.status == status) + + def tests_by_status(self, status: TestStatus) -> List[TestResult]: + """Get all tests with given status.""" + return [test for test in self.tests if test.status == status] + + def group_by_status(self) -> Dict[TestStatus, List[TestResult]]: + """Group all tests by their status.""" + groups: Dict[TestStatus, List[TestResult]] = {} + for test in self.tests: + groups.setdefault(test.status, []).append(test) + return groups diff --git a/devops/scripts/ur_test_tools/parsers/__init__.py b/devops/scripts/ur_test_tools/parsers/__init__.py index 55721e06f10c6..994adc7979455 100644 --- a/devops/scripts/ur_test_tools/parsers/__init__.py +++ b/devops/scripts/ur_test_tools/parsers/__init__.py @@ -9,6 +9,13 @@ ParsedXMLTests, ) from .stats_parser import get_count_from_stats +from .parser_models import ( + ParsedTestObservation, + ParsedLogData, + ParsedXMLData, + LIT_OUTPUT_TO_STATUS, + STATUS_TO_LIT_LABEL, +) __all__ = [ "LITLogParser", @@ -16,4 +23,9 @@ "JUnitXMLParser", "ParsedXMLTests", "get_count_from_stats", + "ParsedTestObservation", + "ParsedLogData", + "ParsedXMLData", + "LIT_OUTPUT_TO_STATUS", + "STATUS_TO_LIT_LABEL", ] diff --git a/devops/scripts/ur_test_tools/parsers/log_parser.py b/devops/scripts/ur_test_tools/parsers/log_parser.py index 51566013a0280..d44e861222403 100644 --- a/devops/scripts/ur_test_tools/parsers/log_parser.py +++ b/devops/scripts/ur_test_tools/parsers/log_parser.py @@ -13,6 +13,11 @@ TEST_TIMES_HEADERS, ) from ..models.test_data import TestLists, TestCounts, TimingSummary +from .parser_models import ( + ParsedLogData, + ParsedTestObservation, + LIT_OUTPUT_TO_STATUS, +) def _read_with_utf8_fallback(path: str, read_func): @@ -149,3 +154,68 @@ def extract_test_lists(self) -> Tuple[TestLists, TestCounts]: declared_counts[current_category] = current_declared_count return categories, declared_counts + + def parse_to_observations(self) -> ParsedLogData: + """Parse LIT output into normalized observations. + + Returns: + ParsedLogData with test observations, counts, and timing info. + """ + # Extract test lists using legacy method + categories, declared_counts = self.extract_test_lists() + + # Convert to observations with normalized status + observations = [] + normalized_counts = {} + + for category_label, test_names in categories.items(): + status = LIT_OUTPUT_TO_STATUS.get(category_label) + if status is None: + print( + f"Warning: Unknown test category '{category_label}', skipping", + file=sys.stderr, + ) + continue + + # Convert test names to observations + for test_name in test_names: + observations.append( + ParsedTestObservation( + name=test_name, + status=status, + duration_ms=None, # Log output doesn't have per-test timing + ) + ) + + # Store declared count for this status + count = declared_counts.get(category_label, 0) + normalized_counts[status] = count + + # Extract statistics (could be used for validation) + stats_lines = self.extract_statistics() + statistics = {} + for line in stats_lines: + # Parse "Stat Name: N" format + parts = line.strip().split(":", 1) + if len(parts) == 2: + stat_name = parts[0].strip() + try: + stat_value = int(parts[1].strip()) + statistics[stat_name] = stat_value + except ValueError: + continue + + # Extract timing info + timing = self.extract_time_summary() + + # Extract error details + errors = self.extract_error_details() + + return ParsedLogData( + tests=observations, + declared_counts=normalized_counts, + statistics=statistics, + error_details=errors, + slowest_tests=timing["slowest"], + time_histogram=timing["histogram"], + ) diff --git a/devops/scripts/ur_test_tools/parsers/parser_models.py b/devops/scripts/ur_test_tools/parsers/parser_models.py new file mode 100644 index 0000000000000..3f23a1f219aa8 --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/parser_models.py @@ -0,0 +1,73 @@ +"""Parser-specific data models - intermediate observations before normalization.""" + +from dataclasses import dataclass, field +from typing import Dict, List, NamedTuple, Optional + +from ..models.test_results import TestStatus + + +class ParsedTestObservation(NamedTuple): + """A single test observation from a parser (log or XML).""" + + name: str + status: TestStatus + duration_ms: Optional[float] = None + + +@dataclass +class ParsedLogData: + """Raw observations and statistics from LIT text output.""" + + tests: List[ParsedTestObservation] = field(default_factory=list) + # Declared counts from "Category Tests (N):" headers + declared_counts: Dict[TestStatus, int] = field(default_factory=dict) + # Statistics from "Stat Name: N" lines + statistics: Dict[str, int] = field(default_factory=dict) + # Raw error/failure details + error_details: List[str] = field(default_factory=list) + # Timing summary sections + slowest_tests: List[str] = field(default_factory=list) + time_histogram: List[str] = field(default_factory=list) + + +@dataclass +class ParsedXMLData: + """Observations from JUnit XML output.""" + + tests: List[ParsedTestObservation] = field(default_factory=list) + # XML may have additional metadata we can extract in the future + total_tests: Optional[int] = None + total_time_seconds: Optional[float] = None + + +# Mapping from LIT output labels to canonical status +LIT_OUTPUT_TO_STATUS = { + # Category headers + "Passed": TestStatus.PASS, + "Passed With Retry": TestStatus.FLAKYPASS, + "Expectedly Failed": TestStatus.XFAIL, + "Unexpectedly Passed": TestStatus.XPASS, + "Failed": TestStatus.FAIL, + "Unresolved": TestStatus.UNRESOLVED, + "Unsupported": TestStatus.UNSUPPORTED, + "Timed Out": TestStatus.TIMEOUT, + "Skipped": TestStatus.SKIPPED, + "Excluded": TestStatus.EXCLUDED, + # Statistics labels (may differ slightly) + "Expected Passes": TestStatus.PASS, +} + + +# Reverse mapping for display +STATUS_TO_LIT_LABEL = { + TestStatus.PASS: "Passed", + TestStatus.FLAKYPASS: "Passed With Retry", + TestStatus.XFAIL: "Expectedly Failed", + TestStatus.XPASS: "Unexpectedly Passed", + TestStatus.FAIL: "Failed", + TestStatus.UNRESOLVED: "Unresolved", + TestStatus.UNSUPPORTED: "Unsupported", + TestStatus.TIMEOUT: "Timed Out", + TestStatus.SKIPPED: "Skipped", + TestStatus.EXCLUDED: "Excluded", +} diff --git a/devops/scripts/ur_test_tools/parsers/xml_parser.py b/devops/scripts/ur_test_tools/parsers/xml_parser.py index ac2eb41e83700..0287b69dfe533 100644 --- a/devops/scripts/ur_test_tools/parsers/xml_parser.py +++ b/devops/scripts/ur_test_tools/parsers/xml_parser.py @@ -7,6 +7,8 @@ import defusedxml.ElementTree as ET from ..constants import TEST_NOT_SELECTED_MSG +from ..models.test_results import TestStatus +from .parser_models import ParsedXMLData, ParsedTestObservation class ParsedXMLTests(NamedTuple): @@ -88,3 +90,87 @@ def extract_skipped_tests(self) -> List[str]: def extract_excluded_tests(self) -> List[str]: _, excluded = self.extract_tests_from_xml() return excluded + + def parse_to_observations(self) -> ParsedXMLData: + """Parse XML into normalized observations. + + Returns: + ParsedXMLData with test observations and metadata. + """ + if not self.parse(): + return ParsedXMLData(tests=[]) + + observations = [] + total_tests = 0 + total_time = 0.0 + + for testcase in self._root.findall(".//testcase"): + test_name = _format_test_name( + testcase.get("classname", ""), testcase.get("name", "") + ) + if not test_name: + continue + + total_tests += 1 + + # Extract duration (in seconds from XML, convert to ms) + duration_ms = None + time_str = testcase.get("time") + if time_str: + try: + duration_sec = float(time_str) + duration_ms = duration_sec * 1000.0 + total_time += duration_sec + except ValueError: + pass + + # Determine status from child elements + status = self._determine_status_from_xml(testcase) + + observations.append( + ParsedTestObservation( + name=test_name, + status=status, + duration_ms=duration_ms, + ) + ) + + return ParsedXMLData( + tests=observations, + total_tests=total_tests, + total_time_seconds=total_time if total_time > 0 else None, + ) + + def _determine_status_from_xml(self, testcase) -> TestStatus: + """Determine test status from XML testcase element. + + JUnit XML structure: + - = FAIL + - = UNRESOLVED or TIMEOUT (depends on message) + - = SKIPPED or EXCLUDED (depends on message) + - No child elements = PASS + """ + # Check for failure + if testcase.find("failure") is not None: + return TestStatus.FAIL + + # Check for error (could be unresolved or timeout) + error_elem = testcase.find("error") + if error_elem is not None: + message = error_elem.get("message", "").lower() + if "timeout" in message: + return TestStatus.TIMEOUT + return TestStatus.UNRESOLVED + + # Check for skipped (could be skipped or excluded) + skipped_elem = testcase.find("skipped") + if skipped_elem is not None: + message = skipped_elem.get("message", "") + if TEST_NOT_SELECTED_MSG in message: + return TestStatus.EXCLUDED + return TestStatus.SKIPPED + + # No special markers = passed + # Note: We can't distinguish PASS/FLAKYPASS/XFAIL/XPASS from XML alone + # That requires log output. XML parser reports PASS for anything that succeeded. + return TestStatus.PASS diff --git a/devops/scripts/ur_test_tools/reconciliation.py b/devops/scripts/ur_test_tools/reconciliation.py new file mode 100644 index 0000000000000..91bf68a12d32b --- /dev/null +++ b/devops/scripts/ur_test_tools/reconciliation.py @@ -0,0 +1,114 @@ +"""Reconciliation layer - combines parser observations into canonical results.""" + +import sys +from typing import List, Optional + +from .models.test_results import TestResult, TestRunResult, TestStatus +from .parsers.parser_models import ParsedLogData, ParsedXMLData + + +def reconcile_test_results( + log_data: ParsedLogData, xml_data: Optional[ParsedXMLData] = None +) -> TestRunResult: + """Reconcile observations from log and XML into canonical test results. + + Reconciliation policy: + 1. Test status comes from log output (authoritative for XFAIL/XPASS/FLAKYPASS) + 2. Test duration comes from XML (more precise than log timing sections) + 3. If test appears in XML but not log, include with XML status + 4. Validate counts and warn on mismatches + + Args: + log_data: Parsed observations from LIT text output + xml_data: Optional parsed observations from JUnit XML + + Returns: + TestRunResult with reconciled test results + """ + # Build index of log results (status is authoritative) + log_by_name = {obs.name: obs for obs in log_data.tests} + + # Build index of XML results (timing is authoritative) + xml_by_name = {} + if xml_data: + xml_by_name = {obs.name: obs for obs in xml_data.tests} + + # Reconcile: start with all tests from log + results = [] + for log_obs in log_data.tests: + # Use log status, but prefer XML timing if available + xml_obs = xml_by_name.get(log_obs.name) + duration_ms = xml_obs.duration_ms if xml_obs else log_obs.duration_ms + + results.append( + TestResult( + name=log_obs.name, + status=log_obs.status, + duration_ms=duration_ms, + ) + ) + + # Add any tests that appear in XML but not in log + # This shouldn't happen normally, but handle it gracefully + xml_only_tests = set(xml_by_name.keys()) - set(log_by_name.keys()) + if xml_only_tests: + print( + f"Warning: {len(xml_only_tests)} test(s) found in XML but not in log", + file=sys.stderr, + ) + for test_name in sorted(xml_only_tests): + xml_obs = xml_by_name[test_name] + results.append( + TestResult( + name=xml_obs.name, + status=xml_obs.status, + duration_ms=xml_obs.duration_ms, + ) + ) + + # Extract total discovered and testing time + total_discovered = log_data.statistics.get("Total Discovered") + testing_time_ms = None + if xml_data and xml_data.total_time_seconds: + testing_time_ms = xml_data.total_time_seconds * 1000.0 + + # Validate counts + _validate_counts(log_data, results) + + return TestRunResult( + tests=results, + total_discovered=total_discovered, + testing_time_ms=testing_time_ms, + ) + + +def _validate_counts(log_data: ParsedLogData, results: List[TestResult]) -> None: + """Validate that reconciled results match declared counts from log. + + Warns if counts don't match but doesn't fail - this helps catch parser bugs. + """ + # Count results by status + actual_counts = {} + for result in results: + actual_counts[result.status] = actual_counts.get(result.status, 0) + 1 + + # Compare with declared counts from log + for status, declared_count in log_data.declared_counts.items(): + actual_count = actual_counts.get(status, 0) + if actual_count != declared_count: + print( + f"Warning: Count mismatch for {status.value}: " + f"declared {declared_count}, found {actual_count}", + file=sys.stderr, + ) + + # Check total discovered if available + if log_data.statistics.get("Total Discovered"): + total_discovered = log_data.statistics["Total Discovered"] + total_results = len(results) + if total_results != total_discovered: + print( + f"Warning: Total test mismatch: " + f"declared {total_discovered}, found {total_results}", + file=sys.stderr, + ) diff --git a/devops/scripts/ur_test_tools/summary_generator.py b/devops/scripts/ur_test_tools/summary_generator.py index 20bbf83cf1f74..96bdf281b97c3 100644 --- a/devops/scripts/ur_test_tools/summary_generator.py +++ b/devops/scripts/ur_test_tools/summary_generator.py @@ -1,215 +1,125 @@ """Generate test summary reports.""" -import sys from typing import List from .models.config import SummaryConfigFromLines -from .models.test_data import ( - TestLists, - TestCounts, - SkippedTestsResult, - ExcludedTestsResult, -) +from .models.test_results import TestResult, TestStatus, TestRunResult from .parsers.log_parser import LITLogParser from .parsers.xml_parser import JUnitXMLParser -from .parsers.stats_parser import get_count_from_stats -from .outputs.console import ConsoleOutput -from .validation.data_validator import validate_test_counts +from .parsers.parser_models import ParsedLogData +from .reconciliation import reconcile_test_results class SummaryReporter: - """Generate comprehensive test summary.""" + """Generate comprehensive test summary using normalized test results.""" def __init__(self, config: SummaryConfigFromLines): self.config = config def generate(self) -> None: - parser = LITLogParser(self.config.log_lines) - stats = parser.extract_statistics() - test_lists, declared_counts = parser.extract_test_lists() - total_discovered = get_count_from_stats(stats, ["Total Discovered"]) - - xml_parser = JUnitXMLParser(self.config.xml_file) - parsed_xml = xml_parser.extract_tests_from_xml() - skipped_xml = parsed_xml.skipped - excluded_xml = parsed_xml.excluded - - ConsoleOutput.print_statistics(stats) - - skipped_result = self._analyze_skipped_tests(test_lists, stats, skipped_xml) - self._validate_skipped_counts(skipped_result, declared_counts, stats) - self._display_skipped_tests(skipped_result) - if skipped_result["count"] > 0: - self._cleanup_skipped_from_test_lists(test_lists) - - excluded_result = self._analyze_excluded_tests(test_lists, stats, excluded_xml) - self._validate_excluded_counts(excluded_result, declared_counts, stats) - self._display_excluded_tests(excluded_result) - if excluded_result["count"] > 0: - self._cleanup_excluded_from_test_lists(test_lists) - - self._display_remaining_categories(test_lists) - - validate_test_counts( - total_discovered, - test_lists, - skipped_result["count"], - excluded_result["count"], - ) - - ConsoleOutput.print_timing_summary(self.config.log_lines) - - def _analyze_skipped_tests( - self, test_lists: TestLists, stats: List[str], skipped_xml: List[str] - ) -> SkippedTestsResult: - """Analyze skipped tests (priority: XML > Log > Stats).""" - skipped_from_log = test_lists.get("Skipped", test_lists.get("Unsupported", [])) - stats_count = get_count_from_stats(stats, ["Skipped", "Unsupported"]) - - if skipped_xml: - return SkippedTestsResult( - tests=skipped_xml, - count=len(skipped_xml), - source="xml", - note="", - ) - - if skipped_from_log: - return SkippedTestsResult( - tests=skipped_from_log, - count=len(skipped_from_log), - source="log", - note="", - ) - - if stats_count: - return SkippedTestsResult( - tests=[], - count=stats_count, - source="stats", - note="Warning: Test names not available", - ) - - return SkippedTestsResult(tests=[], count=0, source="none", note="") - - def _validate_skipped_counts( - self, result: SkippedTestsResult, declared_counts: TestCounts, stats: List[str] - ) -> None: - """Validate skipped counts (warns on mismatch).""" - actual_count = result["count"] - if actual_count == 0: - return # Nothing to validate - - declared_count = declared_counts.get( - "Skipped", declared_counts.get("Unsupported", 0) - ) - stats_count = get_count_from_stats(stats, ["Skipped", "Unsupported"]) - - # Build list of mismatches - mismatches = [] - - if declared_count and declared_count != actual_count: - mismatches.append(f"log header: {declared_count}") - - if stats_count and stats_count != actual_count: - mismatches.append(f"statistics: {stats_count}") - - # Display warning only if mismatches found - if mismatches: - sources_str = ", ".join(mismatches) - print( - f"Warning: Skipped test count mismatch. " - f"Using {actual_count} from {result['source']}, " - f"but found {sources_str}", - file=sys.stderr, - ) - - def _display_skipped_tests(self, result: SkippedTestsResult) -> None: - if result["count"] > 0: - ConsoleOutput.print_test_group( - "Skipped Tests", - result["tests"], - note=result["note"], - count=result["count"] if not result["tests"] else None, - ) - - def _cleanup_skipped_from_test_lists(self, test_lists: TestLists) -> None: - test_lists.pop("Skipped", None) - test_lists.pop("Unsupported", None) - - def _analyze_excluded_tests( - self, test_lists: TestLists, stats: List[str], excluded_xml: List[str] - ) -> ExcludedTestsResult: - """Analyze excluded tests (priority: Log > XML > Stats).""" - excluded_from_log = test_lists.get("Excluded", []) - stats_count = get_count_from_stats(stats, ["Excluded"]) - - if excluded_from_log: - return ExcludedTestsResult( - tests=excluded_from_log, - count=len(excluded_from_log), - source="log", - note="", - ) - - if excluded_xml: - return ExcludedTestsResult( - tests=excluded_xml, count=len(excluded_xml), source="xml", note="" - ) - - if stats_count: - return ExcludedTestsResult( - tests=[], - count=stats_count, - source="stats", - note="Warning: Test names not available", - ) - - return ExcludedTestsResult(tests=[], count=0, source="none", note="") - - def _validate_excluded_counts( - self, result: ExcludedTestsResult, declared_counts: TestCounts, stats: List[str] - ) -> None: - """Validate excluded counts (warns on mismatch).""" - actual_count = result["count"] - if actual_count == 0: - return # Nothing to validate - - declared_count = declared_counts.get("Excluded", 0) - stats_count = get_count_from_stats(stats, ["Excluded"]) - - # Build list of mismatches - mismatches = [] - - if declared_count and declared_count != actual_count: - mismatches.append(f"log header: {declared_count}") - - if stats_count and stats_count != actual_count: - mismatches.append(f"statistics: {stats_count}") - - # Display warning only if mismatches found - if mismatches: - sources_str = ", ".join(mismatches) - print( - f"Warning: Excluded test count mismatch. " - f"Using {actual_count} from {result['source']}, " - f"but found {sources_str}", - file=sys.stderr, - ) - - def _display_excluded_tests(self, result: ExcludedTestsResult) -> None: - if result["count"] > 0: - ConsoleOutput.print_test_group( - "Excluded Tests", - result["tests"], - note=result["note"], - count=result["count"] if not result["tests"] else None, - ) - - def _cleanup_excluded_from_test_lists(self, test_lists: TestLists) -> None: - test_lists.pop("Excluded", None) - - def _display_remaining_categories(self, test_lists: TestLists) -> None: - for category, tests in test_lists.items(): - if tests: - ConsoleOutput.print_test_group(f"{category} Tests", tests) + """Generate and display test summary.""" + # Parse inputs + log_parser = LITLogParser(self.config.log_lines) + log_data = log_parser.parse_to_observations() + + xml_data = None + if self.config.xml_file: + xml_parser = JUnitXMLParser(self.config.xml_file) + xml_data = xml_parser.parse_to_observations() + + # Reconcile into canonical result model + result = reconcile_test_results(log_data, xml_data) + + # Display summary + self._display_statistics(result) + self._display_test_groups(result) + self._display_timing(log_data) + + def _display_statistics(self, result: TestRunResult) -> None: + """Display overall test statistics.""" + print("=" * 70) + print("Test Statistics") + print("=" * 70) + + if result.total_discovered is not None: + print(f"Total Discovered Tests: {result.total_discovered}") + + # Count by status + for status in TestStatus: + count = result.count_by_status(status) + if count > 0: + print(f"{status.display_label}: {count}") + + if result.testing_time_ms is not None: + time_sec = result.testing_time_ms / 1000.0 + print(f"Testing Time: {time_sec:.2f}s") + + print() + + def _display_test_groups(self, result: TestRunResult) -> None: + """Display test groups by status.""" + # Group tests by status + grouped = result.group_by_status() + + # Display groups in priority order (failures first) + priority_order = [ + TestStatus.FAIL, + TestStatus.TIMEOUT, + TestStatus.UNRESOLVED, + TestStatus.XPASS, + TestStatus.SKIPPED, + TestStatus.UNSUPPORTED, + TestStatus.EXCLUDED, + TestStatus.XFAIL, + TestStatus.FLAKYPASS, + TestStatus.PASS, + ] + + for status in priority_order: + tests = grouped.get(status, []) + if not tests: + continue + + # Skip PASS tests by default (too many) + if status == TestStatus.PASS and len(tests) > 10: + print(f"{status.display_label} ({len(tests)} tests) - details omitted") + print() + continue + + # Display test group + self._print_test_group(status, tests) + + def _print_test_group(self, status: TestStatus, tests: List[TestResult]) -> None: + """Print a group of tests with given status.""" + print("=" * 70) + print(f"{status.display_label} ({len(tests)})") + print("=" * 70) + + for test in tests[:100]: # Limit to first 100 + if test.duration_ms is not None: + print(f" {test.name} ({test.duration_ms:.2f}ms)") + else: + print(f" {test.name}") + + if len(tests) > 100: + print(f" ... and {len(tests) - 100} more") + + print() + + def _display_timing(self, log_data: ParsedLogData) -> None: + """Display timing information from log.""" + if log_data.slowest_tests: + print("=" * 70) + print("Slowest Tests") + print("=" * 70) + for line in log_data.slowest_tests: + print(line) + print() + + if log_data.time_histogram: + print("=" * 70) + print("Test Times") + print("=" * 70) + for line in log_data.time_histogram: + print(line) + print() From e24d9f2f21adc42089b7ed20abcddcfbca839aac Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Thu, 3 Sep 2026 18:55:07 +0200 Subject: [PATCH 10/11] [UR][CI] Refine test result reporting This change introduces common test result collection and reporting for Unified Runtime conformance and adapter-specific CI tests. Compared to the initial version, the implementation has been updated based on review feedback to: - keep LIT test statuses such as SKIPPED and UNSUPPORTED distinct, - reconcile LIT and JUnit XML results without losing LIT-specific status information, - separate test execution, parsing, reconciliation, and presentation responsibilities, - simplify the internal API and remove redundant wrappers and helpers, - keep CI reporting options separate from execution-specific LIT options. Focused regression tests were added for the non-trivial result mapping and reconciliation behavior identified during review. --- devops/actions/run-tests/ur/action.yml | 21 +- devops/scripts/ur_test_tools/__init__.py | 38 -- devops/scripts/ur_test_tools/cli.py | 23 +- devops/scripts/ur_test_tools/constants.py | 27 +- .../scripts/ur_test_tools/models/__init__.py | 36 -- devops/scripts/ur_test_tools/models/config.py | 17 +- .../scripts/ur_test_tools/models/test_data.py | 50 --- .../ur_test_tools/models/test_results.py | 57 +-- .../scripts/ur_test_tools/outputs/__init__.py | 9 - .../scripts/ur_test_tools/outputs/console.py | 128 ++---- .../ur_test_tools/outputs/github_actions.py | 22 +- .../scripts/ur_test_tools/parsers/__init__.py | 31 -- .../ur_test_tools/parsers/log_parser.py | 167 +++---- .../ur_test_tools/parsers/parser_models.py | 38 +- .../ur_test_tools/parsers/stats_parser.py | 21 +- .../ur_test_tools/parsers/xml_parser.py | 98 ++--- .../scripts/ur_test_tools/reconciliation.py | 116 +++-- .../scripts/ur_test_tools/result_builder.py | 18 + .../ur_test_tools/summary_generator.py | 126 ++---- devops/scripts/ur_test_tools/test_runner.py | 25 +- .../tests/test_result_reconciliation.py | 409 ++++++++++++++++++ .../ur_test_tools/validation/__init__.py | 9 - .../validation/data_validator.py | 36 -- .../validation/path_validator.py | 93 +--- unified-runtime/test/CMakeLists.txt | 14 +- 25 files changed, 747 insertions(+), 882 deletions(-) delete mode 100644 devops/scripts/ur_test_tools/models/test_data.py create mode 100644 devops/scripts/ur_test_tools/result_builder.py create mode 100644 devops/scripts/ur_test_tools/tests/test_result_reconciliation.py delete mode 100644 devops/scripts/ur_test_tools/validation/data_validator.py diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index 70e7442302cf4..ac7732d963f4e 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -23,24 +23,7 @@ runs: BUILD_DIR: ${{ inputs.build_dir }} TEST_TYPE: ${{ inputs.test_type }} run: | - # Run tests via unified CLI (handles validation, config, cmake execution) - # Use temp file instead of command substitution to preserve outputs even on error - OUTPUTS_FILE=$(mktemp) - set +e # Don't exit on error - devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE" > "$OUTPUTS_FILE" 2>&1 - EXIT_CODE=$? - set -e # Re-enable exit on error - - # Parse Python output and set GitHub Actions outputs - while IFS='=' read -r key value; do - if [ -n "$key" ] && [ -n "$value" ]; then - echo "$key=$value" >> $GITHUB_OUTPUT - echo " Set output: $key=$value" - fi - done < <(grep -E "^(log-file|xml-file|skip-artifacts)=" "$OUTPUTS_FILE") - - rm -f "$OUTPUTS_FILE" - exit $EXIT_CODE + devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE" - name: Filter test log if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }} @@ -102,7 +85,6 @@ runs: devops/scripts/ur-test summary "$LOG_FILE" "${XML_FILE:-}" - # Add link to artifacts if they will be uploaded if [ -n "$ARTIFACT_NAME" ]; then echo "" echo "---" @@ -120,7 +102,6 @@ runs: run: | set -f # Disable glob expansion - # Clean artifacts directory to avoid mixing files from different test runs rm -rf test_artifacts mkdir -p test_artifacts diff --git a/devops/scripts/ur_test_tools/__init__.py b/devops/scripts/ur_test_tools/__init__.py index 45e58ada0bb6a..e69de29bb2d1d 100644 --- a/devops/scripts/ur_test_tools/__init__.py +++ b/devops/scripts/ur_test_tools/__init__.py @@ -1,38 +0,0 @@ -"""UR Test Tools - Modular test orchestration and summary generation.""" - -from .models import ( - # Configuration - TestConfig, - SummaryConfigFromLines, - # Normalized result model - TestStatus, - TestResult, - TestRunResult, - # Legacy (to be removed) - TimingSummary, -) -from .test_runner import TestRunner -from .summary_generator import SummaryReporter -from .validation import ( - PathValidator, -) -from .outputs import ( - ConsoleOutput, - GitHubActionsOutput, -) - -__all__ = [ - "__version__", - "__author__", - "TestConfig", - "SummaryConfigFromLines", - "TestStatus", - "TestResult", - "TestRunResult", - "TimingSummary", # Legacy - "TestRunner", - "SummaryReporter", - "PathValidator", - "ConsoleOutput", - "GitHubActionsOutput", -] diff --git a/devops/scripts/ur_test_tools/cli.py b/devops/scripts/ur_test_tools/cli.py index 0811eaffe920b..d87b1738876e9 100644 --- a/devops/scripts/ur_test_tools/cli.py +++ b/devops/scripts/ur_test_tools/cli.py @@ -4,19 +4,20 @@ import os from pathlib import Path -from .models.config import SummaryConfigFromLines, TestConfig, TestExecutionContext +from .models.config import TestConfig, TestExecutionContext from .validation.path_validator import PathValidator from .parsers.log_parser import ( LITLogParser, read_log_file, ) -from .outputs.console import ConsoleOutput +from .outputs.console import filter_log_for_display +from .result_builder import build_test_run_result from .summary_generator import SummaryReporter from .test_runner import ( TestRunner, get_test_config, ) -from .outputs.github_actions import GitHubActionsOutput +from .outputs import github_actions def main() -> int: @@ -57,15 +58,15 @@ def main_test_summary(command: str) -> int: print(line, end="") elif command == "filter-log": - for line in ConsoleOutput.filter_log_for_display(lines): + for line in filter_log_for_display(lines): print(line, end="") elif command == "show-summary": xml_file = PathValidator.validate_optional_path( sys.argv[3] if len(sys.argv) > 3 else "", "XML", allow_absolute=True ) - config = SummaryConfigFromLines(log_lines=lines, xml_file=xml_file or None) - SummaryReporter(config).generate() + result = build_test_run_result(lines, xml_file or None) + SummaryReporter(result).generate() else: print(f"Error: Unknown command '{command}'", file=sys.stderr) @@ -99,18 +100,16 @@ def _run_tests_command() -> int: build_dir = sys.argv[3] workspace = sys.argv[4] - # Validate inputs if not PathValidator.validate_build_dir(build_dir, workspace): - GitHubActionsOutput.print_error("Invalid build_dir") + github_actions.print_error("Invalid build_dir") return 1 try: config = get_test_config(test_type) except ValueError as e: - GitHubActionsOutput.print_error(str(e)) + github_actions.print_error(str(e)) return 1 - # Convert to paths and create context workspace_path = Path(workspace).resolve() build_dir_path = workspace_path / build_dir @@ -132,13 +131,11 @@ def _run_tests_command() -> int: env=env, ) - # Validate context try: context.validate() except ValueError as e: - GitHubActionsOutput.print_error(str(e)) + github_actions.print_error(str(e)) return 1 - # Run tests runner = TestRunner(context) return runner.run() diff --git a/devops/scripts/ur_test_tools/constants.py b/devops/scripts/ur_test_tools/constants.py index cb03253e130a1..396574d62516c 100644 --- a/devops/scripts/ur_test_tools/constants.py +++ b/devops/scripts/ur_test_tools/constants.py @@ -2,21 +2,13 @@ import re -# File I/O # Conservative upper bound for locating the LIT "Testing:" marker. # Non-verbose CI runs normally emit it much earlier. MAX_LINES_TO_SCAN = 1000 SEPARATOR_WIDTH = 70 -# LIT Configuration -# Following LLVM's model: define common defaults + allow invocation-specific overrides. -# -# LIT_COMMON_REPORTING_OPTIONS describes WHAT test information to show (semantics). -# These align with standalone builds (unified-runtime/test/CMakeLists.txt). -# -# LIT_CI_OPTIONS describes HOW to format output for CI tooling (execution policy). -# Standalone uses --succinct instead for developer-friendly interactive feedback. +# Keep reporting options aligned with unified-runtime/test/CMakeLists.txt. LIT_COMMON_REPORTING_OPTIONS = [ "--show-unsupported", "--show-pass", @@ -24,9 +16,9 @@ "--time-tests", "--show-flakypass", "--show-skipped", + "--show-excluded", ] -# CI-specific options: verbose output for log parsing and reporting tools LIT_CI_OPTIONS = [ "-v", "--no-progress-bar", @@ -35,31 +27,20 @@ DEFAULT_LIT_TIMEOUT = 120 DEFAULT_LIT_JOBS = 50 -# Test Type Identifiers TEST_TYPE_ADAPTER_SPECIFIC = "adapter-specific" TEST_TYPE_CONFORMANCE = "conformance" -# LIT Test Filters # These tests cause timeouts on CI and are excluded from adapter-specific runs LIT_FILTER_OUT_ADAPTER_SPECIFIC = ( "(adapters/level_zero/memcheck.test|" "adapters/level_zero/v2/deferred_kernel_memcheck.test)" ) -# Constants TEST_NOT_SELECTED_MSG = "Test not selected" SLOWEST_TESTS_HEADER = "Slowest Tests:" TEST_TIMES_HEADERS = ("Tests Times:", "Test Times:") -# LIT Output Patterns FAIL_TIMEOUT_PATTERN = re.compile(r"^(FAIL|TIMEOUT):") -TEST_LIST_HEADER_PATTERN = re.compile( - r"^(Passed|Unsupported|Failed|Expectedly Failed|" - r"Timed Out|Unexpectedly Passed|Unresolved) Tests \(" -) -STATS_PATTERN = re.compile( - r"^\s*(Total Discovered|Expected Passes|Expectedly Failed|" - r"Excluded|Unsupported|Skipped|Passed|Passed With Retry|" - r"Failed|Timed Out|Unexpectedly Passed|Unresolved)(\s+Tests)?\s*:" -) TEST_CATEGORY_PATTERN = re.compile(r"^([A-Za-z]+(?: [A-Za-z]+)*) Tests \((\d+)\):") +TESTING_TIME_PATTERN = re.compile(r"^\s*Testing Time:\s*([0-9.]+)s") +STAT_LINE_PATTERN = re.compile(r"^\s*(.+?)\s*:\s*(\d+)(?:\s|$)") diff --git a/devops/scripts/ur_test_tools/models/__init__.py b/devops/scripts/ur_test_tools/models/__init__.py index f4b1b46261e40..e69de29bb2d1d 100644 --- a/devops/scripts/ur_test_tools/models/__init__.py +++ b/devops/scripts/ur_test_tools/models/__init__.py @@ -1,36 +0,0 @@ -"""Models package - Data structures for UR test tools.""" - -from .test_data import ( - TestLists, - TestCounts, - TimingSummary, - SkippedTestsResult, - ExcludedTestsResult, -) -from .config import ( - TestConfig, - TestExecutionContext, - SummaryConfigFromLines, -) -from .test_results import ( - TestStatus, - TestResult, - TestRunResult, -) - -__all__ = [ - # Legacy models (to be deprecated) - "TestLists", - "TestCounts", - "TimingSummary", - "SkippedTestsResult", - "ExcludedTestsResult", - # Configuration - "TestConfig", - "TestExecutionContext", - "SummaryConfigFromLines", - # Normalized result model - "TestStatus", - "TestResult", - "TestRunResult", -] diff --git a/devops/scripts/ur_test_tools/models/config.py b/devops/scripts/ur_test_tools/models/config.py index 9094e4dadee91..0c640fa06751c 100644 --- a/devops/scripts/ur_test_tools/models/config.py +++ b/devops/scripts/ur_test_tools/models/config.py @@ -1,28 +1,21 @@ -"""Configuration dataclasses for UR test tools.""" - from dataclasses import dataclass, field from pathlib import Path -from typing import Optional, Dict, List +from typing import Optional, Dict @dataclass class TestConfig: - """Test execution configuration.""" - target: str log_file: str lit_filter_out: Optional[str] = None def __post_init__(self): - """Validate configuration on creation.""" if not all([self.target, self.log_file]): raise ValueError("target and log_file are required") @dataclass class TestExecutionContext: - """Context for test execution.""" - test_type: str build_dir: Path workspace: Path @@ -38,11 +31,3 @@ def validate(self) -> None: path.resolve().relative_to(workspace_resolved) except ValueError as e: raise ValueError(f"Path outside workspace: {e}") from e - - -@dataclass -class SummaryConfigFromLines: - """Configuration for summary generation from parsed log lines.""" - - log_lines: List[str] - xml_file: Optional[str] = None diff --git a/devops/scripts/ur_test_tools/models/test_data.py b/devops/scripts/ur_test_tools/models/test_data.py deleted file mode 100644 index 782c475609dc8..0000000000000 --- a/devops/scripts/ur_test_tools/models/test_data.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Type definitions for test data structures.""" - -from typing import TypedDict, List - - -class TestLists(TypedDict, total=False): - """Type definition for test list dictionary.""" - - Passed: List[str] - Failed: List[str] - Skipped: List[str] - Unsupported: List[str] - Excluded: List[str] - Unresolved: List[str] - - -class TestCounts(TypedDict, total=False): - """Type definition for test count dictionary.""" - - Passed: int - Failed: int - Skipped: int - Unsupported: int - Excluded: int - Unresolved: int - - -class TimingSummary(TypedDict): - """Type definition for test timing summary.""" - - slowest: List[str] - histogram: List[str] - - -class SkippedTestsResult(TypedDict): - """Result of skipped tests analysis.""" - - tests: List[str] - count: int - source: str - note: str - - -class ExcludedTestsResult(TypedDict): - """Result of excluded tests analysis.""" - - tests: List[str] - count: int - source: str - note: str diff --git a/devops/scripts/ur_test_tools/models/test_results.py b/devops/scripts/ur_test_tools/models/test_results.py index 098305f159efa..60915b54723d0 100644 --- a/devops/scripts/ur_test_tools/models/test_results.py +++ b/devops/scripts/ur_test_tools/models/test_results.py @@ -1,37 +1,14 @@ -"""Canonical test result model - independent of data sources. +"""Test result models.""" -This module defines the normalized representation of test results that is used -throughout the system. Parsers translate source-specific formats (LIT log, -JUnit XML) into this model, and consumers (summary, validation, future DB) -operate only on these normalized types. -""" - -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum from typing import Dict, List, Optional class TestStatus(str, Enum): - """Canonical test result status. - - Maps to LIT result codes. The string values match LIT's internal codes, - not the human-readable output labels. - - LIT Output Label -> TestStatus: - "Passed" -> PASS - "Passed With Retry" -> FLAKYPASS - "Expectedly Failed" -> XFAIL - "Unexpectedly Passed" -> XPASS - "Failed" -> FAIL - "Unresolved" -> UNRESOLVED - "Unsupported" -> UNSUPPORTED - "Timed Out" -> TIMEOUT - "Skipped" -> SKIPPED - "Excluded" -> EXCLUDED - """ - PASS = "PASS" FLAKYPASS = "FLAKYPASS" + FIXED = "FIXED" XFAIL = "XFAIL" XPASS = "XPASS" FAIL = "FAIL" @@ -43,10 +20,6 @@ class TestStatus(str, Enum): @property def is_failure(self) -> bool: - """Check if this status represents a test failure. - - LIT treats FAIL, XPASS, UNRESOLVED, and TIMEOUT as failures. - """ return self in ( TestStatus.FAIL, TestStatus.XPASS, @@ -56,10 +29,10 @@ def is_failure(self) -> bool: @property def display_label(self) -> str: - """Get human-readable label for display.""" return { TestStatus.PASS: "Passed", TestStatus.FLAKYPASS: "Passed With Retry", + TestStatus.FIXED: "Passed After Update", TestStatus.XFAIL: "Expectedly Failed", TestStatus.XPASS: "Unexpectedly Passed", TestStatus.FAIL: "Failed", @@ -73,11 +46,6 @@ def display_label(self) -> str: @dataclass class TestResult: - """Normalized result for a single test. - - Represents test outcome independent of where the information was obtained. - """ - name: str status: TestStatus duration_ms: Optional[float] = None @@ -85,25 +53,14 @@ class TestResult: @dataclass class TestRunResult: - """Complete normalized result of a test run. - - Contains all test results and run-level metadata. - """ - tests: List[TestResult] total_discovered: Optional[int] = None testing_time_ms: Optional[float] = None - - def count_by_status(self, status: TestStatus) -> int: - """Count tests with given status.""" - return sum(1 for test in self.tests if test.status == status) - - def tests_by_status(self, status: TestStatus) -> List[TestResult]: - """Get all tests with given status.""" - return [test for test in self.tests if test.status == status] + statistics_lines: List[str] = field(default_factory=list) + slowest_tests: List[str] = field(default_factory=list) + time_histogram: List[str] = field(default_factory=list) def group_by_status(self) -> Dict[TestStatus, List[TestResult]]: - """Group all tests by their status.""" groups: Dict[TestStatus, List[TestResult]] = {} for test in self.tests: groups.setdefault(test.status, []).append(test) diff --git a/devops/scripts/ur_test_tools/outputs/__init__.py b/devops/scripts/ur_test_tools/outputs/__init__.py index 9273918957af5..e69de29bb2d1d 100644 --- a/devops/scripts/ur_test_tools/outputs/__init__.py +++ b/devops/scripts/ur_test_tools/outputs/__init__.py @@ -1,9 +0,0 @@ -"""Outputs package - Console and GitHub Actions output generation.""" - -from .console import ConsoleOutput -from .github_actions import GitHubActionsOutput - -__all__ = [ - "ConsoleOutput", - "GitHubActionsOutput", -] diff --git a/devops/scripts/ur_test_tools/outputs/console.py b/devops/scripts/ur_test_tools/outputs/console.py index a0b5f98fb2d86..2d88e3a29ca0c 100644 --- a/devops/scripts/ur_test_tools/outputs/console.py +++ b/devops/scripts/ur_test_tools/outputs/console.py @@ -1,119 +1,51 @@ """Console output formatting for test results.""" -from typing import List, Optional +from typing import List from ..constants import ( - SEPARATOR_WIDTH, SLOWEST_TESTS_HEADER, - STATS_PATTERN, TEST_CATEGORY_PATTERN, TEST_TIMES_HEADERS, ) -from ..models.test_data import TimingSummary from ..parsers.log_parser import LITLogParser -class ConsoleOutput: - """Format test results for console output.""" +def filter_log_for_display(lines: List[str]) -> List[str]: + """Remove statistics, test lists, and timing from log.""" + result = [] + statistics = set(LITLogParser(lines).extract_statistics()) + skip_until_empty = False + in_timing = False - @staticmethod - def print_test_group( - title: str, tests: List[str], note: str = "", count: Optional[int] = None - ) -> None: - """Print GitHub Actions collapsible group with test list.""" - test_count = count if count is not None else len(tests) - print(f"::group::{title} ({test_count})") - if note: - print(note) - print() - for test in tests: - print(test) - print("::endgroup::") + for line in lines: + stripped = line.strip() - @staticmethod - def print_statistics(stats: List[str]) -> None: - """Print statistics section.""" - if stats: - print("=== Test Statistics ===") - for stat in stats: - print(stat.rstrip()) - print() + if line in statistics: + continue - @staticmethod - def print_timing_summary(lines: List[str]) -> None: - """Print timing information section.""" - parser = LITLogParser(lines) - time_info = parser.extract_time_summary() + if TEST_CATEGORY_PATTERN.match(line): + skip_until_empty = True + continue - testing_time = None - for line in lines: - if line.strip().startswith("Testing Time:"): - testing_time = line.strip() - break + if stripped == SLOWEST_TESTS_HEADER or stripped in TEST_TIMES_HEADERS: + in_timing = True + continue - if not (time_info["slowest"] or time_info["histogram"] or testing_time): - return + if in_timing and stripped.replace("*", "") == "": + in_timing = False + continue - print("::group::Test Timing Summary") + if in_timing: + continue - if testing_time: - print(testing_time) - print() + if stripped.startswith("Testing Time:"): + continue - if time_info["slowest"]: - print(SLOWEST_TESTS_HEADER) - print("-" * SEPARATOR_WIDTH) - for line in time_info["slowest"]: - print(line) - print() + if skip_until_empty: + if not stripped: + skip_until_empty = False + continue - if time_info["histogram"]: - print("Test Times Distribution:") - print("-" * SEPARATOR_WIDTH) - for line in time_info["histogram"]: - print(line) + result.append(line) - print("::endgroup::") - - @staticmethod - def filter_log_for_display(lines: List[str]) -> List[str]: - """Remove statistics, test lists, and timing from log.""" - result = [] - skip_until_empty = False - in_timing = False - - for line in lines: - stripped = line.strip() - - # Skip statistics lines - if STATS_PATTERN.match(line): - continue - - # Skip test category sections - if TEST_CATEGORY_PATTERN.match(line): - skip_until_empty = True - continue - - # Skip timing sections - if stripped == SLOWEST_TESTS_HEADER or stripped in TEST_TIMES_HEADERS: - in_timing = True - continue - - if in_timing and stripped.replace("*", "") == "": - in_timing = False - continue - - if in_timing: - continue - - if stripped.startswith("Testing Time:"): - continue - - if skip_until_empty: - if not stripped: - skip_until_empty = False - continue - - result.append(line) - - return result + return result diff --git a/devops/scripts/ur_test_tools/outputs/github_actions.py b/devops/scripts/ur_test_tools/outputs/github_actions.py index e892ad3f2f036..acadec9d8e52c 100644 --- a/devops/scripts/ur_test_tools/outputs/github_actions.py +++ b/devops/scripts/ur_test_tools/outputs/github_actions.py @@ -1,20 +1,18 @@ """GitHub Actions-specific output formatting.""" +import os import sys -class GitHubActionsOutput: - """Format output for GitHub Actions.""" +def print_error(message: str) -> None: + print(f"::error::{message}", file=sys.stderr) - @staticmethod - def print_error(message: str) -> None: - print(f"::error::{message}", file=sys.stderr) - @staticmethod - def print_warning(message: str) -> None: - print(f"::warning::{message}", file=sys.stderr) +def print_warning(message: str) -> None: + print(f"::warning::{message}", file=sys.stderr) - @staticmethod - def set_output(name: str, value: str) -> None: - print(f"{name}={value}", flush=True) - sys.stdout.flush() + +def set_output(name: str, value: str) -> None: + """Write name=value to the GITHUB_OUTPUT file for later steps.""" + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + print(f"{name}={value}", file=f) diff --git a/devops/scripts/ur_test_tools/parsers/__init__.py b/devops/scripts/ur_test_tools/parsers/__init__.py index 994adc7979455..e69de29bb2d1d 100644 --- a/devops/scripts/ur_test_tools/parsers/__init__.py +++ b/devops/scripts/ur_test_tools/parsers/__init__.py @@ -1,31 +0,0 @@ -"""Parsers package - Data extraction from logs and XML.""" - -from .log_parser import ( - LITLogParser, - read_log_file, -) -from .xml_parser import ( - JUnitXMLParser, - ParsedXMLTests, -) -from .stats_parser import get_count_from_stats -from .parser_models import ( - ParsedTestObservation, - ParsedLogData, - ParsedXMLData, - LIT_OUTPUT_TO_STATUS, - STATUS_TO_LIT_LABEL, -) - -__all__ = [ - "LITLogParser", - "read_log_file", - "JUnitXMLParser", - "ParsedXMLTests", - "get_count_from_stats", - "ParsedTestObservation", - "ParsedLogData", - "ParsedXMLData", - "LIT_OUTPUT_TO_STATUS", - "STATUS_TO_LIT_LABEL", -] diff --git a/devops/scripts/ur_test_tools/parsers/log_parser.py b/devops/scripts/ur_test_tools/parsers/log_parser.py index d44e861222403..d312c0438b4ff 100644 --- a/devops/scripts/ur_test_tools/parsers/log_parser.py +++ b/devops/scripts/ur_test_tools/parsers/log_parser.py @@ -6,18 +6,18 @@ from ..constants import ( FAIL_TIMEOUT_PATTERN, - TEST_LIST_HEADER_PATTERN, - STATS_PATTERN, + STAT_LINE_PATTERN, TEST_CATEGORY_PATTERN, + TESTING_TIME_PATTERN, SLOWEST_TESTS_HEADER, TEST_TIMES_HEADERS, ) -from ..models.test_data import TestLists, TestCounts, TimingSummary from .parser_models import ( ParsedLogData, ParsedTestObservation, - LIT_OUTPUT_TO_STATUS, + LIT_CATEGORY_TO_STATUS, ) +from .stats_parser import parse_statistics def _read_with_utf8_fallback(path: str, read_func): @@ -35,16 +35,17 @@ def _read_with_utf8_fallback(path: str, read_func): def read_log_file(log_path: str) -> List[str]: path = Path(log_path) - file_size = path.stat().st_size - - if file_size > 10 * 1024 * 1024: # 10 MB - print( - f"Large log file: {file_size / (1024 * 1024):.1f} MB. " - f"This may indicate a test problem.", - file=sys.stderr, - ) try: + file_size = path.stat().st_size + + if file_size > 10 * 1024 * 1024: # 10 MB + print( + f"Large log file: {file_size / (1024 * 1024):.1f} MB. " + f"This may indicate a test problem.", + file=sys.stderr, + ) + return _read_with_utf8_fallback(log_path, lambda f: f.readlines()) except OSError as e: raise OSError(f"Cannot read log file: {e}") from e @@ -64,9 +65,8 @@ def extract_error_details(self) -> List[str]: if FAIL_TIMEOUT_PATTERN.match(line): in_error = True - # Stop at test list headers or timing summaries if in_error and ( - TEST_LIST_HEADER_PATTERN.match(line) + TEST_CATEGORY_PATTERN.match(line) or line.strip() == SLOWEST_TESTS_HEADER or line.strip() in TEST_TIMES_HEADERS ): @@ -78,10 +78,23 @@ def extract_error_details(self) -> List[str]: return result def extract_statistics(self) -> List[str]: - return [line for line in self.lines if STATS_PATTERN.match(line)] + result = [] + in_summary = False + for line in self.lines: + match = STAT_LINE_PATTERN.match(line) + if not in_summary: + if match and match.group(1) == "Total Discovered Tests": + in_summary = True + result.append(line) + elif match: + result.append(line) + elif line.strip(): + break + return result - def extract_time_summary(self) -> TimingSummary: - result: TimingSummary = {"slowest": [], "histogram": []} + def extract_time_summary(self) -> Tuple[List[str], List[str]]: + slowest = [] + histogram = [] current_section = None skip_next_hr = False @@ -105,117 +118,65 @@ def extract_time_summary(self) -> TimingSummary: if not stripped: current_section = None elif not stripped.startswith("---"): - result["slowest"].append(line.rstrip()) + slowest.append(line.rstrip()) elif current_section == "histogram": if not stripped: current_section = None elif stripped.replace("*", "") == "": current_section = None elif stripped.startswith("[") or stripped.replace("-", "") == "": - result["histogram"].append(line.rstrip()) + histogram.append(line.rstrip()) else: current_section = None - return result + return slowest, histogram - def extract_test_lists(self) -> Tuple[TestLists, TestCounts]: - categories: TestLists = {} - declared_counts: TestCounts = {} - current_category = None - current_tests = [] - current_declared_count = 0 + def parse_to_observations(self) -> ParsedLogData: + observations = [] + declared_counts = {} + current_status = None for line in self.lines: match = TEST_CATEGORY_PATTERN.match(line) if match: - if current_category: - categories[current_category] = current_tests - declared_counts[current_category] = current_declared_count - - current_category = match.group(1) - current_declared_count = int(match.group(2)) - current_tests = [] - continue - - if current_category: - if not line.strip(): - categories[current_category] = current_tests - declared_counts[current_category] = current_declared_count - current_category = None - current_tests = [] - current_declared_count = 0 - else: - test_name = line.strip() - if test_name: - current_tests.append(test_name) - - if current_category: - categories[current_category] = current_tests - declared_counts[current_category] = current_declared_count - - return categories, declared_counts - - def parse_to_observations(self) -> ParsedLogData: - """Parse LIT output into normalized observations. - - Returns: - ParsedLogData with test observations, counts, and timing info. - """ - # Extract test lists using legacy method - categories, declared_counts = self.extract_test_lists() + category = match.group(1) + current_status = LIT_CATEGORY_TO_STATUS.get(category) + if current_status is None: + print( + f"Warning: Unknown test category '{category}'", file=sys.stderr + ) + continue - # Convert to observations with normalized status - observations = [] - normalized_counts = {} - - for category_label, test_names in categories.items(): - status = LIT_OUTPUT_TO_STATUS.get(category_label) - if status is None: - print( - f"Warning: Unknown test category '{category_label}', skipping", - file=sys.stderr, - ) + declared_counts[current_status] = declared_counts.get( + current_status, 0 + ) + int(match.group(2)) continue - # Convert test names to observations - for test_name in test_names: + if current_status and not line.strip(): + current_status = None + elif current_status: observations.append( - ParsedTestObservation( - name=test_name, - status=status, - duration_ms=None, # Log output doesn't have per-test timing - ) + ParsedTestObservation(name=line.strip(), status=current_status) ) - # Store declared count for this status - count = declared_counts.get(category_label, 0) - normalized_counts[status] = count - - # Extract statistics (could be used for validation) stats_lines = self.extract_statistics() - statistics = {} - for line in stats_lines: - # Parse "Stat Name: N" format - parts = line.strip().split(":", 1) - if len(parts) == 2: - stat_name = parts[0].strip() - try: - stat_value = int(parts[1].strip()) - statistics[stat_name] = stat_value - except ValueError: - continue + statistics = parse_statistics(stats_lines) - # Extract timing info - timing = self.extract_time_summary() + testing_time_ms = None + for line in self.lines: + match = TESTING_TIME_PATTERN.match(line) + if match: + testing_time_ms = float(match.group(1)) * 1000.0 + break - # Extract error details - errors = self.extract_error_details() + slowest_tests, time_histogram = self.extract_time_summary() return ParsedLogData( tests=observations, - declared_counts=normalized_counts, + declared_counts=declared_counts, statistics=statistics, - error_details=errors, - slowest_tests=timing["slowest"], - time_histogram=timing["histogram"], + statistics_lines=[line.rstrip() for line in stats_lines], + slowest_tests=slowest_tests, + time_histogram=time_histogram, + testing_time_ms=testing_time_ms, ) diff --git a/devops/scripts/ur_test_tools/parsers/parser_models.py b/devops/scripts/ur_test_tools/parsers/parser_models.py index 3f23a1f219aa8..0f071e2eaabbd 100644 --- a/devops/scripts/ur_test_tools/parsers/parser_models.py +++ b/devops/scripts/ur_test_tools/parsers/parser_models.py @@ -1,4 +1,4 @@ -"""Parser-specific data models - intermediate observations before normalization.""" +"""Parser result models.""" from dataclasses import dataclass, field from typing import Dict, List, NamedTuple, Optional @@ -7,8 +7,6 @@ class ParsedTestObservation(NamedTuple): - """A single test observation from a parser (log or XML).""" - name: str status: TestStatus duration_ms: Optional[float] = None @@ -16,35 +14,26 @@ class ParsedTestObservation(NamedTuple): @dataclass class ParsedLogData: - """Raw observations and statistics from LIT text output.""" - tests: List[ParsedTestObservation] = field(default_factory=list) - # Declared counts from "Category Tests (N):" headers declared_counts: Dict[TestStatus, int] = field(default_factory=dict) - # Statistics from "Stat Name: N" lines statistics: Dict[str, int] = field(default_factory=dict) - # Raw error/failure details - error_details: List[str] = field(default_factory=list) - # Timing summary sections + statistics_lines: List[str] = field(default_factory=list) slowest_tests: List[str] = field(default_factory=list) time_histogram: List[str] = field(default_factory=list) + testing_time_ms: Optional[float] = None @dataclass class ParsedXMLData: - """Observations from JUnit XML output.""" - tests: List[ParsedTestObservation] = field(default_factory=list) - # XML may have additional metadata we can extract in the future total_tests: Optional[int] = None total_time_seconds: Optional[float] = None -# Mapping from LIT output labels to canonical status -LIT_OUTPUT_TO_STATUS = { - # Category headers +LIT_CATEGORY_TO_STATUS = { "Passed": TestStatus.PASS, "Passed With Retry": TestStatus.FLAKYPASS, + "Passed After Update": TestStatus.FIXED, "Expectedly Failed": TestStatus.XFAIL, "Unexpectedly Passed": TestStatus.XPASS, "Failed": TestStatus.FAIL, @@ -53,21 +42,10 @@ class ParsedXMLData: "Timed Out": TestStatus.TIMEOUT, "Skipped": TestStatus.SKIPPED, "Excluded": TestStatus.EXCLUDED, - # Statistics labels (may differ slightly) - "Expected Passes": TestStatus.PASS, } -# Reverse mapping for display -STATUS_TO_LIT_LABEL = { - TestStatus.PASS: "Passed", - TestStatus.FLAKYPASS: "Passed With Retry", - TestStatus.XFAIL: "Expectedly Failed", - TestStatus.XPASS: "Unexpectedly Passed", - TestStatus.FAIL: "Failed", - TestStatus.UNRESOLVED: "Unresolved", - TestStatus.UNSUPPORTED: "Unsupported", - TestStatus.TIMEOUT: "Timed Out", - TestStatus.SKIPPED: "Skipped", - TestStatus.EXCLUDED: "Excluded", +LIT_STAT_TO_STATUS = { + **LIT_CATEGORY_TO_STATUS, + "Expected Passes": TestStatus.PASS, } diff --git a/devops/scripts/ur_test_tools/parsers/stats_parser.py b/devops/scripts/ur_test_tools/parsers/stats_parser.py index 5424e6c084405..2bdfed27ec5e1 100644 --- a/devops/scripts/ur_test_tools/parsers/stats_parser.py +++ b/devops/scripts/ur_test_tools/parsers/stats_parser.py @@ -1,16 +1,17 @@ """Parse statistics from LIT output.""" -import re -from typing import List +from typing import Dict, List +from ..constants import STAT_LINE_PATTERN -def get_count_from_stats(stats: List[str], keywords: List[str]) -> int: - # Build regex pattern from keywords (compile once per call) - pattern = re.compile("|".join(re.escape(kw) for kw in keywords)) +def parse_statistics(stats: List[str]) -> Dict[str, int]: + result = {} for stat in stats: - if pattern.search(stat): - match = re.search(r"(\d+)", stat) - if match: - return int(match.group(1)) - return 0 + match = STAT_LINE_PATTERN.match(stat) + if match: + label = match.group(1) + if label.endswith(" Tests"): + label = label[: -len(" Tests")] + result[label] = result.get(label, 0) + int(match.group(2)) + return result diff --git a/devops/scripts/ur_test_tools/parsers/xml_parser.py b/devops/scripts/ur_test_tools/parsers/xml_parser.py index 0287b69dfe533..b804efd4b46ff 100644 --- a/devops/scripts/ur_test_tools/parsers/xml_parser.py +++ b/devops/scripts/ur_test_tools/parsers/xml_parser.py @@ -2,7 +2,7 @@ import sys from pathlib import Path -from typing import List, NamedTuple, Optional +from typing import Optional import defusedxml.ElementTree as ET @@ -11,17 +11,31 @@ from .parser_models import ParsedXMLData, ParsedTestObservation -class ParsedXMLTests(NamedTuple): - """Skipped and excluded tests from XML parsing.""" +def _format_test_name(classname: str, name: str) -> str: + """Convert a JUnit name to LIT's display format.""" + if not classname: + return name + if not name: + return classname - skipped: List[str] - excluded: List[str] + suite_name, separator, path = classname.partition(".") + if not separator or path == suite_name: + return f"{suite_name} :: {name}" + return f"{suite_name} :: {path}/{name}" -def _format_test_name(classname: str, name: str) -> str: - if classname and name: - return f"{classname}.{name}" - return name +def _status_from_skipped_message(message: str) -> TestStatus: + """Recover the LIT status encoded as a JUnit ``skipped`` element.""" + if message.startswith(TEST_NOT_SELECTED_MSG): + return TestStatus.EXCLUDED + if message == "User interrupt": + return TestStatus.SKIPPED + if message.startswith("Missing required feature(s):"): + return TestStatus.UNSUPPORTED + if message == "Unsupported configuration": + return TestStatus.UNSUPPORTED + + return TestStatus.SKIPPED class JUnitXMLParser: @@ -55,48 +69,8 @@ def parse(self) -> bool: ) return False - def extract_tests_from_xml(self) -> ParsedXMLTests: - if not self.parse(): - return ParsedXMLTests([], []) - - skipped = [] - excluded = [] - - for testcase in self._root.findall(".//testcase"): - skipped_elem = testcase.find("skipped") - if skipped_elem is None: - continue - - message = skipped_elem.get("message", "") - test_name = _format_test_name( - testcase.get("classname", ""), testcase.get("name", "") - ) - - if not test_name: - continue - - # Separate by message type - if TEST_NOT_SELECTED_MSG in message: - excluded.append(test_name) - else: - skipped.append(test_name) - - return ParsedXMLTests(skipped=skipped, excluded=excluded) - - def extract_skipped_tests(self) -> List[str]: - skipped, _ = self.extract_tests_from_xml() - return skipped - - def extract_excluded_tests(self) -> List[str]: - _, excluded = self.extract_tests_from_xml() - return excluded - def parse_to_observations(self) -> ParsedXMLData: - """Parse XML into normalized observations. - - Returns: - ParsedXMLData with test observations and metadata. - """ + """Parse XML test results.""" if not self.parse(): return ParsedXMLData(tests=[]) @@ -113,7 +87,6 @@ def parse_to_observations(self) -> ParsedXMLData: total_tests += 1 - # Extract duration (in seconds from XML, convert to ms) duration_ms = None time_str = testcase.get("time") if time_str: @@ -124,7 +97,6 @@ def parse_to_observations(self) -> ParsedXMLData: except ValueError: pass - # Determine status from child elements status = self._determine_status_from_xml(testcase) observations.append( @@ -142,19 +114,9 @@ def parse_to_observations(self) -> ParsedXMLData: ) def _determine_status_from_xml(self, testcase) -> TestStatus: - """Determine test status from XML testcase element. - - JUnit XML structure: - - = FAIL - - = UNRESOLVED or TIMEOUT (depends on message) - - = SKIPPED or EXCLUDED (depends on message) - - No child elements = PASS - """ - # Check for failure if testcase.find("failure") is not None: return TestStatus.FAIL - # Check for error (could be unresolved or timeout) error_elem = testcase.find("error") if error_elem is not None: message = error_elem.get("message", "").lower() @@ -162,15 +124,9 @@ def _determine_status_from_xml(self, testcase) -> TestStatus: return TestStatus.TIMEOUT return TestStatus.UNRESOLVED - # Check for skipped (could be skipped or excluded) skipped_elem = testcase.find("skipped") if skipped_elem is not None: - message = skipped_elem.get("message", "") - if TEST_NOT_SELECTED_MSG in message: - return TestStatus.EXCLUDED - return TestStatus.SKIPPED - - # No special markers = passed - # Note: We can't distinguish PASS/FLAKYPASS/XFAIL/XPASS from XML alone - # That requires log output. XML parser reports PASS for anything that succeeded. + return _status_from_skipped_message(skipped_elem.get("message", "")) + + # JUnit does not distinguish successful LIT statuses. return TestStatus.PASS diff --git a/devops/scripts/ur_test_tools/reconciliation.py b/devops/scripts/ur_test_tools/reconciliation.py index 91bf68a12d32b..e08ef2d833157 100644 --- a/devops/scripts/ur_test_tools/reconciliation.py +++ b/devops/scripts/ur_test_tools/reconciliation.py @@ -1,42 +1,57 @@ -"""Reconciliation layer - combines parser observations into canonical results.""" +"""Combine parsed test results.""" import sys from typing import List, Optional from .models.test_results import TestResult, TestRunResult, TestStatus -from .parsers.parser_models import ParsedLogData, ParsedXMLData +from .parsers.parser_models import ( + LIT_STAT_TO_STATUS, + ParsedLogData, + ParsedTestObservation, + ParsedXMLData, +) + +XML_FALLBACK_STATUSES = { + TestStatus.EXCLUDED, + TestStatus.SKIPPED, + TestStatus.UNSUPPORTED, +} + + +def _index_by_name(observations: List[ParsedTestObservation], source: str): + result = {} + for observation in observations: + if observation.name in result: + print( + f"Warning: Duplicate test name in {source}: {observation.name}", + file=sys.stderr, + ) + continue + result[observation.name] = observation + return result def reconcile_test_results( log_data: ParsedLogData, xml_data: Optional[ParsedXMLData] = None ) -> TestRunResult: - """Reconcile observations from log and XML into canonical test results. - - Reconciliation policy: - 1. Test status comes from log output (authoritative for XFAIL/XPASS/FLAKYPASS) - 2. Test duration comes from XML (more precise than log timing sections) - 3. If test appears in XML but not log, include with XML status - 4. Validate counts and warn on mismatches - - Args: - log_data: Parsed observations from LIT text output - xml_data: Optional parsed observations from JUnit XML + """Merge LIT log results with supplementary JUnit XML data.""" + log_by_name = _index_by_name(log_data.tests, "log") + log_counts = {} + for observation in log_data.tests: + log_counts[observation.status] = log_counts.get(observation.status, 0) + 1 + + complete_log_statuses = { + status + for status, declared_count in log_data.declared_counts.items() + if log_counts.get(status, 0) == declared_count + } - Returns: - TestRunResult with reconciled test results - """ - # Build index of log results (status is authoritative) - log_by_name = {obs.name: obs for obs in log_data.tests} - - # Build index of XML results (timing is authoritative) xml_by_name = {} if xml_data: - xml_by_name = {obs.name: obs for obs in xml_data.tests} + xml_by_name = _index_by_name(xml_data.tests, "XML") - # Reconcile: start with all tests from log results = [] for log_obs in log_data.tests: - # Use log status, but prefer XML timing if available xml_obs = xml_by_name.get(log_obs.name) duration_ms = xml_obs.duration_ms if xml_obs else log_obs.duration_ms @@ -48,16 +63,16 @@ def reconcile_test_results( ) ) - # Add any tests that appear in XML but not in log - # This shouldn't happen normally, but handle it gracefully - xml_only_tests = set(xml_by_name.keys()) - set(log_by_name.keys()) - if xml_only_tests: - print( - f"Warning: {len(xml_only_tests)} test(s) found in XML but not in log", - file=sys.stderr, - ) - for test_name in sorted(xml_only_tests): - xml_obs = xml_by_name[test_name] + result_names = set(log_by_name) + if xml_data: + for xml_obs in xml_data.tests: + if ( + xml_obs.status not in XML_FALLBACK_STATUSES + or xml_obs.status in complete_log_statuses + or xml_obs.name in result_names + ): + continue + results.append( TestResult( name=xml_obs.name, @@ -65,34 +80,30 @@ def reconcile_test_results( duration_ms=xml_obs.duration_ms, ) ) + result_names.add(xml_obs.name) - # Extract total discovered and testing time total_discovered = log_data.statistics.get("Total Discovered") - testing_time_ms = None - if xml_data and xml_data.total_time_seconds: + testing_time_ms = log_data.testing_time_ms + if testing_time_ms is None and xml_data and xml_data.total_time_seconds: testing_time_ms = xml_data.total_time_seconds * 1000.0 - # Validate counts _validate_counts(log_data, results) return TestRunResult( tests=results, total_discovered=total_discovered, testing_time_ms=testing_time_ms, + statistics_lines=log_data.statistics_lines, + slowest_tests=log_data.slowest_tests, + time_histogram=log_data.time_histogram, ) def _validate_counts(log_data: ParsedLogData, results: List[TestResult]) -> None: - """Validate that reconciled results match declared counts from log. - - Warns if counts don't match but doesn't fail - this helps catch parser bugs. - """ - # Count results by status actual_counts = {} for result in results: actual_counts[result.status] = actual_counts.get(result.status, 0) + 1 - # Compare with declared counts from log for status, declared_count in log_data.declared_counts.items(): actual_count = actual_counts.get(status, 0) if actual_count != declared_count: @@ -102,9 +113,22 @@ def _validate_counts(log_data: ParsedLogData, results: List[TestResult]) -> None file=sys.stderr, ) - # Check total discovered if available - if log_data.statistics.get("Total Discovered"): - total_discovered = log_data.statistics["Total Discovered"] + for label, expected_count in log_data.statistics.items(): + status = LIT_STAT_TO_STATUS.get(label) + if status is None: + if label != "Total Discovered": + print(f"Warning: Unknown test statistic '{label}'", file=sys.stderr) + continue + actual_count = actual_counts.get(status, 0) + if actual_count != expected_count: + print( + f"Warning: Count mismatch for {status.value}: " + f"statistics {expected_count}, found {actual_count}", + file=sys.stderr, + ) + + total_discovered = log_data.statistics.get("Total Discovered") + if total_discovered is not None: total_results = len(results) if total_results != total_discovered: print( diff --git a/devops/scripts/ur_test_tools/result_builder.py b/devops/scripts/ur_test_tools/result_builder.py new file mode 100644 index 0000000000000..808654dd9f13b --- /dev/null +++ b/devops/scripts/ur_test_tools/result_builder.py @@ -0,0 +1,18 @@ +"""Orchestrate parsers and reconciliation into a TestRunResult.""" + +from typing import List, Optional + +from .models.test_results import TestRunResult +from .parsers.log_parser import LITLogParser +from .parsers.xml_parser import JUnitXMLParser +from .reconciliation import reconcile_test_results + + +def build_test_run_result( + log_lines: List[str], xml_file: Optional[str] = None +) -> TestRunResult: + log_data = LITLogParser(log_lines).parse_to_observations() + xml_data = None + if xml_file: + xml_data = JUnitXMLParser(xml_file).parse_to_observations() + return reconcile_test_results(log_data, xml_data) diff --git a/devops/scripts/ur_test_tools/summary_generator.py b/devops/scripts/ur_test_tools/summary_generator.py index 96bdf281b97c3..cf9f0e47ecbc8 100644 --- a/devops/scripts/ur_test_tools/summary_generator.py +++ b/devops/scripts/ur_test_tools/summary_generator.py @@ -2,66 +2,28 @@ from typing import List -from .models.config import SummaryConfigFromLines from .models.test_results import TestResult, TestStatus, TestRunResult -from .parsers.log_parser import LITLogParser -from .parsers.xml_parser import JUnitXMLParser -from .parsers.parser_models import ParsedLogData -from .reconciliation import reconcile_test_results class SummaryReporter: - """Generate comprehensive test summary using normalized test results.""" - - def __init__(self, config: SummaryConfigFromLines): - self.config = config + def __init__(self, result: TestRunResult): + self.result = result def generate(self) -> None: - """Generate and display test summary.""" - # Parse inputs - log_parser = LITLogParser(self.config.log_lines) - log_data = log_parser.parse_to_observations() - - xml_data = None - if self.config.xml_file: - xml_parser = JUnitXMLParser(self.config.xml_file) - xml_data = xml_parser.parse_to_observations() - - # Reconcile into canonical result model - result = reconcile_test_results(log_data, xml_data) - - # Display summary - self._display_statistics(result) - self._display_test_groups(result) - self._display_timing(log_data) - - def _display_statistics(self, result: TestRunResult) -> None: - """Display overall test statistics.""" - print("=" * 70) - print("Test Statistics") - print("=" * 70) - - if result.total_discovered is not None: - print(f"Total Discovered Tests: {result.total_discovered}") - - # Count by status - for status in TestStatus: - count = result.count_by_status(status) - if count > 0: - print(f"{status.display_label}: {count}") - - if result.testing_time_ms is not None: - time_sec = result.testing_time_ms / 1000.0 - print(f"Testing Time: {time_sec:.2f}s") - - print() + self._display_statistics() + self._display_test_groups() + self._display_timing() + + def _display_statistics(self) -> None: + if self.result.statistics_lines: + print("=== Test Statistics ===") + for line in self.result.statistics_lines: + print(line) + print() - def _display_test_groups(self, result: TestRunResult) -> None: - """Display test groups by status.""" - # Group tests by status - grouped = result.group_by_status() + def _display_test_groups(self) -> None: + grouped = self.result.group_by_status() - # Display groups in priority order (failures first) priority_order = [ TestStatus.FAIL, TestStatus.TIMEOUT, @@ -71,6 +33,7 @@ def _display_test_groups(self, result: TestRunResult) -> None: TestStatus.UNSUPPORTED, TestStatus.EXCLUDED, TestStatus.XFAIL, + TestStatus.FIXED, TestStatus.FLAKYPASS, TestStatus.PASS, ] @@ -80,46 +43,43 @@ def _display_test_groups(self, result: TestRunResult) -> None: if not tests: continue - # Skip PASS tests by default (too many) - if status == TestStatus.PASS and len(tests) > 10: - print(f"{status.display_label} ({len(tests)} tests) - details omitted") - print() - continue - - # Display test group self._print_test_group(status, tests) def _print_test_group(self, status: TestStatus, tests: List[TestResult]) -> None: - """Print a group of tests with given status.""" - print("=" * 70) - print(f"{status.display_label} ({len(tests)})") - print("=" * 70) + print(f"::group::{status.display_label} ({len(tests)})") - for test in tests[:100]: # Limit to first 100 - if test.duration_ms is not None: - print(f" {test.name} ({test.duration_ms:.2f}ms)") - else: - print(f" {test.name}") + for test in tests[:100]: + print(test.name) if len(tests) > 100: - print(f" ... and {len(tests) - 100} more") + print(f"... and {len(tests) - 100} more") - print() + print("::endgroup::") - def _display_timing(self, log_data: ParsedLogData) -> None: - """Display timing information from log.""" - if log_data.slowest_tests: - print("=" * 70) - print("Slowest Tests") - print("=" * 70) - for line in log_data.slowest_tests: - print(line) + def _display_timing(self) -> None: + result = self.result + if not ( + result.testing_time_ms or result.slowest_tests or result.time_histogram + ): + return + + print("::group::Test Timing Summary") + + if result.testing_time_ms is not None: + print(f"Testing Time: {result.testing_time_ms / 1000.0:.2f}s") print() - if log_data.time_histogram: - print("=" * 70) - print("Test Times") - print("=" * 70) - for line in log_data.time_histogram: + if result.slowest_tests: + print("Slowest Tests:") + print("-" * 70) + for line in result.slowest_tests: print(line) print() + + if result.time_histogram: + print("Test Times Distribution:") + print("-" * 70) + for line in result.time_histogram: + print(line) + + print("::endgroup::") diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py index ef4118daaf3f7..f46f3101b0998 100644 --- a/devops/scripts/ur_test_tools/test_runner.py +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -17,12 +17,11 @@ MAX_LINES_TO_SCAN, ) from .models.config import TestConfig, TestExecutionContext -from .outputs.github_actions import GitHubActionsOutput +from .outputs import github_actions from .parsers.log_parser import _read_with_utf8_fallback def get_test_config(test_type: str) -> TestConfig: - """Get test configuration for test type.""" if test_type == TEST_TYPE_ADAPTER_SPECIFIC: return TestConfig( target="check-unified-runtime-adapter", @@ -48,8 +47,6 @@ def calculate_jobs() -> int: def check_log_has_tests(log_file: str) -> bool: - """Check if log contains test results.""" - def _scan_for_testing(f): for _ in range(MAX_LINES_TO_SCAN): line = f.readline() @@ -66,14 +63,11 @@ def _scan_for_testing(f): class TestRunner: - """Execute UR tests.""" - def __init__(self, context: TestExecutionContext): self.context = context self.jobs = calculate_jobs() def run(self) -> int: - """Run tests and return exit code.""" self._setup_environment() result = self._execute_tests() @@ -87,7 +81,6 @@ def run(self) -> int: return result.returncode def _setup_environment(self) -> None: - # Build LIT options: common reporting + CI-specific + dynamic config lit_opts_parts = [ *LIT_COMMON_REPORTING_OPTIONS, *LIT_CI_OPTIONS, @@ -131,36 +124,34 @@ def _execute_tests(self) -> Optional[subprocess.CompletedProcess]: cwd=self.context.workspace, ) except OSError as e: - GitHubActionsOutput.print_error(f"Test execution failed: {e}") + github_actions.print_error(f"Test execution failed: {e}") return None def _validate_output(self) -> bool: log_path = self.context.log_file_path if not log_path.exists() or log_path.stat().st_size == 0: - GitHubActionsOutput.print_error("No log generated") + github_actions.print_error("No log generated") return False return True def _publish_outputs(self) -> None: - GitHubActionsOutput.set_output("log-file", str(self.context.log_file_path)) + github_actions.set_output("log-file", str(self.context.log_file_path)) if ( self.context.test_type == TEST_TYPE_ADAPTER_SPECIFIC and not check_log_has_tests(str(self.context.log_file_path)) ): print("No adapter-specific tests found", file=sys.stderr) - GitHubActionsOutput.set_output("skip-artifacts", "1") + github_actions.set_output("skip-artifacts", "1") return - GitHubActionsOutput.set_output("skip-artifacts", "0") + github_actions.set_output("skip-artifacts", "0") if self.context.xml_output_path.exists(): - GitHubActionsOutput.set_output( - "xml-file", str(self.context.xml_output_path) - ) + github_actions.set_output("xml-file", str(self.context.xml_output_path)) else: - GitHubActionsOutput.print_warning( + github_actions.print_warning( f"Expected XML file not found at {self.context.xml_output_path}" ) diff --git a/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py b/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py new file mode 100644 index 0000000000000..1ad621a13c57f --- /dev/null +++ b/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py @@ -0,0 +1,409 @@ +"""Tests for parsing and reconciling LIT test results.""" + +import io +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from contextlib import contextmanager +from pathlib import Path +import os + +from devops.scripts.ur_test_tools.models.test_results import TestStatus +from devops.scripts.ur_test_tools.parsers.parser_models import ( + ParsedLogData, + ParsedTestObservation, + ParsedXMLData, +) +from devops.scripts.ur_test_tools.parsers.log_parser import LITLogParser, read_log_file +from devops.scripts.ur_test_tools.parsers.stats_parser import parse_statistics +from devops.scripts.ur_test_tools.parsers.xml_parser import JUnitXMLParser +from devops.scripts.ur_test_tools.reconciliation import reconcile_test_results +from devops.scripts.ur_test_tools.result_builder import build_test_run_result +from devops.scripts.ur_test_tools.summary_generator import SummaryReporter +from devops.scripts.ur_test_tools.validation.path_validator import PathValidator + + +@contextmanager +def temporary_cwd(path): + previous_cwd = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(previous_cwd) + + +def _count_by_status(result, status): + return sum(1 for test in result.tests if test.status == status) + + +class JUnitXMLParserTest(unittest.TestCase): + def test_recovers_lit_statuses_from_skipped_messages(self): + xml = """\ + + + + + + + + + + + + + + + + + + + +""" + with tempfile.TemporaryDirectory() as directory: + xml_path = Path(directory) / "results.xml" + xml_path.write_text(xml, encoding="utf-8") + result = JUnitXMLParser(str(xml_path)).parse_to_observations() + + self.assertEqual( + [test.status for test in result.tests], + [ + TestStatus.EXCLUDED, + TestStatus.SKIPPED, + TestStatus.UNSUPPORTED, + TestStatus.UNSUPPORTED, + TestStatus.SKIPPED, + ], + ) + self.assertEqual(result.tests[0].name, "Suite :: path/excluded.cpp") + + def test_formats_test_at_suite_root(self): + xml = """\ + + + + + +""" + with tempfile.TemporaryDirectory() as directory: + xml_path = Path(directory) / "results.xml" + xml_path.write_text(xml, encoding="utf-8") + result = JUnitXMLParser(str(xml_path)).parse_to_observations() + + self.assertEqual(result.tests[0].name, "Suite :: root.cpp") + + def test_formats_unified_runtime_names(self): + xml = """\ + + + + + + +""" + with tempfile.TemporaryDirectory() as directory: + xml_path = Path(directory) / "results.xml" + xml_path.write_text(xml, encoding="utf-8") + result = JUnitXMLParser(str(xml_path)).parse_to_observations() + + self.assertEqual( + [test.name for test in result.tests], + [ + "Unified Runtime :: conformance/urContext/test.cpp", + "Unified Runtime :: adapters/level_zero/event.cpp", + ], + ) + + +class LITLogParserTest(unittest.TestCase): + def test_read_log_file_wraps_stat_oserror(self): + with tempfile.TemporaryDirectory() as directory: + missing_path = Path(directory) / "missing.log" + with self.assertRaisesRegex(OSError, "Cannot read log file"): + read_log_file(str(missing_path)) + + def test_parses_all_builtin_statuses(self): + categories = [ + ("Passed", TestStatus.PASS), + ("Passed With Retry", TestStatus.FLAKYPASS), + ("Passed After Update", TestStatus.FIXED), + ("Expectedly Failed", TestStatus.XFAIL), + ("Unexpectedly Passed", TestStatus.XPASS), + ("Failed", TestStatus.FAIL), + ("Unresolved", TestStatus.UNRESOLVED), + ("Unsupported", TestStatus.UNSUPPORTED), + ("Timed Out", TestStatus.TIMEOUT), + ("Skipped", TestStatus.SKIPPED), + ("Excluded", TestStatus.EXCLUDED), + ] + lines = [] + for index, (label, _) in enumerate(categories): + lines.extend([f"{label} Tests (1):\n", f"suite :: test-{index}\n", "\n"]) + + result = LITLogParser(lines).parse_to_observations() + + self.assertEqual( + [test.status for test in result.tests], + [status for _, status in categories], + ) + self.assertEqual( + result.declared_counts, + {status: 1 for _, status in categories}, + ) + + def test_warns_about_unknown_category(self): + stderr = io.StringIO() + with redirect_stderr(stderr): + result = LITLogParser( + ["New Fancy Tests (1):\n", "suite :: test\n", "\n"] + ).parse_to_observations() + + self.assertEqual(result.tests, []) + self.assertIn("Unknown test category 'New Fancy'", stderr.getvalue()) + + def test_parses_statistics_by_exact_label(self): + result = parse_statistics( + [ + "Total Discovered Tests: 5\n", + " Passed: 2\n", + " Unexpectedly Passed: 3\n", + ] + ) + + self.assertEqual(result["Total Discovered"], 5) + self.assertEqual(result["Passed"], 2) + self.assertEqual(result["Unexpectedly Passed"], 3) + + def test_sums_repeated_statistics_labels(self): + result = parse_statistics([" Passed: 2\n", " Passed: 3\n"]) + + self.assertEqual(result["Passed"], 5) + + def test_sums_repeated_category_counts(self): + lines = [ + "Passed Tests (1):\n", + "suite :: first\n", + "\n", + "Passed Tests (1):\n", + "suite :: second\n", + "\n", + ] + + result = LITLogParser(lines).parse_to_observations() + + self.assertEqual(result.declared_counts[TestStatus.PASS], 2) + self.assertEqual(len(result.tests), 2) + + +class ReconcileTestResultsTest(unittest.TestCase): + def test_complete_log_category_takes_precedence_over_xml(self): + log_data = ParsedLogData( + tests=[ + ParsedTestObservation("Suite :: pass.cpp", TestStatus.PASS), + ParsedTestObservation( + "Suite :: unsupported.cpp", TestStatus.UNSUPPORTED + ), + ], + declared_counts={TestStatus.PASS: 1, TestStatus.UNSUPPORTED: 1}, + ) + xml_data = ParsedXMLData( + tests=[ + ParsedTestObservation("Suite.pass.cpp", TestStatus.PASS), + ParsedTestObservation("Suite.unsupported.cpp", TestStatus.UNSUPPORTED), + ParsedTestObservation("Suite.skipped.cpp", TestStatus.SKIPPED), + ParsedTestObservation("Suite.excluded.cpp", TestStatus.EXCLUDED), + ] + ) + + result = reconcile_test_results(log_data, xml_data) + + self.assertEqual(_count_by_status(result, TestStatus.PASS), 1) + self.assertEqual(_count_by_status(result, TestStatus.UNSUPPORTED), 1) + self.assertEqual(_count_by_status(result, TestStatus.SKIPPED), 1) + self.assertEqual(_count_by_status(result, TestStatus.EXCLUDED), 1) + + def test_xml_does_not_guess_ambiguous_status(self): + log_data = ParsedLogData(tests=[], declared_counts={TestStatus.PASS: 1}) + xml_data = ParsedXMLData( + tests=[ParsedTestObservation("Suite :: pass.cpp", TestStatus.PASS)] + ) + + stderr = io.StringIO() + with redirect_stderr(stderr): + result = reconcile_test_results(log_data, xml_data) + + self.assertEqual(result.tests, []) + self.assertIn("Count mismatch for PASS", stderr.getvalue()) + + def test_xml_fills_in_unambiguous_status(self): + xml_data = ParsedXMLData( + tests=[ + ParsedTestObservation("Suite :: unsupported", TestStatus.UNSUPPORTED) + ] + ) + + result = reconcile_test_results(ParsedLogData(), xml_data) + + self.assertEqual(_count_by_status(result, TestStatus.UNSUPPORTED), 1) + + def test_log_status_takes_precedence_and_xml_provides_duration(self): + statuses = [ + TestStatus.XFAIL, + TestStatus.XPASS, + TestStatus.FLAKYPASS, + TestStatus.FIXED, + ] + log_data = ParsedLogData( + tests=[ + ParsedTestObservation(f"Suite :: test-{index}", status) + for index, status in enumerate(statuses) + ], + declared_counts={status: 1 for status in statuses}, + ) + xml_data = ParsedXMLData( + tests=[ + ParsedTestObservation( + f"Suite :: test-{index}", TestStatus.PASS, 123.4 + index + ) + for index in range(len(statuses)) + ] + ) + + result = reconcile_test_results(log_data, xml_data) + + self.assertEqual([test.status for test in result.tests], statuses) + self.assertEqual(result.tests[0].duration_ms, 123.4) + + def test_unified_runtime_xml_name_matches_log_for_duration(self): + log_data = ParsedLogData( + tests=[ + ParsedTestObservation( + "Unified Runtime :: adapters/level_zero/event.cpp", + TestStatus.PASS, + ) + ], + declared_counts={TestStatus.PASS: 1}, + ) + xml_data = ParsedXMLData( + tests=[ + ParsedTestObservation( + "Unified Runtime :: adapters/level_zero/event.cpp", + TestStatus.PASS, + 250.0, + ) + ] + ) + + result = reconcile_test_results(log_data, xml_data) + + self.assertEqual(result.tests[0].duration_ms, 250.0) + + def test_warns_about_duplicate_test_names(self): + log_data = ParsedLogData( + tests=[ + ParsedTestObservation("Suite :: duplicate", TestStatus.PASS), + ParsedTestObservation("Suite :: duplicate", TestStatus.PASS), + ], + declared_counts={TestStatus.PASS: 2}, + ) + stderr = io.StringIO() + + with redirect_stderr(stderr): + result = reconcile_test_results(log_data) + + self.assertEqual(len(result.tests), 2) + self.assertIn("Duplicate test name in log", stderr.getvalue()) + + def test_warns_about_statistics_count_mismatch(self): + log_data = ParsedLogData( + tests=[ParsedTestObservation("Suite :: pass", TestStatus.PASS)], + statistics={"Passed": 2}, + ) + stderr = io.StringIO() + + with redirect_stderr(stderr): + reconcile_test_results(log_data) + + self.assertIn( + "Count mismatch for PASS: statistics 2, found 1", stderr.getvalue() + ) + + def test_warns_about_unknown_status_statistic(self): + log_data = ParsedLogData(statistics={"New Fancy": 1}) + stderr = io.StringIO() + + with redirect_stderr(stderr): + reconcile_test_results(log_data) + + self.assertIn("Unknown test statistic 'New Fancy'", stderr.getvalue()) + + def test_validates_total_discovered_of_zero(self): + log_data = ParsedLogData( + tests=[ParsedTestObservation("Suite :: pass", TestStatus.PASS)], + statistics={"Total Discovered": 0}, + ) + stderr = io.StringIO() + + with redirect_stderr(stderr): + reconcile_test_results(log_data) + + self.assertIn("Total test mismatch: declared 0, found 1", stderr.getvalue()) + + +class SummaryReporterTest(unittest.TestCase): + def test_displays_skipped_and_unsupported_separately(self): + lines = [ + "Skipped Tests (2):\n", + "suite :: skipped-1\n", + "suite :: skipped-2\n", + "\n", + "Unsupported Tests (3):\n", + "suite :: unsupported-1\n", + "suite :: unsupported-2\n", + "suite :: unsupported-3\n", + "\n", + "Total Discovered Tests: 5\n", + " Skipped: 2\n", + " Unsupported: 3\n", + "Testing Time: 1.25s\n", + ] + stdout = io.StringIO() + + with redirect_stdout(stdout): + result = build_test_run_result(lines) + SummaryReporter(result).generate() + + output = stdout.getvalue() + self.assertIn("::group::Skipped (2)", output) + self.assertIn("::group::Unsupported (3)", output) + self.assertIn(" Skipped: 2", output) + self.assertIn(" Unsupported: 3", output) + self.assertIn("Testing Time: 1.25s", output) + + +class PathValidatorTest(unittest.TestCase): + def test_validates_build_dir_relative_to_workspace_not_cwd(self): + with tempfile.TemporaryDirectory() as workspace: + (Path(workspace) / "build").mkdir() + with tempfile.TemporaryDirectory() as other_directory: + with temporary_cwd(other_directory): + self.assertTrue( + PathValidator.validate_build_dir("build", workspace) + ) + + def test_rejects_build_dir_escaping_workspace(self): + with tempfile.TemporaryDirectory() as workspace: + self.assertFalse(PathValidator.validate_build_dir("../outside", workspace)) + + def test_rejects_absolute_build_dir(self): + with tempfile.TemporaryDirectory() as workspace: + self.assertFalse(PathValidator.validate_build_dir("/etc/passwd", workspace)) + + def test_does_not_reject_dotted_name_without_traversal(self): + with tempfile.TemporaryDirectory() as workspace: + (Path(workspace) / "my..build").mkdir() + self.assertTrue(PathValidator.validate_build_dir("my..build", workspace)) + + +if __name__ == "__main__": + unittest.main() diff --git a/devops/scripts/ur_test_tools/validation/__init__.py b/devops/scripts/ur_test_tools/validation/__init__.py index 081e0c05ae8f7..e69de29bb2d1d 100644 --- a/devops/scripts/ur_test_tools/validation/__init__.py +++ b/devops/scripts/ur_test_tools/validation/__init__.py @@ -1,9 +0,0 @@ -"""Validation package - Security and data validation.""" - -from .path_validator import PathValidator -from .data_validator import validate_test_counts - -__all__ = [ - "PathValidator", - "validate_test_counts", -] diff --git a/devops/scripts/ur_test_tools/validation/data_validator.py b/devops/scripts/ur_test_tools/validation/data_validator.py deleted file mode 100644 index b63cc715f685c..0000000000000 --- a/devops/scripts/ur_test_tools/validation/data_validator.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Test data validation.""" - -import sys -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from ..models.test_data import TestLists - - -def validate_test_counts( - total_discovered: int, - test_lists: "TestLists", - displayed_skipped: int, - displayed_excluded: int, -) -> None: - """Validate test counts match (prints warnings on mismatch).""" - if not total_discovered: - return - - sum_categories = sum(len(tests) for tests in test_lists.values()) - - # Add skipped tests if they were displayed but not in test_lists - if displayed_skipped > 0 and "Skipped" not in test_lists: - sum_categories += displayed_skipped - - # Add excluded tests if they were displayed but not in test_lists - if displayed_excluded > 0 and "Excluded" not in test_lists: - sum_categories += displayed_excluded - - if total_discovered != sum_categories: - print() - print( - f"::warning::Test count mismatch: Total Discovered = " - f"{total_discovered}, but sum of all categories = {sum_categories}" - ) - print() diff --git a/devops/scripts/ur_test_tools/validation/path_validator.py b/devops/scripts/ur_test_tools/validation/path_validator.py index c2094e142ca62..56e1028034301 100644 --- a/devops/scripts/ur_test_tools/validation/path_validator.py +++ b/devops/scripts/ur_test_tools/validation/path_validator.py @@ -1,96 +1,49 @@ """Path validation for security.""" from pathlib import Path -from typing import Optional class PathValidator: - """Validate paths for security and correctness.""" - @staticmethod - def validate_build_dir(build_dir: str, workspace: Optional[str] = None) -> bool: - """Validate build directory is safe and within workspace.""" - if not build_dir or ".." in build_dir or build_dir.startswith("/"): + def validate_build_dir(build_dir: str, workspace: str) -> bool: + """Check build_dir is relative and resolves within workspace.""" + if not build_dir: return False - # Block shell metacharacters, quotes, and control characters - # to prevent injection in f-strings, env vars, and logs - dangerous_chars = {";", "&", "#", "$", "|", "`", "\\", "'", '"', "\n", "\r"} - if any(c in build_dir for c in dangerous_chars): + path = Path(build_dir) + if path.is_absolute(): return False - if workspace: - try: - build_path = Path(build_dir).resolve(strict=False) - workspace_path = Path(workspace).resolve(strict=False) - build_path.relative_to(workspace_path) - return True - except (ValueError, OSError): - return False + workspace_path = Path(workspace).resolve() + resolved = (workspace_path / path).resolve() + try: + resolved.relative_to(workspace_path) + except ValueError: + return False return True @staticmethod def validate_log_path(path: str) -> None: - """Validate log file path (detects path traversal).""" - try: - # Resolve path to detect encoded forms of path traversal (e.g., %2e%2e) - resolved = Path(path).resolve(strict=False) - - # Check for path traversal in original string (simple check) - if ".." in path: - raise ValueError( - f"Invalid log file path (path traversal not allowed): {path}" - ) - - # Verify file exists - if not resolved.exists(): - raise ValueError(f"Log file not found: {path}") - except (OSError, ValueError) as e: - if isinstance(e, ValueError): - raise - raise OSError(f"Invalid log file path: {path} ({e})") from e + """Validate log file path has no traversal and exists.""" + if ".." in Path(path).parts: + raise ValueError(f"Invalid log file path (path traversal): {path}") + if not Path(path).exists(): + raise ValueError(f"Log file not found: {path}") @staticmethod def validate_optional_path( path: str, path_type: str, allow_absolute: bool = False ) -> str: - """Validate optional file path.""" + """Validate optional file path has no traversal and matches absolute-path policy.""" if not path: return "" - try: - # Resolve path to detect encoded forms of path traversal - Path(path).resolve(strict=False) - - # Check for path traversal in original string - if ".." in path: - raise ValueError( - f"Invalid {path_type} file path (path traversal): {path}" - ) + if ".." in Path(path).parts: + raise ValueError(f"Invalid {path_type} file path (path traversal): {path}") - # Check absolute path restriction - if not allow_absolute and path.startswith("/"): - raise ValueError( - f"Invalid {path_type} file path " - f"(absolute paths not allowed): {path}" - ) - except (OSError, ValueError) as e: - if isinstance(e, ValueError): - raise - raise OSError(f"Invalid {path_type} file path: {path} ({e})") from e + if not allow_absolute and Path(path).is_absolute(): + raise ValueError( + f"Invalid {path_type} file path (absolute paths not allowed): {path}" + ) return path - - @staticmethod - def ensure_within_workspace(path: Path, workspace: Path) -> Path: - """Ensure path is within workspace.""" - resolved = path.resolve() - workspace_resolved = workspace.resolve() - - try: - resolved.relative_to(workspace_resolved) - return resolved - except ValueError as e: - raise ValueError( - f"Path outside workspace: {path} not in {workspace}" - ) from e diff --git a/unified-runtime/test/CMakeLists.txt b/unified-runtime/test/CMakeLists.txt index 3fc44c1a9ef50..3429322b925d8 100644 --- a/unified-runtime/test/CMakeLists.txt +++ b/unified-runtime/test/CMakeLists.txt @@ -86,26 +86,18 @@ function(add_ur_lit_testsuite suite) if(UR_STANDALONE_BUILD) add_custom_target(${TARGET} - # LIT invocation for standalone builds - # - # Keep reporting-related options aligned with LIT_COMMON_REPORTING_OPTIONS - # in devops/scripts/ur_test_tools/constants.py. - # - # The invocation is intentionally not identical to CI: - # - Standalone tests use --succinct for developer-friendly interactive output - # - CI uses -v because detailed output is consumed by reporting tools + # Reporting options kept aligned with LIT_COMMON_REPORTING_OPTIONS in + # devops/scripts/ur_test_tools/constants.py; --succinct differs from CI's -v. COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" - # Reporting options (keep aligned with LIT_COMMON_REPORTING_OPTIONS) --show-unsupported --show-pass --show-xfail --time-tests --show-flakypass --show-skipped - # Standalone-specific: interactive developer feedback + --show-excluded --no-progress-bar --succinct - # Execution policy --timeout 120 -j 50 USES_TERMINAL From ebbd0e705c42926e8d021ac70fa11d79589825f1 Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Thu, 3 Sep 2026 19:45:16 +0200 Subject: [PATCH 11/11] Add compact GitHub step summary for test results --- devops/actions/run-tests/ur/action.yml | 6 ++ devops/scripts/ur_test_tools/cli.py | 14 ++- .../ur_test_tools/summary_generator.py | 86 +++++++++++++++---- .../tests/test_result_reconciliation.py | 50 +++++++++++ 4 files changed, 139 insertions(+), 17 deletions(-) diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml index ac7732d963f4e..0e161f6970846 100644 --- a/devops/actions/run-tests/ur/action.yml +++ b/devops/actions/run-tests/ur/action.yml @@ -66,6 +66,7 @@ runs: LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }} XML_FILE: ${{ steps.run_tests.outputs['xml-file'] }} ARTIFACT_NAME: ${{ inputs.artifact_name }} + TEST_TYPE: ${{ inputs.test_type }} WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | if [ -z "$LOG_FILE" ]; then @@ -93,6 +94,11 @@ runs: echo "$WORKFLOW_URL" fi + if [ -n "$GITHUB_STEP_SUMMARY" ]; then + echo "### ${TEST_TYPE} tests" >> "$GITHUB_STEP_SUMMARY" + devops/scripts/ur-test step-summary "$LOG_FILE" "${XML_FILE:-}" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Copy artifacts if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} shell: bash diff --git a/devops/scripts/ur_test_tools/cli.py b/devops/scripts/ur_test_tools/cli.py index d87b1738876e9..53ffcfc943526 100644 --- a/devops/scripts/ur_test_tools/cli.py +++ b/devops/scripts/ur_test_tools/cli.py @@ -30,8 +30,11 @@ def main() -> int: if command == "run": return main_ci_utils("run-tests") - elif command in ("summary", "extract-errors", "filter-log"): - internal_cmd = "show-summary" if command == "summary" else command + elif command in ("summary", "step-summary", "extract-errors", "filter-log"): + internal_cmd = { + "summary": "show-summary", + "step-summary": "show-step-summary", + }.get(command, command) return main_test_summary(internal_cmd) else: @@ -68,6 +71,13 @@ def main_test_summary(command: str) -> int: result = build_test_run_result(lines, xml_file or None) SummaryReporter(result).generate() + elif command == "show-step-summary": + xml_file = PathValidator.validate_optional_path( + sys.argv[3] if len(sys.argv) > 3 else "", "XML", allow_absolute=True + ) + result = build_test_run_result(lines, xml_file or None) + SummaryReporter(result).generate_github_step_summary() + else: print(f"Error: Unknown command '{command}'", file=sys.stderr) return 1 diff --git a/devops/scripts/ur_test_tools/summary_generator.py b/devops/scripts/ur_test_tools/summary_generator.py index cf9f0e47ecbc8..ed02323e3e23c 100644 --- a/devops/scripts/ur_test_tools/summary_generator.py +++ b/devops/scripts/ur_test_tools/summary_generator.py @@ -4,6 +4,31 @@ from .models.test_results import TestResult, TestStatus, TestRunResult +# Failure-first order used when listing test groups. +_STATUS_DISPLAY_ORDER = [ + TestStatus.FAIL, + TestStatus.TIMEOUT, + TestStatus.UNRESOLVED, + TestStatus.XPASS, + TestStatus.SKIPPED, + TestStatus.UNSUPPORTED, + TestStatus.EXCLUDED, + TestStatus.XFAIL, + TestStatus.FIXED, + TestStatus.FLAKYPASS, + TestStatus.PASS, +] + +# Statuses worth listing by name in the compact GitHub step summary. +_STEP_SUMMARY_LISTED_STATUSES = [ + TestStatus.FAIL, + TestStatus.XPASS, + TestStatus.TIMEOUT, + TestStatus.UNRESOLVED, +] + +_STEP_SUMMARY_MAX_LISTED_TESTS = 50 + class SummaryReporter: def __init__(self, result: TestRunResult): @@ -24,21 +49,7 @@ def _display_statistics(self) -> None: def _display_test_groups(self) -> None: grouped = self.result.group_by_status() - priority_order = [ - TestStatus.FAIL, - TestStatus.TIMEOUT, - TestStatus.UNRESOLVED, - TestStatus.XPASS, - TestStatus.SKIPPED, - TestStatus.UNSUPPORTED, - TestStatus.EXCLUDED, - TestStatus.XFAIL, - TestStatus.FIXED, - TestStatus.FLAKYPASS, - TestStatus.PASS, - ] - - for status in priority_order: + for status in _STATUS_DISPLAY_ORDER: tests = grouped.get(status, []) if not tests: continue @@ -83,3 +94,48 @@ def _display_timing(self) -> None: print(line) print("::endgroup::") + + def generate_github_step_summary(self) -> None: + """Print a compact summary for $GITHUB_STEP_SUMMARY. + + Unlike generate(), this lists only statuses that need attention + (Failed, Unexpectedly Passed, Timed Out, Unresolved) and shows + all other statuses as counts only, to keep the workflow run page + readable regardless of how many tests were discovered. + """ + result = self.result + grouped = result.group_by_status() + + print("```") + + for status in _STEP_SUMMARY_LISTED_STATUSES: + tests = grouped.get(status, []) + if not tests: + continue + + print(f"{status.display_label} Tests ({len(tests)}):") + for test in tests[:_STEP_SUMMARY_MAX_LISTED_TESTS]: + print(test.name) + if len(tests) > _STEP_SUMMARY_MAX_LISTED_TESTS: + remaining = len(tests) - _STEP_SUMMARY_MAX_LISTED_TESTS + print(f"... and {remaining} more") + print() + + if result.testing_time_ms is not None: + print(f"Testing Time: {result.testing_time_ms / 1000.0:.2f}s") + print() + + total = result.total_discovered or len(result.tests) + counted_statuses = [s for s in _STATUS_DISPLAY_ORDER if grouped.get(s)] + if total and counted_statuses: + print(f"Total Discovered Tests: {total}") + label_width = max(len(s.display_label) for s in counted_statuses) + for status in counted_statuses: + count = len(grouped[status]) + percentage = count / total * 100 + print( + f"{status.display_label:<{label_width}} : " + f"{count} ({percentage:.2f}%)" + ) + + print("```") diff --git a/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py b/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py index 1ad621a13c57f..65326233c091c 100644 --- a/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py +++ b/devops/scripts/ur_test_tools/tests/test_result_reconciliation.py @@ -380,6 +380,56 @@ def test_displays_skipped_and_unsupported_separately(self): self.assertIn(" Unsupported: 3", output) self.assertIn("Testing Time: 1.25s", output) + def test_step_summary_lists_only_statuses_needing_attention(self): + lines = [ + "Failed Tests (1):\n", + "suite :: fail.cpp\n", + "\n", + "Passed Tests (2):\n", + "suite :: pass-1.cpp\n", + "suite :: pass-2.cpp\n", + "\n", + "Unsupported Tests (1):\n", + "suite :: unsupported.cpp\n", + "\n", + "Total Discovered Tests: 4\n", + " Failed: 1\n", + " Passed: 2\n", + " Unsupported: 1\n", + "Testing Time: 1.50s\n", + ] + stdout = io.StringIO() + + with redirect_stdout(stdout): + result = build_test_run_result(lines) + SummaryReporter(result).generate_github_step_summary() + + output = stdout.getvalue() + self.assertIn("Failed Tests (1):", output) + self.assertIn("suite :: fail.cpp", output) + self.assertNotIn("suite :: pass-1.cpp", output) + self.assertNotIn("suite :: unsupported.cpp", output) + self.assertIn("Testing Time: 1.50s", output) + self.assertIn("Total Discovered Tests: 4", output) + self.assertIn("(25.00%)", output) + self.assertIn("(50.00%)", output) + + def test_step_summary_caps_listed_tests(self): + lines = ["Failed Tests (60):\n"] + lines.extend(f"suite :: fail-{i}.cpp\n" for i in range(60)) + lines.append("\n") + lines.append("Total Discovered Tests: 60\n") + + stdout = io.StringIO() + with redirect_stdout(stdout): + result = build_test_run_result(lines) + SummaryReporter(result).generate_github_step_summary() + + output = stdout.getvalue() + self.assertIn("suite :: fail-49.cpp", output) + self.assertNotIn("suite :: fail-50.cpp", output) + self.assertIn("... and 10 more", output) + class PathValidatorTest(unittest.TestCase): def test_validates_build_dir_relative_to_workspace_not_cwd(self):