[UR][CI] Add unified test result reporting for UR tests

.github/workflows/ur-build-hw.yml

@@ -157,21 +157,48 @@ jobs:
       # This is to check that install command does not fail
       run: cmake --install build
 
-    - name: Test adapter specific
+    - name: Build artifact name suffix
+      id: artifact_suffix
+      shell: bash
       env:
-        ZE_ENABLE_LOADER_DEBUG_TRACE: 1
-        LIT_OPTS: "--timeout 120 -j 50"
-        # These tests cause timeouts on CI
-        LIT_FILTER_OUT: "(adapters/level_zero/memcheck.test|adapters/level_zero/v2/deferred_kernel_memcheck.test)"
-      run: cmake --build build -j $(($(nproc)/3)) -- check-unified-runtime-adapter
+        RUNNER_NAME: ${{ inputs.runner_name }}
+        STATIC_LOADER: ${{ matrix.adapter.static_Loader }}
+        OTHER_NAME: ${{ matrix.adapter.other_name }}
+      run: |
+        RUNNER_SUFFIX=$(echo "${RUNNER_NAME}" | sed 's/^UR_//')
+        SUFFIX="${RUNNER_SUFFIX}"
+
+        if [ "${STATIC_LOADER}" = "ON" ]; then
+          SUFFIX="${SUFFIX}-static"
+        fi
+
+        if [ -n "${OTHER_NAME}" ]; then
+          SUFFIX="${SUFFIX}-${OTHER_NAME}"
+        fi
+
+        echo "suffix=${SUFFIX}" >> $GITHUB_OUTPUT
+
+    - name: Test adapter specific
       # Don't run adapter specific tests when building multiple adapters
       if: ${{ matrix.adapter.other_name == '' }}
+      env:
+        # Enable Level Zero loader debug trace for L0 adapters only
+        ZE_ENABLE_LOADER_DEBUG_TRACE: ${{ startsWith(matrix.adapter.name, 'L0') && '1' || '' }}
+      uses: ./devops/actions/run-tests/ur
+      with:
+        test_type: 'adapter-specific'
+        build_dir: build
+        artifact_name: 'ur-${{matrix.adapter.name}}-${{steps.artifact_suffix.outputs.suffix}}-adapter-specific'
 
     - name: Test adapters
       env:
-        ZE_ENABLE_LOADER_DEBUG_TRACE: 1
-        LIT_OPTS: "--timeout 120 -j 50"
-      run: cmake --build build -j $(($(nproc)/3)) -- check-unified-runtime-conformance
+        # Enable Level Zero loader debug trace for L0 adapters only
+        ZE_ENABLE_LOADER_DEBUG_TRACE: ${{ startsWith(matrix.adapter.name, 'L0') && '1' || '' }}
+      uses: ./devops/actions/run-tests/ur
+      with:
+        test_type: 'conformance'
+        build_dir: build
+        artifact_name: 'ur-${{matrix.adapter.name}}-${{steps.artifact_suffix.outputs.suffix}}-conformance'
 
     - name: Debug CI platform information
       if: ${{ always() }}

devops/actions/run-tests/ur/action.yml

@@ -0,0 +1,129 @@
+name: 'Run UR Tests'
+
+inputs:
+  test_type:
+    description: 'Type of tests to run (adapter-specific or conformance)'
+    required: true
+  build_dir:
+    required: false
+    default: 'build'
+  artifact_name:
+    description: 'Name for uploaded test artifacts (logs, XML)'
+    required: false
+    default: ''
+
+runs:
+  using: "composite"
+  steps:
+  - name: Run ${{ inputs.test_type }} tests
+    id: run_tests
+    continue-on-error: true
+    shell: bash
+    env:
+      BUILD_DIR: ${{ inputs.build_dir }}
+      TEST_TYPE: ${{ inputs.test_type }}
+    run: |
+      devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE"
+
+  - name: Filter test log
+    if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }}
+    shell: bash
+    env:
+      LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }}
+    run: |
+      if [ -z "$LOG_FILE" ]; then
+        echo "::warning::Test log path not provided by runner; skipping log filtering."
+        exit 0
+      fi
+
+      if [ ! -f "$LOG_FILE" ]; then
+        echo "::warning::Test log '$LOG_FILE' does not exist; skipping log filtering."
+        exit 0
+      fi
+
+      echo "::group::Test Log"
+      devops/scripts/ur-test filter-log "$LOG_FILE" || { echo "::error::Filter failed"; head -1000 "$LOG_FILE"; }
+      echo "::endgroup::"
+
+  - name: Report test failures
+    if: steps.run_tests.outcome != 'success'
+    shell: bash
+    env:
+      LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }}
+    run: |
+      if [ -z "$LOG_FILE" ] || [ ! -f "$LOG_FILE" ]; then
+        echo "::warning::Test execution failed, but no test log is available."
+        exit 1
+      fi
+
+      devops/scripts/ur-test extract-errors "$LOG_FILE"
+      exit 1
+
+  - name: Test summary
+    if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }}
+    shell: bash
+    env:
+      LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }}
+      XML_FILE: ${{ steps.run_tests.outputs['xml-file'] }}
+      ARTIFACT_NAME: ${{ inputs.artifact_name }}
+      TEST_TYPE: ${{ inputs.test_type }}
+      WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    run: |
+      if [ -z "$LOG_FILE" ]; then
+        echo "::warning::Test log path not provided by runner; skipping summary."
+        exit 0
+      fi
+
+      if [ ! -f "$LOG_FILE" ]; then
+        echo "::warning::Test log '$LOG_FILE' does not exist; skipping summary."
+        exit 0
+      fi
+
+      if [ ! -s "$LOG_FILE" ]; then
+        echo "::warning::Test log is empty; skipping summary."
+        exit 0
+      fi
+
+      devops/scripts/ur-test summary "$LOG_FILE" "${XML_FILE:-}"
+
+      if [ -n "$ARTIFACT_NAME" ]; then
+        echo ""
+        echo "---"
+        echo ""
+        echo "Full logs and XML are uploaded as workflow artifacts:"
+        echo "$WORKFLOW_URL"
+      fi
+
+      if [ -n "$GITHUB_STEP_SUMMARY" ]; then
+        echo "### ${TEST_TYPE} tests" >> "$GITHUB_STEP_SUMMARY"
+        devops/scripts/ur-test step-summary "$LOG_FILE" "${XML_FILE:-}" >> "$GITHUB_STEP_SUMMARY"
+      fi
+
+  - name: Copy artifacts
+    if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }}
+    shell: bash
+    env:
+      LOG_FILE: ${{ steps.run_tests.outputs['log-file'] }}
+      XML_FILE: ${{ steps.run_tests.outputs['xml-file'] }}
+    run: |
+      set -f  # Disable glob expansion
+
+      rm -rf test_artifacts
+      mkdir -p test_artifacts
+
+      if [ -f "$LOG_FILE" ]; then
+        cp "$LOG_FILE" test_artifacts/
+      fi
+
+      if [ -n "$XML_FILE" ] && [ -f "$XML_FILE" ]; then
+        cp "$XML_FILE" test_artifacts/
+      fi
+
+  - name: Upload test artifacts
+    if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }}
+    uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+    with:
+      name: ${{ inputs.artifact_name }}
+      path: test_artifacts/
+      retention-days: 7
+      if-no-files-found: warn

devops/scripts/ur-test

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+"""Unified Runtime test management CLI."""
+
+import sys
+from ur_test_tools.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

devops/scripts/ur_test_tools/cli.py

@@ -0,0 +1,151 @@
+"""CLI entry points for UR test tools."""
+
+import sys
+import os
+from pathlib import Path
+
+from .models.config import TestConfig, TestExecutionContext
+from .validation.path_validator import PathValidator
+from .parsers.log_parser import (
+    LITLogParser,
+    read_log_file,
+)
+from .outputs.console import filter_log_for_display
+from .result_builder import build_test_run_result
+from .summary_generator import SummaryReporter
+from .test_runner import (
+    TestRunner,
+    get_test_config,
+)
+from .outputs import github_actions
+
+
+def main() -> int:
+    if len(sys.argv) < 2:
+        print("Error: Missing command", file=sys.stderr)
+        return 1
+
+    command = sys.argv[1]
+
+    if command == "run":
+        return main_ci_utils("run-tests")
+
+    elif command in ("summary", "step-summary", "extract-errors", "filter-log"):
+        internal_cmd = {
+            "summary": "show-summary",
+            "step-summary": "show-step-summary",
+        }.get(command, command)
+        return main_test_summary(internal_cmd)
+
+    else:
+        print(f"Error: Unknown command '{command}'", file=sys.stderr)
+        return 1
+
+
+def main_test_summary(command: str) -> int:
+    try:
+        if len(sys.argv) < 3:
+            print(
+                f"Error: {sys.argv[0]} <command> <log_file> [xml_file]",
+                file=sys.stderr,
+            )
+            return 1
+
+        log_file = sys.argv[2]
+        PathValidator.validate_log_path(log_file)
+        lines = read_log_file(log_file)
+        parser = LITLogParser(lines)
+
+        if command == "extract-errors":
+            for line in parser.extract_error_details():
+                print(line, end="")
+
+        elif command == "filter-log":
+            for line in filter_log_for_display(lines):
+                print(line, end="")
+
+        elif command == "show-summary":
+            xml_file = PathValidator.validate_optional_path(
+                sys.argv[3] if len(sys.argv) > 3 else "", "XML", allow_absolute=True
+            )
+            result = build_test_run_result(lines, xml_file or None)
+            SummaryReporter(result).generate()
+
+        elif command == "show-step-summary":
+            xml_file = PathValidator.validate_optional_path(
+                sys.argv[3] if len(sys.argv) > 3 else "", "XML", allow_absolute=True
+            )
+            result = build_test_run_result(lines, xml_file or None)
+            SummaryReporter(result).generate_github_step_summary()
+
+        else:
+            print(f"Error: Unknown command '{command}'", file=sys.stderr)
+            return 1
+
+        return 0
+
+    except (OSError, ValueError) as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+
+def main_ci_utils(command: str) -> int:
+    if command == "run-tests":
+        return _run_tests_command()
+
+    else:
+        print(f"Error: Unknown command '{command}'", file=sys.stderr)
+        return 1
+
+
+def _run_tests_command() -> int:
+    if len(sys.argv) < 5:
+        print(
+            f"Error: run-tests <test_type> <build_dir> <workspace>",
+            file=sys.stderr,
+        )
+        return 1
+
+    test_type = sys.argv[2]
+    build_dir = sys.argv[3]
+    workspace = sys.argv[4]
+
+    if not PathValidator.validate_build_dir(build_dir, workspace):
+        github_actions.print_error("Invalid build_dir")
+        return 1
+
+    try:
+        config = get_test_config(test_type)
+    except ValueError as e:
+        github_actions.print_error(str(e))
+        return 1
+
+    workspace_path = Path(workspace).resolve()
+    build_dir_path = workspace_path / build_dir
+
+    xml_output_name = f"{test_type.replace('-', '_')}_results.xml"
+    xml_output_path = (build_dir_path / xml_output_name).absolute()
+    xml_output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    log_file_path = workspace_path / config.log_file
+
+    env = os.environ.copy()
+
+    context = TestExecutionContext(
+        test_type=test_type,
+        build_dir=build_dir_path,
+        workspace=workspace_path,
+        xml_output_path=xml_output_path,
+        log_file_path=log_file_path,
+        config=config,
+        env=env,
+    )
+
+    try:
+        context.validate()
+    except ValueError as e:
+        github_actions.print_error(str(e))
+        return 1
+
+    runner = TestRunner(context)
+    return runner.run()

devops/scripts/ur_test_tools/constants.py

@@ -0,0 +1,46 @@
+"""Constants for UR test tools."""
+
+import re
+
+# Conservative upper bound for locating the LIT "Testing:" marker.
+# Non-verbose CI runs normally emit it much earlier.
+MAX_LINES_TO_SCAN = 1000
+
+SEPARATOR_WIDTH = 70
+
+# Keep reporting options aligned with unified-runtime/test/CMakeLists.txt.
+LIT_COMMON_REPORTING_OPTIONS = [
+    "--show-unsupported",
+    "--show-pass",
+    "--show-xfail",
+    "--time-tests",
+    "--show-flakypass",
+    "--show-skipped",
+    "--show-excluded",
+]
+
+LIT_CI_OPTIONS = [
+    "-v",
+    "--no-progress-bar",
+]
+
+DEFAULT_LIT_TIMEOUT = 120
+DEFAULT_LIT_JOBS = 50
+
+TEST_TYPE_ADAPTER_SPECIFIC = "adapter-specific"
+TEST_TYPE_CONFORMANCE = "conformance"
+
+# These tests cause timeouts on CI and are excluded from adapter-specific runs
+LIT_FILTER_OUT_ADAPTER_SPECIFIC = (
+    "(adapters/level_zero/memcheck.test|"
+    "adapters/level_zero/v2/deferred_kernel_memcheck.test)"
+)
+
+TEST_NOT_SELECTED_MSG = "Test not selected"
+SLOWEST_TESTS_HEADER = "Slowest Tests:"
+TEST_TIMES_HEADERS = ("Tests Times:", "Test Times:")
+
+FAIL_TIMEOUT_PATTERN = re.compile(r"^(FAIL|TIMEOUT):")
+TEST_CATEGORY_PATTERN = re.compile(r"^([A-Za-z]+(?: [A-Za-z]+)*) Tests \((\d+)\):")
+TESTING_TIME_PATTERN = re.compile(r"^\s*Testing Time:\s*([0-9.]+)s")
+STAT_LINE_PATTERN = re.compile(r"^\s*(.+?)\s*:\s*(\d+)(?:\s|$)")