diff --git a/.github/actions/download-td-artifacts/action.yml b/.github/actions/download-td-artifacts/action.yml new file mode 100644 index 000000000000..595093abaead --- /dev/null +++ b/.github/actions/download-td-artifacts/action.yml @@ -0,0 +1,29 @@ +name: Download TD Artifacts + +description: Download artifacts from target_determination.yml + +inputs: + use-gha: + description: If set to any value, use GHA to download the artifact. Otherwise use s3. + required: false + +runs: + using: composite + steps: + - name: Download TD Artifacts from S3 + if: ${{ !inputs.use-gha }} + uses: seemethere/download-artifact-s3@v4 + with: + name: td_results + + - name: Download TD Artifacts from GHA + if: inputs.use-gha + uses: actions/download-artifact@v3 + with: + name: td_results.json + + - name: Move artifacts to .additional_ci_files folder + shell: bash + run: | + mkdir -p .additional_ci_files + mv td_results.json .additional_ci_files/td_results.json diff --git a/.github/workflows/_linux-test.yml b/.github/workflows/_linux-test.yml index 1715d0cdcdb9..9e41f8faa46f 100644 --- a/.github/workflows/_linux-test.yml +++ b/.github/workflows/_linux-test.yml @@ -117,6 +117,10 @@ jobs: with: name: ${{ inputs.build-environment }} + - name: Download TD artifacts + continue-on-error: true + uses: ./.github/actions/download-td-artifacts + - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py diff --git a/.github/workflows/_mac-test.yml b/.github/workflows/_mac-test.yml index 8d4091d14a0c..b8e90771ec73 100644 --- a/.github/workflows/_mac-test.yml +++ b/.github/workflows/_mac-test.yml @@ -91,6 +91,12 @@ jobs: name: ${{ inputs.build-environment }} use-gha: true + - name: Download TD artifacts + continue-on-error: true + uses: ./.github/actions/download-td-artifacts + with: + use-gha: true + - name: Setup miniconda uses: pytorch/test-infra/.github/actions/setup-miniconda@main with: diff --git a/.github/workflows/_rocm-test.yml b/.github/workflows/_rocm-test.yml index b807aca6f0f0..1f2d86273ee1 100644 --- a/.github/workflows/_rocm-test.yml +++ b/.github/workflows/_rocm-test.yml @@ -103,6 +103,10 @@ jobs: with: name: ${{ inputs.build-environment }} + - name: Download TD artifacts + continue-on-error: true + uses: ./.github/actions/download-td-artifacts + - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py diff --git a/.github/workflows/_win-test.yml b/.github/workflows/_win-test.yml index a5a2548e0808..ebc8434407a7 100644 --- a/.github/workflows/_win-test.yml +++ b/.github/workflows/_win-test.yml @@ -114,6 +114,10 @@ jobs: run: | tree /F C:\$Env:GITHUB_RUN_ID\build-results + - name: Download TD artifacts + continue-on-error: true + uses: ./.github/actions/download-td-artifacts + - name: Get workflow job id id: get-job-id uses: ./.github/actions/get-workflow-job-id diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml index 18f7409fdf5e..99f4dd99395f 100644 --- a/.github/workflows/periodic.yml +++ b/.github/workflows/periodic.yml @@ -23,6 +23,13 @@ concurrency: permissions: read-all jobs: + target-determination: + name: before-test + uses: ./.github/workflows/target_determination.yml + permissions: + id-token: write + contents: read + parallelnative-linux-jammy-py3_8-gcc11-build: name: parallelnative-linux-jammy-py3.8-gcc11 uses: ./.github/workflows/_linux-build.yml @@ -39,7 +46,9 @@ jobs: parallelnative-linux-jammy-py3_8-gcc11-test: name: parallelnative-linux-jammy-py3.8-gcc11 uses: ./.github/workflows/_linux-test.yml - needs: parallelnative-linux-jammy-py3_8-gcc11-build + needs: + - parallelnative-linux-jammy-py3_8-gcc11-build + - target-determination with: build-environment: parallelnative-linux-jammy-py3.8-gcc11 docker-image: ${{ needs.parallelnative-linux-jammy-py3_8-gcc11-build.outputs.docker-image }} @@ -86,7 +95,9 @@ jobs: linux-focal-cuda11_8-py3_10-gcc9-debug-test: name: linux-focal-cuda11.8-py3.10-gcc9-debug uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda11_8-py3_10-gcc9-debug-build + needs: + - linux-focal-cuda11_8-py3_10-gcc9-debug-build + - target-determination with: build-environment: linux-focal-cuda11.8-py3.10-gcc9-debug docker-image: ${{ needs.linux-focal-cuda11_8-py3_10-gcc9-debug-build.outputs.docker-image }} @@ -110,7 +121,9 @@ jobs: win-vs2019-cuda11_8-py3-test: name: win-vs2019-cuda11.8-py3 uses: ./.github/workflows/_win-test.yml - needs: win-vs2019-cuda11_8-py3-build + needs: + - win-vs2019-cuda11_8-py3-build + - target-determination with: build-environment: win-vs2019-cuda11.8-py3 cuda-version: "11.8" @@ -214,7 +227,9 @@ jobs: contents: read name: linux-focal-rocm6.0-py3.8 uses: ./.github/workflows/_rocm-test.yml - needs: linux-focal-rocm6_0-py3_8-build + needs: + - linux-focal-rocm6_0-py3_8-build + - target-determination with: build-environment: linux-focal-rocm6.0-py3.8 docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }} diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 214463ec8230..1e07c6389752 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -20,6 +20,13 @@ concurrency: permissions: read-all jobs: + target-determination: + name: before-test + uses: ./.github/workflows/target_determination.yml + permissions: + id-token: write + contents: read + linux-jammy-py3_8-gcc11-build: name: linux-jammy-py3.8-gcc11 uses: ./.github/workflows/_linux-build.yml @@ -41,7 +48,9 @@ jobs: linux-jammy-py3_8-gcc11-test: name: linux-jammy-py3.8-gcc11 uses: ./.github/workflows/_linux-test.yml - needs: linux-jammy-py3_8-gcc11-build + needs: + - linux-jammy-py3_8-gcc11-build + - target-determination with: build-environment: linux-jammy-py3.8-gcc11 docker-image: ${{ needs.linux-jammy-py3_8-gcc11-build.outputs.docker-image }} @@ -97,7 +106,9 @@ jobs: linux-jammy-py3_10-clang15-asan-test: name: linux-jammy-py3.10-clang15-asan uses: ./.github/workflows/_linux-test.yml - needs: linux-jammy-py3_10-clang15-asan-build + needs: + - linux-jammy-py3_10-clang15-asan-build + - target-determination with: build-environment: linux-jammy-py3.10-clang15-asan docker-image: ${{ needs.linux-jammy-py3_10-clang15-asan-build.outputs.docker-image }} @@ -119,7 +130,9 @@ jobs: linux-focal-py3_8-clang10-onnx-test: name: linux-focal-py3.8-clang10-onnx uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-py3_8-clang10-onnx-build + needs: + - linux-focal-py3_8-clang10-onnx-build + - target-determination with: build-environment: linux-focal-py3.8-clang10-onnx docker-image: ${{ needs.linux-focal-py3_8-clang10-onnx-build.outputs.docker-image }} @@ -146,7 +159,9 @@ jobs: linux-focal-py3_8-clang10-test: name: linux-focal-py3.8-clang10 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-py3_8-clang10-build + needs: + - linux-focal-py3_8-clang10-build + - target-determination with: build-environment: linux-focal-py3.8-clang10 docker-image: ${{ needs.linux-focal-py3_8-clang10-build.outputs.docker-image }} @@ -173,7 +188,9 @@ jobs: linux-focal-py3_11-clang10-test: name: linux-focal-py3.11-clang10 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-py3_11-clang10-build + needs: + - linux-focal-py3_11-clang10-build + - target-determination with: build-environment: linux-focal-py3.11-clang10 docker-image: ${{ needs.linux-focal-py3_11-clang10-build.outputs.docker-image }} @@ -218,7 +235,9 @@ jobs: linux-focal-cuda11_8-py3_10-gcc9-test: name: linux-focal-cuda11.8-py3.10-gcc9 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda11_8-py3_10-gcc9-build + needs: + - linux-focal-cuda11_8-py3_10-gcc9-build + - target-determination with: timeout-minutes: 360 build-environment: linux-focal-cuda11.8-py3.10-gcc9 @@ -244,7 +263,9 @@ jobs: linux-focal-cuda12_1-py3_10-gcc9-test: name: linux-focal-cuda12.1-py3.10-gcc9 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda12_1-py3_10-gcc9-build + needs: + - linux-focal-cuda12_1-py3_10-gcc9-build + - target-determination with: timeout-minutes: 360 build-environment: linux-focal-cuda12.1-py3.10-gcc9 @@ -415,7 +436,9 @@ jobs: linux-focal-cuda12_1-py3_10-gcc9-sm86-test: name: linux-focal-cuda12.1-py3.10-gcc9-sm86 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda12_1-py3_10-gcc9-sm86-build + needs: + - linux-focal-cuda12_1-py3_10-gcc9-sm86-build + - target-determination with: build-environment: linux-focal-cuda12.1-py3.10-gcc9-sm86 docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-sm86-build.outputs.docker-image }} diff --git a/.github/workflows/rocm.yml b/.github/workflows/rocm.yml index d724fdb37d1e..24542c3ddc47 100644 --- a/.github/workflows/rocm.yml +++ b/.github/workflows/rocm.yml @@ -18,6 +18,13 @@ concurrency: permissions: read-all jobs: + target-determination: + name: before-test + uses: ./.github/workflows/target_determination.yml + permissions: + id-token: write + contents: read + linux-focal-rocm6_0-py3_8-build: name: linux-focal-rocm6.0-py3.8 uses: ./.github/workflows/_linux-build.yml @@ -41,7 +48,9 @@ jobs: contents: read name: linux-focal-rocm6.0-py3.8 uses: ./.github/workflows/_rocm-test.yml - needs: linux-focal-rocm6_0-py3_8-build + needs: + - linux-focal-rocm6_0-py3_8-build + - target-determination with: build-environment: linux-focal-rocm6.0-py3.8 docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }} diff --git a/.github/workflows/slow.yml b/.github/workflows/slow.yml index 9c3a52f7537d..33577986f643 100644 --- a/.github/workflows/slow.yml +++ b/.github/workflows/slow.yml @@ -21,6 +21,13 @@ concurrency: permissions: read-all jobs: + target-determination: + name: before-test + uses: ./.github/workflows/target_determination.yml + permissions: + id-token: write + contents: read + linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build: name: linux-focal-cuda12.1-py3-gcc9-slow-gradcheck uses: ./.github/workflows/_linux-build.yml @@ -39,7 +46,9 @@ jobs: linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-test: name: linux-focal-cuda12.1-py3-gcc9-slow-gradcheck uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build + needs: + - linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build + - target-determination with: build-environment: linux-focal-cuda12.1-py3-gcc9-slow-gradcheck docker-image: ${{ needs.linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build.outputs.docker-image }} @@ -62,7 +71,9 @@ jobs: linux-focal-cuda12_1-py3_10-gcc9-sm86-test: name: linux-focal-cuda12.1-py3.10-gcc9-sm86 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda12_1-py3_10-gcc9-sm86-build + needs: + - linux-focal-cuda12_1-py3_10-gcc9-sm86-build + - target-determination with: build-environment: linux-focal-cuda12.1-py3.10-gcc9-sm86 docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-sm86-build.outputs.docker-image }} @@ -82,7 +93,9 @@ jobs: linux-focal-py3_8-clang10-test: name: linux-focal-py3.8-clang10 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-py3_8-clang10-build + needs: + - linux-focal-py3_8-clang10-build + - target-determination with: build-environment: linux-focal-py3.8-clang10 docker-image: ${{ needs.linux-focal-py3_8-clang10-build.outputs.docker-image }} @@ -105,7 +118,9 @@ jobs: contents: read name: linux-focal-rocm6.0-py3.8 uses: ./.github/workflows/_rocm-test.yml - needs: linux-focal-rocm6_0-py3_8-build + needs: + - linux-focal-rocm6_0-py3_8-build + - target-determination with: build-environment: linux-focal-rocm6.0-py3.8 docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }} @@ -127,7 +142,9 @@ jobs: linux-jammy-py3_10-clang15-asan-test: name: linux-jammy-py3.10-clang15-asan uses: ./.github/workflows/_linux-test.yml - needs: linux-jammy-py3_10-clang15-asan-build + needs: + - linux-jammy-py3_10-clang15-asan-build + - target-determination with: build-environment: linux-jammy-py3.10-clang15-asan docker-image: ${{ needs.linux-jammy-py3_10-clang15-asan-build.outputs.docker-image }} diff --git a/.github/workflows/target_determination.yml b/.github/workflows/target_determination.yml new file mode 100644 index 000000000000..530272598324 --- /dev/null +++ b/.github/workflows/target_determination.yml @@ -0,0 +1,70 @@ +name: target-determination + +on: + workflow_call: + +jobs: + target-determination: + # Don't run on forked repos + if: github.repository_owner == 'pytorch' + runs-on: linux.2xlarge + steps: + # [pytorch repo ref] + # Use a pytorch/pytorch reference instead of a reference to the local + # checkout because when we run this action we don't *have* a local + # checkout. In other cases you should prefer a local checkout. + - name: Checkout PyTorch + uses: pytorch/pytorch/.github/actions/checkout-pytorch@main + with: + submodules: false + + - name: Setup Linux + uses: ./.github/actions/setup-linux + + - name: Get workflow job id + id: get-job-id + uses: ./.github/actions/get-workflow-job-id + if: always() + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download pytest cache + uses: ./.github/actions/pytest-cache-download + continue-on-error: true + with: + cache_dir: .pytest_cache + job_identifier: ${{ github.workflow }} + + - name: Do TD + id: td + continue-on-error: true + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_JOB: ${{ github.job }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + JOB_ID: ${{ steps.get-job-id.outputs.job-id }} + JOB_NAME: ${{ steps.get-job-id.outputs.job-name }} + run: | + python3 -m pip install boto3==1.19.12 + python3 tools/testing/do_target_determination_for_s3.py + + - name: Upload TD results to s3 + uses: seemethere/upload-artifact-s3@v5 + if: steps.td.outcome == 'success' + with: + name: td_results + retention-days: 14 + if-no-files-found: error + path: td_results.json + + - name: Store TD results on GHA + uses: actions/upload-artifact@v3 + if: steps.td.outcome == 'success' + with: + name: td_results.json + retention-days: 14 + if-no-files-found: error + path: td_results.json diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index b6a3f09c86e8..c0538a8600d9 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -19,6 +19,13 @@ concurrency: permissions: read-all jobs: + target-determination: + name: before-test + uses: ./.github/workflows/target_determination.yml + permissions: + id-token: write + contents: read + # Build PyTorch with BUILD_CAFFE2=ON caffe2-linux-jammy-py3_8-gcc11-build: name: caffe2-linux-jammy-py3.8-gcc11 @@ -47,7 +54,9 @@ jobs: linux-focal-cuda12_1-py3_10-gcc9-test: name: linux-focal-cuda12.1-py3.10-gcc9 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda12_1-py3_10-gcc9-build + needs: + - linux-focal-cuda12_1-py3_10-gcc9-build + - target-determination with: build-environment: linux-focal-cuda12.1-py3.10-gcc9 docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-build.outputs.docker-image }} @@ -128,7 +137,9 @@ jobs: macos-12-py3-arm64-test: name: macos-12-py3-arm64 uses: ./.github/workflows/_mac-test.yml - needs: macos-12-py3-arm64-build + needs: + - macos-12-py3-arm64-build + - target-determination with: build-environment: macos-12-py3-arm64 # Same as the build job @@ -153,7 +164,9 @@ jobs: win-vs2019-cpu-py3-test: name: win-vs2019-cpu-py3 uses: ./.github/workflows/_win-test.yml - needs: win-vs2019-cpu-py3-build + needs: + - win-vs2019-cpu-py3-build + - target-determination with: build-environment: win-vs2019-cpu-py3 cuda-version: cpu @@ -195,7 +208,9 @@ jobs: contents: read name: linux-focal-rocm6.0-py3.8 uses: ./.github/workflows/_rocm-test.yml - needs: linux-focal-rocm6_0-py3_8-build + needs: + - linux-focal-rocm6_0-py3_8-build + - target-determination with: build-environment: linux-focal-rocm6.0-py3.8 docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }} diff --git a/test/run_test.py b/test/run_test.py index 33b7d5805feb..4b0fbc872e66 100755 --- a/test/run_test.py +++ b/test/run_test.py @@ -54,11 +54,7 @@ from tools.testing.discover_tests import ( parse_test_module, TESTS, ) -from tools.testing.target_determination.determinator import ( - AggregatedHeuristics, - get_prediction_confidences, - get_test_prioritizations, -) +from tools.testing.do_target_determination_for_s3 import import_results from tools.testing.test_run import TestRun from tools.testing.test_selections import ( @@ -1631,24 +1627,17 @@ def main(): test_directory = str(REPO_ROOT / "test") selected_tests = get_selected_tests(options) + test_prioritizations = import_results() + test_prioritizations.amend_tests(selected_tests) + os.makedirs(REPO_ROOT / "test" / "test-reports", exist_ok=True) if options.coverage and not PYTORCH_COLLECT_COVERAGE: shell(["coverage", "erase"]) - aggregated_heuristics: AggregatedHeuristics = AggregatedHeuristics(selected_tests) - - with open( - REPO_ROOT / "test" / "test-reports" / "td_heuristic_rankings.log", "a" - ) as f: - if IS_CI: - # downloading test cases configuration to local environment - get_test_case_configs(dirpath=test_directory) - aggregated_heuristics = get_test_prioritizations(selected_tests, file=f) - - test_prioritizations = aggregated_heuristics.get_aggregated_priorities() - - f.write(test_prioritizations.get_info_str()) + if IS_CI: + # downloading test cases configuration to local environment + get_test_case_configs(dirpath=test_directory) test_file_times_dict = load_test_file_times() test_class_times_dict = load_test_class_times() @@ -1736,21 +1725,15 @@ def main(): all_failures = test_batch.failures if IS_CI: - num_tests = len(selected_tests) for test, _ in all_failures: - test_stats = aggregated_heuristics.get_test_stats(test) - test_stats["num_total_tests"] = num_tests - - print_to_stderr("Emiting td_test_failure_stats") + test_stats = test_prioritizations.get_test_stats(test) + print_to_stderr("Emiting td_test_failure_stats_v2") emit_metric( - "td_test_failure_stats", + "td_test_failure_stats_v2", { - **test_stats, - "confidence_ratings": get_prediction_confidences( - selected_tests - ), + "selected_tests": selected_tests, "failure": str(test), - "tests": selected_tests, + **test_stats, }, ) diff --git a/tools/stats/export_test_times.py b/tools/stats/export_test_times.py index 8254157f0d76..2b9c0c450683 100644 --- a/tools/stats/export_test_times.py +++ b/tools/stats/export_test_times.py @@ -3,26 +3,13 @@ import sys REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent sys.path.append(str(REPO_ROOT)) -from tools.stats.import_test_stats import ( - copy_pytest_cache, - get_td_heuristic_historial_edited_files_json, - get_td_heuristic_profiling_json, - get_test_class_ratings, - get_test_class_times, - get_test_file_ratings, - get_test_times, -) +from tools.stats.import_test_stats import get_test_class_times, get_test_times def main() -> None: - print("Exporting files from test-infra") + print("Exporting test times from test-infra") get_test_times() get_test_class_times() - get_test_file_ratings() - get_test_class_ratings() - get_td_heuristic_historial_edited_files_json() - get_td_heuristic_profiling_json() - copy_pytest_cache() if __name__ == "__main__": diff --git a/tools/stats/upload_metrics.py b/tools/stats/upload_metrics.py index b4ff5256fb98..8a0e93858b6f 100644 --- a/tools/stats/upload_metrics.py +++ b/tools/stats/upload_metrics.py @@ -105,7 +105,7 @@ def emit_metric( env_var_metrics = [ EnvVarMetric("repo", "GITHUB_REPOSITORY"), EnvVarMetric("workflow", "GITHUB_WORKFLOW"), - EnvVarMetric("build_environment", "BUILD_ENVIRONMENT"), + EnvVarMetric("build_environment", "BUILD_ENVIRONMENT", required=False), EnvVarMetric("job", "GITHUB_JOB"), EnvVarMetric("test_config", "TEST_CONFIG", required=False), EnvVarMetric("pr_number", "PR_NUMBER", required=False, type_conversion_fn=int), @@ -177,6 +177,8 @@ def _convert_float_values_to_decimals(data: Dict[str, Any]) -> Dict[str, Any]: return [_helper(v) for v in o] if isinstance(o, dict): return {_helper(k): _helper(v) for k, v in o.items()} + if isinstance(o, tuple): + return tuple(_helper(v) for v in o) return o return {k: _helper(v) for k, v in data.items()} diff --git a/tools/test/heuristics/test_interface.py b/tools/test/heuristics/test_interface.py index f507640af657..df122ab7d56f 100644 --- a/tools/test/heuristics/test_interface.py +++ b/tools/test/heuristics/test_interface.py @@ -540,5 +540,31 @@ class TestAggregatedHeuristicsTestStats(TestTD): aggregator.get_test_stats(TestRun("test2")) +class TestJsonParsing(TestTD): + def test_json_parsing_matches_TestPrioritizations(self) -> None: + tests = ["test1", "test2", "test3", "test4", "test5"] + tp = interface.TestPrioritizations( + tests, + { + TestRun("test3", included=["ClassA"]): 0.8, + TestRun("test3", excluded=["ClassA"]): 0.2, + TestRun("test4"): 0.7, + TestRun("test5"): 0.6, + }, + ) + tp_json = tp.to_json() + tp_json_to_tp = interface.TestPrioritizations.from_json(tp_json) + + self.assertSetEqual(tp._original_tests, tp_json_to_tp._original_tests) + self.assertDictEqual(tp._test_scores, tp_json_to_tp._test_scores) + + def test_json_parsing_matches_TestRun(self) -> None: + testrun = TestRun("test1", included=["classA", "classB"]) + testrun_json = testrun.to_json() + testrun_json_to_test = TestRun.from_json(testrun_json) + + self.assertTrue(testrun == testrun_json_to_test) + + if __name__ == "__main__": unittest.main() diff --git a/tools/test/test_upload_stats_lib.py b/tools/test/test_upload_stats_lib.py index 8199994d6144..0baf323966e1 100644 --- a/tools/test/test_upload_stats_lib.py +++ b/tools/test/test_upload_stats_lib.py @@ -1,13 +1,18 @@ import decimal import inspect +import pathlib +import sys import unittest from typing import Any, Dict from unittest import mock +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO_ROOT)) from tools.stats.upload_metrics import add_global_metric, emit_metric from tools.stats.upload_stats_lib import BATCH_SIZE, upload_to_rockset +sys.path.remove(str(REPO_ROOT)) # default values REPO = "some/repo" @@ -278,7 +283,7 @@ class TestUploadStats(unittest.TestCase): mock.patch.dict( "os.environ", { - "BUILD_ENVIRONMENT": "", + "GITHUB_JOB": "", }, ).start() diff --git a/tools/testing/do_target_determination_for_s3.py b/tools/testing/do_target_determination_for_s3.py new file mode 100644 index 000000000000..c7691b567922 --- /dev/null +++ b/tools/testing/do_target_determination_for_s3.py @@ -0,0 +1,76 @@ +import json +import os +import pathlib +import sys + + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +from tools.stats.import_test_stats import ( + copy_pytest_cache, + get_td_heuristic_historial_edited_files_json, + get_td_heuristic_profiling_json, + get_test_class_ratings, + get_test_class_times, + get_test_file_ratings, + get_test_times, +) +from tools.stats.upload_metrics import emit_metric + +from tools.testing.discover_tests import TESTS +from tools.testing.target_determination.determinator import ( + AggregatedHeuristics, + get_test_prioritizations, + TestPrioritizations, +) + +sys.path.remove(str(REPO_ROOT)) + + +def import_results() -> TestPrioritizations: + if not (REPO_ROOT / ".additional_ci_files/td_results.json").exists(): + print("No TD results found") + return TestPrioritizations([], {}) + with open(REPO_ROOT / ".additional_ci_files/td_results.json") as f: + td_results = json.load(f) + tp = TestPrioritizations.from_json(td_results) + + return tp + + +def main() -> None: + selected_tests = TESTS + + aggregated_heuristics: AggregatedHeuristics = AggregatedHeuristics(selected_tests) + + get_test_times() + get_test_class_times() + get_test_file_ratings() + get_test_class_ratings() + get_td_heuristic_historial_edited_files_json() + get_td_heuristic_profiling_json() + copy_pytest_cache() + + aggregated_heuristics = get_test_prioritizations(selected_tests) + + test_prioritizations = aggregated_heuristics.get_aggregated_priorities() + + if os.getenv("CI") == "true": + print("Emitting metrics") + # Split into 3 due to size constraints + emit_metric( + "td_results_final_test_prioritizations", + {"test_prioritizations": test_prioritizations.to_json()}, + ) + emit_metric( + "td_results_aggregated_heuristics", + {"aggregated_heuristics": aggregated_heuristics.to_json()}, + ) + + with open(REPO_ROOT / "td_results.json", "w") as f: + f.write(json.dumps(test_prioritizations.to_json())) + + +if __name__ == "__main__": + main() diff --git a/tools/testing/target_determination/determinator.py b/tools/testing/target_determination/determinator.py index ea99942c27dc..35994758f380 100644 --- a/tools/testing/target_determination/determinator.py +++ b/tools/testing/target_determination/determinator.py @@ -1,5 +1,5 @@ import sys -from typing import Any, Dict, List +from typing import Any, List from tools.testing.target_determination.heuristics import ( AggregatedHeuristics as AggregatedHeuristics, @@ -23,11 +23,3 @@ def get_test_prioritizations( print(new_rankings.get_info_str(), file=file) return aggregated_results - - -def get_prediction_confidences(tests: List[str]) -> Dict[str, TestPrioritizations]: - # heuristic name -> test -> rating/confidence - rankings: Dict[str, TestPrioritizations] = {} - for heuristic in HEURISTICS: - rankings[heuristic.name] = heuristic.get_prediction_confidence(tests) - return rankings diff --git a/tools/testing/target_determination/heuristics/interface.py b/tools/testing/target_determination/heuristics/interface.py index c6935643ce58..a5dae08f74cb 100644 --- a/tools/testing/target_determination/heuristics/interface.py +++ b/tools/testing/target_determination/heuristics/interface.py @@ -139,6 +139,66 @@ class TestPrioritizations: return {"position": idx, "score": score} raise AssertionError(f"Test run {test_run} not found") + def get_test_stats(self, test: TestRun) -> Dict[str, Any]: + return { + "test_name": test.test_file, + "test_filters": test.get_pytest_filter(), + **self.get_priority_info_for_test(test), + "max_score": max(score for score, _ in self._traverse_scores()), + "min_score": min(score for score, _ in self._traverse_scores()), + "all_scores": { + str(test): score for test, score in self._test_scores.items() + }, + } + + def to_json(self) -> Dict[str, Any]: + """ + Returns a JSON dict that describes this TestPrioritizations object. + """ + json_dict = { + "_test_scores": [ + (test.to_json(), score) + for test, score in self._test_scores.items() + if score != 0 + ], + "_original_tests": list(self._original_tests), + } + return json_dict + + @staticmethod + def from_json(json_dict: Dict[str, Any]) -> "TestPrioritizations": + """ + Returns a TestPrioritizations object from a JSON dict. + """ + test_prioritizations = TestPrioritizations( + tests_being_ranked=json_dict["_original_tests"], + scores={ + TestRun.from_json(testrun_json): score + for testrun_json, score in json_dict["_test_scores"] + }, + ) + return test_prioritizations + + def amend_tests(self, tests: List[str]) -> None: + """ + Removes tests that are not in the given list from the + TestPrioritizations. Adds tests that are in the list but not in the + TestPrioritizations. + """ + valid_scores = { + test: score + for test, score in self._test_scores.items() + if test.test_file in tests + } + self._test_scores = valid_scores + + for test in tests: + if test not in self._original_tests: + self._test_scores[TestRun(test)] = 0 + self._original_tests = frozenset(tests) + + self.validate() + class AggregatedHeuristics: """ @@ -224,6 +284,16 @@ class AggregatedHeuristics: return stats + def to_json(self) -> Dict[str, Any]: + """ + Returns a JSON dict that describes this AggregatedHeuristics object. + """ + json_dict: Dict[str, Any] = {} + for heuristic, heuristic_results in self._heuristic_results.items(): + json_dict[heuristic.name] = heuristic_results.to_json() + + return json_dict + class HeuristicInterface: """ diff --git a/tools/testing/test_run.py b/tools/testing/test_run.py index c5b04e57d348..25429b3c2c3b 100644 --- a/tools/testing/test_run.py +++ b/tools/testing/test_run.py @@ -1,6 +1,6 @@ from copy import copy from functools import total_ordering -from typing import FrozenSet, Iterable, List, Optional, Union +from typing import Any, Dict, FrozenSet, Iterable, List, Optional, Union class TestRun: @@ -210,6 +210,24 @@ class TestRun: return (self | other) - (self - other) - (other - self) + def to_json(self) -> Dict[str, Any]: + r: Dict[str, Any] = { + "test_file": self.test_file, + } + if self._included: + r["included"] = list(self._included) + if self._excluded: + r["excluded"] = list(self._excluded) + return r + + @staticmethod + def from_json(json: Dict[str, Any]) -> "TestRun": + return TestRun( + json["test_file"], + included=json.get("included", []), + excluded=json.get("excluded", []), + ) + @total_ordering class ShardedTest: