frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 21:14:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

TD outside of test job (#118250)

Browse Source 
Give TD it's own job so that each shard can get the results from this one job artifact and they will always be in sync with each other/no longer need to worry about consistently issues

* Move test discovery to its own file that is not dependent on torch so it can be run without building torch
  * Cannot do cpp test discovery before building pytorch
* Move TD calculation to own file that will create a json file with the final results
* TD is now job/build env agnostic
* TD will rank all tests, including those that test jobs may not want to run (ex it will rank distributed tests along with default tests, even though these tests are never run on the same machine together)
Pull Request resolved: https://github.com/pytorch/pytorch/pull/118250
Approved by: https://github.com/huydhn
This commit is contained in:
Catherine Lee
2024-03-01 23:08:10 +00:00
committed by PyTorch MergeBot
parent d08ce51881
commit 06b52dd103
20 changed files with 433 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
.github/actions/download-td-artifacts/action.yml vendored Normal file
View File

@ -0,0 +1,29 @@
name: Download TD Artifacts
description: Download artifacts from target_determination.yml
inputs:
use-gha:
description: If set to any value, use GHA to download the artifact. Otherwise use s3.
required: false
runs:
using: composite
steps:
- name: Download TD Artifacts from S3
if: ${{ !inputs.use-gha }}
uses: seemethere/download-artifact-s3@v4
with:
name: td_results
- name: Download TD Artifacts from GHA
if: inputs.use-gha
uses: actions/download-artifact@v3
with:
name: td_results.json
- name: Move artifacts to .additional_ci_files folder
shell: bash
run: |
mkdir -p .additional_ci_files
mv td_results.json .additional_ci_files/td_results.json

4
.github/workflows/_linux-test.yml vendored
View File

@ -117,6 +117,10 @@ jobs:
with:
name: ${{ inputs.build-environment }}
- name: Download TD artifacts
continue-on-error: true
uses: ./.github/actions/download-td-artifacts
- name: Parse ref
id: parse-ref
run: .github/scripts/parse_ref.py

6
.github/workflows/_mac-test.yml vendored
View File

@ -91,6 +91,12 @@ jobs:
name: ${{ inputs.build-environment }}
use-gha: true
- name: Download TD artifacts
continue-on-error: true
uses: ./.github/actions/download-td-artifacts
with:
use-gha: true
- name: Setup miniconda
uses: pytorch/test-infra/.github/actions/setup-miniconda@main
with:

4
.github/workflows/_rocm-test.yml vendored
View File

@ -103,6 +103,10 @@ jobs:
with:
name: ${{ inputs.build-environment }}
- name: Download TD artifacts
continue-on-error: true
uses: ./.github/actions/download-td-artifacts
- name: Parse ref
id: parse-ref
run: .github/scripts/parse_ref.py

4
.github/workflows/_win-test.yml vendored
View File

@ -114,6 +114,10 @@ jobs:
run: |
tree /F C:\$Env:GITHUB_RUN_ID\build-results
- name: Download TD artifacts
continue-on-error: true
uses: ./.github/actions/download-td-artifacts
- name: Get workflow job id
id: get-job-id
uses: ./.github/actions/get-workflow-job-id

23
.github/workflows/periodic.yml vendored
View File

@ -23,6 +23,13 @@ concurrency:
permissions: read-all
jobs:
target-determination:
name: before-test
uses: ./.github/workflows/target_determination.yml
permissions:
id-token: write
contents: read
parallelnative-linux-jammy-py3_8-gcc11-build:
name: parallelnative-linux-jammy-py3.8-gcc11
uses: ./.github/workflows/_linux-build.yml
@ -39,7 +46,9 @@ jobs:
parallelnative-linux-jammy-py3_8-gcc11-test:
name: parallelnative-linux-jammy-py3.8-gcc11
uses: ./.github/workflows/_linux-test.yml
needs: parallelnative-linux-jammy-py3_8-gcc11-build
needs:
- parallelnative-linux-jammy-py3_8-gcc11-build
- target-determination
with:
build-environment: parallelnative-linux-jammy-py3.8-gcc11
docker-image: ${{ needs.parallelnative-linux-jammy-py3_8-gcc11-build.outputs.docker-image }}
@ -86,7 +95,9 @@ jobs:
linux-focal-cuda11_8-py3_10-gcc9-debug-test:
name: linux-focal-cuda11.8-py3.10-gcc9-debug
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-cuda11_8-py3_10-gcc9-debug-build
needs:
- linux-focal-cuda11_8-py3_10-gcc9-debug-build
- target-determination
with:
build-environment: linux-focal-cuda11.8-py3.10-gcc9-debug
docker-image: ${{ needs.linux-focal-cuda11_8-py3_10-gcc9-debug-build.outputs.docker-image }}
@ -110,7 +121,9 @@ jobs:
win-vs2019-cuda11_8-py3-test:
name: win-vs2019-cuda11.8-py3
uses: ./.github/workflows/_win-test.yml
needs: win-vs2019-cuda11_8-py3-build
needs:
- win-vs2019-cuda11_8-py3-build
- target-determination
with:
build-environment: win-vs2019-cuda11.8-py3
cuda-version: "11.8"
@ -214,7 +227,9 @@ jobs:
contents: read
name: linux-focal-rocm6.0-py3.8
uses: ./.github/workflows/_rocm-test.yml
needs: linux-focal-rocm6_0-py3_8-build
needs:
- linux-focal-rocm6_0-py3_8-build
- target-determination
with:
build-environment: linux-focal-rocm6.0-py3.8
docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }}

39
.github/workflows/pull.yml vendored
View File

@ -20,6 +20,13 @@ concurrency:
permissions: read-all
jobs:
target-determination:
name: before-test
uses: ./.github/workflows/target_determination.yml
permissions:
id-token: write
contents: read
linux-jammy-py3_8-gcc11-build:
name: linux-jammy-py3.8-gcc11
uses: ./.github/workflows/_linux-build.yml
@ -41,7 +48,9 @@ jobs:
linux-jammy-py3_8-gcc11-test:
name: linux-jammy-py3.8-gcc11
uses: ./.github/workflows/_linux-test.yml
needs: linux-jammy-py3_8-gcc11-build
needs:
- linux-jammy-py3_8-gcc11-build
- target-determination
with:
build-environment: linux-jammy-py3.8-gcc11
docker-image: ${{ needs.linux-jammy-py3_8-gcc11-build.outputs.docker-image }}
@ -97,7 +106,9 @@ jobs:
linux-jammy-py3_10-clang15-asan-test:
name: linux-jammy-py3.10-clang15-asan
uses: ./.github/workflows/_linux-test.yml
needs: linux-jammy-py3_10-clang15-asan-build
needs:
- linux-jammy-py3_10-clang15-asan-build
- target-determination
with:
build-environment: linux-jammy-py3.10-clang15-asan
docker-image: ${{ needs.linux-jammy-py3_10-clang15-asan-build.outputs.docker-image }}
@ -119,7 +130,9 @@ jobs:
linux-focal-py3_8-clang10-onnx-test:
name: linux-focal-py3.8-clang10-onnx
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-py3_8-clang10-onnx-build
needs:
- linux-focal-py3_8-clang10-onnx-build
- target-determination
with:
build-environment: linux-focal-py3.8-clang10-onnx
docker-image: ${{ needs.linux-focal-py3_8-clang10-onnx-build.outputs.docker-image }}
@ -146,7 +159,9 @@ jobs:
linux-focal-py3_8-clang10-test:
name: linux-focal-py3.8-clang10
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-py3_8-clang10-build
needs:
- linux-focal-py3_8-clang10-build
- target-determination
with:
build-environment: linux-focal-py3.8-clang10
docker-image: ${{ needs.linux-focal-py3_8-clang10-build.outputs.docker-image }}
@ -173,7 +188,9 @@ jobs:
linux-focal-py3_11-clang10-test:
name: linux-focal-py3.11-clang10
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-py3_11-clang10-build
needs:
- linux-focal-py3_11-clang10-build
- target-determination
with:
build-environment: linux-focal-py3.11-clang10
docker-image: ${{ needs.linux-focal-py3_11-clang10-build.outputs.docker-image }}
@ -218,7 +235,9 @@ jobs:
linux-focal-cuda11_8-py3_10-gcc9-test:
name: linux-focal-cuda11.8-py3.10-gcc9
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-cuda11_8-py3_10-gcc9-build
needs:
- linux-focal-cuda11_8-py3_10-gcc9-build
- target-determination
with:
timeout-minutes: 360
build-environment: linux-focal-cuda11.8-py3.10-gcc9
@ -244,7 +263,9 @@ jobs:
linux-focal-cuda12_1-py3_10-gcc9-test:
name: linux-focal-cuda12.1-py3.10-gcc9
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-cuda12_1-py3_10-gcc9-build
needs:
- linux-focal-cuda12_1-py3_10-gcc9-build
- target-determination
with:
timeout-minutes: 360
build-environment: linux-focal-cuda12.1-py3.10-gcc9
@ -415,7 +436,9 @@ jobs:
linux-focal-cuda12_1-py3_10-gcc9-sm86-test:
name: linux-focal-cuda12.1-py3.10-gcc9-sm86
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-cuda12_1-py3_10-gcc9-sm86-build
needs:
- linux-focal-cuda12_1-py3_10-gcc9-sm86-build
- target-determination
with:
build-environment: linux-focal-cuda12.1-py3.10-gcc9-sm86
docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-sm86-build.outputs.docker-image }}

11
.github/workflows/rocm.yml vendored
View File

@ -18,6 +18,13 @@ concurrency:
permissions: read-all
jobs:
target-determination:
name: before-test
uses: ./.github/workflows/target_determination.yml
permissions:
id-token: write
contents: read
linux-focal-rocm6_0-py3_8-build:
name: linux-focal-rocm6.0-py3.8
uses: ./.github/workflows/_linux-build.yml
@ -41,7 +48,9 @@ jobs:
contents: read
name: linux-focal-rocm6.0-py3.8
uses: ./.github/workflows/_rocm-test.yml
needs: linux-focal-rocm6_0-py3_8-build
needs:
- linux-focal-rocm6_0-py3_8-build
- target-determination
with:
build-environment: linux-focal-rocm6.0-py3.8
docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }}

27
.github/workflows/slow.yml vendored
View File

@ -21,6 +21,13 @@ concurrency:
permissions: read-all
jobs:
target-determination:
name: before-test
uses: ./.github/workflows/target_determination.yml
permissions:
id-token: write
contents: read
linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build:
name: linux-focal-cuda12.1-py3-gcc9-slow-gradcheck
uses: ./.github/workflows/_linux-build.yml
@ -39,7 +46,9 @@ jobs:
linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-test:
name: linux-focal-cuda12.1-py3-gcc9-slow-gradcheck
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build
needs:
- linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build
- target-determination
with:
build-environment: linux-focal-cuda12.1-py3-gcc9-slow-gradcheck
docker-image: ${{ needs.linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build.outputs.docker-image }}
@ -62,7 +71,9 @@ jobs:
linux-focal-cuda12_1-py3_10-gcc9-sm86-test:
name: linux-focal-cuda12.1-py3.10-gcc9-sm86
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-cuda12_1-py3_10-gcc9-sm86-build
needs:
- linux-focal-cuda12_1-py3_10-gcc9-sm86-build
- target-determination
with:
build-environment: linux-focal-cuda12.1-py3.10-gcc9-sm86
docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-sm86-build.outputs.docker-image }}
@ -82,7 +93,9 @@ jobs:
linux-focal-py3_8-clang10-test:
name: linux-focal-py3.8-clang10
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-py3_8-clang10-build
needs:
- linux-focal-py3_8-clang10-build
- target-determination
with:
build-environment: linux-focal-py3.8-clang10
docker-image: ${{ needs.linux-focal-py3_8-clang10-build.outputs.docker-image }}
@ -105,7 +118,9 @@ jobs:
contents: read
name: linux-focal-rocm6.0-py3.8
uses: ./.github/workflows/_rocm-test.yml
needs: linux-focal-rocm6_0-py3_8-build
needs:
- linux-focal-rocm6_0-py3_8-build
- target-determination
with:
build-environment: linux-focal-rocm6.0-py3.8
docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }}
@ -127,7 +142,9 @@ jobs:
linux-jammy-py3_10-clang15-asan-test:
name: linux-jammy-py3.10-clang15-asan
uses: ./.github/workflows/_linux-test.yml
needs: linux-jammy-py3_10-clang15-asan-build
needs:
- linux-jammy-py3_10-clang15-asan-build
- target-determination
with:
build-environment: linux-jammy-py3.10-clang15-asan
docker-image: ${{ needs.linux-jammy-py3_10-clang15-asan-build.outputs.docker-image }}

70
.github/workflows/target_determination.yml vendored Normal file
View File

@ -0,0 +1,70 @@
name: target-determination
on:
workflow_call:
jobs:
target-determination:
# Don't run on forked repos
if: github.repository_owner == 'pytorch'
runs-on: linux.2xlarge
steps:
# [pytorch repo ref]
# Use a pytorch/pytorch reference instead of a reference to the local
# checkout because when we run this action we don't *have* a local
# checkout. In other cases you should prefer a local checkout.
- name: Checkout PyTorch
uses: pytorch/pytorch/.github/actions/checkout-pytorch@main
with:
submodules: false
- name: Setup Linux
uses: ./.github/actions/setup-linux
- name: Get workflow job id
id: get-job-id
uses: ./.github/actions/get-workflow-job-id
if: always()
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Download pytest cache
uses: ./.github/actions/pytest-cache-download
continue-on-error: true
with:
cache_dir: .pytest_cache
job_identifier: ${{ github.workflow }}
- name: Do TD
id: td
continue-on-error: true
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_WORKFLOW: ${{ github.workflow }}
GITHUB_JOB: ${{ github.job }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
JOB_ID: ${{ steps.get-job-id.outputs.job-id }}
JOB_NAME: ${{ steps.get-job-id.outputs.job-name }}
run: |
python3 -m pip install boto3==1.19.12
python3 tools/testing/do_target_determination_for_s3.py
- name: Upload TD results to s3
uses: seemethere/upload-artifact-s3@v5
if: steps.td.outcome == 'success'
with:
name: td_results
retention-days: 14
if-no-files-found: error
path: td_results.json
- name: Store TD results on GHA
uses: actions/upload-artifact@v3
if: steps.td.outcome == 'success'
with:
name: td_results.json
retention-days: 14
if-no-files-found: error
path: td_results.json

23
.github/workflows/trunk.yml vendored
View File

@ -19,6 +19,13 @@ concurrency:
permissions: read-all
jobs:
target-determination:
name: before-test
uses: ./.github/workflows/target_determination.yml
permissions:
id-token: write
contents: read
# Build PyTorch with BUILD_CAFFE2=ON
caffe2-linux-jammy-py3_8-gcc11-build:
name: caffe2-linux-jammy-py3.8-gcc11
@ -47,7 +54,9 @@ jobs:
linux-focal-cuda12_1-py3_10-gcc9-test:
name: linux-focal-cuda12.1-py3.10-gcc9
uses: ./.github/workflows/_linux-test.yml
needs: linux-focal-cuda12_1-py3_10-gcc9-build
needs:
- linux-focal-cuda12_1-py3_10-gcc9-build
- target-determination
with:
build-environment: linux-focal-cuda12.1-py3.10-gcc9
docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-build.outputs.docker-image }}
@ -128,7 +137,9 @@ jobs:
macos-12-py3-arm64-test:
name: macos-12-py3-arm64
uses: ./.github/workflows/_mac-test.yml
needs: macos-12-py3-arm64-build
needs:
- macos-12-py3-arm64-build
- target-determination
with:
build-environment: macos-12-py3-arm64
# Same as the build job
@ -153,7 +164,9 @@ jobs:
win-vs2019-cpu-py3-test:
name: win-vs2019-cpu-py3
uses: ./.github/workflows/_win-test.yml
needs: win-vs2019-cpu-py3-build
needs:
- win-vs2019-cpu-py3-build
- target-determination
with:
build-environment: win-vs2019-cpu-py3
cuda-version: cpu
@ -195,7 +208,9 @@ jobs:
contents: read
name: linux-focal-rocm6.0-py3.8
uses: ./.github/workflows/_rocm-test.yml
needs: linux-focal-rocm6_0-py3_8-build
needs:
- linux-focal-rocm6_0-py3_8-build
- target-determination
with:
build-environment: linux-focal-rocm6.0-py3.8
docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }}

41
test/run_test.py
View File

@ -54,11 +54,7 @@ from tools.testing.discover_tests import (
parse_test_module,
TESTS,
)
from tools.testing.target_determination.determinator import (
AggregatedHeuristics,
get_prediction_confidences,
get_test_prioritizations,
)
from tools.testing.do_target_determination_for_s3 import import_results
from tools.testing.test_run import TestRun
from tools.testing.test_selections import (
@ -1631,24 +1627,17 @@ def main():
test_directory = str(REPO_ROOT / "test")
selected_tests = get_selected_tests(options)
test_prioritizations = import_results()
test_prioritizations.amend_tests(selected_tests)
os.makedirs(REPO_ROOT / "test" / "test-reports", exist_ok=True)
if options.coverage and not PYTORCH_COLLECT_COVERAGE:
shell(["coverage", "erase"])
aggregated_heuristics: AggregatedHeuristics = AggregatedHeuristics(selected_tests)
with open(
REPO_ROOT / "test" / "test-reports" / "td_heuristic_rankings.log", "a"
) as f:
if IS_CI:
# downloading test cases configuration to local environment
get_test_case_configs(dirpath=test_directory)
aggregated_heuristics = get_test_prioritizations(selected_tests, file=f)
test_prioritizations = aggregated_heuristics.get_aggregated_priorities()
f.write(test_prioritizations.get_info_str())
if IS_CI:
# downloading test cases configuration to local environment
get_test_case_configs(dirpath=test_directory)
test_file_times_dict = load_test_file_times()
test_class_times_dict = load_test_class_times()
@ -1736,21 +1725,15 @@ def main():
all_failures = test_batch.failures
if IS_CI:
num_tests = len(selected_tests)
for test, _ in all_failures:
test_stats = aggregated_heuristics.get_test_stats(test)
test_stats["num_total_tests"] = num_tests
print_to_stderr("Emiting td_test_failure_stats")
test_stats = test_prioritizations.get_test_stats(test)
print_to_stderr("Emiting td_test_failure_stats_v2")
emit_metric(
"td_test_failure_stats",
"td_test_failure_stats_v2",
{
**test_stats,
"confidence_ratings": get_prediction_confidences(
selected_tests
),
"selected_tests": selected_tests,
"failure": str(test),
"tests": selected_tests,
**test_stats,
},
)

17
tools/stats/export_test_times.py
View File

@ -3,26 +3,13 @@ import sys
REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent
sys.path.append(str(REPO_ROOT))
from tools.stats.import_test_stats import (
copy_pytest_cache,
get_td_heuristic_historial_edited_files_json,
get_td_heuristic_profiling_json,
get_test_class_ratings,
get_test_class_times,
get_test_file_ratings,
get_test_times,
)
from tools.stats.import_test_stats import get_test_class_times, get_test_times
def main() -> None:
print("Exporting files from test-infra")
print("Exporting test times from test-infra")
get_test_times()
get_test_class_times()
get_test_file_ratings()
get_test_class_ratings()
get_td_heuristic_historial_edited_files_json()
get_td_heuristic_profiling_json()
copy_pytest_cache()
if __name__ == "__main__":

4
tools/stats/upload_metrics.py
View File

@ -105,7 +105,7 @@ def emit_metric(
env_var_metrics = [
EnvVarMetric("repo", "GITHUB_REPOSITORY"),
EnvVarMetric("workflow", "GITHUB_WORKFLOW"),
EnvVarMetric("build_environment", "BUILD_ENVIRONMENT"),
EnvVarMetric("build_environment", "BUILD_ENVIRONMENT", required=False),
EnvVarMetric("job", "GITHUB_JOB"),
EnvVarMetric("test_config", "TEST_CONFIG", required=False),
EnvVarMetric("pr_number", "PR_NUMBER", required=False, type_conversion_fn=int),
@ -177,6 +177,8 @@ def _convert_float_values_to_decimals(data: Dict[str, Any]) -> Dict[str, Any]:
return [_helper(v) for v in o]
if isinstance(o, dict):
return {_helper(k): _helper(v) for k, v in o.items()}
if isinstance(o, tuple):
return tuple(_helper(v) for v in o)
return o
return {k: _helper(v) for k, v in data.items()}

26
tools/test/heuristics/test_interface.py
View File

@ -540,5 +540,31 @@ class TestAggregatedHeuristicsTestStats(TestTD):
aggregator.get_test_stats(TestRun("test2"))
class TestJsonParsing(TestTD):
def test_json_parsing_matches_TestPrioritizations(self) -> None:
tests = ["test1", "test2", "test3", "test4", "test5"]
tp = interface.TestPrioritizations(
tests,
{
TestRun("test3", included=["ClassA"]): 0.8,
TestRun("test3", excluded=["ClassA"]): 0.2,
TestRun("test4"): 0.7,
TestRun("test5"): 0.6,
},
)
tp_json = tp.to_json()
tp_json_to_tp = interface.TestPrioritizations.from_json(tp_json)
self.assertSetEqual(tp._original_tests, tp_json_to_tp._original_tests)
self.assertDictEqual(tp._test_scores, tp_json_to_tp._test_scores)
def test_json_parsing_matches_TestRun(self) -> None:
testrun = TestRun("test1", included=["classA", "classB"])
testrun_json = testrun.to_json()
testrun_json_to_test = TestRun.from_json(testrun_json)
self.assertTrue(testrun == testrun_json_to_test)
if __name__ == "__main__":
unittest.main()

7
tools/test/test_upload_stats_lib.py
View File

@ -1,13 +1,18 @@
import decimal
import inspect
import pathlib
import sys
import unittest
from typing import Any, Dict
from unittest import mock
REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent
sys.path.insert(0, str(REPO_ROOT))
from tools.stats.upload_metrics import add_global_metric, emit_metric
from tools.stats.upload_stats_lib import BATCH_SIZE, upload_to_rockset
sys.path.remove(str(REPO_ROOT))
# default values
REPO = "some/repo"
@ -278,7 +283,7 @@ class TestUploadStats(unittest.TestCase):
mock.patch.dict(
"os.environ",
{
"BUILD_ENVIRONMENT": "",
"GITHUB_JOB": "",
},
).start()

76
tools/testing/do_target_determination_for_s3.py Normal file
View File

@ -0,0 +1,76 @@
import json
import os
import pathlib
import sys
REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent
sys.path.insert(0, str(REPO_ROOT))
from tools.stats.import_test_stats import (
copy_pytest_cache,
get_td_heuristic_historial_edited_files_json,
get_td_heuristic_profiling_json,
get_test_class_ratings,
get_test_class_times,
get_test_file_ratings,
get_test_times,
)
from tools.stats.upload_metrics import emit_metric
from tools.testing.discover_tests import TESTS
from tools.testing.target_determination.determinator import (
AggregatedHeuristics,
get_test_prioritizations,
TestPrioritizations,
)
sys.path.remove(str(REPO_ROOT))
def import_results() -> TestPrioritizations:
if not (REPO_ROOT / ".additional_ci_files/td_results.json").exists():
print("No TD results found")
return TestPrioritizations([], {})
with open(REPO_ROOT / ".additional_ci_files/td_results.json") as f:
td_results = json.load(f)
tp = TestPrioritizations.from_json(td_results)
return tp
def main() -> None:
selected_tests = TESTS
aggregated_heuristics: AggregatedHeuristics = AggregatedHeuristics(selected_tests)
get_test_times()
get_test_class_times()
get_test_file_ratings()
get_test_class_ratings()
get_td_heuristic_historial_edited_files_json()
get_td_heuristic_profiling_json()
copy_pytest_cache()
aggregated_heuristics = get_test_prioritizations(selected_tests)
test_prioritizations = aggregated_heuristics.get_aggregated_priorities()
if os.getenv("CI") == "true":
print("Emitting metrics")
# Split into 3 due to size constraints
emit_metric(
"td_results_final_test_prioritizations",
{"test_prioritizations": test_prioritizations.to_json()},
)
emit_metric(
"td_results_aggregated_heuristics",
{"aggregated_heuristics": aggregated_heuristics.to_json()},
)
with open(REPO_ROOT / "td_results.json", "w") as f:
f.write(json.dumps(test_prioritizations.to_json()))
if __name__ == "__main__":
main()

10
tools/testing/target_determination/determinator.py
View File

@ -1,5 +1,5 @@
import sys
from typing import Any, Dict, List
from typing import Any, List
from tools.testing.target_determination.heuristics import (
AggregatedHeuristics as AggregatedHeuristics,
@ -23,11 +23,3 @@ def get_test_prioritizations(
print(new_rankings.get_info_str(), file=file)
return aggregated_results
def get_prediction_confidences(tests: List[str]) -> Dict[str, TestPrioritizations]:
# heuristic name -> test -> rating/confidence
rankings: Dict[str, TestPrioritizations] = {}
for heuristic in HEURISTICS:
rankings[heuristic.name] = heuristic.get_prediction_confidence(tests)
return rankings

70
tools/testing/target_determination/heuristics/interface.py
View File

@ -139,6 +139,66 @@ class TestPrioritizations:
return {"position": idx, "score": score}
raise AssertionError(f"Test run {test_run} not found")
def get_test_stats(self, test: TestRun) -> Dict[str, Any]:
return {
"test_name": test.test_file,
"test_filters": test.get_pytest_filter(),
**self.get_priority_info_for_test(test),
"max_score": max(score for score, _ in self._traverse_scores()),
"min_score": min(score for score, _ in self._traverse_scores()),
"all_scores": {
str(test): score for test, score in self._test_scores.items()
},
}
def to_json(self) -> Dict[str, Any]:
"""
Returns a JSON dict that describes this TestPrioritizations object.
"""
json_dict = {
"_test_scores": [
(test.to_json(), score)
for test, score in self._test_scores.items()
if score != 0
],
"_original_tests": list(self._original_tests),
}
return json_dict
@staticmethod
def from_json(json_dict: Dict[str, Any]) -> "TestPrioritizations":
"""
Returns a TestPrioritizations object from a JSON dict.
"""
test_prioritizations = TestPrioritizations(
tests_being_ranked=json_dict["_original_tests"],
scores={
TestRun.from_json(testrun_json): score
for testrun_json, score in json_dict["_test_scores"]
},
)
return test_prioritizations
def amend_tests(self, tests: List[str]) -> None:
"""
Removes tests that are not in the given list from the
TestPrioritizations. Adds tests that are in the list but not in the
TestPrioritizations.
"""
valid_scores = {
test: score
for test, score in self._test_scores.items()
if test.test_file in tests
}
self._test_scores = valid_scores
for test in tests:
if test not in self._original_tests:
self._test_scores[TestRun(test)] = 0
self._original_tests = frozenset(tests)
self.validate()
class AggregatedHeuristics:
"""
@ -224,6 +284,16 @@ class AggregatedHeuristics:
return stats
def to_json(self) -> Dict[str, Any]:
"""
Returns a JSON dict that describes this AggregatedHeuristics object.
"""
json_dict: Dict[str, Any] = {}
for heuristic, heuristic_results in self._heuristic_results.items():
json_dict[heuristic.name] = heuristic_results.to_json()
return json_dict
class HeuristicInterface:
"""

20
tools/testing/test_run.py
View File

@ -1,6 +1,6 @@
from copy import copy
from functools import total_ordering
from typing import FrozenSet, Iterable, List, Optional, Union
from typing import Any, Dict, FrozenSet, Iterable, List, Optional, Union
class TestRun:
@ -210,6 +210,24 @@ class TestRun:
return (self | other) - (self - other) - (other - self)
def to_json(self) -> Dict[str, Any]:
r: Dict[str, Any] = {
"test_file": self.test_file,
}
if self._included:
r["included"] = list(self._included)
if self._excluded:
r["excluded"] = list(self._excluded)
return r
@staticmethod
def from_json(json: Dict[str, Any]) -> "TestRun":
return TestRun(
json["test_file"],
included=json.get("included", []),
excluded=json.get("excluded", []),
)
@total_ordering
class ShardedTest: