frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 21:14:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

[TD] Make test class correlation scores available to heuristics. (#111229)

Browse Source 
https://github.com/pytorch/test-infra/pull/4617 generates `file_test_class_rating.json`. Now we ensure it's available for heuristics to use during the test step.

(Actual heuristics will come in a separate PR)
Pull Request resolved: https://github.com/pytorch/pytorch/pull/111229
Approved by: https://github.com/huydhn
This commit is contained in:
Zain Rizvi
2023-10-17 16:29:30 +00:00
committed by PyTorch MergeBot
parent 973c87b320
commit 2c313880fc
10 changed files with 24 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.ci/pytorch/win-test-helpers/build_pytorch.bat
View File

@ -129,6 +129,7 @@ python -c "import os, glob; os.system('python -mpip install --no-index --no-deps
python tools/stats/export_test_times.py
copy /Y ".pytorch-test-times.json" "%PYTORCH_FINAL_PACKAGE_DIR%"
copy /Y ".pytorch-test-file-ratings.json" "%PYTORCH_FINAL_PACKAGE_DIR%"
copy /Y ".pytorch-test-class-ratings.json" "%PYTORCH_FINAL_PACKAGE_DIR%"
:: Also save build/.ninja_log as an artifact
copy /Y "build\.ninja_log" "%PYTORCH_FINAL_PACKAGE_DIR%\"

1
.ci/pytorch/win-test-helpers/test_python_jit_legacy.bat
View File

@ -3,6 +3,7 @@ call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
echo Copying over test times file
copy /Y "%PYTORCH_FINAL_PACKAGE_DIR_WIN%\.pytorch-test-times.json" "%PROJECT_DIR_WIN%"
copy /Y "%PYTORCH_FINAL_PACKAGE_DIR_WIN%\.pytorch-test-file-ratings.json" "%PROJECT_DIR_WIN%"
copy /Y "%PYTORCH_FINAL_PACKAGE_DIR_WIN%\.pytorch-test-class-ratings.json" "%PROJECT_DIR_WIN%"
pushd test

1
.ci/pytorch/win-test-helpers/test_python_shard.bat
View File

@ -24,6 +24,7 @@ if "%SHARD_NUMBER%" == "1" (
echo Copying over test times file
copy /Y "%PYTORCH_FINAL_PACKAGE_DIR_WIN%\.pytorch-test-times.json" "%PROJECT_DIR_WIN%"
copy /Y "%PYTORCH_FINAL_PACKAGE_DIR_WIN%\.pytorch-test-file-ratings.json" "%PROJECT_DIR_WIN%"
copy /Y "%PYTORCH_FINAL_PACKAGE_DIR_WIN%\.pytorch-test-class-ratings.json" "%PROJECT_DIR_WIN%"
echo Run nn tests
python run_test.py --exclude-jit-executor --exclude-distributed-tests --shard "%SHARD_NUMBER%" "%NUM_TEST_SHARDS%" --verbose

2
.circleci/config.yml generated
View File

@ -623,7 +623,7 @@ jobs:
- run:
name: Archive artifacts into zip
command: |
zip -1 -r artifacts.zip dist/ build/.ninja_log build/compile_commands.json .pytorch-test-times.json .pytorch-test-file-ratings.json
zip -1 -r artifacts.zip dist/ build/.ninja_log build/compile_commands.json .pytorch-test-times.json .pytorch-test-file-ratings.json .pytorch-test-class-ratings.json
cp artifacts.zip /Users/distiller/workspace
- persist_to_workspace:

2
.circleci/verbatim-sources/job-specs/job-specs-custom.yml
View File

@ -177,7 +177,7 @@
- run:
name: Archive artifacts into zip
command: |
zip -1 -r artifacts.zip dist/ build/.ninja_log build/compile_commands.json .pytorch-test-times.json .pytorch-test-file-ratings.json
zip -1 -r artifacts.zip dist/ build/.ninja_log build/compile_commands.json .pytorch-test-times.json .pytorch-test-file-ratings.json .pytorch-test-class-ratings.json
cp artifacts.zip /Users/distiller/workspace
- persist_to_workspace:

2
.github/workflows/_linux-build.yml vendored
View File

@ -170,7 +170,7 @@ jobs:
- name: Archive artifacts into zip
if: inputs.build-generates-artifacts && steps.build.outcome != 'skipped'
run: |
zip -1 -r artifacts.zip dist/ build/custom_test_artifacts build/lib build/bin .pytorch-test-times.json .pytorch-test-file-ratings.json
zip -1 -r artifacts.zip dist/ build/custom_test_artifacts build/lib build/bin .pytorch-test-times.json .pytorch-test-file-ratings.json .pytorch-test-class-ratings.json
- name: Store PyTorch Build Artifacts on S3
uses: seemethere/upload-artifact-s3@v5

2
.github/workflows/_mac-build.yml vendored
View File

@ -182,7 +182,7 @@ jobs:
- name: Archive artifacts into zip
if: inputs.build-generates-artifacts && steps.build.outcome != 'skipped'
run: |
zip -1 -r artifacts.zip dist/ build/.ninja_log build/compile_commands.json .pytorch-test-times.json .pytorch-test-file-ratings.json
zip -1 -r artifacts.zip dist/ build/.ninja_log build/compile_commands.json .pytorch-test-times.json .pytorch-test-file-ratings.json .pytorch-test-class-ratings.json
- name: Store PyTorch Build Artifacts on GHA
uses: actions/upload-artifact@v3

1
.gitignore vendored
View File

@ -20,6 +20,7 @@ coverage.xml
**/.pytorch-slow-tests.json
**/.pytorch-test-times.json
**/.pytorch-test-file-ratings.json
**/.pytorch-test-class-ratings.json
*/*.pyc
*/*.so*
*/**/__pycache__

8
tools/stats/export_test_times.py
View File

@ -3,16 +3,22 @@ import sys
REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent
sys.path.append(str(REPO_ROOT))
from tools.stats.import_test_stats import get_test_file_ratings, get_test_times
from tools.stats.import_test_stats import (
get_test_class_ratings,
get_test_file_ratings,
get_test_times,
)
TEST_TIMES_FILE = ".pytorch-test-times.json"
TEST_FILE_RATINGS_FILE = ".pytorch-test-file-ratings.json"
TEST_CLASS_RATINGS_FILE = ".pytorch-test-class-ratings.json"
def main() -> None:
print(f"Exporting test times from test-infra to {TEST_TIMES_FILE}")
get_test_times(str(REPO_ROOT), filename=TEST_TIMES_FILE)
get_test_file_ratings(str(REPO_ROOT), filename=TEST_FILE_RATINGS_FILE)
get_test_class_ratings(str(REPO_ROOT), filename=TEST_CLASS_RATINGS_FILE)
if __name__ == "__main__":

9
tools/stats/import_test_stats.py
View File

@ -111,3 +111,12 @@ def get_test_file_ratings(dirpath: str, filename: str) -> Optional[Dict[str, Any
except Exception:
print("Couldn't download test file ratings file, not reordering...")
return {}
def get_test_class_ratings(dirpath: str, filename: str) -> Optional[Dict[str, Any]]:
url = "https://raw.githubusercontent.com/pytorch/test-infra/generated-stats/stats/file_test_class_rating.json"
try:
return fetch_and_cache(dirpath, filename, url, lambda x: x)
except Exception:
print("Couldn't download test class ratings file, not reordering...")
return {}