mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
[ez] Discover tests without importing torch (#118574)
Moves test discovery into a file that doesn't have import torch so test listing can be done without having torch installed. Helpful when you don't have torch installed (aka me when I'm feeling lazy) I want to move TD into it's own job that doesn't need to wait for build to finish, so this is part of that. The first commit is a nothing more than a copy paste of the selected functions/vars into a new file, the second commit has various changes that should be checked. Pull Request resolved: https://github.com/pytorch/pytorch/pull/118574 Approved by: https://github.com/huydhn
This commit is contained in:
Catherine Lee
committed by
PyTorch MergeBot
PyTorch MergeBot
parent
eb9905be5d
commit
2eefbc02a0
139
test/run_test.py
139
test/run_test.py
@ -2,7 +2,6 @@
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
@ -46,6 +45,13 @@ from tools.stats.import_test_stats import (
|
||||
TEST_TIMES_FILE,
|
||||
)
|
||||
from tools.stats.upload_metrics import add_global_metric, emit_metric
|
||||
from tools.testing.discover_tests import (
|
||||
CPP_TEST_PATH,
|
||||
CPP_TEST_PREFIX,
|
||||
CPP_TESTS_DIR,
|
||||
parse_test_module,
|
||||
TESTS,
|
||||
)
|
||||
from tools.testing.target_determination.determinator import (
|
||||
AggregatedHeuristics,
|
||||
get_prediction_confidences,
|
||||
@ -67,8 +73,6 @@ sys.path.remove(str(REPO_ROOT))
|
||||
|
||||
|
||||
RERUN_DISABLED_TESTS = os.getenv("PYTORCH_TEST_RERUN_DISABLED_TESTS", "0") == "1"
|
||||
CPP_TEST_PREFIX = "cpp"
|
||||
CPP_TEST_PATH = "build/bin"
|
||||
DISTRIBUTED_TEST_PREFIX = "distributed"
|
||||
INDUCTOR_TEST_PREFIX = "inductor"
|
||||
|
||||
@ -99,10 +103,6 @@ def strtobool(s):
|
||||
return True
|
||||
|
||||
|
||||
def parse_test_module(test):
|
||||
return test.split(".")[0]
|
||||
|
||||
|
||||
class TestChoices(list):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(args[0])
|
||||
@ -111,131 +111,6 @@ class TestChoices(list):
|
||||
return list.__contains__(self, parse_test_module(item))
|
||||
|
||||
|
||||
def discover_tests(
|
||||
base_dir: Optional[pathlib.Path] = None,
|
||||
cpp_tests_dir: Optional[pathlib.Path] = None,
|
||||
blocklisted_patterns: Optional[List[str]] = None,
|
||||
blocklisted_tests: Optional[List[str]] = None,
|
||||
extra_tests: Optional[List[str]] = None,
|
||||
) -> List[str]:
|
||||
"""
|
||||
Searches for all python files starting with test_ excluding one specified by patterns.
|
||||
If cpp_tests_dir is provided, also scan for all C++ tests under that directory. They
|
||||
are usually found in build/bin
|
||||
"""
|
||||
|
||||
def skip_test_p(name: str) -> bool:
|
||||
rc = False
|
||||
if blocklisted_patterns is not None:
|
||||
rc |= any(name.startswith(pattern) for pattern in blocklisted_patterns)
|
||||
if blocklisted_tests is not None:
|
||||
rc |= name in blocklisted_tests
|
||||
return rc
|
||||
|
||||
cwd = pathlib.Path(__file__).resolve().parent if base_dir is None else base_dir
|
||||
# This supports symlinks, so we can link domain library tests to PyTorch test directory
|
||||
all_py_files = [
|
||||
pathlib.Path(p) for p in glob.glob(f"{cwd}/**/test_*.py", recursive=True)
|
||||
]
|
||||
|
||||
cpp_tests_dir = (
|
||||
f"{cwd.parent}/{CPP_TEST_PATH}" if cpp_tests_dir is None else cpp_tests_dir
|
||||
)
|
||||
# CPP test files are located under pytorch/build/bin. Unlike Python test, C++ tests
|
||||
# are just binaries and could have any name, i.e. basic or atest
|
||||
all_cpp_files = [
|
||||
pathlib.Path(p) for p in glob.glob(f"{cpp_tests_dir}/**/*", recursive=True)
|
||||
]
|
||||
|
||||
rc = [str(fname.relative_to(cwd))[:-3] for fname in all_py_files]
|
||||
# Add the cpp prefix for C++ tests so that we can tell them apart
|
||||
rc.extend(
|
||||
[
|
||||
parse_test_module(f"{CPP_TEST_PREFIX}/{fname.relative_to(cpp_tests_dir)}")
|
||||
for fname in all_cpp_files
|
||||
]
|
||||
)
|
||||
|
||||
# Invert slashes on Windows
|
||||
if sys.platform == "win32":
|
||||
rc = [name.replace("\\", "/") for name in rc]
|
||||
rc = [test for test in rc if not skip_test_p(test)]
|
||||
if extra_tests is not None:
|
||||
rc += extra_tests
|
||||
return sorted(rc)
|
||||
|
||||
|
||||
CPP_TESTS_DIR = os.path.abspath(os.getenv("CPP_TESTS_DIR", default=CPP_TEST_PATH))
|
||||
TESTS = discover_tests(
|
||||
cpp_tests_dir=CPP_TESTS_DIR,
|
||||
blocklisted_patterns=[
|
||||
"ao",
|
||||
"bottleneck_test",
|
||||
"custom_backend",
|
||||
"custom_operator",
|
||||
"fx", # executed by test_fx.py
|
||||
"jit", # executed by test_jit.py
|
||||
"mobile",
|
||||
"onnx_caffe2",
|
||||
"package", # executed by test_package.py
|
||||
"quantization", # executed by test_quantization.py
|
||||
"autograd", # executed by test_autograd.py
|
||||
],
|
||||
blocklisted_tests=[
|
||||
"test_bundled_images",
|
||||
"test_cpp_extensions_aot",
|
||||
"test_determination",
|
||||
"test_jit_fuser",
|
||||
"test_jit_simple",
|
||||
"test_jit_string",
|
||||
"test_kernel_launch_checks",
|
||||
"test_nnapi",
|
||||
"test_static_runtime",
|
||||
"test_throughput_benchmark",
|
||||
"distributed/bin/test_script",
|
||||
"distributed/elastic/multiprocessing/bin/test_script",
|
||||
"distributed/launcher/bin/test_script",
|
||||
"distributed/launcher/bin/test_script_init_method",
|
||||
"distributed/launcher/bin/test_script_is_torchelastic_launched",
|
||||
"distributed/launcher/bin/test_script_local_rank",
|
||||
"distributed/test_c10d_spawn",
|
||||
"distributions/test_transforms",
|
||||
"distributions/test_utils",
|
||||
"test/inductor/test_aot_inductor_utils",
|
||||
"onnx/test_pytorch_onnx_onnxruntime_cuda",
|
||||
"onnx/test_models",
|
||||
# These are not C++ tests
|
||||
f"{CPP_TEST_PREFIX}/CMakeFiles",
|
||||
f"{CPP_TEST_PREFIX}/CTestTestfile.cmake",
|
||||
f"{CPP_TEST_PREFIX}/Makefile",
|
||||
f"{CPP_TEST_PREFIX}/cmake_install.cmake",
|
||||
f"{CPP_TEST_PREFIX}/c10_intrusive_ptr_benchmark",
|
||||
f"{CPP_TEST_PREFIX}/example_allreduce",
|
||||
f"{CPP_TEST_PREFIX}/parallel_benchmark",
|
||||
f"{CPP_TEST_PREFIX}/protoc",
|
||||
f"{CPP_TEST_PREFIX}/protoc-3.13.0.0",
|
||||
f"{CPP_TEST_PREFIX}/torch_shm_manager",
|
||||
f"{CPP_TEST_PREFIX}/tutorial_tensorexpr",
|
||||
],
|
||||
extra_tests=[
|
||||
"test_cpp_extensions_aot_ninja",
|
||||
"test_cpp_extensions_aot_no_ninja",
|
||||
"distributed/elastic/timer/api_test",
|
||||
"distributed/elastic/timer/local_timer_example",
|
||||
"distributed/elastic/timer/local_timer_test",
|
||||
"distributed/elastic/events/lib_test",
|
||||
"distributed/elastic/metrics/api_test",
|
||||
"distributed/elastic/utils/logging_test",
|
||||
"distributed/elastic/utils/util_test",
|
||||
"distributed/elastic/utils/distributed_test",
|
||||
"distributed/elastic/multiprocessing/api_test",
|
||||
],
|
||||
)
|
||||
|
||||
# The doctests are a special case that don't correspond to a file that discover
|
||||
# tests can enable.
|
||||
TESTS = TESTS + ["doctests"]
|
||||
|
||||
FSDP_TEST = [test for test in TESTS if test.startswith("distributed/fsdp")]
|
||||
|
||||
WINDOWS_BLOCKLIST = [
|
||||
|
||||
Reference in New Issue
Block a user