diff --git a/tools/amd_build/build_amd.py b/tools/amd_build/build_amd.py index d007547400f3..967f63ae2c8f 100755 --- a/tools/amd_build/build_amd.py +++ b/tools/amd_build/build_amd.py @@ -5,6 +5,7 @@ import argparse import os import sys + sys.path.append( os.path.realpath( os.path.join( @@ -15,6 +16,7 @@ sys.path.append( from hipify import hipify_python # type: ignore[import] + parser = argparse.ArgumentParser( description="Top-level script for HIPifying, filling in most common parameters" ) diff --git a/tools/autograd/gen_inplace_or_view_type.py b/tools/autograd/gen_inplace_or_view_type.py index 7ef97fd0fb7d..e8141658b033 100644 --- a/tools/autograd/gen_inplace_or_view_type.py +++ b/tools/autograd/gen_inplace_or_view_type.py @@ -24,7 +24,7 @@ from torchgen.api.types import ( OptionalCType, symIntArrayRefT, SymIntT, - tensorT, # See Note [Nested Arg Types] + tensorT, ) from torchgen.code_template import CodeTemplate from torchgen.context import with_native_function diff --git a/tools/build_libtorch.py b/tools/build_libtorch.py index 3b85d415d027..4065f39a1baf 100644 --- a/tools/build_libtorch.py +++ b/tools/build_libtorch.py @@ -2,6 +2,7 @@ import argparse import sys from os.path import abspath, dirname + # By appending pytorch_root to sys.path, this module can import other torch # modules even when run as a standalone script. i.e., it's okay either you # do `python build_libtorch.py` or `python -m tools.build_libtorch`. @@ -11,6 +12,7 @@ sys.path.append(pytorch_root) from tools.build_pytorch_libs import build_caffe2 from tools.setup_helpers.cmake import CMake + if __name__ == "__main__": # Placeholder for future interface. For now just gives a nice -h. parser = argparse.ArgumentParser(description="Build libtorch") diff --git a/tools/linter/adapters/s3_init.py b/tools/linter/adapters/s3_init.py index 260a3d4394bd..3f2649bd225c 100644 --- a/tools/linter/adapters/s3_init.py +++ b/tools/linter/adapters/s3_init.py @@ -11,6 +11,7 @@ import urllib.error import urllib.request from pathlib import Path + # String representing the host platform (e.g. Linux, Darwin). HOST_PLATFORM = platform.system() HOST_PLATFORM_ARCH = platform.system() + "-" + platform.processor() diff --git a/tools/linter/adapters/ufmt_linter.py b/tools/linter/adapters/ufmt_linter.py index 07bfe0e6f7a3..e70bda0bf086 100644 --- a/tools/linter/adapters/ufmt_linter.py +++ b/tools/linter/adapters/ufmt_linter.py @@ -36,9 +36,7 @@ ISORT_WHITELIST = re.compile( # functorch/** "functorch/**", # tools/** - "tools/**", # torchgen/** - "torchgen/**", # test/** # test/[a-c]*/** "test/[a-c]*/**", diff --git a/tools/setup_helpers/gen.py b/tools/setup_helpers/gen.py index 3ca9a8787906..d7e63d9ed4ae 100644 --- a/tools/setup_helpers/gen.py +++ b/tools/setup_helpers/gen.py @@ -3,9 +3,11 @@ import os.path import sys + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, root) import torchgen.gen + torchgen.gen.main() diff --git a/tools/setup_helpers/gen_unboxing.py b/tools/setup_helpers/gen_unboxing.py index b876b72c0b82..91c61f0ab204 100644 --- a/tools/setup_helpers/gen_unboxing.py +++ b/tools/setup_helpers/gen_unboxing.py @@ -3,9 +3,11 @@ import os.path import sys + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, root) import tools.jit.gen_unboxing + tools.jit.gen_unboxing.main(sys.argv[1:]) diff --git a/tools/setup_helpers/generate_code.py b/tools/setup_helpers/generate_code.py index cf1e39dac904..9fee1909970d 100644 --- a/tools/setup_helpers/generate_code.py +++ b/tools/setup_helpers/generate_code.py @@ -8,6 +8,7 @@ from typing import Any, cast import yaml + try: # use faster C loader if available from yaml import CSafeLoader as YamlLoader @@ -30,6 +31,7 @@ def generate_code( ) -> None: from tools.autograd.gen_annotated_fn_args import gen_annotated from tools.autograd.gen_autograd import gen_autograd, gen_autograd_python + from torchgen.selective_build.selector import SelectiveBuilder # Build ATen based Variable classes diff --git a/tools/stats/export_test_times.py b/tools/stats/export_test_times.py index 6a82b934eb02..9bfb6b3810b5 100644 --- a/tools/stats/export_test_times.py +++ b/tools/stats/export_test_times.py @@ -1,6 +1,7 @@ import sys from pathlib import Path + REPO_ROOT = Path(__file__).resolve().parent.parent.parent sys.path.append(str(REPO_ROOT)) from tools.stats.import_test_stats import get_test_class_times, get_test_times diff --git a/tools/stats/import_test_stats.py b/tools/stats/import_test_stats.py index 086d08fcf2c8..013fb7ddeba5 100644 --- a/tools/stats/import_test_stats.py +++ b/tools/stats/import_test_stats.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Any, Callable, cast, Dict from urllib.request import urlopen + REPO_ROOT = Path(__file__).resolve().parent.parent.parent diff --git a/tools/stats/upload_stats_lib.py b/tools/stats/upload_stats_lib.py index 2a8ad8138030..cf2030d4f14c 100644 --- a/tools/stats/upload_stats_lib.py +++ b/tools/stats/upload_stats_lib.py @@ -4,7 +4,6 @@ import gzip import io import json import os - import time import zipfile from pathlib import Path diff --git a/tools/test/heuristics/test_heuristics.py b/tools/test/heuristics/test_heuristics.py index 6e9ff8420d7d..50023e1eb042 100644 --- a/tools/test/heuristics/test_heuristics.py +++ b/tools/test/heuristics/test_heuristics.py @@ -9,8 +9,10 @@ from pathlib import Path from typing import Any from unittest import mock + REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent sys.path.append(str(REPO_ROOT)) + from tools.test.heuristics.test_interface import TestTD from tools.testing.target_determination.determinator import TestPrioritizations from tools.testing.target_determination.heuristics.filepath import ( @@ -25,6 +27,7 @@ from tools.testing.target_determination.heuristics.previously_failed_in_pr impor ) from tools.testing.test_run import TestRun + sys.path.remove(str(REPO_ROOT)) HEURISTIC_CLASS = "tools.testing.target_determination.heuristics.historical_class_failure_correlation." diff --git a/tools/test/heuristics/test_interface.py b/tools/test/heuristics/test_interface.py index 4941c8ada6ca..59b6c13e6397 100644 --- a/tools/test/heuristics/test_interface.py +++ b/tools/test/heuristics/test_interface.py @@ -5,11 +5,14 @@ import unittest from pathlib import Path from typing import Any + REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent sys.path.append(str(REPO_ROOT)) + import tools.testing.target_determination.heuristics.interface as interface from tools.testing.test_run import TestRun + sys.path.remove(str(REPO_ROOT)) diff --git a/tools/test/heuristics/test_utils.py b/tools/test/heuristics/test_utils.py index 23e1d155ab0d..6deb797d31c6 100644 --- a/tools/test/heuristics/test_utils.py +++ b/tools/test/heuristics/test_utils.py @@ -8,9 +8,11 @@ from typing import Any REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent sys.path.append(str(REPO_ROOT)) + import tools.testing.target_determination.heuristics.utils as utils from tools.testing.test_run import TestRun + sys.path.remove(str(REPO_ROOT)) diff --git a/tools/test/test_gen_backend_stubs.py b/tools/test/test_gen_backend_stubs.py index e103c573b3ca..303c6f227a86 100644 --- a/tools/test/test_gen_backend_stubs.py +++ b/tools/test/test_gen_backend_stubs.py @@ -9,9 +9,9 @@ import unittest import expecttest from torchgen.gen import _GLOBAL_PARSE_NATIVE_YAML_CACHE # noqa: F401 - from torchgen.gen_backend_stubs import run + path = os.path.dirname(os.path.realpath(__file__)) gen_backend_stubs_path = os.path.join(path, "../torchgen/gen_backend_stubs.py") diff --git a/tools/test/test_test_run.py b/tools/test/test_test_run.py index 1e4f82c22903..7e9a8f6685c6 100644 --- a/tools/test/test_test_run.py +++ b/tools/test/test_test_run.py @@ -2,6 +2,7 @@ import sys import unittest from pathlib import Path + REPO_ROOT = Path(__file__).resolve().parent.parent.parent try: # using tools/ to optimize test run. diff --git a/tools/test/test_test_selections.py b/tools/test/test_test_selections.py index d18f0c50efb0..1dbe6e1f60bd 100644 --- a/tools/test/test_test_selections.py +++ b/tools/test/test_test_selections.py @@ -7,6 +7,7 @@ import unittest from collections import defaultdict from pathlib import Path + REPO_ROOT = Path(__file__).resolve().parent.parent.parent try: # using tools/ to optimize test run. diff --git a/tools/test/test_upload_stats_lib.py b/tools/test/test_upload_stats_lib.py index a1e34566cf7f..d5faa7ac5ea4 100644 --- a/tools/test/test_upload_stats_lib.py +++ b/tools/test/test_upload_stats_lib.py @@ -8,12 +8,14 @@ from pathlib import Path from typing import Any from unittest import mock + REPO_ROOT = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(REPO_ROOT)) -from tools.stats.upload_metrics import add_global_metric, emit_metric +from tools.stats.upload_metrics import add_global_metric, emit_metric from tools.stats.upload_stats_lib import BATCH_SIZE, upload_to_rockset + sys.path.remove(str(REPO_ROOT)) # default values diff --git a/tools/testing/discover_tests.py b/tools/testing/discover_tests.py index beb4e4296397..3cebaf44756e 100644 --- a/tools/testing/discover_tests.py +++ b/tools/testing/discover_tests.py @@ -5,6 +5,7 @@ import os import sys from pathlib import Path + CPP_TEST_PREFIX = "cpp" CPP_TEST_PATH = "build/bin" CPP_TESTS_DIR = os.path.abspath(os.getenv("CPP_TESTS_DIR", default=CPP_TEST_PATH)) diff --git a/tools/testing/do_target_determination_for_s3.py b/tools/testing/do_target_determination_for_s3.py index 15d72174f211..a280e5bfaf25 100644 --- a/tools/testing/do_target_determination_for_s3.py +++ b/tools/testing/do_target_determination_for_s3.py @@ -18,7 +18,6 @@ from tools.stats.import_test_stats import ( get_test_times, ) from tools.stats.upload_metrics import emit_metric - from tools.testing.discover_tests import TESTS from tools.testing.target_determination.determinator import ( AggregatedHeuristics, @@ -26,6 +25,7 @@ from tools.testing.target_determination.determinator import ( TestPrioritizations, ) + sys.path.remove(str(REPO_ROOT)) diff --git a/tools/testing/modulefinder_determinator.py b/tools/testing/modulefinder_determinator.py index b9262e34548d..760fed9ad4ee 100644 --- a/tools/testing/modulefinder_determinator.py +++ b/tools/testing/modulefinder_determinator.py @@ -7,6 +7,7 @@ import warnings from pathlib import Path from typing import Any + REPO_ROOT = Path(__file__).resolve().parent.parent.parent # These tests are slow enough that it's worth calculating whether the patch diff --git a/tools/testing/target_determination/gen_artifact.py b/tools/testing/target_determination/gen_artifact.py index 0c62bd02f4d3..e6576979de93 100644 --- a/tools/testing/target_determination/gen_artifact.py +++ b/tools/testing/target_determination/gen_artifact.py @@ -5,6 +5,7 @@ import os from pathlib import Path from typing import Any + REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent diff --git a/tools/testing/target_determination/heuristics/filepath.py b/tools/testing/target_determination/heuristics/filepath.py index 31f6cd71b25a..110625608d85 100644 --- a/tools/testing/target_determination/heuristics/filepath.py +++ b/tools/testing/target_determination/heuristics/filepath.py @@ -10,13 +10,13 @@ from tools.testing.target_determination.heuristics.interface import ( HeuristicInterface, TestPrioritizations, ) - from tools.testing.target_determination.heuristics.utils import ( normalize_ratings, query_changed_files, ) from tools.testing.test_run import TestRun + REPO_ROOT = Path(__file__).parent.parent.parent.parent keyword_synonyms: dict[str, list[str]] = { diff --git a/tools/testing/target_determination/heuristics/previously_failed_in_pr.py b/tools/testing/target_determination/heuristics/previously_failed_in_pr.py index ed8486227e1f..a17145a7eca1 100644 --- a/tools/testing/target_determination/heuristics/previously_failed_in_pr.py +++ b/tools/testing/target_determination/heuristics/previously_failed_in_pr.py @@ -10,7 +10,6 @@ from tools.stats.import_test_stats import ( TD_HEURISTIC_PREVIOUSLY_FAILED, TD_HEURISTIC_PREVIOUSLY_FAILED_ADDITIONAL, ) - from tools.testing.target_determination.heuristics.interface import ( HeuristicInterface, TestPrioritizations, @@ -20,6 +19,7 @@ from tools.testing.target_determination.heuristics.utils import ( ) from tools.testing.test_run import TestRun + REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent.parent diff --git a/tools/testing/test_selections.py b/tools/testing/test_selections.py index 5ac1d946d7ec..5aff1120c60a 100644 --- a/tools/testing/test_selections.py +++ b/tools/testing/test_selections.py @@ -9,6 +9,7 @@ from typing import Callable, Sequence from tools.stats.import_test_stats import get_disabled_tests, get_slow_tests from tools.testing.test_run import ShardedTest, TestRun + REPO_ROOT = Path(__file__).resolve().parent.parent.parent IS_MEM_LEAK_CHECK = os.getenv("PYTORCH_TEST_CUDA_MEM_LEAK_CHECK", "0") == "1" diff --git a/tools/vscode_settings.py b/tools/vscode_settings.py index 21fddf6caccb..edfaec3ebf4f 100755 --- a/tools/vscode_settings.py +++ b/tools/vscode_settings.py @@ -2,6 +2,7 @@ from pathlib import Path + try: # VS Code settings allow comments and trailing commas, which are not valid JSON. import json5 as json # type: ignore[import] diff --git a/torchgen/api/types/__init__.py b/torchgen/api/types/__init__.py index a190896f9e01..4e98bb8df493 100644 --- a/torchgen/api/types/__init__.py +++ b/torchgen/api/types/__init__.py @@ -1,3 +1,5 @@ from torchgen.api.types.types import * from torchgen.api.types.types_base import * -from torchgen.api.types.signatures import * # usort:skip + + +from torchgen.api.types.signatures import * # usort: skip diff --git a/torchgen/autoheuristic/gen_data_pad_mm.py b/torchgen/autoheuristic/gen_data_pad_mm.py index 1e0dbb339084..653363d8e3cd 100644 --- a/torchgen/autoheuristic/gen_data_pad_mm.py +++ b/torchgen/autoheuristic/gen_data_pad_mm.py @@ -1,18 +1,19 @@ import argparse import random import time - from typing import Any, Tuple from tqdm import tqdm # type: ignore[import-untyped] import torch + torch.set_default_device("cuda") from torch._inductor.fx_passes.pad_mm import get_alignment_size_dtype from torch._inductor.utils import fresh_inductor_cache + # A100: 81920MiB # without a threshold we sometimes run out of memory threshold_memory = 85899345920 / 4 diff --git a/torchgen/autoheuristic/train.py b/torchgen/autoheuristic/train.py index 23c062e4ae8d..d7568de3bc33 100644 --- a/torchgen/autoheuristic/train.py +++ b/torchgen/autoheuristic/train.py @@ -5,15 +5,14 @@ import sys import warnings import numpy as np - import pandas as pd # type: ignore[import-untyped] - from scipy.stats import gmean # type: ignore[import-untyped] from sklearn.model_selection import train_test_split # type: ignore[import-untyped] from sklearn.tree import DecisionTreeRegressor # type: ignore[import-untyped] from torch._inductor.autoheuristic.autoheuristic import deserialize_data + # TODO (AlnisM): Fix these warnings warnings.filterwarnings( "ignore", diff --git a/torchgen/executorch/api/custom_ops.py b/torchgen/executorch/api/custom_ops.py index c74af600d4d7..bbe62c72f688 100644 --- a/torchgen/executorch/api/custom_ops.py +++ b/torchgen/executorch/api/custom_ops.py @@ -6,8 +6,9 @@ from typing import Sequence, TYPE_CHECKING from torchgen import dest + # disable import sorting to avoid circular dependency. -from torchgen.api.types import DispatcherSignature # usort:skip +from torchgen.api.types import DispatcherSignature # usort: skip from torchgen.context import method_with_native_function from torchgen.model import BaseTy, BaseType, DispatchKey, NativeFunction, Variant from torchgen.utils import concatMap, Target diff --git a/torchgen/executorch/api/types/__init__.py b/torchgen/executorch/api/types/__init__.py index 6fc9666768ba..08cb168df737 100644 --- a/torchgen/executorch/api/types/__init__.py +++ b/torchgen/executorch/api/types/__init__.py @@ -1,2 +1,4 @@ from torchgen.executorch.api.types.types import * -from torchgen.executorch.api.types.signatures import * # usort:skip + + +from torchgen.executorch.api.types.signatures import * # usort: skip