frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 21:14:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

[BE][PYFMT] remove black: finish black -> ruff format migration (#144557)

Browse Source 
Pull Request resolved: https://github.com/pytorch/pytorch/pull/144557
Approved by: https://github.com/ezyang
This commit is contained in:
Xuehai Pan
2025-08-08 01:14:36 +08:00
committed by PyTorch MergeBot
parent 3a56237440
commit 178515d0ff
5 changed files with 1 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.lintrunner.toml
View File

@ -1452,8 +1452,6 @@ init_command = [
'python3',
'tools/linter/adapters/pip_init.py',
'--dry-run={{DRYRUN}}',
'--no-black-binary',
'black==23.12.1',
'usort==1.0.8.post1',
'isort==6.0.1',
'ruff==0.12.2', # sync with RUFF

3
pyproject.toml
View File

@ -69,9 +69,6 @@ pyyaml = ["pyyaml"]
# Linter tools #################################################################
[tool.black]
line-length = 88
[tool.isort]
src_paths = ["caffe2", "torch", "torchgen", "functorch", "test"]
extra_standard_library = ["typing_extensions"]

225
tools/linter/adapters/black_linter.py
View File

@ -1,225 +0,0 @@
from __future__ import annotations
import argparse
import concurrent.futures
import json
import logging
import os
import subprocess
import sys
import time
from enum import Enum
from typing import BinaryIO, NamedTuple
IS_WINDOWS: bool = os.name == "nt"
class LintSeverity(str, Enum):
ERROR = "error"
WARNING = "warning"
ADVICE = "advice"
DISABLED = "disabled"
class LintMessage(NamedTuple):
path: str | None
line: int | None
char: int | None
code: str
severity: LintSeverity
name: str
original: str | None
replacement: str | None
description: str | None
def as_posix(name: str) -> str:
return name.replace("\\", "/") if IS_WINDOWS else name
def _run_command(
args: list[str],
*,
stdin: BinaryIO,
timeout: int,
) -> subprocess.CompletedProcess[bytes]:
logging.debug("$ %s", " ".join(args))
start_time = time.monotonic()
try:
return subprocess.run(
args,
stdin=stdin,
capture_output=True,
shell=IS_WINDOWS, # So batch scripts are found.
timeout=timeout,
check=True,
)
finally:
end_time = time.monotonic()
logging.debug("took %dms", (end_time - start_time) * 1000)
def run_command(
args: list[str],
*,
stdin: BinaryIO,
retries: int,
timeout: int,
) -> subprocess.CompletedProcess[bytes]:
remaining_retries = retries
while True:
try:
return _run_command(args, stdin=stdin, timeout=timeout)
except subprocess.TimeoutExpired as err:
if remaining_retries == 0:
raise err
remaining_retries -= 1
logging.warning(
"(%s/%s) Retrying because command failed with: %r",
retries - remaining_retries,
retries,
err,
)
time.sleep(1)
def check_file(
filename: str,
retries: int,
timeout: int,
) -> list[LintMessage]:
try:
with open(filename, "rb") as f:
original = f.read()
with open(filename, "rb") as f:
proc = run_command(
[sys.executable, "-mblack", "--stdin-filename", filename, "-"],
stdin=f,
retries=retries,
timeout=timeout,
)
except subprocess.TimeoutExpired:
return [
LintMessage(
path=filename,
line=None,
char=None,
code="BLACK",
severity=LintSeverity.ERROR,
name="timeout",
original=None,
replacement=None,
description=(
"black timed out while trying to process a file. "
"Please report an issue in pytorch/pytorch with the "
"label 'module: lint'"
),
)
]
except (OSError, subprocess.CalledProcessError) as err:
return [
LintMessage(
path=filename,
line=None,
char=None,
code="BLACK",
severity=LintSeverity.ADVICE,
name="command-failed",
original=None,
replacement=None,
description=(
f"Failed due to {err.__class__.__name__}:\n{err}"
if not isinstance(err, subprocess.CalledProcessError)
else (
"COMMAND (exit code {returncode})\n"
"{command}\n\n"
"STDERR\n{stderr}\n\n"
"STDOUT\n{stdout}"
).format(
returncode=err.returncode,
command=" ".join(as_posix(x) for x in err.cmd),
stderr=err.stderr.decode("utf-8").strip() or "(empty)",
stdout=err.stdout.decode("utf-8").strip() or "(empty)",
)
),
)
]
replacement = proc.stdout
if original == replacement:
return []
return [
LintMessage(
path=filename,
line=None,
char=None,
code="BLACK",
severity=LintSeverity.WARNING,
name="format",
original=original.decode("utf-8"),
replacement=replacement.decode("utf-8"),
description="Run `lintrunner -a` to apply this patch.",
)
]
def main() -> None:
parser = argparse.ArgumentParser(
description="Format files with black.",
fromfile_prefix_chars="@",
)
parser.add_argument(
"--retries",
default=3,
type=int,
help="times to retry timed out black",
)
parser.add_argument(
"--timeout",
default=90,
type=int,
help="seconds to wait for black",
)
parser.add_argument(
"--verbose",
action="store_true",
help="verbose logging",
)
parser.add_argument(
"filenames",
nargs="+",
help="paths to lint",
)
args = parser.parse_args()
logging.basicConfig(
format="<%(threadName)s:%(levelname)s> %(message)s",
level=logging.NOTSET
if args.verbose
else logging.DEBUG
if len(args.filenames) < 1000
else logging.INFO,
stream=sys.stderr,
)
with concurrent.futures.ThreadPoolExecutor(
max_workers=os.cpu_count(),
thread_name_prefix="Thread",
) as executor:
futures = {
executor.submit(check_file, x, args.retries, args.timeout): x
for x in args.filenames
}
for future in concurrent.futures.as_completed(futures):
try:
for lint_message in future.result():
print(json.dumps(lint_message._asdict()), flush=True)
except Exception:
logging.critical('Failed at "%s".', futures[future])
raise
if __name__ == "__main__":
main()

7
tools/linter/adapters/pip_init.py
View File

@ -41,11 +41,6 @@ def main() -> None:
parser.add_argument(
"--dry-run", help="do not install anything, just print what would be done."
)
parser.add_argument(
"--no-black-binary",
help="do not use pre-compiled binaries from pip for black.",
action="store_true",
)
args = parser.parse_args()
@ -97,8 +92,6 @@ def main() -> None:
"Package {package_name} did not have a version specified. "
"Please specify a version to produce a consistent linting experience."
)
if args.no_black_binary and "black" in package_name:
pip_args.append(f"--no-binary={package_name}")
dry_run = args.dry_run == "1"
if dry_run:

61
tools/linter/adapters/pyfmt_linter.py
View File

@ -2,7 +2,6 @@ from __future__ import annotations
import argparse
import concurrent.futures
import fnmatch
import json
import logging
import os
@ -13,7 +12,6 @@ from enum import Enum
from pathlib import Path
from typing import NamedTuple
import black
import isort
import usort
@ -21,43 +19,6 @@ import usort
IS_WINDOWS: bool = os.name == "nt"
REPO_ROOT = Path(__file__).absolute().parents[3]
# TODO: remove this when it gets empty and remove `black` in PYFMT
USE_BLACK_FILELIST = re.compile(
"|".join(
(
r"\A\Z", # empty string
*map(
fnmatch.translate,
[
# **
# .ci/**
# .github/**
# benchmarks/**
# functorch/**
# tools/**
# torchgen/**
# test/**
# test/[a-h]*/**
# test/[i-j]*/**
# test/[k-m]*/**
# test/optim/**
# test/[p-z]*/**,
# torch/**
# torch/_[a-c]*/**
# torch/_[e-h]*/**
# torch/_i*/**
# torch/_[j-z]*/**
# torch/[a-c]*/**
# torch/d*/**
# torch/[e-m]*/**
# torch/optim/**
# torch/[p-z]*/**
],
),
)
)
)
class LintSeverity(str, Enum):
ERROR = "error"
@ -117,23 +78,6 @@ def run_usort(content: str, path: Path) -> str:
return usort.usort_string(content, path=path, config=usort_config)
def run_black(content: str, path: Path) -> str:
black_config = black.parse_pyproject_toml(black.find_pyproject_toml((str(path),))) # type: ignore[attr-defined,arg-type]
# manually patch options that do not have a 1-to-1 match in Mode arguments
black_config["target_versions"] = {
black.TargetVersion[ver.upper()] # type: ignore[attr-defined]
for ver in black_config.pop("target_version", [])
}
black_config["string_normalization"] = not black_config.pop(
"skip_string_normalization", False
)
black_mode = black.Mode(**black_config)
black_mode.is_pyi = path.suffix.lower() == ".pyi"
black_mode.is_ipynb = path.suffix.lower() == ".ipynb"
return black.format_str(content, mode=black_mode)
def run_ruff_format(content: str, path: Path) -> str:
try:
return subprocess.check_output(
@ -165,10 +109,7 @@ def check_file(filename: str) -> list[LintMessage]:
# NB: run isort first to enforce style for blank lines
replacement = run_isort(replacement, path=path)
replacement = run_usort(replacement, path=path)
if USE_BLACK_FILELIST.match(path.absolute().relative_to(REPO_ROOT).as_posix()):
replacement = run_black(replacement, path=path)
else:
replacement = run_ruff_format(replacement, path=path)
replacement = run_ruff_format(replacement, path=path)
if original == replacement:
return []