[lint] add nativefunctions to lintrunner (#67890)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/67890

Adding another linter. I also added a generic initializer that installs
the right pip packages (you can invoke it by running `lintrunner init`).

Differential Revision:
D32197366
D32197366

Test Plan: Imported from OSS

Reviewed By: driazati

Pulled By: suo

fbshipit-source-id: 82844e78f1ee3047220d8444874eab41d7cc0e9e

.lintrunner.toml

@@ -25,6 +25,19 @@ args = [
     '--',
     '@{{PATHSFILE}}'
 ]
+init_args = [
+    'python3',
+    'tools/linter/adapters/pip_init.py',
+    '--dry-run={{DRYRUN}}',
+    'flake8==3.8.2',
+    'flake8-bugbear==20.1.4',
+    'flake8-comprehensions==3.3.0',
+    'flake8-executable==2.0.4',
+    'flake8-pyi==20.5.0',
+    'mccabe==0.6.1',
+    'pycodestyle==2.6.0',
+    'pyflakes==2.2.0',
+]
 
 
 [[linter]]
@@ -129,3 +142,19 @@ args = [
     '--regen-script=generate_config_yml.py',
 ]
 bypass_matched_file_filter = true
+
+[[linter]]
+name = 'NATIVEFUNCTIONS'
+include_patterns=['aten/src/ATen/native/native_functions.yaml']
+args = [
+    'python3',
+    'tools/linter/adapters/nativefunctions_linter.py',
+    '--native-functions-yml=aten/src/ATen/native/native_functions.yaml',
+]
+init_args = [
+    'python3',
+    'tools/linter/adapters/pip_init.py',
+    '--dry-run={{DRYRUN}}',
+    'ruamel.yaml==0.17.4',
+]
+bypass_matched_file_filter = true

requirements-flake8.txt

@@ -4,6 +4,6 @@ flake8-comprehensions==3.3.0
 flake8-executable==2.0.4
 git+https://github.com/malfet/flake8-coding.git
 flake8-pyi==20.5.0
-mccabe
+mccabe==0.6.1
 pycodestyle==2.6.0
 pyflakes==2.2.0

tools/linter/adapters/nativefunctions_linter.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""
+Verify that it is possible to round-trip native_functions.yaml via ruamel under some
+configuration.  Keeping native_functions.yaml consistent in this way allows us to
+run codemods on the file using ruamel without introducing line noise.  Note that we don't
+want to normalize the YAML file, as that would to lots of spurious lint failures.  Anything
+that ruamel understands how to roundtrip, e.g., whitespace and comments, is OK!
+
+ruamel is a bit picky about inconsistent indentation, so you will have to indent your
+file properly.  Also, if you are working on changing the syntax of native_functions.yaml,
+you may find that you want to use some format that is not what ruamel prefers.  If so,
+it is OK to modify this script (instead of reformatting native_functions.yaml)--the point
+is simply to make sure that there is *some* configuration of ruamel that can round trip
+the YAML, not to be prescriptive about it.
+"""
+
+import ruamel.yaml  # type: ignore[import]
+import argparse
+import json
+import sys
+from io import StringIO
+from enum import Enum
+from typing import NamedTuple, Optional
+
+
+class LintSeverity(str, Enum):
+    ERROR = "error"
+    WARNING = "warning"
+    ADVICE = "advice"
+    DISABLED = "disabled"
+
+
+class LintMessage(NamedTuple):
+    path: Optional[str]
+    line: Optional[int]
+    char: Optional[int]
+    code: str
+    severity: LintSeverity
+    name: str
+    original: Optional[str]
+    replacement: Optional[str]
+    description: Optional[str]
+    bypassChangedLineFiltering: Optional[bool]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="native functions linter", fromfile_prefix_chars="@",
+    )
+    parser.add_argument(
+        "--native-functions-yml",
+        required=True,
+        help="location of native_functions.yaml",
+    )
+
+    args = parser.parse_args()
+
+    with open(args.native_functions_yml) as f:
+        contents = f.read()
+
+    yaml = ruamel.yaml.YAML()  # type: ignore[attr-defined]
+    yaml.preserve_quotes = True  # type: ignore[assignment]
+    yaml.width = 1000  # type: ignore[assignment]
+    yaml.boolean_representation = ["False", "True"]  # type: ignore[attr-defined]
+    try:
+        r = yaml.load(contents)
+    except Exception as err:
+        msg = LintMessage(
+            path=None,
+            line=None,
+            char=None,
+            code="NATIVEFUNCTIONS",
+            severity=LintSeverity.ERROR,
+            name="YAML load failure",
+            original=None,
+            replacement=None,
+            description=f"Failed due to {err.__class__.__name__}:\n{err}",
+            bypassChangedLineFiltering=None,
+        )
+
+        print(json.dumps(msg._asdict()), flush=True)
+        sys.exit(0)
+
+    # Cuz ruamel's author intentionally didn't include conversion to string
+    # https://stackoverflow.com/questions/47614862/best-way-to-use-ruamel-yaml-to-dump-to-string-not-to-stream
+    string_stream = StringIO()
+    yaml.dump(r, string_stream)
+    new_contents = string_stream.getvalue()
+    string_stream.close()
+
+    if contents != new_contents:
+        msg = LintMessage(
+            path=args.native_functions_yml,
+            line=1,
+            char=1,
+            code="NATIVEFUNCTIONS",
+            severity=LintSeverity.ERROR,
+            name="roundtrip inconsistency",
+            original=contents,
+            replacement=new_contents,
+            description=(
+                "YAML roundtrip failed; run `lintrunner --take NATIVEFUNCTIONS -a` to apply the suggested changes. "
+                "If you think this is in error, please see tools/linter/adapters/nativefunctions_linter.py"
+            ),
+            bypassChangedLineFiltering=None,
+        )
+
+        print(json.dumps(msg._asdict()), flush=True)

tools/linter/adapters/pip_init.py

@@ -0,0 +1,56 @@
+"""
+Initializer script that installs stuff to pip.
+"""
+import argparse
+import logging
+import subprocess
+import sys
+import time
+
+from typing import List
+
+
+def run_command(args: List[str]) -> "subprocess.CompletedProcess[bytes]":
+    logging.debug("$ %s", " ".join(args))
+    start_time = time.monotonic()
+    try:
+        return subprocess.run(args, check=True)
+    finally:
+        end_time = time.monotonic()
+        logging.debug("took %dms", (end_time - start_time) * 1000)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="pip initializer")
+    parser.add_argument(
+        "packages", nargs="+", help="pip packages to install",
+    )
+    parser.add_argument(
+        "--verbose", action="store_true", help="verbose logging",
+    )
+    parser.add_argument("--dry-run", help="do not install anything, just print what would be done.")
+
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        format="<%(threadName)s:%(levelname)s> %(message)s",
+        level=logging.NOTSET if args.verbose else logging.DEBUG,
+        stream=sys.stderr,
+    )
+
+    for package in args.packages:
+        package_name, _, version = package.partition("=")
+        if version == "":
+            raise RuntimeError(
+                "Package {package_name} did not have a version specified. "
+                "Please specify a version to product a consistent linting experience."
+            )
+    pip_args = ["pip3", "install", "--user"]
+    pip_args.extend(args.packages)
+
+    dry_run = args.dry_run == "1"
+    if dry_run:
+        print(f"Would have run: {pip_args}")
+        sys.exit(0)
+
+    run_command(pip_args)