[lint] add actionlint to lintrunner

Pull Request resolved: https://github.com/pytorch/pytorch/pull/75857

Approved by: https://github.com/malfet

.github/workflows/lint.yml

@@ -96,8 +96,8 @@ jobs:
           python3 -m pip install boto3==1.19.12
           .github/scripts/lint_test_ownership.py
 
-  shellcheck:
+  workflow-checks:
-    name: shellcheck
+    name: workflow-checks
     runs-on: ubuntu-18.04
     steps:
       - name: Setup Python
@@ -134,32 +134,10 @@ jobs:
             echo 'onto a more recent commit from the PyTorch master branch.'
             false
           fi
-      - name: Install ShellCheck
-        id: install_shellcheck
-        if: always()
-        # https://github.com/koalaman/shellcheck/tree/v0.7.2#installing-a-pre-compiled-binary
-        run: |
-          set -x
-          scversion="v0.7.2"
-          wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv
-          mkdir -p ~/.local/bin
-          cp "shellcheck-${scversion}/shellcheck" ~/.local/bin/
-          rm -r "shellcheck-${scversion}"
-          ~/.local/bin/shellcheck --version
       - name: Check that jobs will be cancelled
         if: ${{ always() && steps.generate_workflows.outcome == 'success' }}
         run: |
           .github/scripts/ensure_actions_will_cancel.py
-      - uses: nick-fields/retry@71062288b76e2b6214ebde0e673ce0de1755740a
-        name: Run actionlint
-        with:
-          timeout_minutes: 1
-          max_attempts: 3
-          command: |
-            set -eux
-            bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
-            ./actionlint --color
-            rm actionlint
 
   toc:
     name: toc

.lintrunner.toml

@@ -492,3 +492,29 @@ init_command = [
     '--dry-run={{DRYRUN}}',
     'shellcheck-py==0.7.2.1',
 ]
+
+[[linter]]
+code = 'ACTIONLINT'
+include_patterns = [
+    '.github/workflows/*.yml',
+    '.github/workflows/*.yaml',
+    # actionlint does not support composite actions yet
+    # '.github/actions/**/*.yml',
+    # '.github/actions/**/*.yaml',
+]
+command = [
+    'python3',
+    'tools/linter/adapters/actionlint_linter.py',
+    '--binary=.lintbin/actionlint',
+    '--',
+    '@{{PATHSFILE}}',
+]
+init_command = [
+    'python3',
+    'tools/linter/adapters/s3_init.py',
+    '--config-json=tools/linter/adapters/s3_init_config.json',
+    '--linter=actionlint',
+    '--dry-run={{DRYRUN}}',
+    '--output-dir=.lintbin',
+    '--output-name=actionlint',
+]

Makefile

@@ -16,57 +16,25 @@ ios:
 
 clean: # This will remove ALL build folders.
 	@rm -r build*/
-	@$(RM) -r $(SHELLCHECK_GHA_GENERATED_FOLDER)
 
 linecount:
 	@cloc --read-lang-def=caffe.cloc caffe2 || \
 		echo "Cloc is not available on the machine. You can install cloc with " && \
 		echo "    sudo apt-get install cloc"
 
-SHELLCHECK_GHA_GENERATED_FOLDER=.shellcheck_generated_gha
-shellcheck-gha:
-	@$(RM) -r $(SHELLCHECK_GHA_GENERATED_FOLDER)
-	tools/extract_scripts.py --out=$(SHELLCHECK_GHA_GENERATED_FOLDER)
-	tools/linter/run_shellcheck.sh $(SHELLCHECK_GHA_GENERATED_FOLDER)
-
-generate-gha-workflows:
-	.github/scripts/generate_ci_workflows.py
-	$(MAKE) shellcheck-gha
-
 shellcheck:
 	@$(PYTHON) tools/actions_local_runner.py \
 		--file .github/workflows/lint.yml \
-		--job 'shellcheck' \
+		--job 'workflow-checks' \
 		--step "Regenerate workflows"
 	@$(PYTHON) tools/actions_local_runner.py \
 		--file .github/workflows/lint.yml \
-		--job 'shellcheck' \
+		--job 'workflow-checks' \
 		--step "Assert that regenerating the workflows didn't change them"
-	@$(PYTHON) tools/actions_local_runner.py \
-		--file .github/workflows/lint.yml \
-		--job 'shellcheck' \
-		--step 'Extract scripts from GitHub Actions workflows'
-	@$(PYTHON) tools/actions_local_runner.py \
-		$(CHANGED_ONLY) \
-		$(REF_BRANCH) \
-		--job 'shellcheck'
 
 setup_lint:
 	$(PIP) install lintrunner
 	lintrunner init
-	$(PYTHON) tools/actions_local_runner.py --file .github/workflows/lint.yml \
-		--job 'shellcheck' --step 'Install Jinja2' --no-quiet
-
-	@if [ "$$(uname)" = "Darwin" ]; then \
-		if [ -z "$$(which brew)" ]; then \
-			echo "'brew' is required to install ShellCheck, get it here: https://brew.sh "; \
-			exit 1; \
-		fi; \
-		brew install shellcheck; \
-	else \
-		$(PYTHON) tools/actions_local_runner.py --file .github/workflows/lint.yml \
-		--job 'shellcheck' --step 'Install ShellCheck' --no-quiet; \
-	fi
 	$(PYTHON) -mpip install jinja2 --user
 
 quick_checks:

tools/linter/adapters/actionlint_linter.py

@@ -0,0 +1,139 @@
+import argparse
+import os
+import re
+import json
+import logging
+import subprocess
+import time
+from enum import Enum
+from typing import List, NamedTuple, Optional, Pattern
+
+
+LINTER_CODE = "ACTIONLINT"
+
+
+class LintSeverity(str, Enum):
+    ERROR = "error"
+    WARNING = "warning"
+    ADVICE = "advice"
+    DISABLED = "disabled"
+
+
+class LintMessage(NamedTuple):
+    path: Optional[str]
+    line: Optional[int]
+    char: Optional[int]
+    code: str
+    severity: LintSeverity
+    name: str
+    original: Optional[str]
+    replacement: Optional[str]
+    description: Optional[str]
+
+
+RESULTS_RE: Pattern[str] = re.compile(
+    r"""(?mx)
+    ^
+    (?P<file>.*?):
+    (?P<line>\d+):
+    (?P<char>\d+):
+    \s(?P<message>.*)
+    \s(?P<code>\[.*\])
+    $
+    """
+)
+
+def run_command(
+    args: List[str],
+) -> "subprocess.CompletedProcess[bytes]":
+    logging.debug("$ %s", " ".join(args))
+    start_time = time.monotonic()
+    try:
+        return subprocess.run(
+            args,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+    finally:
+        end_time = time.monotonic()
+        logging.debug("took %dms", (end_time - start_time) * 1000)
+
+
+def check_files(
+    binary: str,
+    files: List[str],
+) -> List[LintMessage]:
+    try:
+        proc = run_command(
+            [binary] + files
+        )
+    except OSError as err:
+        return [
+            LintMessage(
+                path=None,
+                line=None,
+                char=None,
+                code=LINTER_CODE,
+                severity=LintSeverity.ERROR,
+                name="command-failed",
+                original=None,
+                replacement=None,
+                description=(f"Failed due to {err.__class__.__name__}:\n{err}"),
+            )
+        ]
+    stdout = str(proc.stdout, "utf-8").strip()
+    return [
+        LintMessage(
+            path=match["file"],
+            name=match["code"],
+            description=match["message"],
+            line=int(match["line"]),
+            char=int(match["char"]),
+            code=LINTER_CODE,
+            severity=LintSeverity.ERROR,
+            original=None,
+            replacement=None,
+        )
+        for match in RESULTS_RE.finditer(stdout)
+    ]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="actionlint runner",
+        fromfile_prefix_chars="@",
+    )
+    parser.add_argument(
+        "--binary",
+        required=True,
+        help="actionlint binary path",
+    )
+    parser.add_argument(
+        "filenames",
+        nargs="+",
+        help="paths to lint",
+    )
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.binary):
+        err_msg = LintMessage(
+            path="<none>",
+            line=None,
+            char=None,
+            code=LINTER_CODE,
+            severity=LintSeverity.ERROR,
+            name="command-failed",
+            original=None,
+            replacement=None,
+            description=(
+                f"Could not find actionlint binary at {args.binary},"
+                " you may need to run `lintrunner init`."
+            ),
+        )
+        print(json.dumps(err_msg._asdict()), flush=True)
+        exit(0)
+
+    lint_messages = check_files(args.binary, args.filenames)
+    for lint_message in lint_messages:
+        print(json.dumps(lint_message._asdict()), flush=True)

tools/linter/adapters/s3_init_config.json

@@ -2,29 +2,31 @@
     "clang-format": {
         "Darwin": {
             "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/mac/clang-format-mojave",
-            "hash": "1485a242a96c737ba7cdd9f259114f2201accdb46d87ac7a8650b1a814cd4d4d",
+            "hash": "1485a242a96c737ba7cdd9f259114f2201accdb46d87ac7a8650b1a814cd4d4d"
-            "object_name": "mac/clang-format-mojave",
-            "s3_bucket": "oss-clang-format"
         },
         "Linux": {
             "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64",
-            "hash": "e1c8b97b919541a99e0a355df5c3f9e8abebc64259dbee6f8c68e1ef90582856",
+            "hash": "e1c8b97b919541a99e0a355df5c3f9e8abebc64259dbee6f8c68e1ef90582856"
-            "object_name": "linux64/clang-format-linux64",
-            "s3_bucket": "oss-clang-format"
         }
     },
     "clang-tidy": {
         "Darwin": {
             "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos/clang-tidy",
-            "hash": "541797a7b8fa795e2f3c1adcd8236cc336a40aa927028dc5bc79172e1d9eca36",
+            "hash": "541797a7b8fa795e2f3c1adcd8236cc336a40aa927028dc5bc79172e1d9eca36"
-            "object_name": "macos/clang-tidy",
-            "s3_bucket": "oss-clang-format"
         },
         "Linux": {
             "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-tidy",
-            "hash": "49343a448fcb75cd1e0fb9d6b1f6c2ef4b008b6f91d6ff899d4ac6060f5e52a5",
+            "hash": "49343a448fcb75cd1e0fb9d6b1f6c2ef4b008b6f91d6ff899d4ac6060f5e52a5"
-            "object_name": "linx64/clang-tidy",
+        }
-            "s3_bucket": "oss-clang-format"
+    },
+    "actionlint": {
+        "Darwin": {
+            "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos/actionlint",
+            "hash": "3ce2c94280c540e20b270acae60bdd9e72ad17d6cb35b688951b1ec1eb8cbdd6"
+        },
+        "Linux": {
+            "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/actionlint",
+            "hash": "693f464106474760f0edf4a1778215095eacc4bd5f79aab5dc950892f120828b"
         }
     }
 }