diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 91765566b7c9..b096330bcd40 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -96,8 +96,8 @@ jobs: python3 -m pip install boto3==1.19.12 .github/scripts/lint_test_ownership.py - shellcheck: - name: shellcheck + workflow-checks: + name: workflow-checks runs-on: ubuntu-18.04 steps: - name: Setup Python @@ -134,32 +134,10 @@ jobs: echo 'onto a more recent commit from the PyTorch master branch.' false fi - - name: Install ShellCheck - id: install_shellcheck - if: always() - # https://github.com/koalaman/shellcheck/tree/v0.7.2#installing-a-pre-compiled-binary - run: | - set -x - scversion="v0.7.2" - wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv - mkdir -p ~/.local/bin - cp "shellcheck-${scversion}/shellcheck" ~/.local/bin/ - rm -r "shellcheck-${scversion}" - ~/.local/bin/shellcheck --version - name: Check that jobs will be cancelled if: ${{ always() && steps.generate_workflows.outcome == 'success' }} run: | .github/scripts/ensure_actions_will_cancel.py - - uses: nick-fields/retry@71062288b76e2b6214ebde0e673ce0de1755740a - name: Run actionlint - with: - timeout_minutes: 1 - max_attempts: 3 - command: | - set -eux - bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) - ./actionlint --color - rm actionlint toc: name: toc diff --git a/.lintrunner.toml b/.lintrunner.toml index dbf60207252e..1bd516919167 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -492,3 +492,29 @@ init_command = [ '--dry-run={{DRYRUN}}', 'shellcheck-py==0.7.2.1', ] + +[[linter]] +code = 'ACTIONLINT' +include_patterns = [ + '.github/workflows/*.yml', + '.github/workflows/*.yaml', + # actionlint does not support composite actions yet + # '.github/actions/**/*.yml', + # '.github/actions/**/*.yaml', +] +command = [ + 'python3', + 'tools/linter/adapters/actionlint_linter.py', + '--binary=.lintbin/actionlint', + '--', + '@{{PATHSFILE}}', +] +init_command = [ + 'python3', + 'tools/linter/adapters/s3_init.py', + '--config-json=tools/linter/adapters/s3_init_config.json', + '--linter=actionlint', + '--dry-run={{DRYRUN}}', + '--output-dir=.lintbin', + '--output-name=actionlint', +] diff --git a/Makefile b/Makefile index 45fdf3b2e1f8..2d712064c597 100644 --- a/Makefile +++ b/Makefile @@ -16,57 +16,25 @@ ios: clean: # This will remove ALL build folders. @rm -r build*/ - @$(RM) -r $(SHELLCHECK_GHA_GENERATED_FOLDER) linecount: @cloc --read-lang-def=caffe.cloc caffe2 || \ echo "Cloc is not available on the machine. You can install cloc with " && \ echo " sudo apt-get install cloc" -SHELLCHECK_GHA_GENERATED_FOLDER=.shellcheck_generated_gha -shellcheck-gha: - @$(RM) -r $(SHELLCHECK_GHA_GENERATED_FOLDER) - tools/extract_scripts.py --out=$(SHELLCHECK_GHA_GENERATED_FOLDER) - tools/linter/run_shellcheck.sh $(SHELLCHECK_GHA_GENERATED_FOLDER) - -generate-gha-workflows: - .github/scripts/generate_ci_workflows.py - $(MAKE) shellcheck-gha - shellcheck: @$(PYTHON) tools/actions_local_runner.py \ --file .github/workflows/lint.yml \ - --job 'shellcheck' \ + --job 'workflow-checks' \ --step "Regenerate workflows" @$(PYTHON) tools/actions_local_runner.py \ --file .github/workflows/lint.yml \ - --job 'shellcheck' \ + --job 'workflow-checks' \ --step "Assert that regenerating the workflows didn't change them" - @$(PYTHON) tools/actions_local_runner.py \ - --file .github/workflows/lint.yml \ - --job 'shellcheck' \ - --step 'Extract scripts from GitHub Actions workflows' - @$(PYTHON) tools/actions_local_runner.py \ - $(CHANGED_ONLY) \ - $(REF_BRANCH) \ - --job 'shellcheck' setup_lint: $(PIP) install lintrunner lintrunner init - $(PYTHON) tools/actions_local_runner.py --file .github/workflows/lint.yml \ - --job 'shellcheck' --step 'Install Jinja2' --no-quiet - - @if [ "$$(uname)" = "Darwin" ]; then \ - if [ -z "$$(which brew)" ]; then \ - echo "'brew' is required to install ShellCheck, get it here: https://brew.sh "; \ - exit 1; \ - fi; \ - brew install shellcheck; \ - else \ - $(PYTHON) tools/actions_local_runner.py --file .github/workflows/lint.yml \ - --job 'shellcheck' --step 'Install ShellCheck' --no-quiet; \ - fi $(PYTHON) -mpip install jinja2 --user quick_checks: diff --git a/tools/linter/adapters/actionlint_linter.py b/tools/linter/adapters/actionlint_linter.py new file mode 100644 index 000000000000..b78582931b06 --- /dev/null +++ b/tools/linter/adapters/actionlint_linter.py @@ -0,0 +1,139 @@ +import argparse +import os +import re +import json +import logging +import subprocess +import time +from enum import Enum +from typing import List, NamedTuple, Optional, Pattern + + +LINTER_CODE = "ACTIONLINT" + + +class LintSeverity(str, Enum): + ERROR = "error" + WARNING = "warning" + ADVICE = "advice" + DISABLED = "disabled" + + +class LintMessage(NamedTuple): + path: Optional[str] + line: Optional[int] + char: Optional[int] + code: str + severity: LintSeverity + name: str + original: Optional[str] + replacement: Optional[str] + description: Optional[str] + + +RESULTS_RE: Pattern[str] = re.compile( + r"""(?mx) + ^ + (?P.*?): + (?P\d+): + (?P\d+): + \s(?P.*) + \s(?P\[.*\]) + $ + """ +) + +def run_command( + args: List[str], +) -> "subprocess.CompletedProcess[bytes]": + logging.debug("$ %s", " ".join(args)) + start_time = time.monotonic() + try: + return subprocess.run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + finally: + end_time = time.monotonic() + logging.debug("took %dms", (end_time - start_time) * 1000) + + +def check_files( + binary: str, + files: List[str], +) -> List[LintMessage]: + try: + proc = run_command( + [binary] + files + ) + except OSError as err: + return [ + LintMessage( + path=None, + line=None, + char=None, + code=LINTER_CODE, + severity=LintSeverity.ERROR, + name="command-failed", + original=None, + replacement=None, + description=(f"Failed due to {err.__class__.__name__}:\n{err}"), + ) + ] + stdout = str(proc.stdout, "utf-8").strip() + return [ + LintMessage( + path=match["file"], + name=match["code"], + description=match["message"], + line=int(match["line"]), + char=int(match["char"]), + code=LINTER_CODE, + severity=LintSeverity.ERROR, + original=None, + replacement=None, + ) + for match in RESULTS_RE.finditer(stdout) + ] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="actionlint runner", + fromfile_prefix_chars="@", + ) + parser.add_argument( + "--binary", + required=True, + help="actionlint binary path", + ) + parser.add_argument( + "filenames", + nargs="+", + help="paths to lint", + ) + + args = parser.parse_args() + + if not os.path.exists(args.binary): + err_msg = LintMessage( + path="", + line=None, + char=None, + code=LINTER_CODE, + severity=LintSeverity.ERROR, + name="command-failed", + original=None, + replacement=None, + description=( + f"Could not find actionlint binary at {args.binary}," + " you may need to run `lintrunner init`." + ), + ) + print(json.dumps(err_msg._asdict()), flush=True) + exit(0) + + lint_messages = check_files(args.binary, args.filenames) + for lint_message in lint_messages: + print(json.dumps(lint_message._asdict()), flush=True) diff --git a/tools/linter/adapters/s3_init_config.json b/tools/linter/adapters/s3_init_config.json index 0f3619ad0fff..736ab6addb84 100644 --- a/tools/linter/adapters/s3_init_config.json +++ b/tools/linter/adapters/s3_init_config.json @@ -2,29 +2,31 @@ "clang-format": { "Darwin": { "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/mac/clang-format-mojave", - "hash": "1485a242a96c737ba7cdd9f259114f2201accdb46d87ac7a8650b1a814cd4d4d", - "object_name": "mac/clang-format-mojave", - "s3_bucket": "oss-clang-format" + "hash": "1485a242a96c737ba7cdd9f259114f2201accdb46d87ac7a8650b1a814cd4d4d" }, "Linux": { "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64", - "hash": "e1c8b97b919541a99e0a355df5c3f9e8abebc64259dbee6f8c68e1ef90582856", - "object_name": "linux64/clang-format-linux64", - "s3_bucket": "oss-clang-format" + "hash": "e1c8b97b919541a99e0a355df5c3f9e8abebc64259dbee6f8c68e1ef90582856" } }, "clang-tidy": { "Darwin": { "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos/clang-tidy", - "hash": "541797a7b8fa795e2f3c1adcd8236cc336a40aa927028dc5bc79172e1d9eca36", - "object_name": "macos/clang-tidy", - "s3_bucket": "oss-clang-format" + "hash": "541797a7b8fa795e2f3c1adcd8236cc336a40aa927028dc5bc79172e1d9eca36" }, "Linux": { "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-tidy", - "hash": "49343a448fcb75cd1e0fb9d6b1f6c2ef4b008b6f91d6ff899d4ac6060f5e52a5", - "object_name": "linx64/clang-tidy", - "s3_bucket": "oss-clang-format" + "hash": "49343a448fcb75cd1e0fb9d6b1f6c2ef4b008b6f91d6ff899d4ac6060f5e52a5" + } + }, + "actionlint": { + "Darwin": { + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos/actionlint", + "hash": "3ce2c94280c540e20b270acae60bdd9e72ad17d6cb35b688951b1ec1eb8cbdd6" + }, + "Linux": { + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/actionlint", + "hash": "693f464106474760f0edf4a1778215095eacc4bd5f79aab5dc950892f120828b" } } }