mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 12:54:11 +08:00
[lint] add actionlint to lintrunner
Pull Request resolved: https://github.com/pytorch/pytorch/pull/75857 Approved by: https://github.com/malfet
This commit is contained in:
Michael Suo
committed by
PyTorch MergeBot
PyTorch MergeBot
parent
cbbb96c271
commit
356f1478d8
26
.github/workflows/lint.yml
vendored
26
.github/workflows/lint.yml
vendored
@ -96,8 +96,8 @@ jobs:
|
||||
python3 -m pip install boto3==1.19.12
|
||||
.github/scripts/lint_test_ownership.py
|
||||
|
||||
shellcheck:
|
||||
name: shellcheck
|
||||
workflow-checks:
|
||||
name: workflow-checks
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- name: Setup Python
|
||||
@ -134,32 +134,10 @@ jobs:
|
||||
echo 'onto a more recent commit from the PyTorch master branch.'
|
||||
false
|
||||
fi
|
||||
- name: Install ShellCheck
|
||||
id: install_shellcheck
|
||||
if: always()
|
||||
# https://github.com/koalaman/shellcheck/tree/v0.7.2#installing-a-pre-compiled-binary
|
||||
run: |
|
||||
set -x
|
||||
scversion="v0.7.2"
|
||||
wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv
|
||||
mkdir -p ~/.local/bin
|
||||
cp "shellcheck-${scversion}/shellcheck" ~/.local/bin/
|
||||
rm -r "shellcheck-${scversion}"
|
||||
~/.local/bin/shellcheck --version
|
||||
- name: Check that jobs will be cancelled
|
||||
if: ${{ always() && steps.generate_workflows.outcome == 'success' }}
|
||||
run: |
|
||||
.github/scripts/ensure_actions_will_cancel.py
|
||||
- uses: nick-fields/retry@71062288b76e2b6214ebde0e673ce0de1755740a
|
||||
name: Run actionlint
|
||||
with:
|
||||
timeout_minutes: 1
|
||||
max_attempts: 3
|
||||
command: |
|
||||
set -eux
|
||||
bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
|
||||
./actionlint --color
|
||||
rm actionlint
|
||||
|
||||
toc:
|
||||
name: toc
|
||||
|
||||
@ -492,3 +492,29 @@ init_command = [
|
||||
'--dry-run={{DRYRUN}}',
|
||||
'shellcheck-py==0.7.2.1',
|
||||
]
|
||||
|
||||
[[linter]]
|
||||
code = 'ACTIONLINT'
|
||||
include_patterns = [
|
||||
'.github/workflows/*.yml',
|
||||
'.github/workflows/*.yaml',
|
||||
# actionlint does not support composite actions yet
|
||||
# '.github/actions/**/*.yml',
|
||||
# '.github/actions/**/*.yaml',
|
||||
]
|
||||
command = [
|
||||
'python3',
|
||||
'tools/linter/adapters/actionlint_linter.py',
|
||||
'--binary=.lintbin/actionlint',
|
||||
'--',
|
||||
'@{{PATHSFILE}}',
|
||||
]
|
||||
init_command = [
|
||||
'python3',
|
||||
'tools/linter/adapters/s3_init.py',
|
||||
'--config-json=tools/linter/adapters/s3_init_config.json',
|
||||
'--linter=actionlint',
|
||||
'--dry-run={{DRYRUN}}',
|
||||
'--output-dir=.lintbin',
|
||||
'--output-name=actionlint',
|
||||
]
|
||||
|
||||
36
Makefile
36
Makefile
@ -16,57 +16,25 @@ ios:
|
||||
|
||||
clean: # This will remove ALL build folders.
|
||||
@rm -r build*/
|
||||
@$(RM) -r $(SHELLCHECK_GHA_GENERATED_FOLDER)
|
||||
|
||||
linecount:
|
||||
@cloc --read-lang-def=caffe.cloc caffe2 || \
|
||||
echo "Cloc is not available on the machine. You can install cloc with " && \
|
||||
echo " sudo apt-get install cloc"
|
||||
|
||||
SHELLCHECK_GHA_GENERATED_FOLDER=.shellcheck_generated_gha
|
||||
shellcheck-gha:
|
||||
@$(RM) -r $(SHELLCHECK_GHA_GENERATED_FOLDER)
|
||||
tools/extract_scripts.py --out=$(SHELLCHECK_GHA_GENERATED_FOLDER)
|
||||
tools/linter/run_shellcheck.sh $(SHELLCHECK_GHA_GENERATED_FOLDER)
|
||||
|
||||
generate-gha-workflows:
|
||||
.github/scripts/generate_ci_workflows.py
|
||||
$(MAKE) shellcheck-gha
|
||||
|
||||
shellcheck:
|
||||
@$(PYTHON) tools/actions_local_runner.py \
|
||||
--file .github/workflows/lint.yml \
|
||||
--job 'shellcheck' \
|
||||
--job 'workflow-checks' \
|
||||
--step "Regenerate workflows"
|
||||
@$(PYTHON) tools/actions_local_runner.py \
|
||||
--file .github/workflows/lint.yml \
|
||||
--job 'shellcheck' \
|
||||
--job 'workflow-checks' \
|
||||
--step "Assert that regenerating the workflows didn't change them"
|
||||
@$(PYTHON) tools/actions_local_runner.py \
|
||||
--file .github/workflows/lint.yml \
|
||||
--job 'shellcheck' \
|
||||
--step 'Extract scripts from GitHub Actions workflows'
|
||||
@$(PYTHON) tools/actions_local_runner.py \
|
||||
$(CHANGED_ONLY) \
|
||||
$(REF_BRANCH) \
|
||||
--job 'shellcheck'
|
||||
|
||||
setup_lint:
|
||||
$(PIP) install lintrunner
|
||||
lintrunner init
|
||||
$(PYTHON) tools/actions_local_runner.py --file .github/workflows/lint.yml \
|
||||
--job 'shellcheck' --step 'Install Jinja2' --no-quiet
|
||||
|
||||
@if [ "$$(uname)" = "Darwin" ]; then \
|
||||
if [ -z "$$(which brew)" ]; then \
|
||||
echo "'brew' is required to install ShellCheck, get it here: https://brew.sh "; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
brew install shellcheck; \
|
||||
else \
|
||||
$(PYTHON) tools/actions_local_runner.py --file .github/workflows/lint.yml \
|
||||
--job 'shellcheck' --step 'Install ShellCheck' --no-quiet; \
|
||||
fi
|
||||
$(PYTHON) -mpip install jinja2 --user
|
||||
|
||||
quick_checks:
|
||||
|
||||
139
tools/linter/adapters/actionlint_linter.py
Normal file
139
tools/linter/adapters/actionlint_linter.py
Normal file
@ -0,0 +1,139 @@
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import time
|
||||
from enum import Enum
|
||||
from typing import List, NamedTuple, Optional, Pattern
|
||||
|
||||
|
||||
LINTER_CODE = "ACTIONLINT"
|
||||
|
||||
|
||||
class LintSeverity(str, Enum):
|
||||
ERROR = "error"
|
||||
WARNING = "warning"
|
||||
ADVICE = "advice"
|
||||
DISABLED = "disabled"
|
||||
|
||||
|
||||
class LintMessage(NamedTuple):
|
||||
path: Optional[str]
|
||||
line: Optional[int]
|
||||
char: Optional[int]
|
||||
code: str
|
||||
severity: LintSeverity
|
||||
name: str
|
||||
original: Optional[str]
|
||||
replacement: Optional[str]
|
||||
description: Optional[str]
|
||||
|
||||
|
||||
RESULTS_RE: Pattern[str] = re.compile(
|
||||
r"""(?mx)
|
||||
^
|
||||
(?P<file>.*?):
|
||||
(?P<line>\d+):
|
||||
(?P<char>\d+):
|
||||
\s(?P<message>.*)
|
||||
\s(?P<code>\[.*\])
|
||||
$
|
||||
"""
|
||||
)
|
||||
|
||||
def run_command(
|
||||
args: List[str],
|
||||
) -> "subprocess.CompletedProcess[bytes]":
|
||||
logging.debug("$ %s", " ".join(args))
|
||||
start_time = time.monotonic()
|
||||
try:
|
||||
return subprocess.run(
|
||||
args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
finally:
|
||||
end_time = time.monotonic()
|
||||
logging.debug("took %dms", (end_time - start_time) * 1000)
|
||||
|
||||
|
||||
def check_files(
|
||||
binary: str,
|
||||
files: List[str],
|
||||
) -> List[LintMessage]:
|
||||
try:
|
||||
proc = run_command(
|
||||
[binary] + files
|
||||
)
|
||||
except OSError as err:
|
||||
return [
|
||||
LintMessage(
|
||||
path=None,
|
||||
line=None,
|
||||
char=None,
|
||||
code=LINTER_CODE,
|
||||
severity=LintSeverity.ERROR,
|
||||
name="command-failed",
|
||||
original=None,
|
||||
replacement=None,
|
||||
description=(f"Failed due to {err.__class__.__name__}:\n{err}"),
|
||||
)
|
||||
]
|
||||
stdout = str(proc.stdout, "utf-8").strip()
|
||||
return [
|
||||
LintMessage(
|
||||
path=match["file"],
|
||||
name=match["code"],
|
||||
description=match["message"],
|
||||
line=int(match["line"]),
|
||||
char=int(match["char"]),
|
||||
code=LINTER_CODE,
|
||||
severity=LintSeverity.ERROR,
|
||||
original=None,
|
||||
replacement=None,
|
||||
)
|
||||
for match in RESULTS_RE.finditer(stdout)
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description="actionlint runner",
|
||||
fromfile_prefix_chars="@",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--binary",
|
||||
required=True,
|
||||
help="actionlint binary path",
|
||||
)
|
||||
parser.add_argument(
|
||||
"filenames",
|
||||
nargs="+",
|
||||
help="paths to lint",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.binary):
|
||||
err_msg = LintMessage(
|
||||
path="<none>",
|
||||
line=None,
|
||||
char=None,
|
||||
code=LINTER_CODE,
|
||||
severity=LintSeverity.ERROR,
|
||||
name="command-failed",
|
||||
original=None,
|
||||
replacement=None,
|
||||
description=(
|
||||
f"Could not find actionlint binary at {args.binary},"
|
||||
" you may need to run `lintrunner init`."
|
||||
),
|
||||
)
|
||||
print(json.dumps(err_msg._asdict()), flush=True)
|
||||
exit(0)
|
||||
|
||||
lint_messages = check_files(args.binary, args.filenames)
|
||||
for lint_message in lint_messages:
|
||||
print(json.dumps(lint_message._asdict()), flush=True)
|
||||
@ -2,29 +2,31 @@
|
||||
"clang-format": {
|
||||
"Darwin": {
|
||||
"download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/mac/clang-format-mojave",
|
||||
"hash": "1485a242a96c737ba7cdd9f259114f2201accdb46d87ac7a8650b1a814cd4d4d",
|
||||
"object_name": "mac/clang-format-mojave",
|
||||
"s3_bucket": "oss-clang-format"
|
||||
"hash": "1485a242a96c737ba7cdd9f259114f2201accdb46d87ac7a8650b1a814cd4d4d"
|
||||
},
|
||||
"Linux": {
|
||||
"download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64",
|
||||
"hash": "e1c8b97b919541a99e0a355df5c3f9e8abebc64259dbee6f8c68e1ef90582856",
|
||||
"object_name": "linux64/clang-format-linux64",
|
||||
"s3_bucket": "oss-clang-format"
|
||||
"hash": "e1c8b97b919541a99e0a355df5c3f9e8abebc64259dbee6f8c68e1ef90582856"
|
||||
}
|
||||
},
|
||||
"clang-tidy": {
|
||||
"Darwin": {
|
||||
"download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos/clang-tidy",
|
||||
"hash": "541797a7b8fa795e2f3c1adcd8236cc336a40aa927028dc5bc79172e1d9eca36",
|
||||
"object_name": "macos/clang-tidy",
|
||||
"s3_bucket": "oss-clang-format"
|
||||
"hash": "541797a7b8fa795e2f3c1adcd8236cc336a40aa927028dc5bc79172e1d9eca36"
|
||||
},
|
||||
"Linux": {
|
||||
"download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-tidy",
|
||||
"hash": "49343a448fcb75cd1e0fb9d6b1f6c2ef4b008b6f91d6ff899d4ac6060f5e52a5",
|
||||
"object_name": "linx64/clang-tidy",
|
||||
"s3_bucket": "oss-clang-format"
|
||||
"hash": "49343a448fcb75cd1e0fb9d6b1f6c2ef4b008b6f91d6ff899d4ac6060f5e52a5"
|
||||
}
|
||||
},
|
||||
"actionlint": {
|
||||
"Darwin": {
|
||||
"download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos/actionlint",
|
||||
"hash": "3ce2c94280c540e20b270acae60bdd9e72ad17d6cb35b688951b1ec1eb8cbdd6"
|
||||
},
|
||||
"Linux": {
|
||||
"download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/actionlint",
|
||||
"hash": "693f464106474760f0edf4a1778215095eacc4bd5f79aab5dc950892f120828b"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user