pytorch/.github/workflows/lint.yml

name: Lint

on:
  push:
    branches:
    - master
  pull_request:

jobs:
  quick-checks:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Checkout PyTorch
        uses: actions/checkout@v2
      - name: Install requirements
        run: pip install -r requirements.txt
      - name: Ensure consistent CircleCI YAML config
        run: cd .circleci && ./ensure-consistency.py
      - name: Ensure consistent GHA workflows in cancel_redundant_workflows.yml
        run: |
          pip install ruamel.yaml==0.17.4
          echo "Please locally run .github/scripts/regenerate_cancel_redundant_workflow.py and commit if this step fails."
          .github/scripts/regenerate_cancel_redundant_workflow.py
          git diff --exit-code .github/workflows/cancel_redundant_workflows.yml
      - name: Lint native_functions.yaml
        run: |
          pip install ruamel.yaml==0.17.4
          .github/scripts/lint_native_functions.py
      - name: Extract scripts from GitHub Actions workflows
        run: tools/extract_scripts.py --out=.extracted_scripts
      - name: ShellCheck
        # https://github.com/koalaman/shellcheck/tree/v0.7.2#installing-a-pre-compiled-binary
        run: |
          set -x
          scversion="v0.7.2"
          wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv
          sudo cp "shellcheck-${scversion}/shellcheck" /usr/bin/
          rm -r "shellcheck-${scversion}"
          shellcheck --version
          tools/run_shellcheck.sh .jenkins/pytorch .extracted_scripts
      - name: Ensure correct trailing newlines
        run: |
          (! git grep -Il '' -- . ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude)**.expect' ':(exclude)tools/clang_format_hash' | tools/trailing_newlines.py || (echo "The above files do not have correct trailing newlines; please normalize them"; false))
      - name: Ensure no trailing spaces
        run: |
          (! git grep -In '[[:blank:]]$' -- . ':(exclude)**/contrib/**' ':(exclude)third_party' || (echo "The above lines have trailing spaces; please remove them"; false))
      - name: Ensure no tabs
        run: |
          (! git grep -In $'\t' -- . ':(exclude)*.svg' ':(exclude)**Makefile' ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above lines have tabs; please convert them to spaces"; false))
      - name: Ensure no non-breaking spaces
        run: |
          (! git grep -In $'\u00a0' -- . || (echo "The above lines have non-breaking spaces (U+00A0); please convert them to spaces (U+0020)"; false))
      - name: Ensure canonical include
        run: |
          (! git grep -In $'#include "' -- ./c10 ./aten ./torch/csrc ':(exclude)aten/src/ATen/native/quantized/cpu/qnnpack/**' || (echo "The above lines have include with quotes; please convert them to #include <xxxx>"; false))
      - name: Ensure no unqualified noqa
        run: |
          # shellcheck disable=SC2016
          (! git grep -InP '# noqa(?!: [A-Z]+\d{3})' -- '**.py' ':(exclude)caffe2' || (echo 'The above lines have unqualified `noqa`; please convert them to `noqa: XXXX`'; false))
      # note that this next step depends on a clean checkout;
      # if you run it locally then it will likely to complain
      # about all the generated files in torch/test
      - name: Ensure C++ source files are not executable
        run: |
          # shellcheck disable=SC2016
          (! find . \( -path ./third_party -o -path ./.git -o -path ./torch/bin -o -path ./build \) -prune -o -type f -executable -regextype posix-egrep -not -regex '.+(\.(bash|sh|py|so)|git-pre-commit|git-clang-format|gradlew)$' -print | grep . || (echo 'The above files have executable permission; please remove their executable permission by using `chmod -x`'; false))
      - name: C++ docs check
        run: |
          sudo apt-get install -y doxygen && pip install -r requirements.txt
          cd docs/cpp/source && ./check-doxygen.sh
      - name: CUDA kernel launch check
        run: |
          set -eux
          python torch/testing/check_kernel_launches.py |& tee "${GITHUB_WORKSPACE}"/cuda_kernel_launch_checks.txt
      - name: Ensure no direct cub include
        run: |
          (! git grep -I -no $'#include <cub/' --  ./aten  ':(exclude)aten/src/ATen/cuda/cub.cuh' || (echo "The above files have direct cub include; please include ATen/cuda/cub.cuh instead and wrap your cub calls in at::native namespace if necessary"; false))

  python2-setup-compat:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 2.x
          architecture: x64
      - name: Checkout PyTorch
        uses: actions/checkout@v2
      - name: Attempt to run setup.py
        run: |
          python2 setup.py | grep "Python 2 has reached end-of-life and is no longer supported by PyTorch."

  templates:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Install Jinja2
        run: pip install Jinja2
      - name: Checkout PyTorch
        uses: actions/checkout@v2
      - name: Regenerate workflows
        run: .github/scripts/generate_linux_ci_workflows.py
      - name: Assert that regenerating the workflows didn't change them
        run: .github/scripts/report_git_status.sh

  toc:
    runs-on: ubuntu-18.04
    # https://github.com/actions/virtual-environments/issues/599#issuecomment-602754687
    env:
      NPM_CONFIG_PREFIX: ~/.npm-global
    steps:
      - name: Setup Node
        uses: actions/setup-node@v2
      - name: Checkout PyTorch
        uses: actions/checkout@v2
      - name: Install markdown-toc
        run: npm install -g markdown-toc
      - name: Regenerate ToCs
        run: |
          set -eux
          export PATH=~/.npm-global/bin:"$PATH"
          for FILE in $(git grep -Il '<!-- toc -->' -- '**.md'); do
            markdown-toc --bullets='-' -i "$FILE"
          done
      - name: Assert that regenerating the ToCs didn't change them
        run: .github/scripts/report_git_status.sh

  flake8-py3:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Fetch PyTorch
        uses: actions/checkout@v2
        with:
          fetch-depth: 2 # to allow us to use github.event.pull_request.head.sha
      - name: Prepare output dir with HEAD commit SHA
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          mkdir flake8-output
          cd flake8-output
          echo "$HEAD_SHA" > commit-sha.txt
      - name: Run flake8
        run: |
          set -eux
          pip install typing-extensions # for tools/translate_annotations.py
          pip install -r requirements-flake8.txt
          flake8 --version
          flake8 | tee "${GITHUB_WORKSPACE}"/flake8-output.txt
          cp flake8-output.txt flake8-output/annotations.json
      - name: Translate annotations
        if: github.event_name == 'pull_request'
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          tools/translate_annotations.py \
            --file=flake8-output.txt \
            --regex='^(?P<filename>.*?):(?P<lineNumber>\d+):(?P<columnNumber>\d+): (?P<errorCode>\w+\d+) (?P<errorDesc>.*)' \
            --commit="$HEAD_SHA" \
            > flake8-output/annotations.json
      - name: Upload artifact
        uses: actions/upload-artifact@v2
        with:
          name: flake8-py3
          path: flake8-output/
      - name: Fail if there were any warnings
        run: |
          cat flake8-output.txt
          [ ! -s flake8-output.txt ]

  clang-tidy:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Checkout PyTorch
        uses: actions/checkout@v2
        with:
          fetch-depth: 0 # to allow tools/clang_tidy.py to do its thing
      - name: Prepare output dir with HEAD commit SHA
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          mkdir clang-tidy-output
          cd clang-tidy-output
          echo "$HEAD_SHA" > commit-sha.txt
      - name: Install dependencies
        run: |
          set -eux
          # Install CUDA
          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin
          sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600
          sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
          sudo add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /"
          sudo apt-get update
          sudo apt-get --no-install-recommends -y install cuda-toolkit-10-2
          # Install dependencies
          pip install pyyaml typing_extensions
          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
          sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main"
          sudo apt-get update
          sudo apt-get install -y clang-tidy-11
          sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-11 1000
      - name: Run clang-tidy
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          set -eux
          git remote add upstream https://github.com/pytorch/pytorch
          git fetch upstream "$GITHUB_BASE_REF"

          if [[ ! -d build ]]; then
            git submodule update --init --recursive

            export USE_NCCL=0
            # We really only need compile_commands.json, so no need to build!
            time python setup.py --cmake-only build

            # Generate ATen files.
            time python -m tools.codegen.gen \
              -s aten/src/ATen \
              -d build/aten/src/ATen

            # Generate PyTorch files.
            time python tools/setup_helpers/generate_code.py            \
              --declarations-path build/aten/src/ATen/Declarations.yaml \
              --native-functions-path aten/src/ATen/native/native_functions.yaml \
              --nn-path aten/src
          fi

          # Run Clang-Tidy
          # The negative filters below are to exclude files that include onnx_pb.h or
          # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job.
          # FunctionsManual.cpp is excluded to keep this diff clean. It will be fixed
          # in a follow up PR.
          # /torch/csrc/generic/*.cpp is excluded because those files aren't actually built.
          # deploy/interpreter files are excluded due to using macros and other techniquies
          # that are not easily converted to accepted c++
          python tools/clang_tidy.py                               \
            --verbose                                              \
            --paths torch/csrc/                                    \
            --diff "$BASE_SHA"                                   \
            -g"-torch/csrc/jit/passes/onnx/helper.cpp"             \
            -g"-torch/csrc/jit/passes/onnx/shape_type_inference.cpp"\
            -g"-torch/csrc/jit/serialization/onnx.cpp"             \
            -g"-torch/csrc/jit/serialization/export.cpp"           \
            -g"-torch/csrc/jit/serialization/import.cpp"           \
            -g"-torch/csrc/jit/serialization/import_legacy.cpp"    \
            -g"-torch/csrc/onnx/init.cpp"                          \
            -g"-torch/csrc/cuda/nccl.*"                            \
            -g"-torch/csrc/cuda/python_nccl.cpp"                   \
            -g"-torch/csrc/autograd/FunctionsManual.cpp"           \
            -g"-torch/csrc/generic/*.cpp"                          \
            -g"-torch/csrc/jit/codegen/cuda/runtime/*"             \
            -g"-torch/csrc/deploy/interpreter/interpreter.cpp"     \
            -g"-torch/csrc/deploy/interpreter/interpreter.h"  \
            -g"-torch/csrc/deploy/interpreter/interpreter_impl.h"  \
            -g"-torch/csrc/deploy/interpreter/test_main.cpp"  \
            "$@" > "${GITHUB_WORKSPACE}"/clang-tidy-output.txt

          cat "${GITHUB_WORKSPACE}"/clang-tidy-output.txt

          tools/translate_annotations.py \
            --file=clang-tidy-output.txt \
            --regex='^(?P<filename>.*?):(?P<lineNumber>\d+):(?P<columnNumber>\d+): (?P<errorDesc>.*?) \[(?P<errorCode>.*)\]' \
            --commit="$HEAD_SHA" \
            > clang-tidy-output/annotations.json
      - name: Upload artifact
        uses: actions/upload-artifact@v2
        with:
          name: clang-tidy
          path: clang-tidy-output/

  cmakelint:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Fetch PyTorch
        uses: actions/checkout@v2
      - name: Run cmakelint
        run: |
          set -eux
          pip install cmakelint
          cmakelint --version
          git ls-files -z -- bootstrap '*.cmake' '*.cmake.in' '*CMakeLists.txt' | \
          grep -E -z -v '^(cmake/Modules/|cmake/Modules_CUDA_fix/)' | \
          xargs -0 cmakelint --config=.cmakelintrc --spaces=2 --quiet

  mypy:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
          architecture: x64
      - name: Fetch PyTorch
        uses: actions/checkout@v2
      - name: Install dependencies
        run: |
          set -eux
          pip install -r requirements.txt
          pip install mypy==0.812
      - name: Run autogen
        run: |
          set -eux
          time python -mtools.generate_torch_version --is_debug=false
          time python -mtools.codegen.gen -s aten/src/ATen -d build/aten/src/ATen
          time python -mtools.pyi.gen_pyi --native-functions-path aten/src/ATen/native/native_functions.yaml --deprecated-functions-path "tools/autograd/deprecated.yaml"
      - name: Run mypy
        run: |
          set -eux
          for CONFIG in mypy*.ini; do mypy --config="$CONFIG"; done