pytorch/.github/workflows/lint.yml

name: Lint

on:
  push:
    branches:
    - master
  pull_request:

jobs:
  quick-checks:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Checkout PyTorch
        uses: actions/checkout@v2
      - name: Ensure consistent CircleCI YAML config
        run: |
          pip install -r requirements.txt
          cd .circleci && ./ensure-consistency.py
      - name: Shellcheck Jenkins scripts
        # https://github.com/koalaman/shellcheck#installing-a-pre-compiled-binary
        run: |
          scversion="stable"
          wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv
          sudo cp "shellcheck-${scversion}/shellcheck" /usr/bin/
          rm -r "shellcheck-${scversion}"
          shellcheck --version
          .jenkins/run-shellcheck.sh
      - name: Ensure correct trailing newlines
        run: |
          (! git grep -Il '' -- . ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude)**.expect' | tools/trailing_newlines.py || (echo "The above files do not have correct trailing newlines; please normalize them"; false))
      - name: Ensure no trailing spaces
        run: |
          (! git grep -I -no ' $' -- . ':(exclude)**/contrib/**' ':(exclude)third_party' || (echo "The above files have trailing spaces; please remove them"; false))
      - name: Ensure no tabs
        run: |
          (! git grep -I -no $'\t' -- . ':(exclude)*.svg' ':(exclude)**Makefile' ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above files have tabs; please convert them to spaces"; false))
      - name: Ensure canonical include
        run: |
          (! git grep -I -no $'#include "' -- ./c10 ./aten ./torch/csrc ':(exclude)aten/src/ATen/native/quantized/cpu/qnnpack/**' || (echo "The above files have include with quotes; please convert them to #include <xxxx>"; false))
      # note that this next step depends on a clean checkout;
      # if you run it locally then it will likely to complain
      # about all the generated files in torch/test
      - name: Ensure C++ source files are not executable
        run: |
          (! find . \( -path ./third_party -o -path ./.git -o -path ./torch/bin -o -path ./build \) -prune -o -type f -executable -regextype posix-egrep -not -regex '.+(\.(bash|sh|py|so)|git-pre-commit|git-clang-format|gradlew)$' -print | grep . || (echo 'The above files have executable permission; please remove their executable permission by using `chmod -x`'; false))
      - name: C++ docs check
        run: |
          sudo apt-get install -y doxygen && pip install -r requirements.txt
          cd docs/cpp/source && ./check-doxygen.sh
      - name: CUDA kernel launch check
        run: |
          set -eux
          python torch/testing/check_kernel_launches.py |& tee ${GITHUB_WORKSPACE}/cuda_kernel_launch_checks.txt

  flake8-py3:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Fetch PyTorch
        uses: actions/checkout@v2
      - name: Get HEAD commit SHA
        run: echo ::set-output name=commit-sha::$(git rev-parse HEAD)
        id: get-commit-sha
      - name: Run flake8
        run: |
          set -eux
          pip install -r requirements-flake8.txt
          flake8 --version
          flake8 | tee ${GITHUB_WORKSPACE}/flake8-output.txt
      - name: Add annotations
        uses: pytorch/add-annotations-github-action@master
        with:
          check_name: 'flake8-py3'
          linter_output_path: 'flake8-output.txt'
          commit_sha: ${{ steps.get-commit-sha.outputs.commit-sha }}
          regex: '^(?<filename>.*?):(?<lineNumber>\d+):(?<columnNumber>\d+): (?<errorCode>\w+\d+) (?<errorDesc>.*)'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Catch any other warnings
        run: |
          [ ! -s flake8-output.txt ]

  clang-tidy:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Checkout PyTorch
        uses: actions/checkout@v2
        with:
          fetch-depth: 0 # deep clone, to allow us to use git merge-base
      - name: Get HEAD commit SHA
        run: echo ::set-output name=commit-sha::$(git rev-parse HEAD)
        id: get-commit-sha
      - name: Install dependencies
        run: |
          set -eux
          # Install CUDA
          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin
          sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600
          sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
          sudo add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /"
          sudo apt-get update
          sudo apt-get --no-install-recommends -y install cuda-toolkit-10-2
          # Install dependencies
          pip install pyyaml typing_extensions
          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
          sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main"
          sudo apt-get update
          sudo apt-get install -y clang-tidy-11
          sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-11 1000
      - name: Run clang-tidy
        run: |
          set -eux
          git remote add upstream https://github.com/pytorch/pytorch
          git fetch upstream "$GITHUB_BASE_REF"
          BASE_SHA=${{ github.event.pull_request.base.sha }}
          HEAD_SHA=${{ github.event.pull_request.head.sha }}
          MERGE_BASE=$(git merge-base $BASE_SHA $HEAD_SHA)

          if [[ ! -d build ]]; then
            git submodule update --init --recursive

            export USE_NCCL=0
            # We really only need compile_commands.json, so no need to build!
            time python setup.py --cmake-only build

            # Generate ATen files.
            time python -m tools.codegen.gen \
              -s aten/src/ATen \
              -d build/aten/src/ATen

            # Generate PyTorch files.
            time python tools/setup_helpers/generate_code.py            \
              --declarations-path build/aten/src/ATen/Declarations.yaml \
              --native-functions-path aten/src/ATen/native/native_functions.yaml \
              --nn-path aten/src
          fi

          # Run Clang-Tidy
          # The negative filters below are to exclude files that include onnx_pb.h or
          # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job.
          # FunctionsManual.cpp is excluded to keep this diff clean. It will be fixed
          # in a follow up PR.
          # /torch/csrc/generic/*.cpp is excluded because those files aren't actually built.
          # deploy/interpreter files are excluded due to using macros and other techniquies
          # that are not easily converted to accepted c++
          python tools/clang_tidy.py                               \
            --verbose                                              \
            --paths torch/csrc/                                    \
            --diff "$MERGE_BASE"                                   \
            -g"-torch/csrc/jit/passes/onnx/helper.cpp"             \
            -g"-torch/csrc/jit/passes/onnx/shape_type_inference.cpp"\
            -g"-torch/csrc/jit/serialization/onnx.cpp"             \
            -g"-torch/csrc/jit/serialization/export.cpp"           \
            -g"-torch/csrc/jit/serialization/import.cpp"           \
            -g"-torch/csrc/jit/serialization/import_legacy.cpp"    \
            -g"-torch/csrc/onnx/init.cpp"                          \
            -g"-torch/csrc/cuda/nccl.*"                            \
            -g"-torch/csrc/cuda/python_nccl.cpp"                   \
            -g"-torch/csrc/autograd/FunctionsManual.cpp"           \
            -g"-torch/csrc/generic/*.cpp"                          \
            -g"-torch/csrc/jit/codegen/cuda/runtime/*"             \
            -g"-torch/csrc/deploy/interpreter/interpreter.cpp"     \
            -g"-torch/csrc/deploy/interpreter/interpreter.h"  \
            -g"-torch/csrc/deploy/interpreter/interpreter_impl.h"  \
            -g"-torch/csrc/deploy/interpreter/test_main.cpp"  \
            "$@" > ${GITHUB_WORKSPACE}/clang-tidy-output.txt

          cat ${GITHUB_WORKSPACE}/clang-tidy-output.txt
      - name: Add annotations
        uses: pytorch/add-annotations-github-action@master
        with:
          check_name: 'clang-tidy'
          linter_output_path: 'clang-tidy-output.txt'
          commit_sha: ${{ steps.get-commit-sha.outputs.commit-sha }}
          regex: '^(?<filename>.*?):(?<lineNumber>\d+):(?<columnNumber>\d+): (?<errorDesc>.*?) \[(?<errorCode>.*)\]'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  cmakelint:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Fetch PyTorch
        uses: actions/checkout@v2
      - name: Run cmakelint
        run: |
          set -eux
          pip install cmakelint
          cmakelint --version
          git ls-files -z -- bootstrap '*.cmake' '*.cmake.in' '*CMakeLists.txt' | \
          grep -E -z -v '^(cmake/Modules/|cmake/Modules_CUDA_fix/)' | \
          xargs -0 cmakelint --config=.cmakelintrc --spaces=2 --quiet

  mypy:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
          architecture: x64
      - name: Fetch PyTorch
        uses: actions/checkout@v2
      - name: Install dependencies
        run: |
          set -eux
          pip install -r requirements.txt
          pip install mypy==0.770
      - name: Run autogen
        run: |
          set -eux
          time python -mtools.generate_torch_version --is_debug=false
          time python -mtools.codegen.gen -s aten/src/ATen -d build/aten/src/ATen
          time python -mtools.pyi.gen_pyi --native-functions-path aten/src/ATen/native/native_functions.yaml --deprecated-functions-path "tools/autograd/deprecated.yaml"
      - name: Run mypy
        run: |
          set -eux
          for CONFIG in mypy*.ini; do mypy --config="$CONFIG"; done