pytorch/.github/workflows/lint.yml

name: Lint
# Workflow that runs lint checks and also unittests for tools, and scripts.

on:
  pull_request:
    branches-ignore:
      - nightly
  push:
    branches:
      - main
      - release/*
      - landchecks/*
    tags:
      - ciflow/pull/*
      - ciflow/trunk/*
  workflow_dispatch:

permissions: read-all
# The names of steps that actually test the code should be suffixed with `(nonretryable)`.
# When any other step fails, it's job will be retried once by retryBot.
jobs:
  get-label-type:
    if: github.repository_owner == 'pytorch'
    name: get-label-type
    uses: pytorch/pytorch/.github/workflows/_runner-determinator.yml@main
    with:
      triggering_actor: ${{ github.triggering_actor }}
      issue_owner: ${{ github.event.pull_request.user.login || github.event.issue.user.login }}
      curr_branch: ${{ github.head_ref || github.ref_name }}

  get-changed-files:
    if: github.repository_owner == 'pytorch'
    name: Get changed files
    uses: ./.github/workflows/_get-changed-files.yml
    with:
      all_files: ${{ contains(github.event.pull_request.labels.*.name, 'lint-all-files') || contains(github.event.pull_request.labels.*.name, 'Reverted') || github.event_name == 'push' }}

  lintrunner-clang:
    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
    # Needed to prevent deduping on HUD
    name: lintrunner-clang-${{ needs.get-changed-files.outputs.changed-files == '*' && 'all' || 'partial' }}
    needs: [get-label-type, get-changed-files]
    # Only run if there are changed files relevant to clangtidy / clangformat
    if: |
      github.repository_owner == 'pytorch' && (
        needs.get-changed-files.outputs.changed-files == '*' ||
        contains(needs.get-changed-files.outputs.changed-files, '.h') ||
        contains(needs.get-changed-files.outputs.changed-files, '.cpp') ||
        contains(needs.get-changed-files.outputs.changed-files, '.cc') ||
        contains(needs.get-changed-files.outputs.changed-files, '.cxx') ||
        contains(needs.get-changed-files.outputs.changed-files, '.hpp') ||
        contains(needs.get-changed-files.outputs.changed-files, '.hxx') ||
        contains(needs.get-changed-files.outputs.changed-files, '.cu') ||
        contains(needs.get-changed-files.outputs.changed-files, '.cuh') ||
        contains(needs.get-changed-files.outputs.changed-files, '.mm') ||
        contains(needs.get-changed-files.outputs.changed-files, '.metal')
      )
    with:
      timeout: 120
      runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge"
      docker-image: ci-image:pytorch-linux-jammy-cuda12.8-cudnn9-py3.10-linter
      # NB: A shallow checkout won't work here because calculate-docker-image requires a full checkout
      # to run git rev-parse HEAD~:.ci/docker when a new image is needed
      fetch-depth: 0
      submodules: true
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        CHANGED_FILES="${{ needs.get-changed-files.outputs.changed-files }}"
        if [ "$CHANGED_FILES" = "*" ]; then
          export ADDITIONAL_LINTRUNNER_ARGS="--take CLANGTIDY,CLANGFORMAT --all-files"
        else
          export ADDITIONAL_LINTRUNNER_ARGS="--take CLANGTIDY,CLANGFORMAT $CHANGED_FILES"
        fi
        export CLANG=1
        .github/scripts/lintrunner.sh

  # NOTE: mypy needs its own job because it depends on --all-files, without assessing all files it sometimes
  #       fails to find types when it should
  lintrunner-mypy:
    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
    name: lintrunner-mypy-${{ needs.get-changed-files.outputs.changed-files == '*' && 'all' || 'partial' }}
    needs: [get-label-type, get-changed-files]
    # Only run if there are changed files relevant to mypy
    if: |
      github.repository_owner == 'pytorch' && (
        needs.get-changed-files.outputs.changed-files == '*' ||
        contains(needs.get-changed-files.outputs.changed-files, '.py') ||
        contains(needs.get-changed-files.outputs.changed-files, '.pyi')
      )
    with:
      timeout: 120
      runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge"
      docker-image: ci-image:pytorch-linux-jammy-linter
      # NB: A shallow checkout won't work here because calculate-docker-image requires a full checkout
      # to run git rev-parse HEAD~:.ci/docker when a new image is needed
      fetch-depth: 0
      submodules: true
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        CHANGED_FILES="${{ needs.get-changed-files.outputs.changed-files }}"
        echo "Running mypy"
        ADDITIONAL_LINTRUNNER_ARGS="--take MYPY,MYPYSTRICT --all-files" .github/scripts/lintrunner.sh

  lintrunner-noclang:
    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
    name: lintrunner-noclang-${{ needs.get-changed-files.outputs.changed-files == '*' && 'all' || 'partial' }}
    needs: [get-label-type, get-changed-files]
    with:
      timeout: 120
      runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge"
      docker-image: ci-image:pytorch-linux-jammy-linter
      # NB: A shallow checkout won't work here because calculate-docker-image requires a full checkout
      # to run git rev-parse HEAD~:.ci/docker when a new image is needed
      fetch-depth: 0
      submodules: true
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        CHANGED_FILES="${{ needs.get-changed-files.outputs.changed-files }}"
        echo "Running all other linters"
        if [ "$CHANGED_FILES" = '*' ]; then
          ADDITIONAL_LINTRUNNER_ARGS="--skip CLANGTIDY,CLANGFORMAT,MYPY,MYPYSTRICT,PYREFLY --all-files" .github/scripts/lintrunner.sh
        else
          ADDITIONAL_LINTRUNNER_ARGS="--skip CLANGTIDY,CLANGFORMAT,MYPY,MYPYSTRICT,PYREFLY ${CHANGED_FILES}" .github/scripts/lintrunner.sh
        fi

  quick-checks:
    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
    needs: get-label-type
    with:
      timeout: 120
      runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge"
      docker-image: ci-image:pytorch-linux-jammy-linter
      fetch-depth: 0
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        # Ensure no non-breaking spaces
        # NB: We use 'printf' below rather than '\u000a' since bash pre-4.2
        # does not support the '\u000a' syntax (which is relevant for local linters)
        (! git --no-pager grep -In "$(printf '\xC2\xA0')" -- . || (echo "The above lines have non-breaking spaces (U+00A0); please convert them to spaces (U+0020)"; false))

        # Ensure cross-OS compatible file names
        (! git ls-files | grep -E '([<>:"|?*]|[ .]$)' || (echo "The above file names are not valid across all operating systems. Please ensure they don't contain the characters '<>:""|?*' and don't end with a white space or a '.' "; false))

        # Ensure no versionless Python shebangs
        (! git --no-pager grep -In '#!.*python$' -- . || (echo "The above lines have versionless Python shebangs; please specify either python2 or python3"; false))

        # Ensure ciflow tags mentioned in config
        python3 .github/scripts/collect_ciflow_labels.py --validate-tags

        # C++ docs check
        pushd docs/cpp/source
        ./check-doxygen.sh
        popd

        # CUDA kernel launch check
        set -eux
        python3 torch/testing/_internal/check_kernel_launches.py |& tee cuda_kernel_launch_checks.txt

  pr-sanity-checks:
    name: pr-sanity-checks
    needs: get-label-type
    runs-on: [self-hosted, "${{ needs.get-label-type.outputs.label-type }}linux.large"]
    # Only run this on pull requests. This check is simple enough to be done without a Docker image
    if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'skip-pr-sanity-checks')
    steps:
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@main
        with:
          submodules: false
          fetch-depth: -1

      - name: PR size check (nonretryable)
        env:
          BASE: ${{ github.event.pull_request.base.sha }}
          HEAD: ${{ github.event.pull_request.head.sha }}
        run: |
          bash .github/scripts/pr-sanity-check.sh

  workflow-checks:
    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
    needs: get-label-type
    with:
      timeout: 120
      runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge"
      docker-image: ci-image:pytorch-linux-jammy-linter
      fetch-depth: -1
      submodules: true
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        # Regenerate workflows
        .github/scripts/generate_ci_workflows.py

        RC=0
        # Assert that regenerating the workflows didn't change them
        if ! .github/scripts/report_git_status.sh .github/workflows; then
          echo
          echo 'As shown by the above diff, the committed .github/workflows'
          echo 'are not up to date according to .github/templates.'
          echo 'Please run this command, commit, and push again to your PR:'
          echo
          echo '    .github/scripts/generate_ci_workflows.py'
          echo
          echo 'If running that command does nothing, you may need to rebase'
          echo 'onto a more recent commit from the PyTorch main branch.'
          RC=1
        fi

        # Check that jobs will be cancelled
        .github/scripts/ensure_actions_will_cancel.py

        exit $RC

  toc:
    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
    needs: get-label-type
    with:
      timeout: 120
      runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge"
      docker-image: ci-image:pytorch-linux-jammy-linter
      fetch-depth: 0
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        # Regenerate ToCs and check that they didn't change
        set -eu

        export PATH=~/.npm-global/bin:"$PATH"
        for FILE in $(git grep -Il '<!-- toc -->' -- '**.md'); do
          markdown-toc --bullets='-' -i "$FILE"
        done

        if ! .github/scripts/report_git_status.sh .; then
          echo
          echo 'As shown by the above diff, the table of contents in one or'
          echo 'more Markdown files is not up to date with the file contents.'
          echo 'You can either apply that Git diff directly to correct the'
          echo 'table of contents, or if you have npm installed, you can'
          echo 'install the npm package markdown-toc and run the following'
          # shellcheck disable=SC2016
          echo 'command (replacing $FILE with the filename for which you want'
          echo 'to regenerate the table of contents):'
          echo
          # shellcheck disable=SC2016
          echo "    markdown-toc --bullets='-' -i \"\$FILE\""
          false
        fi

  test-tools:
    name: Test tools
    if: ${{ github.repository == 'pytorch/pytorch' }}
    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
    needs: get-label-type
    with:
      timeout: 120
      runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge"
      docker-image: ci-image:pytorch-linux-jammy-linter
      fetch-depth: 0
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        # Test tools
        PYTHONPATH=$(pwd) pytest tools/stats
        PYTHONPATH=$(pwd) pytest tools/test -o "python_files=test*.py"
        PYTHONPATH=$(pwd) pytest .github/scripts -o "python_files=test*.py"

  test_run_test:
    name: Test `run_test.py` is usable without boto3
    if: ${{ github.repository == 'pytorch/pytorch' }}
    runs-on: linux.24_04.4x
    steps:
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@main
        with:
          submodules: false
          fetch-depth: 1
      - name: Setup Python 3.10
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: '3.10'
          architecture: x64
          cache: pip
      - name: Install dependencies
        run: |
          python3 -m pip install --upgrade pip
          pip install pytest-rerunfailures==11.1.* pytest-flakefinder==1.1.* pytest-xdist==3.3.* expecttest==0.3.* fbscribelogger==0.1.* numpy==1.24.*
          pip install torch --pre --index-url https://download.pytorch.org/whl/nightly/cpu/
      - name: Run run_test.py (nonretryable)
        run: |
          # Run test_vulkan, which is a fast noop on Linux
          python3 test/run_test.py --include test_vulkan --verbose

  test_collect_env:
    if: ${{ github.repository == 'pytorch/pytorch' }}
    name: Test collect_env
    runs-on: ${{ matrix.runner }}
    strategy:
      matrix:
        include:
          - test_type: with_torch
            runner: linux.24_04.4x
          - test_type: without_torch
            runner: linux.24_04.4x
          - test_type: older_python_version
            runner: linux.24_04.4x
    steps:
      # [see note: pytorch repo ref]
      # deep clone (fetch-depth 0) required, to allow us to use git log
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@main
        with:
          submodules: false
          fetch-depth: 1
      - name: Get min python version
        id: get-min-python-version
        if: matrix.test_type == 'older_python_version'
        run: |
          set -eou pipefail
          # Generate PyTorch version to use
          echo "MIN_PYTHON_VERSION=$(python3 .github/scripts/get_ci_variable.py --min-python-version)" >> "${GITHUB_OUTPUT}"
      - name: Setup Old Python version
        if: matrix.test_type == 'older_python_version'
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: 3.8
          architecture: x64
          check-latest: false
          cache: pip
          cache-dependency-path: |
            **/requirements-build.txt
            **/requirements.txt
      - name: Setup Min Python version
        if: matrix.test_type != 'older_python_version'
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: ${{ steps.get-min-python-version.outputs.MIN_PYTHON_VERSION }}
          architecture: x64
          check-latest: false
          cache: pip
          cache-dependency-path: |
            **/requirements-build.txt
            **/requirements.txt
      - name: Install torch
        if: matrix.test_type == 'with_torch'
        run: |
          pip install -r requirements.txt
          # Doesn't really matter what torch version, we just need ANY torch installed
          pip install 'torch==2.*'
      - name: Run collect_env.py (nonretryable)
        run: |
          # All we need to see is that it passes
          python3 torch/utils/collect_env.py

  link-check:
    name: Link checks
    needs: get-label-type
    uses: ./.github/workflows/_link_check.yml
    with:
      runner: ${{ needs.get-label-type.outputs.label-type }}
      ref:    ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
    secrets: inherit

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
  cancel-in-progress: true