switch back to azure pipelines

Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/29740 Test Plan: Imported from OSS Differential Revision: D18482697 Pulled By: suo fbshipit-source-id: 72a454457a005f82683079b79a77343e20c34021
2025-10-20 21:14:14 +08:00 · 2019-11-13 11:48:39 -08:00
parent 73a926fd5d
commit 6de1016f9d
3 changed files with 164 additions and 210 deletions
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@ -1,210 +0,0 @@
-name: Lint
-
-on:
-  push:
-    branches:
-    - master
-  pull_request:
-
-jobs:
-  quick-checks:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Setup Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.x
-          architecture: x64
-      - name: Checkout PyTorch
-        uses: actions/checkout@master
-      - name: Ensure consistent CircleCI YAML config
-        run: |
-          pip install -r requirements.txt
-          cd .circleci && ./ensure-consistency.py
-      - name: Ensure Docker version is correctly deployed
-        run: .circleci/validate-docker-version.py
-      - name: Shellcheck Jenkins scripts
-        run: |
-          sudo apt-get install -y shellcheck
-          .jenkins/run-shellcheck.sh
-      - name: Ensure no tabs
-        run: |
-          (! git grep -I -l $'\t' -- . ':(exclude)*.svg' ':(exclude)**Makefile' ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above files have tabs; please convert them to spaces"; false))
-      - name: Ensure C++ source files are not executable
-        run: |
-          (! find . \( -path ./third_party -o -path ./.git -o -path ./torch/bin -o -path ./build \) -prune -o -type f -executable -regextype posix-egrep -not -regex '.+(\.(bash|sh|py|so)|git-pre-commit)$' -print | grep . || (echo 'The above files have executable permission; please remove their executable permission by using `chmod -x`'; false))
-      - name: MyPy typecheck
-        run: |
-          pip install mypy mypy-extensions
-          mypy @mypy-files.txt
-      - name: C++ docs check
-        run: |
-          sudo apt-get install -y doxygen && pip install -r requirements.txt
-          cd docs/cpp/source && ./check-doxygen.sh
-
-  flake8-py3:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Setup Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.x
-          architecture: x64
-      - name: Fetch PyTorch
-        uses: actions/checkout@master
-      - name: Checkout PR tip
-        run: |
-          set -eux
-          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-            # We are on a PR, so actions/checkout leaves us on a merge commit.
-            # Check out the actual tip of the branch.
-            git checkout ${{ github.event.pull_request.head.sha }}
-          fi
-          echo ::set-output name=commit_sha::$(git rev-parse HEAD)
-        id: get_pr_tip
-      - name: Run flake8
-        run: |
-          set -eux
-          pip install flake8
-          flake8 --exit-zero > ${GITHUB_WORKSPACE}/flake8-output.txt
-          cat ${GITHUB_WORKSPACE}/flake8-output.txt
-      - name: Add annotations
-        uses: pytorch/add-annotations-github-action@master
-        with:
-          check_name: 'flake8-py3'
-          linter_output_path: 'flake8-output.txt'
-          commit_sha: ${{ steps.get_pr_tip.outputs.commit_sha }}
-          regex: '^(?<filename>.*?):(?<lineNumber>\d+):(?<columnNumber>\d+): (?<errorCode>\w\d+) (?<errorDesc>.*)'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  flake8-py2:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Setup Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 2.x
-          architecture: x64
-      - name: Fetch PyTorch
-        uses: actions/checkout@master
-      - name: Checkout PR tip
-        run: |
-          set -eux
-          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-            # We are on a PR, so actions/checkout leaves us on a merge commit.
-            # Check out the actual tip of the branch.
-            git checkout ${{ github.event.pull_request.head.sha }}
-          fi
-          echo ::set-output name=commit_sha::$(git rev-parse HEAD)
-        id: get_pr_tip
-      - name: Run flake8
-        run: |
-          set -eux
-          pip install flake8
-          rm -rf .circleci
-          flake8 --exit-zero > ${GITHUB_WORKSPACE}/flake8-output.txt
-          cat ${GITHUB_WORKSPACE}/flake8-output.txt
-      - name: Add annotations
-        uses: pytorch/add-annotations-github-action@master
-        with:
-          check_name: 'flake8-py2'
-          linter_output_path: 'flake8-output.txt'
-          commit_sha: ${{ steps.get_pr_tip.outputs.commit_sha }}
-          regex: '^(?<filename>.*?):(?<lineNumber>\d+):(?<columnNumber>\d+): (?<errorCode>\w\d+) (?<errorDesc>.*)'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-
-  clang-tidy:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Setup Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.x
-          architecture: x64
-      - name: Checkout PyTorch
-        uses: actions/checkout@master
-      - name: Checkout PR tip
-        run: |
-          set -eux
-          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-            # We are on a PR, so actions/checkout leaves us on a merge commit.
-            # Check out the actual tip of the branch.
-            git checkout ${{ github.event.pull_request.head.sha }}
-          fi
-          echo ::set-output name=commit_sha::$(git rev-parse HEAD)
-        id: get_pr_tip
-      - name: Install dependencies
-        run: |
-          set -eux
-          # Install CUDA
-          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin
-          sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600
-          sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
-          sudo add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /"
-          sudo apt-get update
-          sudo apt-get --no-install-recommends -y install cuda
-          # Install dependencies
-          pip install pyyaml
-          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-          sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main"
-          sudo apt-get update
-          sudo apt-get install -y clang-tidy-8
-          sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-8 1000
-      - name: Run clang-tidy
-        run: |
-          set -eux
-          git remote add upstream https://github.com/pytorch/pytorch
-          git fetch upstream "$GITHUB_BASE_REF"
-          BASE_SHA=${{ github.event.pull_request.base.sha }}
-          HEAD_SHA=${{ github.event.pull_request.head.sha }}
-          MERGE_BASE=$(git merge-base $BASE_SHA $HEAD_SHA)
-
-          if [[ ! -d build ]]; then
-            git submodule update --init --recursive
-
-            export USE_NCCL=0
-            # We really only need compile_commands.json, so no need to build!
-            time python setup.py --cmake-only build
-
-            # Generate ATen files.
-            time python aten/src/ATen/gen.py \
-              -s aten/src/ATen \
-              -d build/aten/src/ATen \
-              aten/src/ATen/Declarations.cwrap \
-              aten/src/THNN/generic/THNN.h \
-              aten/src/THCUNN/generic/THCUNN.h \
-              aten/src/ATen/nn.yaml \
-              aten/src/ATen/native/native_functions.yaml
-
-            # Generate PyTorch files.
-            time python tools/setup_helpers/generate_code.py            \
-              --declarations-path build/aten/src/ATen/Declarations.yaml \
-              --nn-path aten/src
-          fi
-
-          # Run Clang-Tidy
-          # The negative filters below are to exclude files that include onnx_pb.h or
-          # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job.
-          python tools/clang_tidy.py                  \
-            --verbose                                 \
-            --paths torch/csrc/                       \
-            --diff "$MERGE_BASE"                      \
-            -g"-torch/csrc/jit/export.cpp"            \
-            -g"-torch/csrc/jit/import.cpp"            \
-            -g"-torch/csrc/jit/netdef_converter.cpp"  \
-            "$@" > ${GITHUB_WORKSPACE}/clang-tidy-output.txt
-
-          cat ${GITHUB_WORKSPACE}/clang-tidy-output.txt
-      - name: Add annotations
-        uses: suo/add-annotations-github-action@master
-        with:
-          check_name: 'clang-tidy'
-          linter_output_path: 'clang-tidy-output.txt'
-          commit_sha: ${{ steps.get_pr_tip.outputs.commit_sha }}
-          regex: '^(?<filename>.*?):(?<lineNumber>\d+):(?<columnNumber>\d+): (?<errorDesc>.*?) \[(?<errorCode>.*)\]'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -0,0 +1,116 @@
+trigger:
+- master
+
+jobs:
+- job: consistent_circleci
+  displayName: "Ensure consistent CircleCI YAML"
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: "3.7"
+  - script: pip install -r requirements.txt
+  - script: cd .circleci && ./ensure-consistency.py
+
+- job: validate_docker_version
+  displayName: "Ensure Docker version is correctly deployed"
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: "3.7"
+  - script: .circleci/validate-docker-version.py
+
+- job: shellcheck_jenkins
+  displayName: "Shellcheck Jenkins scripts"
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+  - script: sudo apt-get install -y shellcheck
+  - script: .jenkins/run-shellcheck.sh
+
+- job: ensure_no_tabs
+  displayName: "Ensure no tabs"
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: "2.7"
+  - script: (! git grep -I -l $'\t' -- . ':(exclude)*.svg' ':(exclude)**Makefile' ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above files have tabs; please convert them to spaces"; false))
+
+- job: ensure_not_executable
+  displayName: "Ensure C++ source files are not executable"
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: "2.7"
+  - script: (! find . \( -path ./third_party -o -path ./.git -o -path ./torch/bin -o -path ./build \) -prune -o -type f -executable -regextype posix-egrep -not -regex '.+(\.(bash|sh|py|so)|git-pre-commit)$' -print | grep . || (echo 'The above files have executable permission; please remove their executable permission by using `chmod -x`'; false))
+
+- job: python_27_lint
+  displayName: "Python 2.7 Lint"
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: "2.7"
+  - script: pip install flake8
+  - script: |
+      rm -rf .circleci
+      flake8
+- job: python_37_lint
+  displayName: "Python 3.7 Lint"
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: "3.7"
+  - script: pip install flake8
+  - script: flake8
+
+- job: mypy_typecheck
+  displayName: "MyPy typecheck"
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: "3.6"
+  - script: pip install mypy mypy-extensions
+  - script: mypy @mypy-files.txt
+
+- job: cpp_doc_check
+  displayName: "CPP doc check"
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: "3.6"
+  - script: sudo apt-get install -y doxygen && pip install -r requirements.txt
+  - script: cd docs/cpp/source && ./check-doxygen.sh
+
+- job: clang_tidy
+  displayName: "clang tidy"
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: "3.6"
+  - script: |
+      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+      sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main"
+      sudo apt-get update
+      sudo apt-get install -y clang-tidy-8
+      sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-8 1000
+  - script: git branch master origin/master
+  - script: pip install pyyaml
+  - script: tools/run-clang-tidy-in-ci.sh
+
--- a/tools/run-clang-tidy-in-ci.sh
+++ b/tools/run-clang-tidy-in-ci.sh
@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -ex
+
+BASE_BRANCH=master
+# From https://docs.travis-ci.com/user/environment-variables
+if [[ $TRAVIS ]]; then
+  git remote add upstream https://github.com/pytorch/pytorch
+  git fetch upstream "$TRAVIS_BRANCH"
+  BASE_BRANCH="upstream/$TRAVIS_BRANCH"
+fi
+
+if [[ ! -d build ]]; then
+  git submodule update --init --recursive
+
+  mkdir build
+  pushd build
+  # We really only need compile_commands.json, so no need to build!
+  time cmake ..
+  popd
+
+  # Generate ATen files.
+  time python aten/src/ATen/gen.py \
+    -s aten/src/ATen \
+    -d build/aten/src/ATen \
+    aten/src/ATen/Declarations.cwrap \
+    aten/src/THNN/generic/THNN.h \
+    aten/src/THCUNN/generic/THCUNN.h \
+    aten/src/ATen/nn.yaml \
+    aten/src/ATen/native/native_functions.yaml
+
+  # Generate PyTorch files.
+  time python tools/setup_helpers/generate_code.py            \
+    --declarations-path build/aten/src/ATen/Declarations.yaml \
+    --nn-path aten/src
+fi
+
+# Run Clang-Tidy
+# The negative filters below are to exclude files that include onnx_pb.h or
+# caffe2_pb.h, otherwise we'd have to build protos as part of this CI job.
+time python tools/clang_tidy.py             \
+  --verbose                                 \
+  --paths torch/csrc/                       \
+  --diff "$BASE_BRANCH"                     \
+  -g"-torch/csrc/jit/export.cpp"            \
+  -g"-torch/csrc/jit/import.cpp"            \
+  -g"-torch/csrc/jit/netdef_converter.cpp"  \
+  "$@"