From a22c936b6398f5cfd959b3e09622db4d90d61050 Mon Sep 17 00:00:00 2001 From: driazati Date: Tue, 7 Sep 2021 15:15:32 -0700 Subject: [PATCH] Add lint to ensure .github/ pypi dependencies are pinned (#64463) Summary: Example failing run: https://github.com/pytorch/pytorch/pull/64463/checks?check_run_id=3501249102 Pull Request resolved: https://github.com/pytorch/pytorch/pull/64463 Reviewed By: janeyx99 Differential Revision: D30744930 Pulled By: driazati fbshipit-source-id: 4dd97054db1d4c776a4512bc3d664987cd7b6d23 --- .github/templates/bazel_ci_workflow.yml.j2 | 2 +- .github/templates/linux_ci_workflow.yml.j2 | 6 +++--- .github/templates/windows_ci_workflow.yml.j2 | 2 +- .github/workflows/build_linux_conda.yml | 2 +- .github/workflows/build_linux_libtorch.yml | 2 +- .github/workflows/build_linux_wheels.yml | 2 +- ...btorch-linux-xenial-cuda10.2-py3.6-gcc7.yml | 2 +- ...btorch-linux-xenial-cuda11.3-py3.6-gcc7.yml | 2 +- ...erated-linux-bionic-cuda10.2-py3.9-gcc7.yml | 4 ++-- .../generated-linux-bionic-py3.6-clang9.yml | 4 ++-- ...erated-linux-bionic-py3.8-gcc9-coverage.yml | 6 +++--- ...erated-linux-xenial-cuda10.2-py3.6-gcc7.yml | 4 ++-- ...erated-linux-xenial-cuda11.3-py3.6-gcc7.yml | 4 ++-- .../generated-linux-xenial-py3.6-gcc5.4.yml | 4 ++-- ...ated-linux-xenial-py3.6-gcc7-bazel-test.yml | 2 +- ...btorch-linux-xenial-cuda11.1-py3.6-gcc7.yml | 2 +- ...riodic-linux-xenial-cuda11.1-py3.6-gcc7.yml | 4 ++-- ...erated-periodic-win-vs2019-cuda11.1-py3.yml | 2 +- .../workflows/generated-win-vs2019-cpu-py3.yml | 2 +- .../generated-win-vs2019-cuda10.1-py3.yml | 2 +- .../generated-win-vs2019-cuda11.3-py3.yml | 2 +- .github/workflows/lint.yml | 18 +++++++++++++++--- Makefile | 1 + 23 files changed, 47 insertions(+), 34 deletions(-) diff --git a/.github/templates/bazel_ci_workflow.yml.j2 b/.github/templates/bazel_ci_workflow.yml.j2 index 9f982cdd5cb6..9bddb89433c6 100644 --- a/.github/templates/bazel_ci_workflow.yml.j2 +++ b/.github/templates/bazel_ci_workflow.yml.j2 @@ -90,7 +90,7 @@ on: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Test PyTorch run: | diff --git a/.github/templates/linux_ci_workflow.yml.j2 b/.github/templates/linux_ci_workflow.yml.j2 index 520a6a00a19f..f704a3228eb9 100644 --- a/.github/templates/linux_ci_workflow.yml.j2 +++ b/.github/templates/linux_ci_workflow.yml.j2 @@ -170,7 +170,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | @@ -232,7 +232,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix @@ -338,7 +338,7 @@ jobs: {%- if is_coverage %} - name: Report coverage run: | - python3 -mpip install codecov + python3 -mpip install codecov==2.1.12 python3 -mcodecov {%- endif %} - name: Zip test reports for upload diff --git a/.github/templates/windows_ci_workflow.yml.j2 b/.github/templates/windows_ci_workflow.yml.j2 index 20fe72238ffe..22913a331b7a 100644 --- a/.github/templates/windows_ci_workflow.yml.j2 +++ b/.github/templates/windows_ci_workflow.yml.j2 @@ -170,7 +170,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/build_linux_conda.yml b/.github/workflows/build_linux_conda.yml index b43c2013327b..d491e2923e72 100644 --- a/.github/workflows/build_linux_conda.yml +++ b/.github/workflows/build_linux_conda.yml @@ -107,7 +107,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 concurrency: diff --git a/.github/workflows/build_linux_libtorch.yml b/.github/workflows/build_linux_libtorch.yml index 0a1c653375f9..8861cdfcb692 100644 --- a/.github/workflows/build_linux_libtorch.yml +++ b/.github/workflows/build_linux_libtorch.yml @@ -106,7 +106,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 concurrency: diff --git a/.github/workflows/build_linux_wheels.yml b/.github/workflows/build_linux_wheels.yml index 1f8e5f02e222..255fa3024dbf 100644 --- a/.github/workflows/build_linux_wheels.yml +++ b/.github/workflows/build_linux_wheels.yml @@ -105,7 +105,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 concurrency: diff --git a/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml b/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml index 477fe1bac6fe..b3ab1961f892 100644 --- a/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml +++ b/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml @@ -229,7 +229,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | diff --git a/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml b/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml index 9fd6d7ff8d14..50622fe57702 100644 --- a/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml +++ b/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml @@ -229,7 +229,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | diff --git a/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml b/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml index ee0ca4cf76ce..9c2b1948bb5e 100644 --- a/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml +++ b/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml @@ -229,7 +229,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | @@ -302,7 +302,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/generated-linux-bionic-py3.6-clang9.yml b/.github/workflows/generated-linux-bionic-py3.6-clang9.yml index 3aedb76b3e66..b5856edb56fc 100644 --- a/.github/workflows/generated-linux-bionic-py3.6-clang9.yml +++ b/.github/workflows/generated-linux-bionic-py3.6-clang9.yml @@ -229,7 +229,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | @@ -302,7 +302,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml index 2103f2b66bdb..eceab58082e5 100644 --- a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml +++ b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml @@ -229,7 +229,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | @@ -302,7 +302,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix @@ -446,7 +446,7 @@ jobs: docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . - name: Report coverage run: | - python3 -mpip install codecov + python3 -mpip install codecov==2.1.12 python3 -mcodecov - name: Zip test reports for upload if: always() diff --git a/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml index 187f9c1ccfdf..288288e33162 100644 --- a/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml @@ -229,7 +229,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | @@ -302,7 +302,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml index 9fff700c56e8..15bee851a5d9 100644 --- a/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml @@ -229,7 +229,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | @@ -302,7 +302,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml index d1187de624f1..7a1714601a7e 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml @@ -229,7 +229,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | @@ -302,7 +302,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml index 49d2cd2f2267..f0c142c8416d 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml @@ -249,7 +249,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Test PyTorch run: | diff --git a/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml b/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml index 620e4c3d2d31..1e6ad8b5b4c0 100644 --- a/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml @@ -227,7 +227,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | diff --git a/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml b/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml index e318e665c915..5490714eb716 100644 --- a/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml @@ -227,7 +227,7 @@ jobs: run: | COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME - pip3 install requests + pip3 install requests==2.26 python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Chown workspace run: | @@ -300,7 +300,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/generated-periodic-win-vs2019-cuda11.1-py3.yml b/.github/workflows/generated-periodic-win-vs2019-cuda11.1-py3.yml index 360fdc38c86a..94b3ec329c52 100644 --- a/.github/workflows/generated-periodic-win-vs2019-cuda11.1-py3.yml +++ b/.github/workflows/generated-periodic-win-vs2019-cuda11.1-py3.yml @@ -150,7 +150,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/generated-win-vs2019-cpu-py3.yml b/.github/workflows/generated-win-vs2019-cpu-py3.yml index 1277a69f1d13..3c1e7a1771b7 100644 --- a/.github/workflows/generated-win-vs2019-cpu-py3.yml +++ b/.github/workflows/generated-win-vs2019-cpu-py3.yml @@ -142,7 +142,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/generated-win-vs2019-cuda10.1-py3.yml b/.github/workflows/generated-win-vs2019-cuda10.1-py3.yml index 185cb5903e18..bd696817732e 100644 --- a/.github/workflows/generated-win-vs2019-cuda10.1-py3.yml +++ b/.github/workflows/generated-win-vs2019-cuda10.1-py3.yml @@ -152,7 +152,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/generated-win-vs2019-cuda11.3-py3.yml b/.github/workflows/generated-win-vs2019-cuda11.3-py3.yml index b339e79926f5..064b36970f3a 100644 --- a/.github/workflows/generated-win-vs2019-cuda11.3-py3.yml +++ b/.github/workflows/generated-win-vs2019-cuda11.3-py3.yml @@ -152,7 +152,7 @@ jobs: image: python:3.9 steps: - name: Install dependencies - run: pip install typing-extensions + run: pip install typing-extensions==3.10 - name: Clone pytorch/pytorch uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 - name: Generating test matrix diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a1b6182aedaf..7b725e5af41c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -64,6 +64,18 @@ jobs: run: | # shellcheck disable=SC2016 (! git --no-pager grep -InP '# type:\s*ignore(?!\[)' -- '**.py' '**.pyi' ':(exclude)test/test_jit.py' || (echo 'The above lines have unqualified `type: ignore`; please convert them to `type: ignore[xxxx]`'; false)) + - name: Ensure GitHub PyPi dependencies are pinned + if: always() + run: | + (! git --no-pager grep --color=always -InP \ + '(pip|pip3|python -m pip|python3 -m pip|python3 -mpip|python -mpip) install ([a-z][\.a-z-0-9]*+(?!(=|.*\.whl))([[:blank:]]|))+' \ + -- .github \ + ':(exclude)**.rst' \ + ':(exclude)**.py' \ + ':(exclude)**.md' \ + ':(exclude)**.diff' \ + ':(exclude)third_party' || + (echo "The above lines have unpinned PyPi installs; please pin them to a specific version: e.g. 'thepackage==1.2'"; false)) # note that this next step depends on a clean checkout; # if you run it locally then it will likely to complain # about all the generated files in torch/test @@ -287,7 +299,7 @@ jobs: - name: Install dependencies run: | set -eux - pip3 install typing-extensions --user # for tools/linter/translate_annotations.py + pip3 install typing-extensions==3.10 --user # for tools/linter/translate_annotations.py pip3 install -r requirements-flake8.txt --user flake8 --version - name: Run flake8 @@ -416,7 +428,7 @@ jobs: - name: Install dependencies run: | set -eux - pip3 install cmakelint --user + pip3 install cmakelint==1.4.1 --user cmakelint --version - name: Run cmakelint run: | @@ -442,7 +454,7 @@ jobs: pip3 install numpy==1.20 --user # https://github.com/pytorch/pytorch/pull/60472 pip3 install expecttest==0.1.3 mypy==0.812 --user # Needed to check tools/render_junit.py - pip3 install junitparser rich --user + pip3 install junitparser==2.1.1 rich==10.9.0 --user - name: Run autogen run: | set -eux diff --git a/Makefile b/Makefile index f2c835b46b61..3d18c2b46381 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,7 @@ quick_checks: --step 'Ensure canonical include' \ --step 'Ensure no versionless Python shebangs' \ --step 'Ensure no unqualified noqa' \ + --step 'Ensure GitHub PyPi dependencies are pinned' \ --step 'Ensure no unqualified type ignore' \ --step 'Ensure no direct cub include' \ --step 'Ensure correct trailing newlines' \