From bd0907dc4cb45045e1e3ad3309b540faa080a496 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Tue, 30 Sep 2025 20:22:24 -0700 Subject: [PATCH] [BE][CI] Unify requirments (#163396) Both Linux, Windows and MacOS CI workflows should use `.ci/docker/requirements-ci.txt` TODOS: - Investigate why `choco install cmake` is needed to successfully detect MKL - Move `psutil` installation from specific scripts into requirements-ci.txt Pull Request resolved: https://github.com/pytorch/pytorch/pull/163396 Approved by: https://github.com/Skylion007 --- .ci/docker/requirements-ci.txt | 22 ++++++----- .../win-test-helpers/build_pytorch.bat | 2 + .ci/pytorch/win-test.sh | 23 +----------- .../requirements/pip-requirements-macOS.txt | 37 ------------------- .github/workflows/_mac-build.yml | 2 +- .github/workflows/_mac-test.yml | 2 +- .github/workflows/_win-test.yml | 12 ------ 7 files changed, 19 insertions(+), 81 deletions(-) delete mode 100644 .github/requirements/pip-requirements-macOS.txt diff --git a/.ci/docker/requirements-ci.txt b/.ci/docker/requirements-ci.txt index c9c1799d09b6..9a5166ea5fc3 100644 --- a/.ci/docker/requirements-ci.txt +++ b/.ci/docker/requirements-ci.txt @@ -52,10 +52,10 @@ flatbuffers==24.12.23 #Pinned versions: 24.12.23 #test that import: -hypothesis==5.35.1 +hypothesis==6.56.4 # Pin hypothesis to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136 #Description: advanced library for generating parametrized tests -#Pinned versions: 5.35.1 +#Pinned versions: 6.56.4 #test that import: test_xnnpack_integration.py, test_pruning_op.py, test_nn.py junitparser==2.1.1 @@ -98,7 +98,7 @@ librosa==0.10.2 ; python_version == "3.12" and platform_machine != "s390x" #Pinned versions: #test that import: -mypy==1.16.0 ; platform_system != "Windows" +mypy==1.16.0 ; platform_system == "Linux" # Pin MyPy version because new errors are likely to appear with each release # Skip on Windows as lots of type annotations are POSIX specific #Description: linter @@ -169,7 +169,7 @@ optree==0.13.0 pillow==11.0.0 #Description: Python Imaging Library fork -#Pinned versions: 10.3.0 +#Pinned versions: 11.0.0 #test that import: protobuf==5.29.5 @@ -217,7 +217,7 @@ pytest-subtests==0.13.1 #Pinned versions: #test that import: -xdoctest==1.1.0 +xdoctest==1.3.0 #Description: runs doctests in pytest #Pinned versions: 1.1.0 #test that import: @@ -268,7 +268,7 @@ scipy==1.14.1 ; python_version >= "3.12" #test that import: # needed by torchgen utils -typing-extensions>=4.10.0 +typing-extensions==4.12.2 #Description: type hints for python #Pinned versions: #test that import: @@ -361,9 +361,10 @@ pwlf==2.2.1 #test that import: test_sac_estimator.py # To build PyTorch itself -pyyaml +pyyaml==6.0.2 pyzstd -setuptools>=70.1.0 +setuptools==78.1.1 +packaging==23.1 six scons==4.5.2 ; platform_machine == "aarch64" @@ -384,7 +385,10 @@ cmake==3.31.6 tlparse==0.4.0 #Description: required for log parsing -cuda-bindings>=12.0,<13.0 ; platform_machine != "s390x" +filelock==3.18.0 +#Description: required for inductor testing + +cuda-bindings>=12.0,<13.0 ; platform_machine != "s390x" and platform_system != "Darwin" #Description: required for testing CUDAGraph::raw_cuda_graph(). See https://nvidia.github.io/cuda-python/cuda-bindings/latest/support.html for how this version was chosen. Note "Any fix in the latest bindings would be backported to the prior major version" means that only the newest version of cuda-bindings will get fixes. Depending on the latest version of 12.x is okay because all 12.y versions will be supported via "CUDA minor version compatibility". Pytorch builds against 13.z versions of cuda toolkit work with 12.x versions of cuda-bindings as well because newer drivers work with old toolkits. #test that import: test_cuda.py diff --git a/.ci/pytorch/win-test-helpers/build_pytorch.bat b/.ci/pytorch/win-test-helpers/build_pytorch.bat index 427566ff7823..240cc8b55932 100644 --- a/.ci/pytorch/win-test-helpers/build_pytorch.bat +++ b/.ci/pytorch/win-test-helpers/build_pytorch.bat @@ -38,10 +38,12 @@ if errorlevel 1 goto fail if not errorlevel 0 goto fail :: Update CMake +:: TODO: Investigate why this helps MKL detection, even when CMake from choco is not used call choco upgrade -y cmake --no-progress --installargs 'ADD_CMAKE_TO_PATH=System' --apply-install-arguments-to-dependencies --version=3.27.9 if errorlevel 1 goto fail if not errorlevel 0 goto fail +:: TODO: Move to .ci/docker/requirements-ci.txt call pip install mkl==2024.2.0 mkl-static==2024.2.0 mkl-include==2024.2.0 if errorlevel 1 goto fail if not errorlevel 0 goto fail diff --git a/.ci/pytorch/win-test.sh b/.ci/pytorch/win-test.sh index c96d5c331c9f..1a16b166e805 100755 --- a/.ci/pytorch/win-test.sh +++ b/.ci/pytorch/win-test.sh @@ -37,27 +37,8 @@ if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then export PYTORCH_TESTING_DEVICE_ONLY_FOR="cuda" fi -# TODO: Move both of them to Windows AMI -python -m pip install tensorboard==2.13.0 protobuf==5.29.4 pytest-subtests==0.13.1 - -# Copied from https://github.com/pytorch/test-infra/blob/be01a40157c36cd5a48391fdf44a7bc3ebd4c7e3/aws/ami/windows/scripts/Installers/Install-Pip-Dependencies.ps1#L16 with some adjustments -# pytest-rerunfailures==10.3 as 10.2 fails with INTERNALERROR> pluggy._manager.PluginValidationError: unknown hook 'pytest_configure_node' -# scipy from 1.6.3 to 1.10 -# expecttest from 0.1.3 to 0.3.0 -# xdoctest from 1.0.2 to 1.3.0 -python -m pip install "future==0.18.2" "hypothesis==5.35.1" "expecttest==0.3.0" "librosa>=0.6.2" "scipy==1.10.1" "psutil==5.9.1" "pynvml==11.4.1" "pillow==9.2.0" "unittest-xml-reporting<=3.2.0,>=2.0.0" "pytest==7.1.3" "pytest-xdist==2.5.0" "pytest-flakefinder==1.1.0" "pytest-rerunfailures==10.3" "pytest-shard==0.1.2" "sympy==1.11.1" "xdoctest==1.3.0" "pygments==2.12.0" "opt-einsum>=3.3" "networkx==2.8.8" "mpmath==1.2.1" "pytest-cpp==2.3.0" "boto3==1.35.42" - -# Install Z3 optional dependency for Windows builds. -python -m pip install z3-solver==4.15.1.0 - -# Install tlparse for test\dynamo\test_structured_trace.py UTs. -python -m pip install tlparse==0.4.0 - -# Install parameterized -python -m pip install parameterized==0.8.1 - -# Install pulp for testing ilps under torch\distributed\_tools -python -m pip install pulp==2.9.0 +# TODO: Move this to .ci/docker/requirements-ci.txt +python -m pip install "psutil==5.9.1" "pynvml==11.4.1" "pytest-shard==0.1.2" run_tests() { # Run nvidia-smi if available diff --git a/.github/requirements/pip-requirements-macOS.txt b/.github/requirements/pip-requirements-macOS.txt deleted file mode 100644 index ef27f2ea20b4..000000000000 --- a/.github/requirements/pip-requirements-macOS.txt +++ /dev/null @@ -1,37 +0,0 @@ -boto3==1.35.42 -build==1.2.2.post1 -cmake==3.27.* -expecttest==0.3.0 -fbscribelogger==0.1.7 -filelock==3.18.0 -hypothesis==6.56.4 -librosa>=0.6.2 -mpmath==1.3.0 -networkx==2.8.7 -ninja==1.10.2.4 -numba==0.59.0 -numpy==1.26.4 -opt-einsum>=3.3 -optree==0.13.0 -packaging==23.1 -parameterized==0.8.1 -pillow==10.3.0 -protobuf==5.29.5 -psutil==5.9.8 -pygments==2.15.0 -pytest-cpp==2.3.0 -pytest-flakefinder==1.1.0 -pytest-rerunfailures==10.3 -pytest-subtests==0.13.1 -pytest-xdist==3.3.1 -pytest==7.3.2 -pyyaml==6.0.2 -scipy==1.12.0 -setuptools==78.1.1 -sympy==1.13.3 -tlparse==0.4.0 -tensorboard==2.13.0 -typing-extensions==4.12.2 -unittest-xml-reporting<=3.2.0,>=2.0.0 -xdoctest==1.1.0 -z3-solver==4.15.1.0 diff --git a/.github/workflows/_mac-build.yml b/.github/workflows/_mac-build.yml index a2a5f8dd9111..24fe510f0fb5 100644 --- a/.github/workflows/_mac-build.yml +++ b/.github/workflows/_mac-build.yml @@ -85,7 +85,7 @@ jobs: uses: pytorch/test-infra/.github/actions/setup-python@main with: python-version: ${{ inputs.python-version }} - pip-requirements-file: .github/requirements/pip-requirements-macOS.txt + pip-requirements-file: .ci/docker/requirements-ci.txt - name: Install sccache (only for non-forked PRs, and pushes to trunk) uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 diff --git a/.github/workflows/_mac-test.yml b/.github/workflows/_mac-test.yml index 086e25b4868e..82eb3c4bf2c7 100644 --- a/.github/workflows/_mac-test.yml +++ b/.github/workflows/_mac-test.yml @@ -122,7 +122,7 @@ jobs: uses: pytorch/test-infra/.github/actions/setup-python@main with: python-version: ${{ inputs.python-version }} - pip-requirements-file: .github/requirements/pip-requirements-macOS.txt + pip-requirements-file: .ci/docker/requirements-ci.txt - name: Start monitoring script id: monitor-script diff --git a/.github/workflows/_win-test.yml b/.github/workflows/_win-test.yml index a93f10c123aa..7f8e06ece555 100644 --- a/.github/workflows/_win-test.yml +++ b/.github/workflows/_win-test.yml @@ -106,18 +106,6 @@ jobs: with: cuda-version: ${{ inputs.cuda-version }} - # TODO: Move to a requirements.txt file for windows - - name: Install pip dependencies - uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 - with: - shell: bash - timeout_minutes: 5 - max_attempts: 5 - retry_wait_seconds: 30 - command: | - set -eu - python3 -m pip install 'xdoctest>=1.1.0' - - name: Get workflow job id id: get-job-id uses: ./.github/actions/get-workflow-job-id