Re-enable Windows debug libtorch (#73900)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/73422

Fixes https://github.com/pytorch/pytorch/issues/73421
ghstack-source-id: 149978120

Test Plan: None

Reviewed By: cbalioglu

Differential Revision: D34475711

fbshipit-source-id: 9e4d1d57021cbff51f53762b32bbfffbf3f81c4c

.azure_pipelines/build-pipeline.yml

@@ -0,0 +1,63 @@
+# PyTorch CI Builds Pipeline on Azure DevOps
+#
+# This pipeline:
+#   1) builds PyTorch on select configurations
+#   2) runs only TestTorch unit tests.
+
+stages:
+- stage: 'Build'
+  displayName: 'Build PyTorch'
+  jobs:
+  - template: job_templates/build-verify-publish-template-unix.yml
+    parameters:
+      name: ubuntu_1804_CPU_docker
+      pool: 'PyTorch-Linux-CPU'
+      container_endpoint: pytorchms.azurecr.io
+      build_stage: True
+      is_ci_build: True
+      os: ubuntu
+      cuda: cpu
+      customMatrixes:
+        Py_38:
+          configuration: ubuntu_1804_py_38_cpu
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cpu_dev
+
+  - template: job_templates/build-verify-publish-template-unix.yml
+    parameters:
+      name: ubuntu_1804_GPU_docker
+      pool: 'PyTorch-Linux-GPU'
+      container_endpoint: pytorchms.azurecr.io
+      build_stage: True
+      is_ci_build: True
+      os: ubuntu
+      cuda: gpu
+      customMatrixes:
+        Py_39_CUDA_112_cuDNN_810:
+          configuration: ubuntu_1804_py_39_cuda_112_cudnn_810
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_39_cuda_112_cudnn_8_dev
+          CUDA_VERSION: 112
+
+  - template: job_templates/build-verify-publish-template-win.yml
+    parameters:
+      name: windows_2019_CPU
+      pool: 'PyTorch-Win-CPU'
+      build_stage: True
+      is_ci_build: True
+      os: windows
+      cuda: cpu
+      customMatrixes:
+        Py_37:
+          configuration: windows_2019_py_37_cpu
+
+  - template: job_templates/build-verify-publish-template-win.yml
+    parameters:
+      name: windows_2019_GPU
+      pool: 'PyTorch-Win-GPU'
+      build_stage: True
+      is_ci_build: True
+      os: windows
+      cuda: gpu
+      customMatrixes:
+        Py_38_CUDA_102_cuDNN_765:
+          configuration: windows_2019_py_38_cuda_102_cudnn_765
+          CUDA_VERSION: 102

.azure_pipelines/daily-pipeline.yml

@@ -0,0 +1,82 @@
+# PyTorch Daily Builds Pipeline on Azure DevOps
+#
+# This pipeline:
+#   1) builds PyTorch on all available configurations
+#   2) runs all PyTorch unit tests
+
+stages:
+- stage: 'BuildTest'
+  displayName: 'Build and Test PyTorch'
+  jobs:
+  - template: job_templates/build-verify-publish-template-unix.yml
+    parameters:
+      name: ubuntu_1804_CPU_docker
+      pool: 'PyTorch-Linux-CPU'
+      container_endpoint: pytorchms.azurecr.io
+      build_stage: True
+      is_daily_build: True
+      os: ubuntu
+      cuda: cpu
+      customMatrixes:
+        Py_38:
+          configuration: ubuntu_1804_py_38_cpu
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cpu_dev
+        Py_37:
+          configuration: ubuntu_1804_py_37_cpu
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_37_cpu_dev
+
+  - template: job_templates/build-verify-publish-template-unix.yml
+    parameters:
+      name: ubuntu_1804_GPU_docker
+      pool: 'PyTorch-Linux-GPU'
+      container_endpoint: pytorchms.azurecr.io
+      build_stage: True
+      is_daily_build: True
+      os: ubuntu
+      cuda: gpu
+      customMatrixes:
+        Py_39_CUDA_112_cuDNN_810:
+          configuration: ubuntu_1804_py_39_cuda_112_cudnn_810
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_39_cuda_112_cudnn_8_dev
+          CUDA_VERSION: 112
+        Py_38_CUDA_102_cuDNN_810:
+          configuration: ubuntu_1804_py_38_cuda_102_cudnn_810
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cuda_102_cudnn_8_dev
+          CUDA_VERSION: 102
+        Py_37_CUDA_101_cuDNN_765:
+          configuration: ubuntu_1804_py_37_cuda_101_cudnn_765
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_37_cuda_101_cudnn_7_dev
+          CUDA_VERSION: 101
+
+  - template: job_templates/build-verify-publish-template-win.yml
+    parameters:
+      name: windows_2019_CPU
+      pool: 'PyTorch-Win-CPU'
+      build_stage: True
+      is_daily_build: True
+      os: windows
+      cuda: cpu
+      customMatrixes:
+        Py_38:
+          configuration: windows_2019_py_38_cpu
+        Py_37:
+          configuration: windows_2019_py_37_cpu
+
+  - template: job_templates/build-verify-publish-template-win.yml
+    parameters:
+      name: windows_2019_GPU
+      pool: 'PyTorch-Win-GPU'
+      build_stage: True
+      is_daily_build: True
+      os: windows
+      cuda: gpu
+      customMatrixes:
+        Py_39_CUDA_112_cuDNN_810:
+          configuration: windows_2019_py_39_cuda_112_cudnn_810
+          CUDA_VERSION: 112
+        Py_38_CUDA_102_cuDNN_765:
+          configuration: windows_2019_py_38_cuda_102_cudnn_765
+          CUDA_VERSION: 102
+        Py_37_CUDA_101_cuDNN_764:
+          configuration: windows_2019_py_37_cuda_101_cudnn_764
+          CUDA_VERSION: 101

.azure_pipelines/job_templates/build-verify-publish-template-unix.yml

@@ -0,0 +1,134 @@
+# PyTorch build steps template with Unix images Azure DevOps Instances
+#
+# This build depends on 3 parameters set as environment variables in the pipeline:
+#   - AZURE_DEVOPS_CLI_PAT: Secret var for authenticating to Azure DevOps
+#   - AZURE_DEVOPS_ARTIFACTS_ORGANIZATION: Azure Artifacts Organization name to publish artifacts
+#   - AZURE_DEVOPS_ARTIFACTS_PROJECT: Azure Artifacts Project name to publish artifacts
+
+parameters:
+  name: ''
+  pool: ''
+  container_endpoint: ''
+  os: ''
+  cuda: ''
+  is_ci_build: False
+  is_official_build: False
+  is_daily_build: False
+  build_stage: False
+  verify_stage: False
+  publish_stage: False
+  customMatrixes: ''
+
+jobs:
+- job: ${{parameters.name}}
+  timeoutInMinutes: 300
+  strategy:
+    matrix:
+      ${{ insert }}: ${{parameters.customMatrixes}}
+  pool:
+    name: ${{ parameters.pool}}
+  variables:
+    DECODE_PERCENTS: false
+  container:
+    image: $[variables['container_image']]
+    endpoint: ${{parameters.container_endpoint}}
+
+  steps:
+  # Build stage
+  - ${{ if eq(parameters.build_stage, 'True') }}:
+    # Set up environment variables for specific pipeline build
+    - template: set-environment-variables.yml
+      parameters:
+        os: ${{ parameters.os}}
+        cuda: ${{ parameters.cuda}}
+        is_official_build: ${{ parameters.is_official_build}}
+
+    # Sync and update PyTorch submodules
+    - bash: git submodule update --init --recursive --jobs 0
+      displayName: Update PyTorch submodules
+
+    # Build PyTorch and run unit tests - no packaging
+    - ${{ if or(eq(parameters.is_ci_build, 'True'), eq(parameters.is_daily_build, 'True')) }}:
+      # Build PyTorch from source in develop mode
+      - bash: python setup.py develop
+        displayName: Build PyTorch from source
+
+      - ${{ if eq(parameters.is_ci_build, 'True') }}:
+        # Run TestTorch unit tests to demonstrate successful PyTorch build
+        - bash: python test/test_torch.py TestTorch
+          displayName: Run TestTorch unit tests
+
+      - ${{ if eq(parameters.is_daily_build, 'True') }}:
+        # Run all unit tests to demonstrate successful PyTorch build
+        - bash: python test/run_test.py --continue-through-error --exclude-jit-executor --verbose
+          displayName: Run all unit tests
+
+      # Run ComponentGovernance
+      - task: ComponentGovernanceComponentDetection@0
+        inputs:
+          scanType: 'Register'
+          verbosity: 'Verbose'
+          alertWarningLevel: 'High'
+
+    # Build PyTorch and produce artifacts for verification stage
+    - ${{ if eq(parameters.is_official_build, 'True') }}:
+      # Build PyTorch from source in install mode and exclude test binaries
+      - bash: python setup.py install
+        displayName: Build PyTorch from source without test binaries
+
+      # Package PyTorch Wheel
+      - bash: python setup.py bdist_wheel
+        displayName: Package PyTorch Wheel
+
+      # Publish PyTorch Wheel
+      - task: PublishPipelineArtifact@1
+        inputs:
+          targetPath: $(Build.SourcesDirectory)/dist/
+          artifactName: Build_$(Build.BuildNumber)_$(configuration)
+        displayName: Publish PyTorch Wheel to Pipeline Artifacts
+
+  # Verification stage
+  - ${{ if eq(parameters.verify_stage, 'True') }}:
+    # Download PyTorch Wheel
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: Build_$(Build.BuildNumber)_$(configuration)
+        path: $(Build.SourcesDirectory)/verify
+      displayName: Download PyTorch Wheel
+
+    # Install PyTorch Wheel on Windows
+    - bash: python -m pip install $(Build.SourcesDirectory)/verify/torch*linux*.whl
+      displayName: Install PyTorch Wheel
+
+    # Ensure PyTorch installed correctly from produced wheel
+    - bash: |
+        cd $(Build.SourcesDirectory)/verify
+        python -c "import torch; print('Installed Torch version: ' + torch.__version__)"
+      displayName: Check PyTorch correctly installed from wheel
+
+  # Publishing stage
+  - ${{ if eq(parameters.publish_stage, 'True') }}:
+    # Download PyTorch Wheel
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: Build_$(Build.BuildNumber)_$(configuration)
+        path: $(Build.SourcesDirectory)/publish
+      displayName: Download PyTorch Wheel
+
+    # Publish wheel to Azure Artifacts
+    # The flag continueOnError=true is needed as the artifact to be published
+    # may already exist, because the artifact is differentiated based on the
+    # last commit date.
+    - bash: |
+        export TORCH_VERSION=$(head -c 5 ./version.txt)
+        export LAST_COMMIT=$(git rev-parse --short HEAD)
+        export LAST_COMMIT_DATE=$(git log -1 --pretty=%ad --date=format:%Y%m%d)
+        cd $(Build.SourcesDirectory)/publish
+        export TORCH_WHEEL=$(echo torch*linux*whl)
+        az extension add -n azure-devops
+        echo $ADOTOKEN | az devops login
+        az artifacts universal publish --organization $AZURE_DEVOPS_ARTIFACTS_ORGANIZATION --project $AZURE_DEVOPS_ARTIFACTS_PROJECT --scope project --feed "PyTorch" --name $TORCH_WHEEL --description "PyTorch Official Build Artifact" --version $TORCH_VERSION-$LAST_COMMIT_DATE-$LAST_COMMIT --path .
+      env:
+        ADOTOKEN: $(AZURE_DEVOPS_CLI_PAT)
+      continueOnError: true
+      displayName: Upload PyTorch Official Build package to Azure Artifacts

.azure_pipelines/job_templates/build-verify-publish-template-win.yml

@@ -0,0 +1,150 @@
+# PyTorch build steps template with Windows images Azure DevOps Instances
+#
+# This build depends on 3 parameters set as environment variables in the pipeline:
+#   - AZURE_DEVOPS_CLI_PAT: Secret var for authenticating to Azure DevOps
+#   - AZURE_DEVOPS_ARTIFACTS_ORGANIZATION: Azure Artifacts Organization name to publish artifacts
+#   - AZURE_DEVOPS_ARTIFACTS_PROJECT: Azure Artifacts Project name to publish artifacts
+
+parameters:
+  name: ''
+  pool: ''
+  os: ''
+  cuda: ''
+  is_ci_build: False
+  is_official_build: False
+  is_daily_build: False
+  build_stage: False
+  verify_stage: False
+  publish_stage: False
+  customMatrixes: ''
+
+jobs:
+- job: ${{parameters.name}}
+  timeoutInMinutes: 300
+  strategy:
+    matrix:
+      ${{ insert }}: ${{parameters.customMatrixes}}
+  pool:
+    name: ${{ parameters.pool}}
+  variables:
+    CMAKE_GENERATOR: Ninja
+    PACKAGE_PDBS: 0
+
+  steps:
+  # Prepare for PyTorch build on Windows
+  - template: prepare-build-template.yml
+    parameters:
+      configuration: $(configuration)
+      build_stage: ${{ parameters.build_stage}}
+
+  # Build Stage
+  - ${{ if eq(parameters.build_stage, 'True') }}:
+    # Set up environment variables for specific pipeline build
+    - template: set-environment-variables.yml
+      parameters:
+        os: ${{ parameters.os}}
+        cuda: ${{ parameters.cuda}}
+        is_official_build: ${{ parameters.is_official_build}}
+
+    # Sync and update PyTorch submodules
+    - script: git submodule update --init --recursive --jobs 0
+      displayName: Update PyTorch submodules
+
+    # Build PyTorch and run unit tests - no packaging
+    - ${{ if or(eq(parameters.is_ci_build, 'True'), eq(parameters.is_daily_build, 'True')) }}:
+      # Build PyTorch from source in develop mode with Ninja
+      - script: call activate $(configuration) && python setup.py develop
+        displayName: Build PyTorch from source
+
+      - ${{ if eq(parameters.is_ci_build, 'True') }}:
+        # Run TestTorch unit tests to demonstrate successful PyTorch build
+        - script: call activate $(configuration) && python test\test_torch.py TestTorch
+          displayName: Run TestTorch unit tests
+
+      - ${{ if eq(parameters.is_daily_build, 'True') }}:
+        # Run all unit tests to demonstrate successful PyTorch build
+        - script: call activate $(configuration) && python test/run_test.py --continue-through-error --exclude-jit-executor --verbose
+          displayName: Run all unit tests
+
+      # Run ComponentGovernance
+      - task: ComponentGovernanceComponentDetection@0
+        inputs:
+          scanType: 'Register'
+          verbosity: 'Verbose'
+          alertWarningLevel: 'High'
+
+    # Build PyTorch and produce artifacts for verification stage
+    - ${{ if eq(parameters.is_official_build, 'True') }}:
+      # Build PyTorch from source in install mode with Ninja and exclude test binaries
+      - script: call activate $(configuration) && python setup.py install
+        displayName: Build PyTorch from source without test binaries
+
+      # Package PyTorch Wheel
+      - script: call activate $(configuration) && python setup.py bdist_wheel
+        displayName: Package PyTorch Wheel
+
+      # Publish PyTorch Wheel
+      - task: PublishPipelineArtifact@1
+        inputs:
+          targetPath: $(Build.SourcesDirectory)\dist\
+          artifactName: Build_$(Build.BuildNumber)_$(configuration)
+        displayName: Publish PyTorch Wheel to Pipeline Artifacts
+
+  # Verification Stage
+  - ${{ if eq(parameters.verify_stage, 'True') }}:
+    # Download PyTorch Wheel
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: Build_$(Build.BuildNumber)_$(configuration)
+        path: $(Build.SourcesDirectory)\verify
+      displayName: Download PyTorch Wheel
+
+    # Install PyTorch Wheel on Windows
+    - script: |
+        call activate $(configuration)
+        cd $(Build.SourcesDirectory)\verify
+        dir torch*win*.whl /b > whl.txt
+        set /p whl= < whl.txt
+        python -m pip install %whl%
+      displayName: Install PyTorch Wheel
+
+    # Ensure PyTorch installed correctly from produced wheel
+    - script: |
+        call activate $(configuration)
+        cd $(Build.SourcesDirectory)\verify
+        python -c "import torch; print('Installed Torch version: ' + torch.__version__)"
+      displayName: Check PyTorch correctly installed from wheel
+
+  # Publishing stage
+  - ${{ if eq(parameters.publish_stage, 'True') }}:
+    # Download PyTorch Wheel
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: Build_$(Build.BuildNumber)_$(configuration)
+        path: $(Build.SourcesDirectory)\publish
+      displayName: Download PyTorch Wheel
+
+    # Set up Azure Artifacts for Windows
+    # The pip install --upgrade command is a bug fix for Azure CLI on Windows
+    # More info: https://github.com/Azure/azure-cli/issues/16858
+    - script: |
+        pip install --upgrade pip --target \opt\az\lib\python3.6\site-packages\
+        az extension add -n azure-devops
+      displayName: Set up Azure Artifacts download on Windows
+
+    # Publish wheel to Azure Artifacts
+    # The flag continueOnError=true is needed as the artifact to be published
+    # may already exist, because the artifact is differentiated based on the
+    # last commit date.
+    - script: |
+        set /p TORCH_VERSION= < version.txt
+        cd $(Build.SourcesDirectory)\publish
+        git rev-parse --short HEAD > last_commit.txt && set /p LAST_COMMIT= < last_commit.txt
+        git log -1 --pretty=%ad --date=format:%Y%m%d > last_commit_date.txt && set /p LAST_COMMIT_DATE= < last_commit_date.txt
+        dir torch*win*.whl /b > whl.txt && set /p TORCH_WHEEL= < whl.txt
+        echo %ADOTOKEN% | az devops login
+        az artifacts universal publish --organization %AZURE_DEVOPS_ARTIFACTS_ORGANIZATION% --project %AZURE_DEVOPS_ARTIFACTS_PROJECT% --scope project --feed "PyTorch" --name %TORCH_WHEEL% --description "PyTorch Official Build Artifact" --version %TORCH_VERSION:~0,5%-%LAST_COMMIT_DATE%-%LAST_COMMIT% --path .
+      env:
+        ADOTOKEN: $(AZURE_DEVOPS_CLI_PAT)
+      continueOnError: true
+      displayName: Upload PyTorch nigthly package to Azure Artifacts

.azure_pipelines/job_templates/common-packages.yml

@@ -0,0 +1,17 @@
+dependencies:
+  - python=PYTHON_VERSION
+  - numpy
+  - ninja
+  - pyyaml
+  - mkl
+  - mkl-include
+  - setuptools
+  - cmake
+  - cffi
+  - typing_extensions
+  - future
+  - six
+  - requests
+  - dataclasses
+  - pip:
+    - -r ../../requirements.txt

.azure_pipelines/job_templates/notify-webapp-template.yml

@@ -0,0 +1,26 @@
+parameters:
+  name: ''
+  pool: ''
+  customMatrixes: ''
+
+jobs:
+- job: ${{parameters.name}}
+  timeoutInMinutes: 600
+  strategy:
+    matrix:
+      ${{ insert }}: ${{parameters.customMatrixes}}
+  pool:
+    name: ${{ parameters.pool}}
+  steps:
+  # Clone PyTorch Tests repository
+  - bash: |
+      B64_PAT=$(echo -n ":$_ADOTOKEN" | base64)
+      git -c http.extraHeader="Authorization: Basic ${B64_PAT}" clone $(AZURE_DEVOPS_PYTORCH_TESTS_REPO_URL)
+      cd pytorch_tests
+      git checkout $(PYTORCH_TESTS_CHECKOUT_BRANCH)
+    env:
+      _ADOTOKEN: $(AZURE_DEVOPS_CLI_PAT)
+    displayName: Clone PyTorch Tests repo
+  - bash: |
+      bash $(Build.SourcesDirectory)/pytorch_tests/webapp/notify_webapp.sh
+    displayName: Notify Webapp

.azure_pipelines/job_templates/prepare-build-template.yml

@@ -0,0 +1,62 @@
+# Build prepare steps for PyTorch on Azure DevOps to build from source.
+# These steps share between normal build process and semmle security scan tasks
+
+parameters:
+  build_stage: False
+  configuration: ''
+
+steps:
+# End Python tasks that may be lingering over from previous runs
+# Note: If python.exe isn't currently running, exit code becomes 128,
+# which fails the run. Here exit code is set to 0 to avoid failed run.
+- script: |
+    taskkill /f /im python.exe
+    IF %ERRORLEVEL% EQU 128 exit 0
+  displayName: End previous Python processes
+
+# Clean up env directory in conda for fresh builds and set up conda environment YAML
+- powershell: |
+    Remove-Item 'C:\Miniconda\envs' -Recurse -ErrorAction Ignore
+    $env:PYTHON_VERSION = $env:SYSTEM_JOBNAME.Substring(3,1) + '.' + $env:SYSTEM_JOBNAME.Substring(4,1)
+    (Get-Content .azure_pipelines\job_templates\common-packages.yml) -replace 'PYTHON_VERSION', $env:PYTHON_VERSION | Out-File -encoding ASCII .azure_pipelines\job_templates\common-packages.yml
+  displayName: Clean up previous environments and Set up conda environment YAML
+
+# Make conda environment and install required packages
+- script: |
+    call conda clean --all -y
+    call conda env create -n $(configuration) --file .azure_pipelines\job_templates\common-packages.yml
+    call activate $(configuration)
+    call conda install -c conda-forge libuv=1.39
+  displayName: Set up conda environment for building from source
+
+- ${{ if eq(parameters.build_stage, 'True') }}:
+  # Install MKL
+  - script: |
+      rmdir /s /q mkl
+      del mkl_2020.2.254.7z
+      curl https://s3.amazonaws.com/ossci-windows/mkl_2020.2.254.7z -k -O
+      7z x -aoa mkl_2020.2.254.7z -omkl
+    displayName: Install MKL
+
+  # Install sccache and randomtemp
+  # Related PyTorch GitHub issue: https://github.com/pytorch/pytorch/issues/25393
+  # Related fix: https://github.com/pytorch/builder/pull/448/
+  - script: |
+      mkdir .\tmp_bin
+      curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output .\tmp_bin\sccache.exe
+      curl -k https://s3.amazonaws.com/ossci-windows/sccache-cl.exe --output .\tmp_bin\sccache-cl.exe
+      copy .\tmp_bin\sccache.exe .\tmp_bin\nvcc.exe
+      curl -kL https://github.com/peterjc123/randomtemp-rust/releases/download/v0.4/randomtemp.exe --output .\tmp_bin\randomtemp.exe
+    displayName: Install sccache and randomtemp
+    condition: not(eq(variables.CUDA_VERSION, ''))
+
+  # CUDA 11.2's CUB directory conflicts with CUDA 10.2 and 10.1
+  # builds, where CUDA 11.2's CUB is injected into non-CUDA
+  # 11.2 builds.
+  - powershell: Remove-Item "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\include\cub" -Recurse -ErrorAction Ignore
+    displayName: Remove conflicting CUB from CUDA installation
+    condition: not(eq(variables.CUDA_VERSION, ''))
+
+  - powershell: Copy-Item -Path "F:\cuda_11_2\cub\" -Destination "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\include" -Recurse
+    displayName: Copy CUDA CUB for CUDA 11.2 build
+    condition: eq(variables.CUDA_VERSION, '112')

.azure_pipelines/job_templates/pytorch-template-unix.yml

@@ -0,0 +1,61 @@
+# PyTorch build steps template with Unix images Azure DevOps Instances
+#
+# This build depends on 5 parameters set as an environment variables in the pipeline:
+#   - AZURE_DEVOPS_CLI_PAT: Secret var for authenticating to Azure DevOps
+#   - AZURE_STORAGE_KEY: Secret var for authenticating to Azure Storage
+#   - _TS_CLONE_P, _TS_P, _TS_SM_P: Secret vars for specific unit tests
+
+parameters:
+  name: ''
+  pool: ''
+  container_endpoint: ''
+  customMatrixes: ''
+
+jobs:
+- job: ${{parameters.name}}
+  timeoutInMinutes: 600
+  strategy:
+    matrix:
+      ${{ insert }}: ${{parameters.customMatrixes}}
+  pool:
+    name: ${{ parameters.pool}}
+  variables:
+    DECODE_PERCENTS: false
+
+  steps:
+  # Don't checkout repo contents to save time and CPU compute. Environment variables
+  # related to checkout branch such as $(BUILD_SOURCEBRANCH) are still available.
+  - checkout: none
+
+  # Delete pytorch_tests repo from previous builds if exists
+  - bash: rm -rf pytorch_tests/
+    displayName: Delete pytorch_tests repo from previous builds if exists
+
+  # Clone PyTorch Tests repository
+  - bash: |
+      B64_PAT=$(echo -n ":$_ADOTOKEN" | base64)
+      git -c http.extraHeader="Authorization: Basic ${B64_PAT}" clone $(AZURE_DEVOPS_PYTORCH_TESTS_REPO_URL)
+      cd pytorch_tests
+      git checkout $(PYTORCH_TESTS_CHECKOUT_BRANCH)
+    env:
+      _ADOTOKEN: $(AZURE_DEVOPS_CLI_PAT)
+    displayName: Clone PyTorch Tests repo
+
+  # Run PyTorch Unit Tests
+  - bash: bash $(Build.SourcesDirectory)/pytorch_tests/scripts/linux/run.sh
+    env:
+      _AZURE_STORAGE_KEY: $(AZURE_STORAGE_KEY)
+      _TS_CLONE_P: $(TS_CLONE_PASSWORD)
+      _TS_P: $(TS_PAT)
+      _TS_SM_P: $(TS_SM_PAT)
+      _AZUREML_CLONE_PASSWORD: $(AZUREML_CLONE_PASSWORD)
+      _SPPASSWORD: $(SPPASSWORD)
+    displayName: Run PyTorch Unit Tests
+
+  # Tests results are available outside the docker container since
+  # the current directory is mounted as a volume of the container.
+  - task: PublishTestResults@2
+    condition: always()
+    inputs:
+      testResultsFiles: '**/test-*.xml'
+      testRunTitle: 'Publish test results for Python'

.azure_pipelines/job_templates/pytorch-template-win.yml

@@ -0,0 +1,57 @@
+# PyTorch build steps template with Windows images Azure DevOps Instances
+#
+# This build depends on 5 parameters set as an environment variables in the pipeline:
+#   - AZURE_DEVOPS_CLI_PAT: Secret var for authenticating to Azure DevOps
+#   - AZURE_STORAGE_KEY: Secret var for authenticating to Azure Storage
+#   - _TS_CLONE_P, _TS_P, _TS_SM_P: Secret vars for specific unit tests
+
+parameters:
+  name: ''
+  pool: ''
+  customMatrixes: ''
+
+jobs:
+- job: ${{parameters.name}}
+  timeoutInMinutes: 600
+  strategy:
+    matrix:
+      ${{ insert }}: ${{parameters.customMatrixes}}
+  pool:
+    name: ${{ parameters.pool}}
+
+  steps:
+  # Don't checkout repo contents to save time and CPU compute. Environment variables
+  # related to checkout branch such as $(BUILD_SOURCEBRANCH) are still available.
+  - checkout: none
+
+  # Delete pytorch_tests repo from previous builds if exists
+  - script: if exist "pytorch_tests/" rmdir "pytorch_tests/" /q /s
+    displayName: Delete pytorch_tests repo from previous builds if exists
+
+  # Clone PyTorch Tests repository
+  - powershell: |
+      $env:B64Pat = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes(":$env:_ADOTOKEN"))
+      git -c http.extraHeader="Authorization: Basic $env:B64Pat" clone $env:AZURE_DEVOPS_pytorch_tests_REPO_URL
+      cd pytorch_tests
+      git checkout $(PYTORCH_TESTS_CHECKOUT_BRANCH)
+    env:
+      _ADOTOKEN: $(AZURE_DEVOPS_CLI_PAT)
+    displayName: Clone PyTorch Tests repo
+
+  # Run PyTorch Unit Tests
+  - script: call $(Build.SourcesDirectory)\pytorch_tests\scripts\windows\run.bat
+    env:
+      _ADOTOKEN: $(AZURE_DEVOPS_CLI_PAT)
+      _AZURE_STORAGE_KEY: $(AZURE_STORAGE_KEY)
+      _TS_CLONE_P: $(TS_CLONE_PASSWORD)
+      _TS_P: $(TS_PAT)
+      _TS_SM_P: $(TS_SM_PAT)
+    displayName: Run PyTorch Unit Tests
+
+  # Tests results are available outside the docker container since
+  # the current directory is mounted as a volume of the container.
+  - task: PublishTestResults@2
+    condition: always()
+    inputs:
+      testResultsFiles: '**\test-*.xml'
+      testRunTitle: 'Publish test results for Python'

.azure_pipelines/job_templates/set-environment-variables.yml

@@ -0,0 +1,129 @@
+# Set environment variables for specific configurations
+
+parameters:
+  is_official_build: False
+  os: ''
+  cuda: ''
+
+steps:
+  # Environment configuration steps for Ubuntu builds
+  - ${{ if contains(parameters.os, 'ubuntu') }}:
+    # Set configuration specific build flags
+    - ${{ if eq(parameters.is_official_build, True) }}:
+      - bash: |
+          echo "##vso[task.setvariable variable=INSTALL_TEST;]0"
+          echo "##vso[task.setvariable variable=PYTORCH_BUILD_NUMBER;]1"
+          export PYTORCH_VERSION=$(head -c 5 ./version.txt)
+          echo "##vso[task.setvariable variable=PYTORCH_BUILD_VERSION;]$PYTORCH_VERSION.dev"
+        displayName: Set configuration-specific build flags
+
+      # Set PyTorch CPU/GPU build flags.
+      - ${{ if contains(parameters.cuda, 'cpu') }}:
+        - bash: |
+            echo "##vso[task.setvariable variable=USE_CUDA;]0"
+            echo "##vso[task.setvariable variable=PYTORCH_BUILD_VERSION;]$(PYTORCH_BUILD_VERSION).cpu"
+          displayName: Set CUDA-specific build flag for CPU builds
+
+      - ${{ if contains(parameters.cuda, 'gpu') }}:
+        - bash: |
+            echo "##vso[task.setvariable variable=USE_CUDA;]1"
+            echo "##vso[task.setvariable variable=PYTORCH_BUILD_VERSION;]$(PYTORCH_BUILD_VERSION).cu$(CUDA_VERSION)"
+          displayName: Set CUDA-specific build flag for GPU builds
+
+    # Set MKL environment variables
+    - bash: |
+        echo "##vso[task.setvariable variable=CMAKE_LIBRARY_PATH;]/opt/intel/lib:$CMAKE_LIBRARY_PATH"
+        echo "##vso[task.setvariable variable=CMAKE_INCLUDE_PATH;]/opt/intel/include:$CMAKE_INCLUDE_PATH"
+      displayName: Set MKL paths
+
+    # View current environment variables
+    - bash:
+        printenv
+      displayName: Show environment variables
+
+  # Environment configuration steps for Windows builds
+  - ${{ if contains(parameters.os, 'windows') }}:
+    # Set Conda Lib Path
+    - powershell: Write-Host "##vso[task.setvariable variable=CONDA_LIB_PATH;]C:\Miniconda\envs\$(configuration)\Library\bin"
+      displayName: Set Conda Lib Path
+
+    # Set configuration specific build flags
+    - ${{ if eq(parameters.is_official_build, True) }}:
+      - powershell: |
+          Write-Host "##vso[task.setvariable variable=INSTALL_TEST;]0"
+          Write-Host "##vso[task.setvariable variable=PYTORCH_BUILD_NUMBER;]1"
+          Set-Variable -Name PYTORCH_VERSION -Value (Get-Content .\version.txt).Substring(0,5)
+          Write-Host "##vso[task.setvariable variable=PYTORCH_BUILD_VERSION;]$PYTORCH_VERSION.dev"
+        displayName: Set configuration-specific build flags
+
+      # Set PyTorch CPU/GPU build flags..
+      - ${{ if contains(parameters.cuda, 'cpu') }}:
+        - powershell: |
+            Write-Host "##vso[task.setvariable variable=USE_CUDA;]0"
+            Write-Host "##vso[task.setvariable variable=PYTORCH_BUILD_VERSION;]$(PYTORCH_BUILD_VERSION).cpu"
+          displayName: Set CUDA-specific build flag for CPU build
+
+      - ${{ if contains(parameters.cuda, 'gpu') }}:
+        - powershell: |
+            Write-Host "##vso[task.setvariable variable=USE_CUDA;]1"
+            Write-Host "##vso[task.setvariable variable=PYTORCH_BUILD_VERSION;]$(PYTORCH_BUILD_VERSION).cu$(CUDA_VERSION)"
+          displayName: Set CUDA-specific build flag for GPU build
+
+    # Set CUDA 11.2, 10.2 or 10.1 specific build flags
+    - ${{ if eq(parameters.cuda, 'gpu') }}:
+      - powershell: |
+          Write-Host "##vso[task.setvariable variable=TORCH_CUDA_ARCH_LIST;]3.7+PTX;5.0;6.0;6.1;7.0;7.5;8.0;8.6"
+          Write-Host "##vso[task.setvariable variable=CUDA_PATH;]C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\"
+        displayName: Set CUDA 11.2 specific build flags
+        condition: eq(variables.CUDA_VERSION, '112')
+
+      - powershell: |
+          Write-Host "##vso[task.setvariable variable=TORCH_CUDA_ARCH_LIST;]3.7+PTX;5.0;6.0;6.1;7.0;7.5"
+          Write-Host "##vso[task.setvariable variable=CUDA_PATH;]C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\"
+        displayName: Set CUDA 10.2 specific build flags
+        condition: eq(variables.CUDA_VERSION, '102')
+
+      - powershell: |
+          Write-Host "##vso[task.setvariable variable=TORCH_CUDA_ARCH_LIST;]3.7+PTX;5.0;6.0;6.1;7.0;7.5"
+          Write-Host "##vso[task.setvariable variable=CUDA_PATH;]C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1\"
+        displayName: Set CUDA 10.1 specific build flags
+        condition: eq(variables.CUDA_VERSION, '101')
+
+      - powershell: |
+          Write-Host "##vso[task.setvariable variable=CUDA_BIN_PATH;]$env:CUDA_PATH\bin\"
+          Write-Host "##vso[task.setvariable variable=CUDNN_ROOT;]$env:CUDA_PATH"
+          Write-Host "##vso[task.setvariable variable=CUDNN_INCLUDE_DIR;]$env:CUDA_PATH\include\"
+          Write-Host "##vso[task.setvariable variable=CUDNN_LIBRARY;]$env:CUDA_PATH\lib\x64\"
+          Write-Host "##vso[task.prependpath]$env:CUDA_PATH\bin"
+          Write-Host "##vso[task.setvariable variable=TORCH_NVCC_FLAGS;]-Xfatbin -compress-all --no-host-device-move-forward"
+          Write-Host "##vso[task.setvariable variable=THRUST_IGNORE_CUB_VERSION_CHECK;]1"
+          Write-Host "##vso[task.setvariable variable=NVTOOLSEXT_PATH;]C:\Program Files\NVIDIA Corporation\NvToolsExt\"
+        displayName: Set CUDA environment variables
+
+      - powershell: |
+          copy "$(CUDA_BIN_PATH)\cusparse*64_*.dll*" $(Build.SourcesDirectory)\torch\lib
+          copy "$(CUDA_BIN_PATH)\cublas*64_*.dll*" $(Build.SourcesDirectory)\torch\lib
+          copy "$(CUDA_BIN_PATH)\cudart*64_*.dll*" $(Build.SourcesDirectory)\torch\lib
+          copy "$(CUDA_BIN_PATH)\curand*64_*.dll*" $(Build.SourcesDirectory)\torch\lib
+          copy "$(CUDA_BIN_PATH)\cufft*64_*.dll*" $(Build.SourcesDirectory)\torch\lib
+          copy "$(CUDA_BIN_PATH)\cusolver*64_*.dll*" $(Build.SourcesDirectory)\torch\lib
+          copy "$(CUDA_BIN_PATH)\cudnn*64_*.dll*" $(Build.SourcesDirectory)\torch\lib
+          copy "$(CUDA_BIN_PATH)\nvrtc*64_*.dll*" $(Build.SourcesDirectory)\torch\lib
+          copy "C:\Program Files\NVIDIA Corporation\NvToolsExt\bin\x64\nvToolsExt64_1.dll*" $(Build.SourcesDirectory)\torch\lib
+          copy "$(CONDA_LIB_PATH)\libiomp*5md.dll" $(Build.SourcesDirectory)\torch\lib
+          copy "$(CONDA_LIB_PATH)\uv.dll" $(Build.SourcesDirectory)\torch\lib
+        displayName: Copy CUDA/cuDNN/libomp/libuv dlls to torch\lib
+
+    # Set MKL, sccache and randomtemp environment variables
+    - powershell: |
+        Write-Host "##vso[task.setvariable variable=CMAKE_INCLUDE_PATH;]$(Build.SourcesDirectory)\mkl\include"
+        Write-Host "##vso[task.setvariable variable=CMAKE_LIBRARY_PATH;]$(Build.SourcesDirectory)\mkl\lib;$env:CMAKE_LIBRARY_PATH"
+        Write-Host "##vso[task.setvariable variable=ADDITIONAL_PATH;]$(Build.SourcesDirectory)\tmp_bin"
+        Write-Host "##vso[task.setvariable variable=SCCACHE_IDLE_TIMEOUT;]1500"
+        Write-Host "##vso[task.setvariable variable=CMAKE_CUDA_COMPILER_LAUNCHER;]$(Build.SourcesDirectory)/tmp_bin/randomtemp.exe;$(Build.SourcesDirectory)/tmp_bin/sccache.exe"
+      displayName: Set MKL, sccache and randomtemp environment variables
+
+    # View current environment variables
+    - script:
+        set
+      displayName: Show environment variables

.azure_pipelines/job_templates/wheel-wait-job-template.yml

@@ -0,0 +1,14 @@
+# Main logic to initiate wait for PR artifact to be ready
+
+steps:
+- task: InvokeRESTAPI@1
+  displayName: 'Wait for job success and wheel ready'
+  timeoutInMinutes: 60
+  inputs:
+    connectionType: 'connectedServiceName'
+    serviceConnection: circleciconn
+    method: 'POST'
+    headers: '{"Content-Type":"application/json", "BranchName":"$(_TARGET_BRANCH_TO_CHECK)", "JobName":"$(TARGET_CIRCLECI_BUILD_PR)", "PRNumber":"$(_TARGET_PR_NUMBER)", "TargetCommit":"$(_TARGET_COMMIT)", "PlanUrl":"$(System.CollectionUri)", "ProjectId":"$(System.TeamProjectId)", "HubName":"$(System.HostType)", "PlanId":"$(System.PlanId)", "JobId":"$(System.JobId)", "TimelineId":"$(System.TimelineId)", "TaskInstanceId":"$(System.TaskInstanceId)", "AuthToken":"$(System.AccessToken)"}'
+    body: ''
+    urlSuffix: 'api/JobStatus'
+    waitForCompletion: true

.azure_pipelines/job_templates/wheel-wait-template.yml

@@ -0,0 +1,92 @@
+# Initiate 5 agentless-server waiting jobs to check on the
+# status of PR artifact builds, for a maximum wait time of
+# 11*60 min=660 mins. These jobs will pass immediately
+# once targeted CircleCI build is ready.
+
+jobs:
+- job: checkjob1
+  pool: server
+  timeoutInMinutes: 60
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml
+
+- job: checkjob2
+  pool: server
+  timeoutInMinutes: 60
+  dependsOn: checkjob1
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml
+
+- job: checkjob3
+  pool: server
+  timeoutInMinutes: 60
+  dependsOn: checkjob2
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml
+
+- job: checkjob4
+  pool: server
+  timeoutInMinutes: 60
+  dependsOn: checkjob3
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml
+
+- job: checkjob5
+  pool: server
+  timeoutInMinutes: 60
+  dependsOn: checkjob4
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml
+
+- job: checkjob6
+  pool: server
+  timeoutInMinutes: 60
+  dependsOn: checkjob5
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml
+
+- job: checkjob7
+  pool: server
+  timeoutInMinutes: 60
+  dependsOn: checkjob6
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml
+
+- job: checkjob8
+  pool: server
+  timeoutInMinutes: 60
+  dependsOn: checkjob7
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml
+
+- job: checkjob9
+  pool: server
+  timeoutInMinutes: 60
+  dependsOn: checkjob8
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml
+
+- job: checkjob10
+  pool: server
+  timeoutInMinutes: 60
+  dependsOn: checkjob9
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml
+
+- job: checkjob11
+  pool: server
+  timeoutInMinutes: 60
+  dependsOn: checkjob10
+  continueOnError: true
+  steps:
+  - template: wheel-wait-job-template.yml

.azure_pipelines/nightly-pytorch-tests-pipeline.yml

@@ -0,0 +1,60 @@
+# PyTorch Nightly PyTorch Tests Builds Pipeline on Azure DevOps
+#
+# This pipeline runs custom PyTorch unit-tests on nightly
+# PyTorch wheels.
+
+stages:
+- stage: 'NightlyCustomTests'
+  displayName: 'Run custom unit tests on PyTorch wheels'
+  jobs:
+  - template: job_templates/pytorch-template-unix.yml
+    parameters:
+      name: ubuntu_1804_CPU_docker
+      pool: $(BUILD_POOL_LIN_1)
+      customMatrixes:
+        Nightly_Custom_Tests:
+          _DOCKER_IMAGE: $(DOCKER_IMAGE_LIN_1)
+          _PYTHON_VERSION: $(PYTHON_VERSION_LIN_1)
+          _CUDA_BUILD_VERSION: $(CUDA_BUILD_VERSION_LIN_1)
+          _RUN_TESTS: $(RUN_TESTS_LIN)
+
+  - template: job_templates/pytorch-template-unix.yml
+    parameters:
+      name: ubuntu_1804_GPU_docker
+      pool: $(BUILD_POOL_LIN_2)
+      customMatrixes:
+        Nightly_Custom_Tests:
+          _DOCKER_IMAGE: $(DOCKER_IMAGE_LIN_2)
+          _PYTHON_VERSION: $(PYTHON_VERSION_LIN_2)
+          _CUDA_BUILD_VERSION: $(CUDA_BUILD_VERSION_LIN_2)
+          _RUN_TESTS: $(RUN_TESTS_LIN)
+
+  - template: job_templates/pytorch-template-win.yml
+    parameters:
+      name: windows_2019_CPU
+      pool: $(BUILD_POOL_WIN_1)
+      customMatrixes:
+        Nightly_Custom_Tests:
+          _PYTHON_VERSION: $(PYTHON_VERSION_WIN_1)
+          _CUDA_BUILD_VERSION: $(CUDA_BUILD_VERSION_WIN_1)
+          _RUN_TESTS: $(RUN_TESTS_WIN)
+
+  - template: job_templates/pytorch-template-win.yml
+    parameters:
+      name: windows_2019_GPU
+      pool: $(BUILD_POOL_WIN_2)
+      customMatrixes:
+        Nightly_Custom_Tests:
+          _PYTHON_VERSION: $(PYTHON_VERSION_WIN_2)
+          _CUDA_BUILD_VERSION: $(CUDA_BUILD_VERSION_WIN_2)
+          _RUN_TESTS: $(RUN_TESTS_WIN)
+
+- stage: 'NotifyWebapp'
+  displayName: 'Notify Webapp that pipeline is finished'
+  dependsOn: NightlyCustomTests
+  condition: succeededOrFailed()
+  jobs:
+  - template: job_templates/notify-webapp-template.yml
+    parameters:
+      name: ubuntu_1804_CPU
+      pool: $(BUILD_POOL_LIN_1)

.azure_pipelines/pytorch-tests-pipeline.yml

@@ -0,0 +1,62 @@
+# PyTorch PR PyTorch Tests Builds Pipeline on Azure DevOps
+#
+# This pipeline:
+#   1) ensures that CircleCI builds for a given PR
+#      have finished, and that its artifacts are
+#      ready for download
+#   2) runs custom PyTorch unit-tests on PyTorch
+#      wheels generated during PR builds.
+
+resources:
+  webhooks:
+    - webhook: GitHubPyTorchPRTrigger
+      connection: GitHubPyTorchPRTriggerConnection
+      filters:
+        - path: repositoryName
+          value: pytorch_tests
+
+stages:
+- stage: 'EnsureArtifactsReady'
+  displayName: 'Ensure PyTorch PR Artifacts are ready'
+  jobs:
+  - template: job_templates/wheel-wait-template.yml
+  variables:
+    _TARGET_BRANCH_TO_CHECK: ${{parameters.GitHubPyTorchPRTrigger.TARGET_BRANCH_TO_CHECK_AZ_DEVOPS_PR}}
+    _TARGET_PR_NUMBER: ${{parameters.GitHubPyTorchPRTrigger.PR_NUMBER}}
+    _TARGET_COMMIT: ${{parameters.GitHubPyTorchPRTrigger.TARGET_COMMIT}}
+
+- stage: 'PRCustomTests'
+  displayName: 'Run custom unit tests on PyTorch wheels'
+  dependsOn: EnsureArtifactsReady
+  condition: succeeded()
+  jobs:
+  - template: job_templates/pytorch-template-unix.yml
+    parameters:
+      name: ubuntu_1804_GPU_docker
+      pool: $(BUILD_POOL_PR)
+      customMatrixes:
+        PR_Custom_Tests:
+          _PYTHON_VERSION: $(PYTHON_VERSION_PR)
+          _CUDA_BUILD_VERSION: $(CUDA_BUILD_VERSION_PR)
+          _TARGET_CIRCLECI_BUILD: $(TARGET_CIRCLECI_BUILD_PR)
+          _TARGET_BRANCH_TO_CHECK: ${{parameters.GitHubPyTorchPRTrigger.TARGET_BRANCH_TO_CHECK_AZ_DEVOPS_PR}}
+          _TARGET_PR_NUMBER: ${{parameters.GitHubPyTorchPRTrigger.PR_NUMBER}}
+          _TARGET_COMMIT: ${{parameters.GitHubPyTorchPRTrigger.TARGET_COMMIT}}
+          _DOCKER_IMAGE: $(DOCKER_IMAGE_PR)
+          _RUN_TESTS: $(RUN_TESTS_PR)
+
+- stage: 'NotifyWebapp'
+  displayName: 'Notify Webapp that pipeline is finished'
+  dependsOn: PRCustomTests
+  condition: succeededOrFailed()
+  jobs:
+  - template: job_templates/notify-webapp-template.yml
+    parameters:
+      name: ubuntu_1804_CPU
+      pool: $(BUILD_POOL_LIN_1)
+      customMatrixes:
+        PR_Notify_WebApp:
+          _TARGET_CIRCLECI_BUILD: $(TARGET_CIRCLECI_BUILD_PR)
+          _TARGET_BRANCH_TO_CHECK: ${{parameters.GitHubPyTorchPRTrigger.TARGET_BRANCH_TO_CHECK_AZ_DEVOPS_PR}}
+          _TARGET_PR_NUMBER: ${{parameters.GitHubPyTorchPRTrigger.PR_NUMBER}}
+          _TARGET_COMMIT: ${{parameters.GitHubPyTorchPRTrigger.TARGET_COMMIT}}

.azure_pipelines/verify-pipeline.yml

@@ -0,0 +1,224 @@
+# PyTorch Official Builds Pipeline on Azure DevOps
+#
+# This pipeline:
+#   1) builds PyTorch on all available configurations
+#   2) verifies PyTorch artifacts by installing them in a clean environment
+#      and checking torch.__version_
+#   3) publishes official PyTorch artifacts to Azure DevOps Artifacts for consumption
+
+stages:
+- stage: 'Build'
+  displayName: 'Build PyTorch'
+  jobs:
+  - template: job_templates/build-verify-publish-template-unix.yml
+    parameters:
+      name: ubuntu_1804_CPU_docker
+      pool: 'PyTorch-Linux-CPU'
+      container_endpoint: pytorchms.azurecr.io
+      build_stage: True
+      is_official_build: True
+      os: ubuntu
+      cuda: cpu
+      customMatrixes:
+        Py_38:
+          configuration: ubuntu_1804_py_38_cpu
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cpu_dev
+        Py_37:
+          configuration: ubuntu_1804_py_37_cpu
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_37_cpu_dev
+
+  - template: job_templates/build-verify-publish-template-unix.yml
+    parameters:
+      name: ubuntu_1804_GPU_docker
+      pool: 'PyTorch-Linux-GPU'
+      container_endpoint: pytorchms.azurecr.io
+      build_stage: True
+      is_official_build: True
+      os: ubuntu
+      cuda: gpu
+      customMatrixes:
+        Py_39_CUDA_112_cuDNN_810:
+          configuration: ubuntu_1804_py_39_cuda_112_cudnn_810
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_39_cuda_112_cudnn_8_dev
+          CUDA_VERSION: 112
+        Py_38_CUDA_102_cuDNN_810:
+          configuration: ubuntu_1804_py_38_cuda_102_cudnn_810
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cuda_102_cudnn_8_dev
+          CUDA_VERSION: 102
+        Py_37_CUDA_101_cuDNN_765:
+          configuration: ubuntu_1804_py_37_cuda_101_cudnn_765
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_37_cuda_101_cudnn_7_dev
+          CUDA_VERSION: 101
+
+  - template: job_templates/build-verify-publish-template-win.yml
+    parameters:
+      name: windows_2019_CPU
+      pool: 'PyTorch-Win-CPU'
+      build_stage: True
+      is_official_build: True
+      os: windows
+      cuda: cpu
+      customMatrixes:
+        Py_38:
+          configuration: windows_2019_py_38_cpu
+        Py_37:
+          configuration: windows_2019_py_37_cpu
+
+  - template: job_templates/build-verify-publish-template-win.yml
+    parameters:
+      name: windows_2019_GPU
+      pool: 'PyTorch-Win-GPU'
+      build_stage: True
+      is_official_build: True
+      os: windows
+      cuda: gpu
+      customMatrixes:
+        Py_39_CUDA_112_cuDNN_810:
+          configuration: windows_2019_py_39_cuda_112_cudnn_810
+          CUDA_VERSION: 112
+        Py_38_CUDA_102_cuDNN_765:
+          configuration: windows_2019_py_38_cuda_102_cudnn_765
+          CUDA_VERSION: 102
+        Py_37_CUDA_101_cuDNN_764:
+          configuration: windows_2019_py_37_cuda_101_cudnn_764
+          CUDA_VERSION: 101
+
+- stage: 'Verify'
+  displayName: 'Verify PyTorch wheels'
+  dependsOn: Build
+  condition: succeeded()
+  jobs:
+  - template: job_templates/build-verify-publish-template-unix.yml
+    parameters:
+      name: ubuntu_1804_CPU_docker
+      pool: 'PyTorch-Linux-CPU'
+      container_endpoint: pytorchms.azurecr.io
+      verify_stage: True
+      is_official_build: True
+      customMatrixes:
+        Py_38:
+          configuration: ubuntu_1804_py_38_cpu
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cpu_dev
+        Py_37:
+          configuration: ubuntu_1804_py_37_cpu
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_37_cpu_dev
+
+  - template: job_templates/build-verify-publish-template-unix.yml
+    parameters:
+      name: ubuntu_1804_GPU_docker
+      pool: 'PyTorch-Linux-GPU'
+      container_endpoint: pytorchms.azurecr.io
+      verify_stage: True
+      is_official_build: True
+      customMatrixes:
+        Py_39_CUDA_112_cuDNN_810:
+          configuration: ubuntu_1804_py_39_cuda_112_cudnn_810
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_39_cuda_112_cudnn_8_dev
+          CUDA_VERSION: 112
+        Py_38_CUDA_102_cuDNN_810:
+          configuration: ubuntu_1804_py_38_cuda_102_cudnn_810
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cuda_102_cudnn_8_dev
+          CUDA_VERSION: 102
+        Py_37_CUDA_101_cuDNN_765:
+          configuration: ubuntu_1804_py_37_cuda_101_cudnn_765
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_37_cuda_101_cudnn_7_dev
+          CUDA_VERSION: 101
+
+  - template: job_templates/build-verify-publish-template-win.yml
+    parameters:
+      name: windows_2019_CPU
+      pool: 'PyTorch-Win-CPU'
+      verify_stage: True
+      is_official_build: True
+      customMatrixes:
+        Py_38:
+          configuration: windows_2019_py_38_cpu
+        Py_37:
+          configuration: windows_2019_py_37_cpu
+
+  - template: job_templates/build-verify-publish-template-win.yml
+    parameters:
+      name: windows_2019_GPU
+      pool: 'PyTorch-Win-GPU'
+      verify_stage: True
+      is_official_build: True
+      customMatrixes:
+        Py_39_CUDA_112_cuDNN_810:
+          configuration: windows_2019_py_39_cuda_112_cudnn_810
+          CUDA_VERSION: 112
+        Py_38_CUDA_102_cuDNN_765:
+          configuration: windows_2019_py_38_cuda_102_cudnn_765
+          CUDA_VERSION: 102
+        Py_37_CUDA_101_cuDNN_764:
+          configuration: windows_2019_py_37_cuda_101_cudnn_764
+          CUDA_VERSION: 101
+
+- stage: 'Publish'
+  displayName: 'Publish PyTorch wheels'
+  dependsOn: Verify
+  condition: succeeded()
+  jobs:
+  - template: job_templates/build-verify-publish-template-unix.yml
+    parameters:
+      name: ubuntu_1804_CPU_docker
+      pool: 'PyTorch-Linux-CPU'
+      container_endpoint: pytorchms.azurecr.io
+      publish_stage: True
+      is_official_build: True
+      customMatrixes:
+        Py_38:
+          configuration: ubuntu_1804_py_38_cpu
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cpu_dev
+        Py_37:
+          configuration: ubuntu_1804_py_37_cpu
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_37_cpu_dev
+
+  - template: job_templates/build-verify-publish-template-unix.yml
+    parameters:
+      name: ubuntu_1804_GPU_docker
+      pool: 'PyTorch-Linux-GPU'
+      container_endpoint: pytorchms.azurecr.io
+      publish_stage: True
+      is_official_build: True
+      customMatrixes:
+        Py_39_CUDA_112_cuDNN_810:
+          configuration: ubuntu_1804_py_39_cuda_112_cudnn_810
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_39_cuda_112_cudnn_8_dev
+          CUDA_VERSION: 112
+        Py_38_CUDA_102_cuDNN_810:
+          configuration: ubuntu_1804_py_38_cuda_102_cudnn_810
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cuda_102_cudnn_8_dev
+          CUDA_VERSION: 102
+        Py_37_CUDA_101_cuDNN_765:
+          configuration: ubuntu_1804_py_37_cuda_101_cudnn_765
+          container_image: pytorchms.azurecr.io/ubuntu_1804_py_37_cuda_101_cudnn_7_dev
+          CUDA_VERSION: 101
+
+  - template: job_templates/build-verify-publish-template-win.yml
+    parameters:
+      name: windows_2019_CPU
+      pool: 'PyTorch-Win-CPU'
+      publish_stage: True
+      is_official_build: True
+      customMatrixes:
+        Py_38:
+          configuration: windows_2019_py_38_cpu
+        Py_37:
+          configuration: windows_2019_py_37_cpu
+
+  - template: job_templates/build-verify-publish-template-win.yml
+    parameters:
+      name: windows_2019_GPU
+      pool: 'PyTorch-Win-GPU'
+      publish_stage: True
+      is_official_build: True
+      customMatrixes:
+        Py_39_CUDA_112_cuDNN_810:
+          configuration: windows_2019_py_39_cuda_112_cudnn_810
+          CUDA_VERSION: 112
+        Py_38_CUDA_102_cuDNN_765:
+          configuration: windows_2019_py_38_cuda_102_cudnn_765
+          CUDA_VERSION: 102
+        Py_37_CUDA_101_cuDNN_764:
+          configuration: windows_2019_py_37_cuda_101_cudnn_764
+          CUDA_VERSION: 101

.bazelignore

@@ -1,4 +0,0 @@
-# We do not use this library in our Bazel build. It contains an
-# infinitely recursing symlink that makes Bazel very unhappy.
-third_party/ittapi/
-third_party/opentelemetry-cpp

.bazelrc

@@ -1,11 +1,10 @@
-build --cxxopt=--std=c++17
+build --copt=--std=c++14
 build --copt=-I.
 # Bazel does not support including its cc_library targets as system
 # headers. We work around this for generated code
 # (e.g. c10/macros/cmake_macros.h) by making the generated directory a
 # system include path.
 build --copt=-isystem --copt bazel-out/k8-fastbuild/bin
-build --copt=-isystem --copt bazel-out/darwin-fastbuild/bin
 build --experimental_ui_max_stdouterr_bytes=2048576
 
 # Configuration to disable tty features for environments like CI
@@ -13,102 +12,15 @@ build:no-tty --curses no
 build:no-tty --progress_report_interval 10
 build:no-tty --show_progress_rate_limit 10
 
-# Build with GPU support by default.
-build --define=cuda=true
-# rules_cuda configuration
-build --@rules_cuda//cuda:enable_cuda
-build --@rules_cuda//cuda:cuda_targets=sm_52
-build --@rules_cuda//cuda:compiler=nvcc
-build --repo_env=CUDA_PATH=/usr/local/cuda
-
-# Configuration to build without GPU support
-build:cpu-only --define=cuda=false
+# Configuration to build with GPU support
+build:gpu --define=cuda=true
 # define a separate build folder for faster switching between configs
-build:cpu-only --platform_suffix=-cpu-only
+build:gpu --platform_suffix=-gpu
 # See the note on the config-less build for details about why we are
-# doing this. We must also do it for the "-cpu-only" platform suffix.
-build --copt=-isystem --copt=bazel-out/k8-fastbuild-cpu-only/bin
+# doing this. We must also do it for the "-gpu" platform suffix.
+build --copt=-isystem --copt=bazel-out/k8-fastbuild-gpu/bin
 # rules_cuda configuration
-build:cpu-only --@rules_cuda//cuda:enable_cuda=False
-
-# Definition of --config=shell
-# interactive shell immediately before execution
-build:shell --run_under="//tools/bazel_tools:shellwrap"
-
-# Disable all warnings for external repositories. We don't care about
-# their warnings.
-build --per_file_copt=^external/@-w
-
-# Set additional warnings to error level.
-#
-# Implementation notes:
-#  * we use file extensions to determine if we are using the C++
-#    compiler or the cuda compiler
-#  * we use ^// at the start of the regex to only permit matching
-#    PyTorch files. This excludes external repos.
-#
-# Note that because this is logically a command-line flag, it is
-# considered the word on what warnings are enabled. This has the
-# unfortunate consequence of preventing us from disabling an error at
-# the target level because those flags will come before these flags in
-# the action invocation. Instead we provide per-file exceptions after
-# this.
-#
-# On the bright side, this means we don't have to more broadly apply
-# the exceptions to an entire target.
-#
-# Looking for CUDA flags? We have a cu_library macro that we can edit
-# directly. Look in //tools/rules:cu.bzl for details. Editing the
-# macro over this has the following advantages:
-#  * making changes does not require discarding the Bazel analysis
-#    cache
-#  * it allows for selective overrides on individual targets since the
-#    macro-level opts will come earlier than target level overrides
-
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Werror=all
-# The following warnings come from -Wall. We downgrade them from error
-# to warnings here.
-#
-# We intentionally use #pragma unroll, which is compiler specific.
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-error=unknown-pragmas
-
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Werror=extra
-# The following warnings come from -Wextra. We downgrade them from error
-# to warnings here.
-#
-# unused-parameter-compare has a tremendous amount of violations in the
-# codebase. It will be a lot of work to fix them, just disable it for
-# now.
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-unused-parameter
-# missing-field-parameters has both a large number of violations in
-# the codebase, but it also is used pervasively in the Python C
-# API. There are a couple of catches though:
-# * we use multiple versions of the Python API and hence have
-#   potentially multiple different versions of each relevant
-#   struct. They may have different numbers of fields. It will be
-#   unwieldy to support multiple versions in the same source file.
-# * Python itself for many of these structs recommends only
-#   initializing a subset of the fields. We should respect the API
-#   usage conventions of our dependencies.
-#
-# Hence, we just disable this warning altogether. We may want to clean
-# up some of the clear-cut cases that could be risky, but we still
-# likely want to have this disabled for the most part.
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-missing-field-initializers
-
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-unused-function
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-unused-variable
-
-build --per_file_copt='//:aten/src/ATen/RegisterCompositeExplicitAutograd\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterCompositeImplicitAutograd\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterMkldnnCPU\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterNestedTensorCPU\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterQuantizedCPU\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterSparseCPU\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterSparseCsrCPU\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterNestedTensorMeta\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterSparseMeta\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterQuantizedMeta\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterZeroTensor\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:torch/csrc/lazy/generated/RegisterAutogradLazy\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:torch/csrc/lazy/generated/RegisterLazy\.cpp$'@-Wno-error=unused-function
+build:gpu --@rules_cuda//cuda:enable_cuda
+build:gpu --@rules_cuda//cuda:cuda_targets=sm_52
+build:gpu --@rules_cuda//cuda:compiler=nvcc
+build:gpu --repo_env=CUDA_PATH=/usr/local/cuda

.bazelversion

@@ -1 +1 @@
-6.1.1
+4.2.1

.buckconfig.oss

@@ -1,26 +0,0 @@
-[pt]
-  is_oss=1
-
-[buildfile]
-  name = BUCK.oss
-  includes = //tools/build_defs/select.bzl
-
-[repositories]
-  bazel_skylib = third_party/bazel-skylib/
-  ovr_config = .
-
-[download]
-  in_build = true
-
-[cxx]
-  cxxflags = -std=c++17
-  ldflags = -Wl,--no-undefined
-  should_remap_host_platform = true
-  cpp = /usr/bin/clang
-  cc = /usr/bin/clang
-  cxx = /usr/bin/clang++
-  cxxpp = /usr/bin/clang++
-  ld = /usr/bin/clang++
-
-[project]
-  default_flavors_mode=all

.ci/caffe2/common.sh

@@ -1,36 +0,0 @@
-set -ex
-
-LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
-TEST_DIR="$ROOT_DIR/test"
-gtest_reports_dir="${TEST_DIR}/test-reports/cpp"
-pytest_reports_dir="${TEST_DIR}/test-reports/python"
-
-# Figure out which Python to use
-PYTHON="$(which python)"
-if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
-  PYTHON=$(which "python${BASH_REMATCH[1]}")
-fi
-
-if [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
-    # HIP_PLATFORM is auto-detected by hipcc; unset to avoid build errors
-    unset HIP_PLATFORM
-    if which sccache > /dev/null; then
-        # Save sccache logs to file
-        sccache --stop-server || true
-        rm -f ~/sccache_error.log || true
-        SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=0 sccache --start-server
-
-        # Report sccache stats for easier debugging
-        sccache --zero-stats
-    fi
-fi
-
-# /usr/local/caffe2 is where the cpp bits are installed to in cmake-only
-# builds. In +python builds the cpp tests are copied to /usr/local/caffe2 so
-# that the test code in .ci/test.sh is the same
-INSTALL_PREFIX="/usr/local/caffe2"
-
-mkdir -p "$gtest_reports_dir" || true
-mkdir -p "$pytest_reports_dir" || true
-mkdir -p "$INSTALL_PREFIX" || true

.ci/caffe2/test.sh

@@ -1,172 +0,0 @@
-#!/bin/bash
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-
-if [[ ${BUILD_ENVIRONMENT} == *onnx* ]]; then
-  pip install click mock tabulate networkx==2.0
-  pip -q install --user "file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx"
-fi
-
-# Skip tests in environments where they are not built/applicable
-if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
-  echo 'Skipping tests'
-  exit 0
-fi
-if [[ "${BUILD_ENVIRONMENT}" == *-rocm* ]]; then
-  # temporary to locate some kernel issues on the CI nodes
-  export HSAKMT_DEBUG_LEVEL=4
-fi
-# These additional packages are needed for circleci ROCm builds.
-if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
-    # Need networkx 2.0 because bellmand_ford was moved in 2.1 . Scikit-image by
-    # defaults installs the most recent networkx version, so we install this lower
-    # version explicitly before scikit-image pulls it in as a dependency
-    pip install networkx==2.0
-    # click - onnx
-    pip install --progress-bar off click protobuf tabulate virtualenv mock typing-extensions
-fi
-
-# Find where cpp tests and Caffe2 itself are installed
-if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
-  # For cmake only build we install everything into /usr/local
-  cpp_test_dir="$INSTALL_PREFIX/cpp_test"
-  ld_library_path="$INSTALL_PREFIX/lib"
-else
-  # For Python builds we install into python
-  # cd to /usr first so the python import doesn't get confused by any 'caffe2'
-  # directory in cwd
-  python_installation="$(dirname $(dirname $(cd /usr && $PYTHON -c 'import os; import caffe2; print(os.path.realpath(caffe2.__file__))')))"
-  caffe2_pypath="$python_installation/caffe2"
-  cpp_test_dir="$python_installation/torch/test"
-  ld_library_path="$python_installation/torch/lib"
-fi
-
-################################################################################
-# C++ tests #
-################################################################################
-# Only run cpp tests in the first shard, don't run cpp tests a second time in the second shard
-if [[ "${SHARD_NUMBER:-1}" == "1" ]]; then
-  echo "Running C++ tests.."
-  for test in $(find "$cpp_test_dir" -executable -type f); do
-    case "$test" in
-      # skip tests we know are hanging or bad
-      */mkl_utils_test|*/aten/integer_divider_test)
-        continue
-        ;;
-      */scalar_tensor_test|*/basic|*/native_test)
-        if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
-          continue
-        else
-          LD_LIBRARY_PATH="$ld_library_path" "$test"
-        fi
-        ;;
-      */*_benchmark)
-        LD_LIBRARY_PATH="$ld_library_path" "$test" --benchmark_color=false
-        ;;
-      *)
-        # Currently, we use a mixture of gtest (caffe2) and Catch2 (ATen). While
-        # planning to migrate to gtest as the common PyTorch c++ test suite, we
-        # currently do NOT use the xml test reporter, because Catch doesn't
-        # support multiple reporters
-        # c.f. https://github.com/catchorg/Catch2/blob/master/docs/release-notes.md#223
-        # which means that enabling XML output means you lose useful stdout
-        # output for Jenkins.  It's more important to have useful console
-        # output than it is to have XML output for Jenkins.
-        # Note: in the future, if we want to use xml test reporter once we switch
-        # to all gtest, one can simply do:
-        LD_LIBRARY_PATH="$ld_library_path" \
-            "$test" --gtest_output=xml:"$gtest_reports_dir/$(basename $test).xml"
-        ;;
-    esac
-  done
-fi
-
-################################################################################
-# Python tests #
-################################################################################
-if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
-  exit 0
-fi
-
-# If pip is installed as root, we must use sudo.
-# CircleCI docker images could install conda as jenkins user, or use the OS's python package.
-PIP=$(which pip)
-PIP_USER=$(stat --format '%U' $PIP)
-CURRENT_USER=$(id -u -n)
-if [[ "$PIP_USER" = root && "$CURRENT_USER" != root ]]; then
-  MAYBE_SUDO=sudo
-fi
-
-# Uninstall pre-installed hypothesis and coverage to use an older version as newer
-# versions remove the timeout parameter from settings which ideep/conv_transpose_test.py uses
-$MAYBE_SUDO pip -q uninstall -y hypothesis
-$MAYBE_SUDO pip -q uninstall -y coverage
-
-# "pip install hypothesis==3.44.6" from official server is unreliable on
-# CircleCI, so we host a copy on S3 instead
-$MAYBE_SUDO pip -q install attrs==18.1.0 -f https://s3.amazonaws.com/ossci-linux/wheels/attrs-18.1.0-py2.py3-none-any.whl
-$MAYBE_SUDO pip -q install coverage==4.5.1 -f https://s3.amazonaws.com/ossci-linux/wheels/coverage-4.5.1-cp36-cp36m-macosx_10_12_x86_64.whl
-$MAYBE_SUDO pip -q install hypothesis==3.44.6 -f https://s3.amazonaws.com/ossci-linux/wheels/hypothesis-3.44.6-py3-none-any.whl
-
-# Collect additional tests to run (outside caffe2/python)
-EXTRA_TESTS=()
-
-# CUDA builds always include NCCL support
-if [[ "$BUILD_ENVIRONMENT" == *-cuda* ]] || [[ "$BUILD_ENVIRONMENT" == *-rocm* ]]; then
-  EXTRA_TESTS+=("$caffe2_pypath/contrib/nccl")
-fi
-
-rocm_ignore_test=()
-if [[ $BUILD_ENVIRONMENT == *-rocm* ]]; then
-  # Currently these tests are failing on ROCM platform:
-
-  # On ROCm, RCCL (distributed) development isn't complete.
-  # https://github.com/ROCmSoftwarePlatform/rccl
-  rocm_ignore_test+=("--ignore $caffe2_pypath/python/data_parallel_model_test.py")
-
-  # This test has been flaky in ROCm CI (but note the tests are
-  # cpu-only so should be unrelated to ROCm)
-  rocm_ignore_test+=("--ignore $caffe2_pypath/python/operator_test/blobs_queue_db_test.py")
-  # This test is skipped on Jenkins(compiled without MKL) and otherwise known flaky
-  rocm_ignore_test+=("--ignore $caffe2_pypath/python/ideep/convfusion_op_test.py")
-  # This test is skipped on Jenkins(compiled without MKL) and causing segfault on Circle
-  rocm_ignore_test+=("--ignore $caffe2_pypath/python/ideep/pool_op_test.py")
-fi
-
-echo "Running Python tests.."
-# locale setting is required by click package
-for loc in "en_US.utf8" "C.UTF-8"; do
-  if locale -a | grep "$loc" >/dev/null 2>&1; then
-    export LC_ALL="$loc"
-    export LANG="$loc"
-    break;
-  fi
-done
-
-# Some Caffe2 tests fail when run using AVX512 ISA, see https://github.com/pytorch/pytorch/issues/66111
-export DNNL_MAX_CPU_ISA=AVX2
-
-# Should still run even in the absence of SHARD_NUMBER
-if [[ "${SHARD_NUMBER:-1}" == "1" ]]; then
-  # TODO(sdym@meta.com) remove this when the linked issue resolved.
-  # py is temporary until https://github.com/Teemu/pytest-sugar/issues/241 is fixed
-  pip install --user py==1.11.0
-  pip install --user pytest-sugar
-  # NB: Warnings are disabled because they make it harder to see what
-  # the actual erroring test is
-  "$PYTHON" \
-    -m pytest \
-    -x \
-    -v \
-    --disable-warnings \
-    --junit-xml="$pytest_reports_dir/result.xml" \
-    --ignore "$caffe2_pypath/python/test/executor_test.py" \
-    --ignore "$caffe2_pypath/python/operator_test/matmul_op_test.py" \
-    --ignore "$caffe2_pypath/python/operator_test/pack_ops_test.py" \
-    --ignore "$caffe2_pypath/python/mkl/mkl_sbn_speed_test.py" \
-    --ignore "$caffe2_pypath/python/trt/test_pt_onnx_trt.py" \
-    ${rocm_ignore_test[@]} \
-    "$caffe2_pypath/python" \
-    "${EXTRA_TESTS[@]}"
-fi

.ci/docker/README.md

@@ -1,38 +0,0 @@
-# Docker images for GitHub CI and CD
-
-This directory contains everything needed to build the Docker images
-that are used in our CI.
-
-The Dockerfiles located in subdirectories are parameterized to
-conditionally run build stages depending on build arguments passed to
-`docker build`. This lets us use only a few Dockerfiles for many
-images. The different configurations are identified by a freeform
-string that we call a _build environment_. This string is persisted in
-each image as the `BUILD_ENVIRONMENT` environment variable.
-
-See `build.sh` for valid build environments (it's the giant switch).
-
-## Docker CI builds
-
-* `build.sh` -- dispatch script to launch all builds
-* `common` -- scripts used to execute individual Docker build stages
-* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
-* `ubuntu-cuda` -- Dockerfile for Ubuntu image with CUDA support for nvidia-docker
-* `ubuntu-rocm` -- Dockerfile for Ubuntu image with ROCm support
-* `ubuntu-xpu` -- Dockerfile for Ubuntu image with XPU support
-
-### Docker CD builds
-
-* `conda` - Dockerfile and build.sh to build Docker images used in nightly conda builds
-* `manywheel` - Dockerfile and build.sh to build Docker images used in nightly manywheel builds
-* `libtorch` - Dockerfile and build.sh to build Docker images used in nightly libtorch builds
-
-## Usage
-
-```bash
-# Build a specific image
-./build.sh pytorch-linux-bionic-py3.8-gcc9 -t myimage:latest
-
-# Set flags (see build.sh) and build image
-sudo bash -c 'PROTOBUF=1 ./build.sh pytorch-linux-bionic-py3.8-gcc9 -t myimage:latest
-```

.ci/docker/aotriton_version.txt

@@ -1,5 +0,0 @@
-0.6b
-manylinux_2_17
-rocm6.2
-7f07e8a1cb1f99627eb6d77f5c0e9295c775f3c7
-e4ab195d2bd19e939c675a13280c29714c6ef9f2cf420690da150fa0cac043b1

.ci/docker/build.sh

@@ -1,582 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-image="$1"
-shift
-
-if [ -z "${image}" ]; then
-  echo "Usage: $0 IMAGE"
-  exit 1
-fi
-
-function extract_version_from_image_name() {
-  eval export $2=$(echo "${image}" | perl -n -e"/$1(\d+(\.\d+)?(\.\d+)?)/ && print \$1")
-  if [ "x${!2}" = x ]; then
-    echo "variable '$2' not correctly parsed from image='$image'"
-    exit 1
-  fi
-}
-
-function extract_all_from_image_name() {
-  # parts $image into array, splitting on '-'
-  keep_IFS="$IFS"
-  IFS="-"
-  declare -a parts=($image)
-  IFS="$keep_IFS"
-  unset keep_IFS
-
-  for part in "${parts[@]}"; do
-    name=$(echo "${part}" | perl -n -e"/([a-zA-Z]+)\d+(\.\d+)?(\.\d+)?/ && print \$1")
-    vername="${name^^}_VERSION"
-    # "py" is the odd one out, needs this special case
-    if [ "x${name}" = xpy ]; then
-      vername=ANACONDA_PYTHON_VERSION
-    fi
-    # skip non-conforming fields such as "pytorch", "linux" or "bionic" without version string
-    if [ -n "${name}" ]; then
-      extract_version_from_image_name "${name}" "${vername}"
-    fi
-  done
-}
-
-# Use the same pre-built XLA test image from PyTorch/XLA
-if [[ "$image" == *xla* ]]; then
-  echo "Using pre-built XLA test image..."
-  exit 0
-fi
-
-if [[ "$image" == *-focal* ]]; then
-  UBUNTU_VERSION=20.04
-elif [[ "$image" == *-jammy* ]]; then
-  UBUNTU_VERSION=22.04
-elif [[ "$image" == *ubuntu* ]]; then
-  extract_version_from_image_name ubuntu UBUNTU_VERSION
-elif [[ "$image" == *centos* ]]; then
-  extract_version_from_image_name centos CENTOS_VERSION
-fi
-
-if [ -n "${UBUNTU_VERSION}" ]; then
-  OS="ubuntu"
-elif [ -n "${CENTOS_VERSION}" ]; then
-  OS="centos"
-else
-  echo "Unable to derive operating system base..."
-  exit 1
-fi
-
-DOCKERFILE="${OS}/Dockerfile"
-# When using ubuntu - 22.04, start from Ubuntu docker image, instead of nvidia/cuda docker image.
-if [[ "$image" == *cuda* && "$UBUNTU_VERSION" != "22.04" ]]; then
-  DOCKERFILE="${OS}-cuda/Dockerfile"
-elif [[ "$image" == *rocm* ]]; then
-  DOCKERFILE="${OS}-rocm/Dockerfile"
-elif [[ "$image" == *xpu* ]]; then
-  DOCKERFILE="${OS}-xpu/Dockerfile"
-elif [[ "$image" == *cuda*linter* ]]; then
-  # Use a separate Dockerfile for linter to keep a small image size
-  DOCKERFILE="linter-cuda/Dockerfile"
-elif [[ "$image" == *linter* ]]; then
-  # Use a separate Dockerfile for linter to keep a small image size
-  DOCKERFILE="linter/Dockerfile"
-fi
-
-# CMake 3.18 is needed to support CUDA17 language variant
-CMAKE_VERSION=3.18.5
-
-_UCX_COMMIT=7bb2722ff2187a0cad557ae4a6afa090569f83fb
-_UCC_COMMIT=20eae37090a4ce1b32bcce6144ccad0b49943e0b
-
-# It's annoying to rename jobs every time you want to rewrite a
-# configuration, so we hardcode everything here rather than do it
-# from scratch
-case "$image" in
-  pytorch-linux-focal-cuda12.4-cudnn9-py3-gcc9)
-    CUDA_VERSION=12.4.1
-    CUDNN_VERSION=9
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-focal-cuda12.1-cudnn9-py3-gcc9)
-    CUDA_VERSION=12.1.1
-    CUDNN_VERSION=9
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-focal-cuda12.4-cudnn9-py3-gcc9-inductor-benchmarks)
-    CUDA_VERSION=12.4.1
-    CUDNN_VERSION=9
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    TRITON=yes
-    INDUCTOR_BENCHMARKS=yes
-    ;;
-  pytorch-linux-focal-cuda12.1-cudnn9-py3-gcc9-inductor-benchmarks)
-    CUDA_VERSION=12.1.1
-    CUDNN_VERSION=9
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    TRITON=yes
-    INDUCTOR_BENCHMARKS=yes
-    ;;
-  pytorch-linux-focal-cuda12.1-cudnn9-py3.12-gcc9-inductor-benchmarks)
-    CUDA_VERSION=12.1.1
-    CUDNN_VERSION=9
-    ANACONDA_PYTHON_VERSION=3.12
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    TRITON=yes
-    INDUCTOR_BENCHMARKS=yes
-    ;;
-  pytorch-linux-focal-cuda12.4-cudnn9-py3.12-gcc9-inductor-benchmarks)
-    CUDA_VERSION=12.4.1
-    CUDNN_VERSION=9
-    ANACONDA_PYTHON_VERSION=3.12
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    TRITON=yes
-    INDUCTOR_BENCHMARKS=yes
-    ;;
-  pytorch-linux-focal-cuda11.8-cudnn9-py3-gcc9)
-    CUDA_VERSION=11.8.0
-    CUDNN_VERSION=9
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-focal-cuda12.4-cudnn9-py3-gcc9)
-    CUDA_VERSION=12.4.1
-    CUDNN_VERSION=9
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-focal-cuda12.1-cudnn9-py3-gcc9)
-    CUDA_VERSION=12.1.1
-    CUDNN_VERSION=9
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-focal-cuda12.4-cudnn9-py3-gcc9)
-    CUDA_VERSION=12.4.1
-    CUDNN_VERSION=9
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-focal-py3-clang10-onnx)
-    ANACONDA_PYTHON_VERSION=3.8
-    CLANG_VERSION=10
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    CONDA_CMAKE=yes
-    ONNX=yes
-    ;;
-  pytorch-linux-focal-py3-clang9-android-ndk-r21e)
-    ANACONDA_PYTHON_VERSION=3.8
-    CLANG_VERSION=9
-    LLVMDEV=yes
-    PROTOBUF=yes
-    ANDROID=yes
-    ANDROID_NDK_VERSION=r21e
-    GRADLE_VERSION=6.8.3
-    NINJA_VERSION=1.9.0
-    ;;
-  pytorch-linux-focal-py3.8-clang10)
-    ANACONDA_PYTHON_VERSION=3.8
-    CLANG_VERSION=10
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    VULKAN_SDK_VERSION=1.2.162.1
-    SWIFTSHADER=yes
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-focal-py3.11-clang10)
-    ANACONDA_PYTHON_VERSION=3.11
-    CLANG_VERSION=10
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    VULKAN_SDK_VERSION=1.2.162.1
-    SWIFTSHADER=yes
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-focal-py3.8-gcc9)
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-focal-rocm-n-1-py3)
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ROCM_VERSION=6.0
-    NINJA_VERSION=1.9.0
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-focal-rocm-n-py3)
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ROCM_VERSION=6.1
-    NINJA_VERSION=1.9.0
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-jammy-xpu-2024.0-py3)
-    ANACONDA_PYTHON_VERSION=3.9
-    GCC_VERSION=11
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    XPU_VERSION=0.5
-    NINJA_VERSION=1.9.0
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-    pytorch-linux-jammy-py3.8-gcc11-inductor-benchmarks)
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=11
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    CONDA_CMAKE=yes
-    TRITON=yes
-    DOCS=yes
-    INDUCTOR_BENCHMARKS=yes
-    ;;
-  pytorch-linux-jammy-cuda11.8-cudnn9-py3.8-clang12)
-    ANACONDA_PYTHON_VERSION=3.8
-    CUDA_VERSION=11.8
-    CUDNN_VERSION=9
-    CLANG_VERSION=12
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-jammy-py3-clang12-asan)
-    ANACONDA_PYTHON_VERSION=3.9
-    CLANG_VERSION=12
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    CONDA_CMAKE=yes
-    TRITON=yes
-    ;;
-  pytorch-linux-jammy-py3-clang15-asan)
-    ANACONDA_PYTHON_VERSION=3.10
-    CLANG_VERSION=15
-    CONDA_CMAKE=yes
-    VISION=yes
-    ;;
-  pytorch-linux-jammy-py3.8-gcc11)
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=11
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    CONDA_CMAKE=yes
-    TRITON=yes
-    DOCS=yes
-    UNINSTALL_DILL=yes
-    ;;
-  pytorch-linux-jammy-py3-clang12-executorch)
-    ANACONDA_PYTHON_VERSION=3.10
-    CLANG_VERSION=12
-    CONDA_CMAKE=yes
-    EXECUTORCH=yes
-    ;;
-  pytorch-linux-jammy-py3.12-halide)
-    CUDA_VERSION=12.4
-    ANACONDA_PYTHON_VERSION=3.12
-    GCC_VERSION=11
-    CONDA_CMAKE=yes
-    HALIDE=yes
-    ;;
-  pytorch-linux-focal-linter)
-    # TODO: Use 3.9 here because of this issue https://github.com/python/mypy/issues/13627.
-    # We will need to update mypy version eventually, but that's for another day. The task
-    # would be to upgrade mypy to 1.0.0 with Python 3.11
-    ANACONDA_PYTHON_VERSION=3.9
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-jammy-cuda11.8-cudnn9-py3.9-linter)
-    ANACONDA_PYTHON_VERSION=3.9
-    CUDA_VERSION=11.8
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-jammy-aarch64-py3.10-gcc11)
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=11
-    ACL=yes
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    CONDA_CMAKE=yes
-    # snadampal: skipping sccache due to the following issue
-    # https://github.com/pytorch/pytorch/issues/121559
-    SKIP_SCCACHE_INSTALL=yes
-    # snadampal: skipping llvm src build install because the current version
-    # from pytorch/llvm:9.0.1 is x86 specific
-    SKIP_LLVM_SRC_BUILD_INSTALL=yes
-    ;;
-  pytorch-linux-jammy-aarch64-py3.10-gcc11-inductor-benchmarks)
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=11
-    ACL=yes
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    CONDA_CMAKE=yes
-    # snadampal: skipping sccache due to the following issue
-    # https://github.com/pytorch/pytorch/issues/121559
-    SKIP_SCCACHE_INSTALL=yes
-    # snadampal: skipping llvm src build install because the current version
-    # from pytorch/llvm:9.0.1 is x86 specific
-    SKIP_LLVM_SRC_BUILD_INSTALL=yes
-    INDUCTOR_BENCHMARKS=yes
-    ;;
-  *)
-    # Catch-all for builds that are not hardcoded.
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    echo "image '$image' did not match an existing build configuration"
-    if [[ "$image" == *py* ]]; then
-      extract_version_from_image_name py ANACONDA_PYTHON_VERSION
-    fi
-    if [[ "$image" == *cuda* ]]; then
-      extract_version_from_image_name cuda CUDA_VERSION
-      extract_version_from_image_name cudnn CUDNN_VERSION
-    fi
-    if [[ "$image" == *rocm* ]]; then
-      extract_version_from_image_name rocm ROCM_VERSION
-      NINJA_VERSION=1.9.0
-      TRITON=yes
-      # To ensure that any ROCm config will build using conda cmake
-      # and thus have LAPACK/MKL enabled
-      CONDA_CMAKE=yes
-    fi
-    if [[ "$image" == *centos7* ]]; then
-      NINJA_VERSION=1.10.2
-    fi
-    if [[ "$image" == *gcc* ]]; then
-      extract_version_from_image_name gcc GCC_VERSION
-    fi
-    if [[ "$image" == *clang* ]]; then
-      extract_version_from_image_name clang CLANG_VERSION
-    fi
-    if [[ "$image" == *devtoolset* ]]; then
-      extract_version_from_image_name devtoolset DEVTOOLSET_VERSION
-    fi
-    if [[ "$image" == *glibc* ]]; then
-      extract_version_from_image_name glibc GLIBC_VERSION
-    fi
-    if [[ "$image" == *cmake* ]]; then
-      extract_version_from_image_name cmake CMAKE_VERSION
-    fi
-  ;;
-esac
-
-tmp_tag=$(basename "$(mktemp -u)" | tr '[:upper:]' '[:lower:]')
-
-#when using cudnn version 8 install it separately from cuda
-if [[ "$image" == *cuda*  && ${OS} == "ubuntu" ]]; then
-  IMAGE_NAME="nvidia/cuda:${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel-ubuntu${UBUNTU_VERSION}"
-  if [[ ${CUDNN_VERSION} == 9 ]]; then
-    IMAGE_NAME="nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}"
-  fi
-fi
-
-# Build image
-docker build \
-       --no-cache \
-       --progress=plain \
-       --build-arg "BUILD_ENVIRONMENT=${image}" \
-       --build-arg "PROTOBUF=${PROTOBUF:-}" \
-       --build-arg "LLVMDEV=${LLVMDEV:-}" \
-       --build-arg "DB=${DB:-}" \
-       --build-arg "VISION=${VISION:-}" \
-       --build-arg "UBUNTU_VERSION=${UBUNTU_VERSION}" \
-       --build-arg "CENTOS_VERSION=${CENTOS_VERSION}" \
-       --build-arg "DEVTOOLSET_VERSION=${DEVTOOLSET_VERSION}" \
-       --build-arg "GLIBC_VERSION=${GLIBC_VERSION}" \
-       --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
-       --build-arg "ANACONDA_PYTHON_VERSION=${ANACONDA_PYTHON_VERSION}" \
-       --build-arg "GCC_VERSION=${GCC_VERSION}" \
-       --build-arg "CUDA_VERSION=${CUDA_VERSION}" \
-       --build-arg "CUDNN_VERSION=${CUDNN_VERSION}" \
-       --build-arg "TENSORRT_VERSION=${TENSORRT_VERSION}" \
-       --build-arg "ANDROID=${ANDROID}" \
-       --build-arg "ANDROID_NDK=${ANDROID_NDK_VERSION}" \
-       --build-arg "GRADLE_VERSION=${GRADLE_VERSION}" \
-       --build-arg "VULKAN_SDK_VERSION=${VULKAN_SDK_VERSION}" \
-       --build-arg "SWIFTSHADER=${SWIFTSHADER}" \
-       --build-arg "CMAKE_VERSION=${CMAKE_VERSION:-}" \
-       --build-arg "NINJA_VERSION=${NINJA_VERSION:-}" \
-       --build-arg "KATEX=${KATEX:-}" \
-       --build-arg "ROCM_VERSION=${ROCM_VERSION:-}" \
-       --build-arg "PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH:-gfx906;gfx90a}" \
-       --build-arg "IMAGE_NAME=${IMAGE_NAME}" \
-       --build-arg "UCX_COMMIT=${UCX_COMMIT}" \
-       --build-arg "UCC_COMMIT=${UCC_COMMIT}" \
-       --build-arg "CONDA_CMAKE=${CONDA_CMAKE}" \
-       --build-arg "TRITON=${TRITON}" \
-       --build-arg "ONNX=${ONNX}" \
-       --build-arg "DOCS=${DOCS}" \
-       --build-arg "INDUCTOR_BENCHMARKS=${INDUCTOR_BENCHMARKS}" \
-       --build-arg "EXECUTORCH=${EXECUTORCH}" \
-       --build-arg "HALIDE=${HALIDE}" \
-       --build-arg "XPU_VERSION=${XPU_VERSION}" \
-       --build-arg "ACL=${ACL:-}" \
-       --build-arg "SKIP_SCCACHE_INSTALL=${SKIP_SCCACHE_INSTALL:-}" \
-       --build-arg "SKIP_LLVM_SRC_BUILD_INSTALL=${SKIP_LLVM_SRC_BUILD_INSTALL:-}" \
-       -f $(dirname ${DOCKERFILE})/Dockerfile \
-       -t "$tmp_tag" \
-       "$@" \
-       .
-
-# NVIDIA dockers for RC releases use tag names like `11.0-cudnn9-devel-ubuntu18.04-rc`,
-# for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
-# find the correct image. As a result, here we have to replace the
-#   "$UBUNTU_VERSION" == "18.04-rc"
-# with
-#   "$UBUNTU_VERSION" == "18.04"
-UBUNTU_VERSION=$(echo ${UBUNTU_VERSION} | sed 's/-rc$//')
-
-function drun() {
-  docker run --rm "$tmp_tag" $*
-}
-
-if [[ "$OS" == "ubuntu" ]]; then
-
-  if !(drun lsb_release -a 2>&1 | grep -qF Ubuntu); then
-    echo "OS=ubuntu, but:"
-    drun lsb_release -a
-    exit 1
-  fi
-  if !(drun lsb_release -a 2>&1 | grep -qF "$UBUNTU_VERSION"); then
-    echo "UBUNTU_VERSION=$UBUNTU_VERSION, but:"
-    drun lsb_release -a
-    exit 1
-  fi
-fi
-
-if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
-  if !(drun python --version 2>&1 | grep -qF "Python $ANACONDA_PYTHON_VERSION"); then
-    echo "ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION, but:"
-    drun python --version
-    exit 1
-  fi
-fi
-
-if [ -n "$GCC_VERSION" ]; then
-  if !(drun gcc --version 2>&1 | grep -q " $GCC_VERSION\\W"); then
-    echo "GCC_VERSION=$GCC_VERSION, but:"
-    drun gcc --version
-    exit 1
-  fi
-fi
-
-if [ -n "$CLANG_VERSION" ]; then
-  if !(drun clang --version 2>&1 | grep -qF "clang version $CLANG_VERSION"); then
-    echo "CLANG_VERSION=$CLANG_VERSION, but:"
-    drun clang --version
-    exit 1
-  fi
-fi
-
-if [ -n "$KATEX" ]; then
-  if !(drun katex --version); then
-    echo "KATEX=$KATEX, but:"
-    drun katex --version
-    exit 1
-  fi
-fi

.ci/docker/centos-rocm/Dockerfile

@@ -1,133 +0,0 @@
-ARG CENTOS_VERSION
-
-FROM centos:${CENTOS_VERSION}
-
-ARG CENTOS_VERSION
-
-# Set AMD gpu targets to build for
-ARG PYTORCH_ROCM_ARCH
-ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
-
-# Install required packages to build Caffe2
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Update CentOS git version
-RUN yum -y remove git
-RUN yum -y remove git-*
-RUN yum -y install https://packages.endpoint.com/rhel/7/os/x86_64/endpoint-repo-1.9-1.x86_64.rpm || \
-    (yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo-1.9-1.x86_64.rpm && \
-    sed -i "s/packages.endpoint/packages.endpointdev/" /etc/yum.repos.d/endpoint.repo)
-RUN yum install -y git
-
-# Install devtoolset
-ARG DEVTOOLSET_VERSION
-COPY ./common/install_devtoolset.sh install_devtoolset.sh
-RUN bash ./install_devtoolset.sh && rm install_devtoolset.sh
-ENV BASH_ENV "/etc/profile"
-
-# (optional) Install non-default glibc version
-ARG GLIBC_VERSION
-COPY ./common/install_glibc.sh install_glibc.sh
-RUN if [ -n "${GLIBC_VERSION}" ]; then bash ./install_glibc.sh; fi
-RUN rm install_glibc.sh
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ARG CONDA_CMAKE
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-COPY requirements-ci.txt /opt/conda/requirements-ci.txt
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
-
-# (optional) Install protobuf for ONNX
-ARG PROTOBUF
-COPY ./common/install_protobuf.sh install_protobuf.sh
-RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
-RUN rm install_protobuf.sh
-ENV INSTALLED_PROTOBUF ${PROTOBUF}
-
-# (optional) Install database packages like LMDB and LevelDB
-ARG DB
-COPY ./common/install_db.sh install_db.sh
-RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
-RUN rm install_db.sh
-ENV INSTALLED_DB ${DB}
-
-# (optional) Install vision packages like OpenCV
-ARG VISION
-COPY ./common/install_vision.sh ./common/cache_vision_models.sh ./common/common_utils.sh ./
-RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
-RUN rm install_vision.sh cache_vision_models.sh common_utils.sh
-ENV INSTALLED_VISION ${VISION}
-
-# Install rocm
-ARG ROCM_VERSION
-COPY ./common/install_rocm.sh install_rocm.sh
-RUN bash ./install_rocm.sh
-RUN rm install_rocm.sh
-COPY ./common/install_rocm_magma.sh install_rocm_magma.sh
-RUN bash ./install_rocm_magma.sh
-RUN rm install_rocm_magma.sh
-COPY ./common/install_amdsmi.sh install_amdsmi.sh
-RUN bash ./install_amdsmi.sh
-RUN rm install_amdsmi.sh
-ENV PATH /opt/rocm/bin:$PATH
-ENV PATH /opt/rocm/hcc/bin:$PATH
-ENV PATH /opt/rocm/hip/bin:$PATH
-ENV PATH /opt/rocm/opencl/bin:$PATH
-ENV PATH /opt/rocm/llvm/bin:$PATH
-ENV MAGMA_HOME /opt/rocm/magma
-ENV LANG en_US.utf8
-ENV LC_ALL en_US.utf8
-
-# (optional) Install non-default CMake version
-ARG CMAKE_VERSION
-COPY ./common/install_cmake.sh install_cmake.sh
-RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
-RUN rm install_cmake.sh
-
-# (optional) Install non-default Ninja version
-ARG NINJA_VERSION
-COPY ./common/install_ninja.sh install_ninja.sh
-RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
-RUN rm install_ninja.sh
-
-ARG TRITON
-# Install triton, this needs to be done before sccache because the latter will
-# try to reach out to S3, which docker build runners don't have access
-ENV CMAKE_C_COMPILER cc
-ENV CMAKE_CXX_COMPILER c++
-COPY ./common/install_triton.sh install_triton.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/triton-rocm.txt triton-rocm.txt
-COPY triton_version.txt triton_version.txt
-RUN if [ -n "${TRITON}" ]; then bash ./install_triton.sh; fi
-RUN rm install_triton.sh common_utils.sh triton-rocm.txt triton_version.txt
-
-# Install AOTriton (Early fail)
-COPY ./aotriton_version.txt aotriton_version.txt
-COPY ./common/common_utils.sh common_utils.sh
-COPY ./common/install_aotriton.sh install_aotriton.sh
-RUN ["/bin/bash", "-c", "./install_aotriton.sh /opt/rocm && rm -rf install_aotriton.sh aotriton_version.txt common_utils.sh"]
-ENV AOTRITON_INSTALLED_PREFIX /opt/rocm/aotriton
-
-# Install ccache/sccache (do this last, so we get priority in PATH)
-COPY ./common/install_cache.sh install_cache.sh
-ENV PATH /opt/cache/bin:$PATH
-RUN bash ./install_cache.sh && rm install_cache.sh
-
-# Include BUILD_ENVIRONMENT environment variable in image
-ARG BUILD_ENVIRONMENT
-ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
-
-USER jenkins
-CMD ["bash"]

.ci/docker/ci_commit_pins/executorch.txt

@@ -1 +0,0 @@
-69472e5c43481324ad923ceb29392ab72830acee

.ci/docker/ci_commit_pins/halide.txt

@@ -1 +0,0 @@
-340136fec6d3ebc73e7a19eba1663e9b0ba8ab2d

.ci/docker/ci_commit_pins/huggingface.txt

@@ -1 +0,0 @@
-243e186efbf7fb93328dd6b34927a4e8c8f24395

.ci/docker/ci_commit_pins/timm.txt

@@ -1 +0,0 @@
-ac3470188b914c5d7a5058a7e28b9eb685a62427

.ci/docker/ci_commit_pins/triton-rocm.txt

@@ -1 +0,0 @@
-21eae954efa5bf584da70324b640288c3ee7aede

.ci/docker/ci_commit_pins/triton-xpu.txt

@@ -1 +0,0 @@
-1b2f15840e0d70eec50d84c7a0575cb835524def

.ci/docker/ci_commit_pins/triton.txt

@@ -1 +0,0 @@
-dedb7bdf339a3546896d4820366ca562c586bfa0

.ci/docker/common/cache_vision_models.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-# Cache the test models at ~/.cache/torch/hub/
-IMPORT_SCRIPT_FILENAME="/tmp/torchvision_import_script.py"
-as_jenkins echo 'import torchvision; torchvision.models.mobilenet_v2(pretrained=True); torchvision.models.mobilenet_v3_large(pretrained=True);' > "${IMPORT_SCRIPT_FILENAME}"
-
-pip_install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
-# Very weird quoting behavior here https://github.com/conda/conda/issues/10972,
-# so echo the command to a file and run the file instead
-conda_run python "${IMPORT_SCRIPT_FILENAME}"
-
-# Cleaning up
-conda_run pip uninstall -y torch torchvision
-rm "${IMPORT_SCRIPT_FILENAME}" || true

.ci/docker/common/common_utils.sh

@@ -1,36 +0,0 @@
-#!/bin/bash
-
-# Work around bug where devtoolset replaces sudo and breaks it.
-if [ -n "$DEVTOOLSET_VERSION" ]; then
-  export SUDO=/bin/sudo
-else
-  export SUDO=sudo
-fi
-
-as_jenkins() {
-  # NB: unsetting the environment variables works around a conda bug
-  # https://github.com/conda/conda/issues/6576
-  # NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
-  # NB: This must be run from a directory that jenkins has access to,
-  # works around https://github.com/conda/conda-package-handling/pull/34
-  $SUDO -E -H -u jenkins env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" $*
-}
-
-conda_install() {
-  # Ensure that the install command don't upgrade/downgrade Python
-  # This should be called as
-  #   conda_install pkg1 pkg2 ... [-c channel]
-  as_jenkins conda install -q -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION" $*
-}
-
-conda_run() {
-  as_jenkins conda run -n py_$ANACONDA_PYTHON_VERSION --no-capture-output $*
-}
-
-pip_install() {
-  as_jenkins conda run -n py_$ANACONDA_PYTHON_VERSION pip install --progress-bar off $*
-}
-
-get_pinned_commit() {
-  cat "${1}".txt
-}

.ci/docker/common/install_acl.sh

@@ -1,16 +0,0 @@
-set -euo pipefail
-
-readonly version=v24.04
-readonly src_host=https://review.mlplatform.org/ml
-readonly src_repo=ComputeLibrary
-
-# Clone ACL
-[[ ! -d ${src_repo} ]] && git clone ${src_host}/${src_repo}.git
-cd ${src_repo}
-
-git checkout $version
-
-# Build with scons
-scons -j8  Werror=0 debug=0 neon=1 opencl=0 embed_kernels=0 \
-  os=linux arch=armv8a build=native multi_isa=1 \
-  fixed_format_kernels=1 openmp=1 cppthreads=0

.ci/docker/common/install_amdsmi.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-cd /opt/rocm/share/amd_smi && pip install .

.ci/docker/common/install_aotriton.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-TARBALL='aotriton.tar.bz2'
-# This read command alwasy returns with exit code 1
-read -d "\n" VER MANYLINUX ROCMBASE PINNED_COMMIT SHA256 < aotriton_version.txt || true
-ARCH=$(uname -m)
-AOTRITON_INSTALL_PREFIX="$1"
-AOTRITON_URL="https://github.com/ROCm/aotriton/releases/download/${VER}/aotriton-${VER}-${MANYLINUX}_${ARCH}-${ROCMBASE}-shared.tar.bz2"
-
-cd "${AOTRITON_INSTALL_PREFIX}"
-# Must use -L to follow redirects
-curl -L --retry 3 -o "${TARBALL}" "${AOTRITON_URL}"
-ACTUAL_SHA256=$(sha256sum "${TARBALL}" | cut -d " " -f 1)
-if [ "${SHA256}" != "${ACTUAL_SHA256}" ]; then
-  echo -n "Error: The SHA256 of downloaded tarball is ${ACTUAL_SHA256},"
-  echo " which does not match the expected value ${SHA256}."
-  exit
-fi
-tar xf "${TARBALL}" && rm -rf "${TARBALL}"

.ci/docker/common/install_base.sh

@@ -1,159 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-install_ubuntu() {
-  # NVIDIA dockers for RC releases use tag names like `11.0-cudnn9-devel-ubuntu18.04-rc`,
-  # for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
-  # find the correct image. As a result, here we have to check for
-  #   "$UBUNTU_VERSION" == "18.04"*
-  # instead of
-  #   "$UBUNTU_VERSION" == "18.04"
-  if [[ "$UBUNTU_VERSION" == "20.04"* ]]; then
-    cmake3="cmake=3.16*"
-    maybe_libiomp_dev=""
-  elif [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
-    cmake3="cmake=3.22*"
-    maybe_libiomp_dev=""
-  else
-    cmake3="cmake=3.5*"
-    maybe_libiomp_dev="libiomp-dev"
-  fi
-
-  if [[ "$CLANG_VERSION" == 15 ]]; then
-    maybe_libomp_dev="libomp-15-dev"
-  elif [[ "$CLANG_VERSION" == 12 ]]; then
-    maybe_libomp_dev="libomp-12-dev"
-  elif [[ "$CLANG_VERSION" == 10 ]]; then
-    maybe_libomp_dev="libomp-10-dev"
-  else
-    maybe_libomp_dev=""
-  fi
-
-  # HACK: UCC testing relies on libnccl library from NVIDIA repo, and version 2.16 crashes
-  # See https://github.com/pytorch/pytorch/pull/105260#issuecomment-1673399729
-  if [[ "$UBUNTU_VERSION" == "20.04"* && "$CUDA_VERSION" == "11.8"* ]]; then
-    maybe_libnccl_dev="libnccl2=2.15.5-1+cuda11.8 libnccl-dev=2.15.5-1+cuda11.8 --allow-downgrades --allow-change-held-packages"
-  else
-    maybe_libnccl_dev=""
-  fi
-
-  # Install common dependencies
-  apt-get update
-  # TODO: Some of these may not be necessary
-  ccache_deps="asciidoc docbook-xml docbook-xsl xsltproc"
-  deploy_deps="libffi-dev libbz2-dev libreadline-dev libncurses5-dev libncursesw5-dev libgdbm-dev libsqlite3-dev uuid-dev tk-dev"
-  numpy_deps="gfortran"
-  apt-get install -y --no-install-recommends \
-    $ccache_deps \
-    $numpy_deps \
-    ${deploy_deps} \
-    ${cmake3} \
-    apt-transport-https \
-    autoconf \
-    automake \
-    build-essential \
-    ca-certificates \
-    curl \
-    git \
-    libatlas-base-dev \
-    libc6-dbg \
-    ${maybe_libiomp_dev} \
-    libyaml-dev \
-    libz-dev \
-    libjemalloc2 \
-    libjpeg-dev \
-    libasound2-dev \
-    libsndfile-dev \
-    ${maybe_libomp_dev} \
-    ${maybe_libnccl_dev} \
-    software-properties-common \
-    wget \
-    sudo \
-    vim \
-    jq \
-    libtool \
-    vim \
-    unzip \
-    gpg-agent \
-    gdb
-
-  # Should resolve issues related to various apt package repository cert issues
-  # see: https://github.com/pytorch/pytorch/issues/65931
-  apt-get install -y libgnutls30
-
-  # Cleanup package manager
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-}
-
-install_centos() {
-  # Need EPEL for many packages we depend on.
-  # See http://fedoraproject.org/wiki/EPEL
-  yum --enablerepo=extras install -y epel-release
-
-  ccache_deps="asciidoc docbook-dtds docbook-style-xsl libxslt"
-  numpy_deps="gcc-gfortran"
-  # Note: protobuf-c-{compiler,devel} on CentOS are too old to be used
-  # for Caffe2. That said, we still install them to make sure the build
-  # system opts to build/use protoc and libprotobuf from third-party.
-  yum install -y \
-    $ccache_deps \
-    $numpy_deps \
-    autoconf \
-    automake \
-    bzip2 \
-    cmake \
-    cmake3 \
-    curl \
-    gcc \
-    gcc-c++ \
-    gflags-devel \
-    git \
-    glibc-devel \
-    glibc-headers \
-    glog-devel \
-    libstdc++-devel \
-    libsndfile-devel \
-    make \
-    opencv-devel \
-    sudo \
-    wget \
-    vim \
-    unzip \
-    gdb
-
-  # Cleanup
-  yum clean all
-  rm -rf /var/cache/yum
-  rm -rf /var/lib/yum/yumdb
-  rm -rf /var/lib/yum/history
-}
-
-# Install base packages depending on the base OS
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    install_ubuntu
-    ;;
-  centos)
-    install_centos
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac
-
-# Install Valgrind separately since the apt-get version is too old.
-mkdir valgrind_build && cd valgrind_build
-VALGRIND_VERSION=3.20.0
-wget https://ossci-linux.s3.amazonaws.com/valgrind-${VALGRIND_VERSION}.tar.bz2
-tar -xjf valgrind-${VALGRIND_VERSION}.tar.bz2
-cd valgrind-${VALGRIND_VERSION}
-./configure --prefix=/usr/local
-make -j$[$(nproc) - 2]
-sudo make install
-cd ../../
-rm -rf valgrind_build
-alias valgrind="/usr/local/bin/valgrind"

.ci/docker/common/install_clang.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-if [ -n "$CLANG_VERSION" ]; then
-
-  if [[ $CLANG_VERSION == 9 && $UBUNTU_VERSION == 18.04 ]]; then
-    sudo apt-get update
-    # gpg-agent is not available by default on 18.04
-    sudo apt-get install  -y --no-install-recommends gpg-agent
-    wget --no-check-certificate -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add  -
-    apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-${CLANG_VERSION} main"
-  elif [[ $UBUNTU_VERSION == 22.04 ]]; then
-    # work around ubuntu apt-get conflicts
-    sudo apt-get -y -f install
-  fi
-
-  sudo apt-get update
-  apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
-  apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
-
-  # Install dev version of LLVM.
-  if [ -n "$LLVMDEV" ]; then
-    sudo apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"-dev
-  fi
-
-  # Use update-alternatives to make this version the default
-  update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
-  update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
-  # Override cc/c++ to clang as well
-  update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
-  update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
-
-  # clang's packaging is a little messed up (the runtime libs aren't
-  # added into the linker path), so give it a little help
-  clang_lib=("/usr/lib/llvm-$CLANG_VERSION/lib/clang/"*"/lib/linux")
-  echo "$clang_lib" > /etc/ld.so.conf.d/clang.conf
-  ldconfig
-
-  # Cleanup package manager
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-fi

.ci/docker/common/install_cmake.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-[ -n "$CMAKE_VERSION" ]
-
-# Remove system cmake install so it won't get used instead
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    apt-get remove cmake -y
-    ;;
-  centos)
-    yum remove cmake -y
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac
-
-# Turn 3.6.3 into v3.6
-path=$(echo "${CMAKE_VERSION}" | sed -e 's/\([0-9].[0-9]\+\).*/v\1/')
-file="cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz"
-
-# Download and install specific CMake version in /usr/local
-pushd /tmp
-curl -Os --retry 3 "https://cmake.org/files/${path}/${file}"
-tar -C /usr/local --strip-components 1 --no-same-owner -zxf cmake-*.tar.gz
-rm -f cmake-*.tar.gz
-popd

.ci/docker/common/install_conda.sh

@@ -1,118 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# Optionally install conda
-if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
-  BASE_URL="https://repo.anaconda.com/miniconda"
-  CONDA_FILE="Miniconda3-latest-Linux-x86_64.sh"
-  if [[ $(uname -m) == "aarch64" ]] || [[ "$BUILD_ENVIRONMENT" == *xpu* ]]; then
-    BASE_URL="https://github.com/conda-forge/miniforge/releases/latest/download"
-    CONDA_FILE="Miniforge3-Linux-$(uname -m).sh"
-  fi
-
-  MAJOR_PYTHON_VERSION=$(echo "$ANACONDA_PYTHON_VERSION" | cut -d . -f 1)
-  MINOR_PYTHON_VERSION=$(echo "$ANACONDA_PYTHON_VERSION" | cut -d . -f 2)
-
-  case "$MAJOR_PYTHON_VERSION" in
-    3);;
-    *)
-      echo "Unsupported ANACONDA_PYTHON_VERSION: $ANACONDA_PYTHON_VERSION"
-      exit 1
-      ;;
-  esac
-
-  mkdir -p /opt/conda
-  chown jenkins:jenkins /opt/conda
-
-  source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-  pushd /tmp
-  wget -q "${BASE_URL}/${CONDA_FILE}"
-  # NB: Manually invoke bash per https://github.com/conda/conda/issues/10431
-  as_jenkins bash "${CONDA_FILE}" -b -f -p "/opt/conda"
-  popd
-
-  # NB: Don't do this, rely on the rpath to get it right
-  #echo "/opt/conda/lib" > /etc/ld.so.conf.d/conda-python.conf
-  #ldconfig
-  sed -e 's|PATH="\(.*\)"|PATH="/opt/conda/bin:\1"|g' -i /etc/environment
-  export PATH="/opt/conda/bin:$PATH"
-
-  # Ensure we run conda in a directory that jenkins has write access to
-  pushd /opt/conda
-
-  # Prevent conda from updating to 4.14.0, which causes docker build failures
-  # See https://hud.pytorch.org/pytorch/pytorch/commit/754d7f05b6841e555cea5a4b2c505dd9e0baec1d
-  # Uncomment the below when resolved to track the latest conda update
-  # as_jenkins conda update -y -n base conda
-
-  if [[ $(uname -m) == "aarch64" ]]; then
-    export SYSROOT_DEP="sysroot_linux-aarch64=2.17"
-  else
-    export SYSROOT_DEP="sysroot_linux-64=2.17"
-  fi
-
-  # Install correct Python version
-  # Also ensure sysroot is using a modern GLIBC to match system compilers
-  as_jenkins conda create -n py_$ANACONDA_PYTHON_VERSION -y\
-             python="$ANACONDA_PYTHON_VERSION" \
-             ${SYSROOT_DEP}
-
-  # libstdcxx from conda default channels are too old, we need GLIBCXX_3.4.30
-  # which is provided in libstdcxx 12 and up.
-  conda_install libstdcxx-ng=12.3.0 -c conda-forge
-
-  # Install PyTorch conda deps, as per https://github.com/pytorch/pytorch README
-  if [[ $(uname -m) == "aarch64" ]]; then
-    CONDA_COMMON_DEPS="astunparse pyyaml setuptools openblas==0.3.25=*openmp* ninja==1.11.1 scons==4.5.2"
-
-    if [ "$ANACONDA_PYTHON_VERSION" = "3.8" ]; then
-      NUMPY_VERSION=1.24.4
-    else
-      NUMPY_VERSION=1.26.2
-    fi
-  else
-    CONDA_COMMON_DEPS="astunparse pyyaml mkl=2021.4.0 mkl-include=2021.4.0 setuptools"
-
-    if [ "$ANACONDA_PYTHON_VERSION" = "3.11" ] || [ "$ANACONDA_PYTHON_VERSION" = "3.12" ] || [ "$ANACONDA_PYTHON_VERSION" = "3.13" ]; then
-      NUMPY_VERSION=1.26.0
-    else
-      NUMPY_VERSION=1.21.2
-    fi
-  fi
-  conda_install ${CONDA_COMMON_DEPS}
-
-  # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
-  # and libpython-static for torch deploy
-  conda_install llvmdev=8.0.0 "libpython-static=${ANACONDA_PYTHON_VERSION}"
-
-  # Use conda cmake in some cases. Conda cmake will be newer than our supported
-  # min version (3.5 for xenial and 3.10 for bionic), so we only do it in those
-  # following builds that we know should use conda. Specifically, Ubuntu bionic
-  # and focal cannot find conda mkl with stock cmake, so we need a cmake from conda
-  if [ -n "${CONDA_CMAKE}" ]; then
-    conda_install cmake
-  fi
-
-  # Magma package names are concatenation of CUDA major and minor ignoring revision
-  # I.e. magma-cuda102 package corresponds to CUDA_VERSION=10.2 and CUDA_VERSION=10.2.89
-  if [ -n "$CUDA_VERSION" ]; then
-    conda_install magma-cuda$(TMP=${CUDA_VERSION/./};echo ${TMP%.*[0-9]}) -c pytorch
-  fi
-
-  # Install some other packages, including those needed for Python test reporting
-  pip_install -r /opt/conda/requirements-ci.txt
-  pip_install numpy=="$NUMPY_VERSION"
-  pip_install -U scikit-learn
-
-  if [ -n "$DOCS" ]; then
-    apt-get update
-    apt-get -y install expect-dev
-
-    # We are currently building docs with python 3.8 (min support version)
-    pip_install -r /opt/conda/requirements-docs.txt
-  fi
-
-  popd
-fi

.ci/docker/common/install_conda_docker.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-# Script used only in CD pipeline
-set -ex
-
-# Anaconda
-# Latest anaconda is using openssl-3 which is incompatible with all currently published versions of git
-# Which are using openssl-1.1.1, see https://anaconda.org/anaconda/git/files?version=2.40.1 for example
-MINICONDA_URL=https://repo.anaconda.com/miniconda/Miniconda3-py311_23.5.2-0-Linux-x86_64.sh
-wget -q $MINICONDA_URL
-# NB: Manually invoke bash per https://github.com/conda/conda/issues/10431
-bash $(basename "$MINICONDA_URL") -b -p /opt/conda
-rm $(basename "$MINICONDA_URL")
-export PATH=/opt/conda/bin:$PATH
-# See https://github.com/pytorch/builder/issues/1473
-# Pin conda to 23.5.2 as it's the last one compatible with openssl-1.1.1
-conda install -y conda=23.5.2 conda-build anaconda-client git ninja
-# The cmake version here needs to match with the minimum version of cmake
-# supported by PyTorch (3.18). There is only 3.18.2 on anaconda
-/opt/conda/bin/pip3 install cmake==3.18.2
-conda remove -y --force patchelf

.ci/docker/common/install_cpython.sh

@@ -1,96 +0,0 @@
-#!/bin/bash
-# Script used only in CD pipeline
-set -uex -o pipefail
-
-PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python
-PYTHON_DOWNLOAD_GITHUB_BRANCH=https://github.com/python/cpython/archive/refs/heads
-GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
-
-# Python versions to be installed in /opt/$VERSION_NO
-CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"3.8.1 3.9.0 3.10.1 3.11.0 3.12.0 3.13.0"}
-
-function check_var {
-    if [ -z "$1" ]; then
-        echo "required variable not defined"
-        exit 1
-    fi
-}
-
-function do_cpython_build {
-    local py_ver=$1
-    local py_folder=$2
-    check_var $py_ver
-    check_var $py_folder
-    tar -xzf Python-$py_ver.tgz
-    pushd $py_folder
-
-    local prefix="/opt/_internal/cpython-${py_ver}"
-    mkdir -p ${prefix}/lib
-    if [[ -n $(which patchelf) ]]; then
-        local shared_flags="--enable-shared"
-    else
-        local shared_flags="--disable-shared"
-    fi
-    if [[ -z  "${WITH_OPENSSL+x}" ]]; then
-        local openssl_flags=""
-    else
-        local openssl_flags="--with-openssl=${WITH_OPENSSL} --with-openssl-rpath=auto"
-    fi
-
-    # -Wformat added for https://bugs.python.org/issue17547 on Python 2.6
-    CFLAGS="-Wformat" ./configure --prefix=${prefix} ${openssl_flags} ${shared_flags} > /dev/null
-
-    make -j40 > /dev/null
-    make install > /dev/null
-
-    if [[ "${shared_flags}" == "--enable-shared" ]]; then
-        patchelf --set-rpath '$ORIGIN/../lib' ${prefix}/bin/python3
-    fi
-
-    popd
-    rm -rf $py_folder
-    # Some python's install as bin/python3. Make them available as
-    # bin/python.
-    if [ -e ${prefix}/bin/python3 ]; then
-        ln -s python3 ${prefix}/bin/python
-    fi
-    ${prefix}/bin/python get-pip.py
-    if [ -e ${prefix}/bin/pip3 ] && [ ! -e ${prefix}/bin/pip ]; then
-        ln -s pip3 ${prefix}/bin/pip
-    fi
-    # install setuptools since python 3.12 is required to use distutils
-    ${prefix}/bin/pip install wheel==0.34.2 setuptools==68.2.2
-    local abi_tag=$(${prefix}/bin/python -c "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag; print('{0}{1}-{2}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag()))")
-    ln -s ${prefix} /opt/python/${abi_tag}
-}
-
-function build_cpython {
-    local py_ver=$1
-    check_var $py_ver
-    check_var $PYTHON_DOWNLOAD_URL
-    local py_ver_folder=$py_ver
-    if [ "$py_ver" = "3.13.0" ]; then
-        PY_VER_SHORT="3.13"
-        check_var $PYTHON_DOWNLOAD_GITHUB_BRANCH
-        wget $PYTHON_DOWNLOAD_GITHUB_BRANCH/$PY_VER_SHORT.tar.gz -O Python-$py_ver.tgz
-        do_cpython_build $py_ver cpython-$PY_VER_SHORT
-    else
-        wget -q $PYTHON_DOWNLOAD_URL/$py_ver_folder/Python-$py_ver.tgz
-        do_cpython_build $py_ver Python-$py_ver
-    fi
-
-    rm -f Python-$py_ver.tgz
-}
-
-function build_cpythons {
-    check_var $GET_PIP_URL
-    curl -sLO $GET_PIP_URL
-    for py_ver in $@; do
-        build_cpython $py_ver
-    done
-    rm -f get-pip.py
-}
-
-mkdir -p /opt/python
-mkdir -p /opt/_internal
-build_cpythons $CPYTHON_VERSIONS

.ci/docker/common/install_cuda.sh

@@ -1,250 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-NCCL_VERSION=v2.21.5-1
-CUDNN_VERSION=9.1.0.70
-
-function install_cusparselt_040 {
-    # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html
-    mkdir tmp_cusparselt && pushd tmp_cusparselt
-    wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.4.0.7-archive.tar.xz
-    tar xf libcusparse_lt-linux-x86_64-0.4.0.7-archive.tar.xz
-    cp -a libcusparse_lt-linux-x86_64-0.4.0.7-archive/include/* /usr/local/cuda/include/
-    cp -a libcusparse_lt-linux-x86_64-0.4.0.7-archive/lib/* /usr/local/cuda/lib64/
-    popd
-    rm -rf tmp_cusparselt
-}
-
-function install_cusparselt_052 {
-    # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html
-    mkdir tmp_cusparselt && pushd tmp_cusparselt
-    wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.5.2.1-archive.tar.xz
-    tar xf libcusparse_lt-linux-x86_64-0.5.2.1-archive.tar.xz
-    cp -a libcusparse_lt-linux-x86_64-0.5.2.1-archive/include/* /usr/local/cuda/include/
-    cp -a libcusparse_lt-linux-x86_64-0.5.2.1-archive/lib/* /usr/local/cuda/lib64/
-    popd
-    rm -rf tmp_cusparselt
-}
-
-function install_cusparselt_062 {
-    # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html
-    mkdir tmp_cusparselt && pushd tmp_cusparselt
-    wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.6.2.3-archive.tar.xz
-    tar xf libcusparse_lt-linux-x86_64-0.6.2.3-archive.tar.xz
-    cp -a libcusparse_lt-linux-x86_64-0.6.2.3-archive/include/* /usr/local/cuda/include/
-    cp -a libcusparse_lt-linux-x86_64-0.6.2.3-archive/lib/* /usr/local/cuda/lib64/
-    popd
-    rm -rf tmp_cusparselt
-}
-
-function install_118 {
-    echo "Installing CUDA 11.8 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and cuSparseLt-0.4.0"
-    rm -rf /usr/local/cuda-11.8 /usr/local/cuda
-    # install CUDA 11.8.0 in the same container
-    wget -q https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run
-    chmod +x cuda_11.8.0_520.61.05_linux.run
-    ./cuda_11.8.0_520.61.05_linux.run --toolkit --silent
-    rm -f cuda_11.8.0_520.61.05_linux.run
-    rm -f /usr/local/cuda && ln -s /usr/local/cuda-11.8 /usr/local/cuda
-
-    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
-    mkdir tmp_cudnn && cd tmp_cudnn
-    wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive.tar.xz
-    tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive.tar.xz
-    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive/include/* /usr/local/cuda/include/
-    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive/lib/* /usr/local/cuda/lib64/
-    cd ..
-    rm -rf tmp_cudnn
-
-    # NCCL license: https://docs.nvidia.com/deeplearning/nccl/#licenses
-    # Follow build: https://github.com/NVIDIA/nccl/tree/master?tab=readme-ov-file#build
-    git clone -b $NCCL_VERSION --depth 1 https://github.com/NVIDIA/nccl.git
-    cd nccl && make -j src.build
-    cp -a build/include/* /usr/local/cuda/include/
-    cp -a build/lib/* /usr/local/cuda/lib64/
-    cd ..
-    rm -rf nccl
-
-    install_cusparselt_040
-
-    ldconfig
-}
-
-function install_121 {
-    echo "Installing CUDA 12.1 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and cuSparseLt-0.5.2"
-    rm -rf /usr/local/cuda-12.1 /usr/local/cuda
-    # install CUDA 12.1.0 in the same container
-    wget -q https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_530.30.02_linux.run
-    chmod +x cuda_12.1.1_530.30.02_linux.run
-    ./cuda_12.1.1_530.30.02_linux.run --toolkit --silent
-    rm -f cuda_12.1.1_530.30.02_linux.run
-    rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.1 /usr/local/cuda
-
-    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
-    mkdir tmp_cudnn && cd tmp_cudnn
-    wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
-    tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
-    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/include/* /usr/local/cuda/include/
-    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/lib/* /usr/local/cuda/lib64/
-    cd ..
-    rm -rf tmp_cudnn
-
-    # NCCL license: https://docs.nvidia.com/deeplearning/nccl/#licenses
-    # Follow build: https://github.com/NVIDIA/nccl/tree/master?tab=readme-ov-file#build
-    git clone -b $NCCL_VERSION --depth 1 https://github.com/NVIDIA/nccl.git
-    cd nccl && make -j src.build
-    cp -a build/include/* /usr/local/cuda/include/
-    cp -a build/lib/* /usr/local/cuda/lib64/
-    cd ..
-    rm -rf nccl
-
-    install_cusparselt_052
-
-    ldconfig
-}
-
-function install_124 {
-  echo "Installing CUDA 12.4.1 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and cuSparseLt-0.5.2"
-  rm -rf /usr/local/cuda-12.4 /usr/local/cuda
-  # install CUDA 12.4.1 in the same container
-  wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-  chmod +x cuda_12.4.1_550.54.15_linux.run
-  ./cuda_12.4.1_550.54.15_linux.run --toolkit --silent
-  rm -f cuda_12.4.1_550.54.15_linux.run
-  rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.4 /usr/local/cuda
-
-  # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
-  mkdir tmp_cudnn && cd tmp_cudnn
-  wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
-  tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
-  cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/include/* /usr/local/cuda/include/
-  cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/lib/* /usr/local/cuda/lib64/
-  cd ..
-  rm -rf tmp_cudnn
-
-  # NCCL license: https://docs.nvidia.com/deeplearning/nccl/#licenses
-  # Follow build: https://github.com/NVIDIA/nccl/tree/master?tab=readme-ov-file#build
-  git clone -b $NCCL_VERSION --depth 1 https://github.com/NVIDIA/nccl.git
-  cd nccl && make -j src.build
-  cp -a build/include/* /usr/local/cuda/include/
-  cp -a build/lib/* /usr/local/cuda/lib64/
-  cd ..
-  rm -rf nccl
-
-  install_cusparselt_062
-
-  ldconfig
-}
-
-function prune_118 {
-    echo "Pruning CUDA 11.8 and cuDNN"
-    #####################################################################################
-    # CUDA 11.8 prune static libs
-    #####################################################################################
-    export NVPRUNE="/usr/local/cuda-11.8/bin/nvprune"
-    export CUDA_LIB_DIR="/usr/local/cuda-11.8/lib64"
-
-    export GENCODE="-gencode arch=compute_35,code=sm_35 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
-    export GENCODE_CUDNN="-gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
-
-    if [[ -n "$OVERRIDE_GENCODE" ]]; then
-        export GENCODE=$OVERRIDE_GENCODE
-    fi
-
-    # all CUDA libs except CuDNN and CuBLAS (cudnn and cublas need arch 3.7 included)
-    ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis"  \
-      | xargs -I {} bash -c \
-                "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}"
-
-    # prune CuDNN and CuBLAS
-    $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a
-    $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a
-
-    #####################################################################################
-    # CUDA 11.8 prune visual tools
-    #####################################################################################
-    export CUDA_BASE="/usr/local/cuda-11.8/"
-    rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2022.3.0 $CUDA_BASE/nsight-systems-2022.4.2/
-}
-
-function prune_121 {
-  echo "Pruning CUDA 12.1"
-  #####################################################################################
-  # CUDA 12.1 prune static libs
-  #####################################################################################
-    export NVPRUNE="/usr/local/cuda-12.1/bin/nvprune"
-    export CUDA_LIB_DIR="/usr/local/cuda-12.1/lib64"
-
-    export GENCODE="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
-    export GENCODE_CUDNN="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
-
-    if [[ -n "$OVERRIDE_GENCODE" ]]; then
-        export GENCODE=$OVERRIDE_GENCODE
-    fi
-
-    # all CUDA libs except CuDNN and CuBLAS
-    ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis"  \
-      | xargs -I {} bash -c \
-                "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}"
-
-    # prune CuDNN and CuBLAS
-    $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a
-    $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a
-
-    #####################################################################################
-    # CUDA 12.1 prune visual tools
-    #####################################################################################
-    export CUDA_BASE="/usr/local/cuda-12.1/"
-    rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2023.1.0 $CUDA_BASE/nsight-systems-2023.1.2/
-}
-
-function prune_124 {
-  echo "Pruning CUDA 12.4"
-  #####################################################################################
-  # CUDA 12.4 prune static libs
-  #####################################################################################
-  export NVPRUNE="/usr/local/cuda-12.4/bin/nvprune"
-  export CUDA_LIB_DIR="/usr/local/cuda-12.4/lib64"
-
-  export GENCODE="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
-  export GENCODE_CUDNN="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
-
-  if [[ -n "$OVERRIDE_GENCODE" ]]; then
-      export GENCODE=$OVERRIDE_GENCODE
-  fi
-  if [[ -n "$OVERRIDE_GENCODE_CUDNN" ]]; then
-      export GENCODE_CUDNN=$OVERRIDE_GENCODE_CUDNN
-  fi
-
-  # all CUDA libs except CuDNN and CuBLAS
-  ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis"  \
-      | xargs -I {} bash -c \
-                "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}"
-
-  # prune CuDNN and CuBLAS
-  $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a
-  $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a
-
-  #####################################################################################
-  # CUDA 12.1 prune visual tools
-  #####################################################################################
-  export CUDA_BASE="/usr/local/cuda-12.4/"
-  rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2024.1.0 $CUDA_BASE/nsight-systems-2023.4.4/
-}
-
-# idiomatic parameter and option handling in sh
-while test $# -gt 0
-do
-    case "$1" in
-    11.8) install_118; prune_118
-        ;;
-    12.1) install_121; prune_121
-        ;;
-    12.4) install_124; prune_124
-        ;;
-    *) echo "bad argument $1"; exit 1
-        ;;
-    esac
-    shift
-done

.ci/docker/common/install_cuda_aarch64.sh

@@ -1,93 +0,0 @@
-#!/bin/bash
-# Script used only in CD pipeline
-
-set -ex
-
-NCCL_VERSION=v2.21.5-1
-
-function install_cusparselt_052 {
-    # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html
-    mkdir tmp_cusparselt && pushd tmp_cusparselt
-    wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-sbsa/libcusparse_lt-linux-sbsa-0.5.2.1-archive.tar.xz
-    tar xf libcusparse_lt-linux-sbsa-0.5.2.1-archive.tar.xz
-    cp -a libcusparse_lt-linux-sbsa-0.5.2.1-archive/include/* /usr/local/cuda/include/
-    cp -a libcusparse_lt-linux-sbsa-0.5.2.1-archive/lib/* /usr/local/cuda/lib64/
-    popd
-    rm -rf tmp_cusparselt
-}
-
-function install_124 {
-  echo "Installing CUDA 12.4.1 and cuDNN 9.1 and NCCL ${NCCL_VERSION} and cuSparseLt-0.5.2"
-  rm -rf /usr/local/cuda-12.4 /usr/local/cuda
-  # install CUDA 12.4.1 in the same container
-  wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux_sbsa.run
-  chmod +x cuda_12.4.1_550.54.15_linux_sbsa.run
-  ./cuda_12.4.1_550.54.15_linux_sbsa.run --toolkit --silent
-  rm -f cuda_12.4.1_550.54.15_linux_sbsa.run
-  rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.4 /usr/local/cuda
-
-  # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
-  mkdir tmp_cudnn && cd tmp_cudnn
-  wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-9.1.0.70_cuda12-archive.tar.xz -O cudnn-linux-sbsa-9.1.0.70_cuda12-archive.tar.xz
-  tar xf cudnn-linux-sbsa-9.1.0.70_cuda12-archive.tar.xz
-  cp -a cudnn-linux-sbsa-9.1.0.70_cuda12-archive/include/* /usr/local/cuda/include/
-  cp -a cudnn-linux-sbsa-9.1.0.70_cuda12-archive/lib/* /usr/local/cuda/lib64/
-  cd ..
-  rm -rf tmp_cudnn
-
-  # NCCL license: https://docs.nvidia.com/deeplearning/nccl/#licenses
-  # Follow build: https://github.com/NVIDIA/nccl/tree/master?tab=readme-ov-file#build
-  git clone -b ${NCCL_VERSION} --depth 1 https://github.com/NVIDIA/nccl.git
-  cd nccl && make -j src.build
-  cp -a build/include/* /usr/local/cuda/include/
-  cp -a build/lib/* /usr/local/cuda/lib64/
-  cd ..
-  rm -rf nccl
-
-  install_cusparselt_052
-
-  ldconfig
-}
-
-function prune_124 {
-  echo "Pruning CUDA 12.4"
-  #####################################################################################
-  # CUDA 12.4 prune static libs
-  #####################################################################################
-  export NVPRUNE="/usr/local/cuda-12.4/bin/nvprune"
-  export CUDA_LIB_DIR="/usr/local/cuda-12.4/lib64"
-
-  export GENCODE="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
-  export GENCODE_CUDNN="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
-
-  if [[ -n "$OVERRIDE_GENCODE" ]]; then
-      export GENCODE=$OVERRIDE_GENCODE
-  fi
-
-  # all CUDA libs except CuDNN and CuBLAS
-  ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis"  \
-      | xargs -I {} bash -c \
-                "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}"
-
-  # prune CuDNN and CuBLAS
-  $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a
-  $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a
-
-  #####################################################################################
-  # CUDA 12.1 prune visual tools
-  #####################################################################################
-  export CUDA_BASE="/usr/local/cuda-12.4/"
-  rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2024.1.0 $CUDA_BASE/nsight-systems-2023.4.4/
-}
-
-# idiomatic parameter and option handling in sh
-while test $# -gt 0
-do
-    case "$1" in
-    12.4) install_124; prune_124
-        ;;
-    *) echo "bad argument $1"; exit 1
-        ;;
-    esac
-    shift
-done

.ci/docker/common/install_cudnn.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-if [[ -n "${CUDNN_VERSION}" ]]; then
-    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
-    mkdir tmp_cudnn
-    pushd tmp_cudnn
-    if [[ ${CUDA_VERSION:0:2} == "12" ]]; then
-        CUDNN_NAME="cudnn-linux-x86_64-9.1.0.70_cuda12-archive"
-    elif [[ ${CUDA_VERSION:0:2} == "11" ]]; then
-        CUDNN_NAME="cudnn-linux-x86_64-9.1.0.70_cuda11-archive"
-    else
-        print "Unsupported CUDA version ${CUDA_VERSION}"
-        exit 1
-    fi
-    curl --retry 3 -OLs https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/${CUDNN_NAME}.tar.xz
-    tar xf ${CUDNN_NAME}.tar.xz
-    cp -a ${CUDNN_NAME}/include/* /usr/local/cuda/include/
-    cp -a ${CUDNN_NAME}/lib/* /usr/local/cuda/lib64/
-    popd
-    rm -rf tmp_cudnn
-    ldconfig
-fi

.ci/docker/common/install_cudss.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# cudss license: https://docs.nvidia.com/cuda/cudss/license.html
-mkdir tmp_cudss && cd tmp_cudss
-
-if [[ ${CUDA_VERSION:0:4} =~ ^12\.[1-4]$ ]]; then
-    arch_path='sbsa'
-    export TARGETARCH=${TARGETARCH:-$(uname -m)}
-    if [ ${TARGETARCH} = 'amd64' ] || [ "${TARGETARCH}" = 'x86_64' ]; then
-        arch_path='x86_64'
-    fi
-    CUDSS_NAME="libcudss-linux-${arch_path}-0.3.0.9_cuda12-archive"
-    curl --retry 3 -OLs https://developer.download.nvidia.com/compute/cudss/redist/libcudss/linux-${arch_path}/${CUDSS_NAME}.tar.xz
-
-    # only for cuda 12
-    tar xf ${CUDSS_NAME}.tar.xz
-    cp -a ${CUDSS_NAME}/include/* /usr/local/cuda/include/
-    cp -a ${CUDSS_NAME}/lib/* /usr/local/cuda/lib64/
-fi
-
-cd ..
-rm -rf tmp_cudss
-ldconfig

.ci/docker/common/install_cusparselt.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# cuSPARSELt license: https://docs.nvidia.com/cuda/cusparselt/license.html
-mkdir tmp_cusparselt && cd tmp_cusparselt
-
-if [[ ${CUDA_VERSION:0:4} =~ ^12\.[2-4]$ ]]; then
-    arch_path='sbsa'
-    export TARGETARCH=${TARGETARCH:-$(uname -m)}
-    if [ ${TARGETARCH} = 'amd64' ] || [ "${TARGETARCH}" = 'x86_64' ]; then
-        arch_path='x86_64'
-    fi
-    CUSPARSELT_NAME="libcusparse_lt-linux-${arch_path}-0.6.2.3-archive"
-    curl --retry 3 -OLs https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-${arch_path}/${CUSPARSELT_NAME}.tar.xz
-elif [[ ${CUDA_VERSION:0:4} == "12.1" ]]; then
-    arch_path='sbsa'
-    export TARGETARCH=${TARGETARCH:-$(uname -m)}
-    if [ ${TARGETARCH} = 'amd64' ] || [ "${TARGETARCH}" = 'x86_64' ]; then
-        arch_path='x86_64'
-    fi
-    CUSPARSELT_NAME="libcusparse_lt-linux-${arch_path}-0.5.2.1-archive"
-    curl --retry 3 -OLs https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-${arch_path}/${CUSPARSELT_NAME}.tar.xz
-elif [[ ${CUDA_VERSION:0:4} == "11.8" ]]; then
-    CUSPARSELT_NAME="libcusparse_lt-linux-x86_64-0.4.0.7-archive"
-    curl --retry 3 -OLs https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/${CUSPARSELT_NAME}.tar.xz
-fi
-
-tar xf ${CUSPARSELT_NAME}.tar.xz
-cp -a ${CUSPARSELT_NAME}/include/* /usr/local/cuda/include/
-cp -a ${CUSPARSELT_NAME}/lib/* /usr/local/cuda/lib64/
-cd ..
-rm -rf tmp_cusparselt
-ldconfig

.ci/docker/common/install_db.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-install_ubuntu() {
-  apt-get update
-
-  # Cleanup
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-}
-
-install_centos() {
-  # Need EPEL for many packages we depend on.
-  # See http://fedoraproject.org/wiki/EPEL
-  yum --enablerepo=extras install -y epel-release
-
-  # Cleanup
-  yum clean all
-  rm -rf /var/cache/yum
-  rm -rf /var/lib/yum/yumdb
-  rm -rf /var/lib/yum/history
-}
-
-# Install base packages depending on the base OS
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    install_ubuntu
-    ;;
-  centos)
-    install_centos
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac

.ci/docker/common/install_docs_reqs.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-if [ -n "$KATEX" ]; then
-  apt-get update
-  # Ignore error if gpg-agent doesn't exist (for Ubuntu 16.04)
-  apt-get install -y gpg-agent || :
-
-  curl --retry 3 -sL https://deb.nodesource.com/setup_16.x | sudo -E bash -
-  sudo apt-get install -y nodejs
-
-  curl --retry 3 -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
-  echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
-
-  apt-get update
-  apt-get install -y --no-install-recommends yarn
-  yarn global add katex --prefix /usr/local
-
-  sudo apt-get -y install doxygen
-
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-fi

.ci/docker/common/install_executorch.sh

@@ -1,65 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-clone_executorch() {
-  EXECUTORCH_PINNED_COMMIT=$(get_pinned_commit executorch)
-
-  # Clone the Executorch
-  git clone https://github.com/pytorch/executorch.git
-
-  # and fetch the target commit
-  pushd executorch
-  git checkout "${EXECUTORCH_PINNED_COMMIT}"
-  git submodule update --init
-  popd
-
-  chown -R jenkins executorch
-}
-
-install_buck2() {
-  pushd executorch/.ci/docker
-
-  BUCK2_VERSION=$(cat ci_commit_pins/buck2.txt)
-  source common/install_buck.sh
-
-  popd
-}
-
-install_conda_dependencies() {
-  pushd executorch/.ci/docker
-  # Install conda dependencies like flatbuffer
-  conda_install --file conda-env-ci.txt
-  popd
-}
-
-install_pip_dependencies() {
-  pushd executorch/.ci/docker
-  # Install PyTorch CPU build beforehand to avoid installing the much bigger CUDA
-  # binaries later, ExecuTorch only needs CPU
-  pip_install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-  # Install all Python dependencies
-  pip_install -r requirements-ci.txt
-  popd
-}
-
-setup_executorch() {
-  pushd executorch
-  # Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
-  as_jenkins bash .ci/scripts/setup-vulkan-linux-deps.sh
-
-  export PYTHON_EXECUTABLE=python
-  export EXECUTORCH_BUILD_PYBIND=ON
-  export CMAKE_ARGS="-DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON"
-
-  as_jenkins .ci/scripts/setup-linux.sh cmake
-  popd
-}
-
-clone_executorch
-install_buck2
-install_conda_dependencies
-install_pip_dependencies
-setup_executorch

.ci/docker/common/install_gcc.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-if [ -n "$GCC_VERSION" ]; then
-
-  # Need the official toolchain repo to get alternate packages
-  add-apt-repository ppa:ubuntu-toolchain-r/test
-  apt-get update
-  apt-get install -y g++-$GCC_VERSION
-  update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-"$GCC_VERSION" 50
-  update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-"$GCC_VERSION" 50
-  update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-"$GCC_VERSION" 50
-
-
-  # Cleanup package manager
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-fi

.ci/docker/common/install_halide.sh

@@ -1,46 +0,0 @@
-#!/bin/bash
-set -ex
-
-source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-COMMIT=$(get_pinned_commit halide)
-test -n "$COMMIT"
-
-# activate conda to populate CONDA_PREFIX
-test -n "$ANACONDA_PYTHON_VERSION"
-eval "$(conda shell.bash hook)"
-conda activate py_$ANACONDA_PYTHON_VERSION
-
-if [ -n "${UBUNTU_VERSION}" ];then
-    apt update
-    apt-get install -y lld liblld-15-dev libpng-dev libjpeg-dev libgl-dev \
-                  libopenblas-dev libeigen3-dev libatlas-base-dev libzstd-dev
-fi
-
-conda_install numpy scipy imageio cmake ninja
-
-git clone --depth 1 --branch release/16.x --recursive https://github.com/llvm/llvm-project.git
-cmake -DCMAKE_BUILD_TYPE=Release \
-        -DLLVM_ENABLE_PROJECTS="clang" \
-        -DLLVM_TARGETS_TO_BUILD="X86;NVPTX" \
-        -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_ASSERTIONS=ON \
-        -DLLVM_ENABLE_EH=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_BUILD_32_BITS=OFF \
-        -S llvm-project/llvm -B llvm-build -G Ninja
-cmake --build llvm-build
-cmake --install llvm-build --prefix llvm-install
-export LLVM_ROOT=`pwd`/llvm-install
-export LLVM_CONFIG=$LLVM_ROOT/bin/llvm-config
-
-git clone https://github.com/halide/Halide.git
-pushd Halide
-git checkout ${COMMIT} && git submodule update --init --recursive
-pip_install -r requirements.txt
-cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -S . -B build
-cmake --build build
-test -e ${CONDA_PREFIX}/lib/python3 || ln -s python${ANACONDA_PYTHON_VERSION} ${CONDA_PREFIX}/lib/python3
-cmake --install build --prefix ${CONDA_PREFIX}
-chown -R jenkins ${CONDA_PREFIX}
-popd
-rm -rf Halide llvm-build llvm-project llvm-install
-
-python -c "import halide"  # check for errors

.ci/docker/common/install_inductor_benchmark_deps.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-function install_huggingface() {
-  local version
-  commit=$(get_pinned_commit huggingface)
-  pip_install pandas==2.0.3
-  pip_install "git+https://github.com/huggingface/transformers@${commit}"
-}
-
-function install_timm() {
-  local commit
-  commit=$(get_pinned_commit timm)
-  pip_install pandas==2.0.3
-  pip_install "git+https://github.com/huggingface/pytorch-image-models@${commit}"
-  # Clean up
-  conda_run pip uninstall -y cmake torch torchvision triton
-}
-
-# Pango is needed for weasyprint which is needed for doctr
-conda_install pango
-install_huggingface
-install_timm

.ci/docker/common/install_libpng.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-# Script used only in CD pipeline
-
-set -ex
-
-LIBPNG_VERSION=1.6.37
-
-mkdir -p libpng
-pushd libpng
-
-wget http://download.sourceforge.net/libpng/libpng-$LIBPNG_VERSION.tar.gz
-tar -xvzf libpng-$LIBPNG_VERSION.tar.gz
-
-pushd libpng-$LIBPNG_VERSION
-
-./configure
-make
-make install
-
-popd
-
-popd
-rm -rf libpng

.ci/docker/common/install_linter.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-if [ -n "${UBUNTU_VERSION}" ]; then
-  apt update
-  apt-get install -y clang doxygen git graphviz nodejs npm libtinfo5
-fi
-
-# Do shallow clone of PyTorch so that we can init lintrunner in Docker build context
-git clone https://github.com/pytorch/pytorch.git --depth 1
-chown -R jenkins pytorch
-
-pushd pytorch
-# Install all linter dependencies
-pip_install -r requirements.txt
-conda_run lintrunner init
-
-# Cache .lintbin directory as part of the Docker image
-cp -r .lintbin /tmp
-popd
-
-# Node dependencies required by toc linter job
-npm install -g markdown-toc
-
-# Cleaning up
-rm -rf pytorch

.ci/docker/common/install_magma.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-# Script used only in CD pipeline
-
-set -eou pipefail
-
-MAGMA_VERSION="2.5.2"
-
-function do_install() {
-    cuda_version=$1
-    cuda_version_nodot=${1/./}
-
-    MAGMA_VERSION="2.6.1"
-    magma_archive="magma-cuda${cuda_version_nodot}-${MAGMA_VERSION}-1.tar.bz2"
-
-    cuda_dir="/usr/local/cuda-${cuda_version}"
-    (
-        set -x
-        tmp_dir=$(mktemp -d)
-        pushd ${tmp_dir}
-        curl -OLs https://anaconda.org/pytorch/magma-cuda${cuda_version_nodot}/${MAGMA_VERSION}/download/linux-64/${magma_archive}
-        tar -xvf "${magma_archive}"
-        mkdir -p "${cuda_dir}/magma"
-        mv include "${cuda_dir}/magma/include"
-        mv lib "${cuda_dir}/magma/lib"
-        popd
-    )
-}
-
-do_install $1

.ci/docker/common/install_miopen.sh

@@ -1,137 +0,0 @@
-#!/bin/bash
-# Script used only in CD pipeline
-
-set -ex
-
-ROCM_VERSION=$1
-
-if [[ -z $ROCM_VERSION ]]; then
-    echo "missing ROCM_VERSION"
-    exit 1;
-fi
-
-# To make version comparison easier, create an integer representation.
-save_IFS="$IFS"
-IFS=. ROCM_VERSION_ARRAY=(${ROCM_VERSION})
-IFS="$save_IFS"
-if [[ ${#ROCM_VERSION_ARRAY[@]} == 2 ]]; then
-    ROCM_VERSION_MAJOR=${ROCM_VERSION_ARRAY[0]}
-    ROCM_VERSION_MINOR=${ROCM_VERSION_ARRAY[1]}
-    ROCM_VERSION_PATCH=0
-elif [[ ${#ROCM_VERSION_ARRAY[@]} == 3 ]]; then
-    ROCM_VERSION_MAJOR=${ROCM_VERSION_ARRAY[0]}
-    ROCM_VERSION_MINOR=${ROCM_VERSION_ARRAY[1]}
-    ROCM_VERSION_PATCH=${ROCM_VERSION_ARRAY[2]}
-else
-    echo "Unhandled ROCM_VERSION ${ROCM_VERSION}"
-    exit 1
-fi
-ROCM_INT=$(($ROCM_VERSION_MAJOR * 10000 + $ROCM_VERSION_MINOR * 100 + $ROCM_VERSION_PATCH))
-
-# Install custom MIOpen + COMgr for ROCm >= 4.0.1
-if [[ $ROCM_INT -lt 40001 ]]; then
-    echo "ROCm version < 4.0.1; will not install custom MIOpen"
-    exit 0
-fi
-
-# Function to retry functions that sometimes timeout or have flaky failures
-retry () {
-    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
-}
-
-# Build custom MIOpen to use comgr for offline compilation.
-
-## Need a sanitized ROCM_VERSION without patchlevel; patchlevel version 0 must be added to paths.
-ROCM_DOTS=$(echo ${ROCM_VERSION} | tr -d -c '.' | wc -c)
-if [[ ${ROCM_DOTS} == 1 ]]; then
-    ROCM_VERSION_NOPATCH="${ROCM_VERSION}"
-    ROCM_INSTALL_PATH="/opt/rocm-${ROCM_VERSION}.0"
-else
-    ROCM_VERSION_NOPATCH="${ROCM_VERSION%.*}"
-    ROCM_INSTALL_PATH="/opt/rocm-${ROCM_VERSION}"
-fi
-
-# MIOPEN_USE_HIP_KERNELS is a Workaround for COMgr issues
-MIOPEN_CMAKE_COMMON_FLAGS="
--DMIOPEN_USE_COMGR=ON
--DMIOPEN_BUILD_DRIVER=OFF
-"
-# Pull MIOpen repo and set DMIOPEN_EMBED_DB based on ROCm version
-if [[ $ROCM_INT -ge 60200 ]] && [[ $ROCM_INT -lt 60300 ]]; then
-    echo "ROCm 6.2 MIOpen does not need any patches, do not build from source"
-    exit 0
-elif [[ $ROCM_INT -ge 60100 ]] && [[ $ROCM_INT -lt 60200 ]]; then
-    echo "ROCm 6.1 MIOpen does not need any patches, do not build from source"
-    exit 0
-elif [[ $ROCM_INT -ge 60000 ]] && [[ $ROCM_INT -lt 60100 ]]; then
-    echo "ROCm 6.0 MIOpen does not need any patches, do not build from source"
-    exit 0
-elif [[ $ROCM_INT -ge 50700 ]] && [[ $ROCM_INT -lt 60000 ]]; then
-    echo "ROCm 5.7 MIOpen does not need any patches, do not build from source"
-    exit 0
-elif [[ $ROCM_INT -ge 50600 ]] && [[ $ROCM_INT -lt 50700 ]]; then
-    MIOPEN_BRANCH="release/rocm-rel-5.6-staging"
-elif [[ $ROCM_INT -ge 50500 ]] && [[ $ROCM_INT -lt 50600 ]]; then
-    MIOPEN_BRANCH="release/rocm-rel-5.5-gfx11"
-elif [[ $ROCM_INT -ge 50400 ]] && [[ $ROCM_INT -lt 50500 ]]; then
-    MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx900_56;gfx906_60;gfx90878;gfx90a6e;gfx1030_36 -DMIOPEN_USE_MLIR=Off"
-    MIOPEN_BRANCH="release/rocm-rel-5.4-staging"
-elif [[ $ROCM_INT -ge 50300 ]] && [[ $ROCM_INT -lt 50400 ]]; then
-    MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx900_56;gfx906_60;gfx90878;gfx90a6e;gfx1030_36 -DMIOPEN_USE_MLIR=Off"
-    MIOPEN_BRANCH="release/rocm-rel-5.3-staging"
-elif [[ $ROCM_INT -ge 50200 ]] && [[ $ROCM_INT -lt 50300 ]]; then
-    MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx900_56;gfx906_60;gfx90878;gfx90a6e;gfx1030_36 -DMIOPEN_USE_MLIR=Off"
-    MIOPEN_BRANCH="release/rocm-rel-5.2-staging"
-elif [[ $ROCM_INT -ge 50100 ]] && [[ $ROCM_INT -lt 50200 ]]; then
-    MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx900_56;gfx906_60;gfx90878;gfx90a6e;gfx1030_36"
-    MIOPEN_BRANCH="release/rocm-rel-5.1-staging"
-elif [[ $ROCM_INT -ge 50000 ]] && [[ $ROCM_INT -lt 50100 ]]; then
-    MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx900_56;gfx906_60;gfx90878;gfx90a6e;gfx1030_36"
-    MIOPEN_BRANCH="release/rocm-rel-5.0-staging"
-else
-    echo "Unhandled ROCM_VERSION ${ROCM_VERSION}"
-    exit 1
-fi
-
-yum remove -y miopen-hip
-
-git clone https://github.com/ROCm/MIOpen -b ${MIOPEN_BRANCH}
-pushd MIOpen
-# remove .git to save disk space since CI runner was running out
-rm -rf .git
-# Don't build MLIR to save docker build time
-# since we are disabling MLIR backend for MIOpen anyway
-if [[ $ROCM_INT -ge 50400 ]] && [[ $ROCM_INT -lt 50500 ]]; then
-    sed -i '/rocMLIR/d' requirements.txt
-elif [[ $ROCM_INT -ge 50200 ]] && [[ $ROCM_INT -lt 50400 ]]; then
-    sed -i '/llvm-project-mlir/d' requirements.txt
-fi
-## MIOpen minimum requirements
-cmake -P install_deps.cmake --minimum
-
-# clean up since CI runner was running out of disk space
-rm -rf /tmp/*
-yum clean all
-rm -rf /var/cache/yum
-rm -rf /var/lib/yum/yumdb
-rm -rf /var/lib/yum/history
-
-## Build MIOpen
-mkdir -p build
-cd build
-PKG_CONFIG_PATH=/usr/local/lib/pkgconfig CXX=${ROCM_INSTALL_PATH}/llvm/bin/clang++ cmake .. \
-    ${MIOPEN_CMAKE_COMMON_FLAGS} \
-    ${MIOPEN_CMAKE_DB_FLAGS} \
-    -DCMAKE_PREFIX_PATH="${ROCM_INSTALL_PATH}/hip;${ROCM_INSTALL_PATH}"
-make MIOpen -j $(nproc)
-
-# Build MIOpen package
-make -j $(nproc) package
-
-# clean up since CI runner was running out of disk space
-rm -rf /usr/local/cget
-
-yum install -y miopen-*.rpm
-
-popd
-rm -rf MIOpen

.ci/docker/common/install_mkl.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-set -ex
-
-# MKL
-MKL_VERSION=2024.2.0
-
-MKLROOT=/opt/intel
-mkdir -p ${MKLROOT}
-pushd /tmp
-
-python3 -mpip install wheel
-python3 -mpip download -d . mkl-static==${MKL_VERSION}
-python3 -m wheel unpack mkl_static-${MKL_VERSION}-py2.py3-none-manylinux1_x86_64.whl
-python3 -m wheel unpack mkl_include-${MKL_VERSION}-py2.py3-none-manylinux1_x86_64.whl
-mv mkl_static-${MKL_VERSION}/mkl_static-${MKL_VERSION}.data/data/lib ${MKLROOT}
-mv mkl_include-${MKL_VERSION}/mkl_include-${MKL_VERSION}.data/data/include ${MKLROOT}

.ci/docker/common/install_mnist.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-# Script used only in CD pipeline
-
-set -ex
-
-mkdir -p /usr/local/mnist/
-
-cd /usr/local/mnist
-
-for img in train-images-idx3-ubyte.gz train-labels-idx1-ubyte.gz t10k-images-idx3-ubyte.gz t10k-labels-idx1-ubyte.gz; do
-  wget -q https://ossci-datasets.s3.amazonaws.com/mnist/$img
-  gzip -d $img
-done

.ci/docker/common/install_nvpl.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-function install_nvpl {
-
-    mkdir -p /opt/nvpl/lib /opt/nvpl/include
-
-    wget https://developer.download.nvidia.com/compute/nvpl/redist/nvpl_blas/linux-sbsa/nvpl_blas-linux-sbsa-0.3.0-archive.tar.xz
-    tar xf nvpl_blas-linux-sbsa-0.3.0-archive.tar.xz
-    cp -r nvpl_blas-linux-sbsa-0.3.0-archive/lib/* /opt/nvpl/lib/
-    cp -r nvpl_blas-linux-sbsa-0.3.0-archive/include/* /opt/nvpl/include/
-
-    wget https://developer.download.nvidia.com/compute/nvpl/redist/nvpl_lapack/linux-sbsa/nvpl_lapack-linux-sbsa-0.2.3.1-archive.tar.xz
-    tar xf nvpl_lapack-linux-sbsa-0.2.3.1-archive.tar.xz
-    cp -r nvpl_lapack-linux-sbsa-0.2.3.1-archive/lib/* /opt/nvpl/lib/
-    cp -r nvpl_lapack-linux-sbsa-0.2.3.1-archive/include/* /opt/nvpl/include/
-}
-
-install_nvpl

.ci/docker/common/install_onnx.sh

@@ -1,53 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-retry () {
-    "$@" || (sleep 10 && "$@") || (sleep 20 && "$@") || (sleep 40 && "$@")
-}
-
-# A bunch of custom pip dependencies for ONNX
-pip_install \
-  beartype==0.15.0 \
-  filelock==3.9.0 \
-  flatbuffers==2.0 \
-  mock==5.0.1 \
-  ninja==1.10.2 \
-  networkx==2.0 \
-  numpy==1.24.2
-
-# ONNXRuntime should be installed before installing
-# onnx-weekly. Otherwise, onnx-weekly could be
-# overwritten by onnx.
-pip_install \
-  parameterized==0.8.1 \
-  pytest-cov==4.0.0 \
-  pytest-subtests==0.10.0 \
-  tabulate==0.9.0 \
-  transformers==4.36.2
-
-pip_install coloredlogs packaging
-
-pip_install onnxruntime==1.18
-pip_install onnx==1.16.0
-# pip_install "onnxscript@git+https://github.com/microsoft/onnxscript@3e869ef8ccf19b5ebd21c10d3e9c267c9a9fa729" --no-deps
-pip_install onnxscript==0.1.0.dev20240613 --no-deps
-# required by onnxscript
-pip_install ml_dtypes
-
-# Cache the transformers model to be used later by ONNX tests. We need to run the transformers
-# package to download the model. By default, the model is cached at ~/.cache/huggingface/hub/
-IMPORT_SCRIPT_FILENAME="/tmp/onnx_import_script.py"
-as_jenkins echo 'import transformers; transformers.AutoModel.from_pretrained("sshleifer/tiny-gpt2"); transformers.AutoTokenizer.from_pretrained("sshleifer/tiny-gpt2"); transformers.AutoModelForSpeechSeq2Seq.from_pretrained("openai/whisper-large-v3");' > "${IMPORT_SCRIPT_FILENAME}"
-
-# Need a PyTorch version for transformers to work
-pip_install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
-# Very weird quoting behavior here https://github.com/conda/conda/issues/10972,
-# so echo the command to a file and run the file instead
-conda_run python "${IMPORT_SCRIPT_FILENAME}"
-
-# Cleaning up
-conda_run pip uninstall -y torch
-rm "${IMPORT_SCRIPT_FILENAME}" || true

.ci/docker/common/install_openblas.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-# Script used only in CD pipeline
-
-set -ex
-
-cd /
-git clone https://github.com/OpenMathLib/OpenBLAS.git -b v0.3.25 --depth 1 --shallow-submodules
-
-
-OPENBLAS_BUILD_FLAGS="
-NUM_THREADS=128
-USE_OPENMP=1
-NO_SHARED=0
-DYNAMIC_ARCH=1
-TARGET=ARMV8
-CFLAGS=-O3
-"
-
-OPENBLAS_CHECKOUT_DIR="OpenBLAS"
-
-make -j8 ${OPENBLAS_BUILD_FLAGS} -C ${OPENBLAS_CHECKOUT_DIR}
-make -j8 ${OPENBLAS_BUILD_FLAGS} install -C ${OPENBLAS_CHECKOUT_DIR}

.ci/docker/common/install_openssl.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-OPENSSL=openssl-1.1.1k
-
-wget -q -O "${OPENSSL}.tar.gz" "https://ossci-linux.s3.amazonaws.com/${OPENSSL}.tar.gz"
-tar xf "${OPENSSL}.tar.gz"
-cd "${OPENSSL}"
-./config --prefix=/opt/openssl -d '-Wl,--enable-new-dtags,-rpath,$(LIBRPATH)'
-# NOTE: openssl install errors out when built with the -j option
-NPROC=$[$(nproc) - 2]
-make -j${NPROC}; make install_sw
-# Link the ssl libraries to the /usr/lib folder.
-sudo ln -s /opt/openssl/lib/lib* /usr/lib
-cd ..
-rm -rf "${OPENSSL}"

.ci/docker/common/install_patchelf.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-# Script used only in CD pipeline
-
-set -ex
-
-# Pin the version to latest release 0.17.2, building newer commit starts
-# to fail on the current image
-git clone -b 0.17.2 --single-branch https://github.com/NixOS/patchelf
-cd patchelf
-sed -i 's/serial/parallel/g' configure.ac
-./bootstrap.sh
-./configure
-make
-make install
-cd ..
-rm -rf patchelf

.ci/docker/common/install_protobuf.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-pb_dir="/usr/temp_pb_install_dir"
-mkdir -p $pb_dir
-
-# On the nvidia/cuda:9-cudnn7-devel-centos7 image we need this symlink or
-# else it will fail with
-#   g++: error: ./../lib64/crti.o: No such file or directory
-ln -s /usr/lib64 "$pb_dir/lib64"
-
-curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-all-3.17.3.tar.gz" --retry 3
-
-tar -xvz --no-same-owner -C "$pb_dir" --strip-components 1 -f protobuf-all-3.17.3.tar.gz
-NPROC=$[$(nproc) - 2]
-pushd "$pb_dir" && ./configure && make -j${NPROC} && make -j${NPROC} check && sudo make -j${NRPOC} install && sudo ldconfig
-popd
-rm -rf $pb_dir

.ci/docker/common/install_rocm.sh

@@ -1,148 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-ver() {
-    printf "%3d%03d%03d%03d" $(echo "$1" | tr '.' ' ');
-}
-
-install_ubuntu() {
-    apt-get update
-    if [[ $UBUNTU_VERSION == 18.04 ]]; then
-      # gpg-agent is not available by default on 18.04
-      apt-get install -y --no-install-recommends gpg-agent
-    fi
-    if [[ $UBUNTU_VERSION == 20.04 ]]; then
-      # gpg-agent is not available by default on 20.04
-      apt-get install -y --no-install-recommends gpg-agent
-    fi
-    apt-get install -y kmod
-    apt-get install -y wget
-
-    # Need the libc++1 and libc++abi1 libraries to allow torch._C to load at runtime
-    apt-get install -y libc++1
-    apt-get install -y libc++abi1
-
-    # Add amdgpu repository
-    UBUNTU_VERSION_NAME=`cat /etc/os-release | grep UBUNTU_CODENAME | awk -F= '{print $2}'`
-    echo "deb [arch=amd64] https://repo.radeon.com/amdgpu/${ROCM_VERSION}/ubuntu ${UBUNTU_VERSION_NAME} main" > /etc/apt/sources.list.d/amdgpu.list
-
-    # Add rocm repository
-    wget -qO - http://repo.radeon.com/rocm/rocm.gpg.key | apt-key add -
-    local rocm_baseurl="http://repo.radeon.com/rocm/apt/${ROCM_VERSION}"
-    echo "deb [arch=amd64] ${rocm_baseurl} ${UBUNTU_VERSION_NAME} main" > /etc/apt/sources.list.d/rocm.list
-    apt-get update --allow-insecure-repositories
-
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
-                   rocm-dev \
-                   rocm-utils \
-                   rocm-libs \
-                   rccl \
-                   rocprofiler-dev \
-                   roctracer-dev \
-                   amd-smi-lib
-
-    if [[ $(ver $ROCM_VERSION) -ge $(ver 6.1) ]]; then
-        DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated rocm-llvm-dev
-    fi
-
-    # precompiled miopen kernels added in ROCm 3.5, renamed in ROCm 5.5
-    # search for all unversioned packages
-    # if search fails it will abort this script; use true to avoid case where search fails
-    MIOPENHIPGFX=$(apt-cache search --names-only miopen-hip-gfx | awk '{print $1}' | grep -F -v . || true)
-    if [[ "x${MIOPENHIPGFX}" = x ]]; then
-      echo "miopen-hip-gfx package not available" && exit 1
-    else
-      DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated ${MIOPENHIPGFX}
-    fi
-
-    # ROCm 6.0 had a regression where journal_mode was enabled on the kdb files resulting in permission errors at runtime
-    for kdb in /opt/rocm/share/miopen/db/*.kdb
-    do
-        sqlite3 $kdb "PRAGMA journal_mode=off; PRAGMA VACUUM;"
-    done
-
-    # Cleanup
-    apt-get autoclean && apt-get clean
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-}
-
-install_centos() {
-
-  yum update -y
-  yum install -y kmod
-  yum install -y wget
-  yum install -y openblas-devel
-
-  yum install -y epel-release
-  yum install -y dkms kernel-headers-`uname -r` kernel-devel-`uname -r`
-
-  # Add amdgpu repository
-  local amdgpu_baseurl
-  if [[ $OS_VERSION == 9 ]]; then
-      amdgpu_baseurl="https://repo.radeon.com/amdgpu/${ROCM_VERSION}/rhel/9.0/main/x86_64"
-  else
-      amdgpu_baseurl="https://repo.radeon.com/amdgpu/${ROCM_VERSION}/rhel/7.9/main/x86_64"
-  fi
-  echo "[AMDGPU]" > /etc/yum.repos.d/amdgpu.repo
-  echo "name=AMDGPU" >> /etc/yum.repos.d/amdgpu.repo
-  echo "baseurl=${amdgpu_baseurl}" >> /etc/yum.repos.d/amdgpu.repo
-  echo "enabled=1" >> /etc/yum.repos.d/amdgpu.repo
-  echo "gpgcheck=1" >> /etc/yum.repos.d/amdgpu.repo
-  echo "gpgkey=http://repo.radeon.com/rocm/rocm.gpg.key" >> /etc/yum.repos.d/amdgpu.repo
-
-  local rocm_baseurl="http://repo.radeon.com/rocm/yum/${ROCM_VERSION}"
-  echo "[ROCm]" > /etc/yum.repos.d/rocm.repo
-  echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo
-  echo "baseurl=${rocm_baseurl}" >> /etc/yum.repos.d/rocm.repo
-  echo "enabled=1" >> /etc/yum.repos.d/rocm.repo
-  echo "gpgcheck=1" >> /etc/yum.repos.d/rocm.repo
-  echo "gpgkey=http://repo.radeon.com/rocm/rocm.gpg.key" >> /etc/yum.repos.d/rocm.repo
-
-  yum update -y
-
-  yum install -y \
-                   rocm-dev \
-                   rocm-utils \
-                   rocm-libs \
-                   rccl \
-                   rocprofiler-dev \
-                   roctracer-dev \
-                   amd-smi-lib
-
-  # precompiled miopen kernels; search for all unversioned packages
-  # if search fails it will abort this script; use true to avoid case where search fails
-  MIOPENHIPGFX=$(yum -q search miopen-hip-gfx | grep miopen-hip-gfx | awk '{print $1}'| grep -F kdb. || true)
-  if [[ "x${MIOPENHIPGFX}" = x ]]; then
-    echo "miopen-hip-gfx package not available" && exit 1
-  else
-    yum install -y ${MIOPENHIPGFX}
-  fi
-
-  # ROCm 6.0 had a regression where journal_mode was enabled on the kdb files resulting in permission errors at runtime
-  for kdb in /opt/rocm/share/miopen/db/*.kdb
-  do
-      sqlite3 $kdb "PRAGMA journal_mode=off; PRAGMA VACUUM;"
-  done
-
-  # Cleanup
-  yum clean all
-  rm -rf /var/cache/yum
-  rm -rf /var/lib/yum/yumdb
-  rm -rf /var/lib/yum/history
-}
-
-# Install Python packages depending on the base OS
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    install_ubuntu
-    ;;
-  centos)
-    install_centos
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac

.ci/docker/common/install_rocm_drm.sh

@@ -1,150 +0,0 @@
-#!/bin/bash
-# Script used only in CD pipeline
-
-###########################
-### prereqs
-###########################
-# Install Python packages depending on the base OS
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    apt-get update -y
-    apt-get install -y libpciaccess-dev pkg-config
-    apt-get clean
-    ;;
-  centos)
-    yum install -y libpciaccess-devel pkgconfig
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac
-python3 -m pip install meson ninja
-
-###########################
-### clone repo
-###########################
-GIT_SSL_NO_VERIFY=true git clone https://gitlab.freedesktop.org/mesa/drm.git
-pushd drm
-
-###########################
-### patch
-###########################
-patch -p1 <<'EOF'
-diff --git a/amdgpu/amdgpu_asic_id.c b/amdgpu/amdgpu_asic_id.c
-index a5007ffc..13fa07fc 100644
---- a/amdgpu/amdgpu_asic_id.c
-+++ b/amdgpu/amdgpu_asic_id.c
-@@ -22,6 +22,13 @@
-  *
-  */
-
-+#define _XOPEN_SOURCE 700
-+#define _LARGEFILE64_SOURCE
-+#define _FILE_OFFSET_BITS 64
-+#include <ftw.h>
-+#include <link.h>
-+#include <limits.h>
-+
- #include <ctype.h>
- #include <stdio.h>
- #include <stdlib.h>
-@@ -34,6 +41,19 @@
- #include "amdgpu_drm.h"
- #include "amdgpu_internal.h"
-
-+static char *amdgpuids_path = NULL;
-+static const char* amdgpuids_path_msg = NULL;
-+
-+static int check_for_location_of_amdgpuids(const char *filepath, const struct stat *info, const int typeflag, struct FTW *pathinfo)
-+{
-+	if (typeflag == FTW_F && strstr(filepath, "amdgpu.ids")) {
-+		amdgpuids_path = strdup(filepath);
-+		return 1;
-+	}
-+
-+	return 0;
-+}
-+
- static int parse_one_line(struct amdgpu_device *dev, const char *line)
- {
- 	char *buf, *saveptr;
-@@ -113,10 +133,46 @@ void amdgpu_parse_asic_ids(struct amdgpu_device *dev)
- 	int line_num = 1;
- 	int r = 0;
-
-+	// attempt to find typical location for amdgpu.ids file
- 	fp = fopen(AMDGPU_ASIC_ID_TABLE, "r");
-+
-+	// if it doesn't exist, search
-+	if (!fp) {
-+
-+	char self_path[ PATH_MAX ];
-+	ssize_t count;
-+	ssize_t i;
-+
-+	count = readlink( "/proc/self/exe", self_path, PATH_MAX );
-+	if (count > 0) {
-+		self_path[count] = '\0';
-+
-+		// remove '/bin/python' from self_path
-+		for (i=count; i>0; --i) {
-+			if (self_path[i] == '/') break;
-+			self_path[i] = '\0';
-+		}
-+		self_path[i] = '\0';
-+		for (; i>0; --i) {
-+			if (self_path[i] == '/') break;
-+			self_path[i] = '\0';
-+		}
-+		self_path[i] = '\0';
-+
-+		if (1 == nftw(self_path, check_for_location_of_amdgpuids, 5, FTW_PHYS)) {
-+			fp = fopen(amdgpuids_path, "r");
-+			amdgpuids_path_msg = amdgpuids_path;
-+		}
-+	}
-+
-+	}
-+	else {
-+		amdgpuids_path_msg = AMDGPU_ASIC_ID_TABLE;
-+	}
-+
-+	// both hard-coded location and search have failed
- 	if (!fp) {
--		fprintf(stderr, "%s: %s\n", AMDGPU_ASIC_ID_TABLE,
--			strerror(errno));
-+		fprintf(stderr, "amdgpu.ids: No such file or directory\n");
- 		return;
- 	}
-
-@@ -132,7 +188,7 @@ void amdgpu_parse_asic_ids(struct amdgpu_device *dev)
- 			continue;
- 		}
-
--		drmMsg("%s version: %s\n", AMDGPU_ASIC_ID_TABLE, line);
-+		drmMsg("%s version: %s\n", amdgpuids_path_msg, line);
- 		break;
- 	}
-
-@@ -150,7 +206,7 @@ void amdgpu_parse_asic_ids(struct amdgpu_device *dev)
-
- 	if (r == -EINVAL) {
- 		fprintf(stderr, "Invalid format: %s: line %d: %s\n",
--			AMDGPU_ASIC_ID_TABLE, line_num, line);
-+			amdgpuids_path_msg, line_num, line);
- 	} else if (r && r != -EAGAIN) {
- 		fprintf(stderr, "%s: Cannot parse ASIC IDs: %s\n",
- 			__func__, strerror(-r));
-EOF
-
-###########################
-### build
-###########################
-meson builddir --prefix=/opt/amdgpu
-pushd builddir
-ninja install
-
-popd
-popd

.ci/docker/common/install_rocm_magma.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-# Script used in CI and CD pipeline
-
-set -ex
-
-
-MKLROOT=${MKLROOT:-/opt/conda/envs/py_$ANACONDA_PYTHON_VERSION}
-
-# "install" hipMAGMA into /opt/rocm/magma by copying after build
-git clone https://bitbucket.org/icl/magma.git
-pushd magma
-
-# Version 2.7.2 + ROCm related updates
-git checkout a1625ff4d9bc362906bd01f805dbbe12612953f6
-
-cp make.inc-examples/make.inc.hip-gcc-mkl make.inc
-echo 'LIBDIR += -L$(MKLROOT)/lib' >> make.inc
-if [[ -f "${MKLROOT}/lib/libmkl_core.a" ]]; then
-    echo 'LIB = -Wl,--start-group -lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -Wl,--end-group -lpthread -lstdc++ -lm -lgomp -lhipblas -lhipsparse' >> make.inc
-fi
-echo 'LIB += -Wl,--enable-new-dtags -Wl,--rpath,/opt/rocm/lib -Wl,--rpath,$(MKLROOT)/lib -Wl,--rpath,/opt/rocm/magma/lib -ldl' >> make.inc
-echo 'DEVCCFLAGS += --gpu-max-threads-per-block=256' >> make.inc
-export PATH="${PATH}:/opt/rocm/bin"
-if [[ -n "$PYTORCH_ROCM_ARCH" ]]; then
-  amdgpu_targets=`echo $PYTORCH_ROCM_ARCH | sed 's/;/ /g'`
-else
-  amdgpu_targets=`rocm_agent_enumerator | grep -v gfx000 | sort -u | xargs`
-fi
-for arch in $amdgpu_targets; do
-  echo "DEVCCFLAGS += --offload-arch=$arch" >> make.inc
-done
-# hipcc with openmp flag may cause isnan() on __device__ not to be found; depending on context, compiler may attempt to match with host definition
-sed -i 's/^FOPENMP/#FOPENMP/g' make.inc
-make -f make.gen.hipMAGMA -j $(nproc)
-LANG=C.UTF-8 make lib/libmagma.so -j $(nproc) MKLROOT="${MKLROOT}"
-make testing/testing_dgemm -j $(nproc) MKLROOT="${MKLROOT}"
-popd
-mv magma /opt/rocm

.ci/docker/common/install_triton.sh

@@ -1,86 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-get_conda_version() {
-  as_jenkins conda list -n py_$ANACONDA_PYTHON_VERSION | grep -w $* | head -n 1 | awk '{print $2}'
-}
-
-conda_reinstall() {
-  as_jenkins conda install -q -n py_$ANACONDA_PYTHON_VERSION -y --force-reinstall $*
-}
-
-if [ -n "${ROCM_VERSION}" ]; then
-  TRITON_REPO="https://github.com/openai/triton"
-  TRITON_TEXT_FILE="triton-rocm"
-elif [ -n "${XPU_VERSION}" ]; then
-  TRITON_REPO="https://github.com/intel/intel-xpu-backend-for-triton"
-  TRITON_TEXT_FILE="triton-xpu"
-else
-  TRITON_REPO="https://github.com/openai/triton"
-  TRITON_TEXT_FILE="triton"
-fi
-
-# The logic here is copied from .ci/pytorch/common_utils.sh
-TRITON_PINNED_COMMIT=$(get_pinned_commit ${TRITON_TEXT_FILE})
-
-if [ -n "${UBUNTU_VERSION}" ];then
-    apt update
-    apt-get install -y gpg-agent
-fi
-
-if [ -n "${CONDA_CMAKE}" ]; then
-  # Keep the current cmake and numpy version here, so we can reinstall them later
-  CMAKE_VERSION=$(get_conda_version cmake)
-  NUMPY_VERSION=$(get_conda_version numpy)
-fi
-
-if [ -z "${MAX_JOBS}" ]; then
-    export MAX_JOBS=$(nproc)
-fi
-
-# Git checkout triton
-mkdir /var/lib/jenkins/triton
-chown -R jenkins /var/lib/jenkins/triton
-chgrp -R jenkins /var/lib/jenkins/triton
-pushd /var/lib/jenkins/
-
-as_jenkins git clone ${TRITON_REPO} triton
-cd triton
-as_jenkins git checkout ${TRITON_PINNED_COMMIT}
-cd python
-
-# TODO: remove patch setup.py once we have a proper fix for https://github.com/triton-lang/triton/issues/4527
-as_jenkins sed -i -e 's/https:\/\/tritonlang.blob.core.windows.net\/llvm-builds/https:\/\/oaitriton.blob.core.windows.net\/public\/llvm-builds/g' setup.py
-
-if [ -n "${UBUNTU_VERSION}" ] && [ -n "${GCC_VERSION}" ] && [[ "${GCC_VERSION}" == "7" ]]; then
-  # Triton needs at least gcc-9 to build
-  apt-get install -y g++-9
-
-  CXX=g++-9 pip_install -e .
-elif [ -n "${UBUNTU_VERSION}" ] && [ -n "${CLANG_VERSION}" ]; then
-  # Triton needs <filesystem> which surprisingly is not available with clang-9 toolchain
-  add-apt-repository -y ppa:ubuntu-toolchain-r/test
-  apt-get install -y g++-9
-
-  CXX=g++-9 pip_install -e .
-else
-  pip_install -e .
-fi
-
-if [ -n "${CONDA_CMAKE}" ]; then
-  # TODO: This is to make sure that the same cmake and numpy version from install conda
-  # script is used. Without this step, the newer cmake version (3.25.2) downloaded by
-  # triton build step via pip will fail to detect conda MKL. Once that issue is fixed,
-  # this can be removed.
-  #
-  # The correct numpy version also needs to be set here because conda claims that it
-  # causes inconsistent environment.  Without this, conda will attempt to install the
-  # latest numpy version, which fails ASAN tests with the following import error: Numba
-  # needs NumPy 1.20 or less.
-  conda_reinstall cmake="${CMAKE_VERSION}"
-  # Note that we install numpy with pip as conda might not have the version we want
-  pip_install --force-reinstall numpy=="${NUMPY_VERSION}"
-fi

.ci/docker/common/install_ucc.sh

@@ -1,53 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-if [[ -d "/usr/local/cuda/" ]];  then
-  with_cuda=/usr/local/cuda/
-else
-  with_cuda=no
-fi
-
-function install_ucx() {
-  set -ex
-  git clone --recursive https://github.com/openucx/ucx.git
-  pushd ucx
-  git checkout ${UCX_COMMIT}
-  git submodule update --init --recursive
-
-  ./autogen.sh
-  ./configure --prefix=$UCX_HOME      \
-      --enable-mt                     \
-      --with-cuda=$with_cuda          \
-      --enable-profiling              \
-      --enable-stats
-  time make -j
-  sudo make install
-
-  popd
-  rm -rf ucx
-}
-
-function install_ucc() {
-  set -ex
-  git clone --recursive https://github.com/openucx/ucc.git
-  pushd ucc
-  git checkout ${UCC_COMMIT}
-  git submodule update --init --recursive
-
-  ./autogen.sh
-  # We only run distributed tests on Tesla M60 and A10G
-  NVCC_GENCODE="-gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_86,code=compute_86"
-  ./configure --prefix=$UCC_HOME          \
-    --with-ucx=$UCX_HOME                  \
-    --with-cuda=$with_cuda                \
-    --with-nvcc-gencode="${NVCC_GENCODE}"
-  time make -j
-  sudo make install
-
-  popd
-  rm -rf ucc
-}
-
-install_ucx
-install_ucc

.ci/docker/common/install_user.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# Mirror jenkins user in container
-# jenkins user as ec2-user should have the same user-id
-echo "jenkins:x:1000:1000::/var/lib/jenkins:" >> /etc/passwd
-echo "jenkins:x:1000:" >> /etc/group
-# Needed on focal or newer
-echo "jenkins:*:19110:0:99999:7:::" >>/etc/shadow
-
-# Create $HOME
-mkdir -p /var/lib/jenkins
-chown jenkins:jenkins /var/lib/jenkins
-mkdir -p /var/lib/jenkins/.ccache
-chown jenkins:jenkins /var/lib/jenkins/.ccache
-
-# Allow writing to /usr/local (for make install)
-chown jenkins:jenkins /usr/local
-
-# Allow sudo
-# TODO: Maybe we shouldn't
-echo 'jenkins ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/jenkins
-
-# Work around bug where devtoolset replaces sudo and breaks it.
-if [ -n "$DEVTOOLSET_VERSION" ]; then
-  SUDO=/bin/sudo
-else
-  SUDO=sudo
-fi
-
-# Test that sudo works
-$SUDO -u jenkins $SUDO -v

.ci/docker/common/install_xpu.sh

@@ -1,161 +0,0 @@
-#!/bin/bash
-set -xe
-# Script used in CI and CD pipeline
-
-# Intel® software for general purpose GPU capabilities.
-# Refer to https://www.intel.com/content/www/us/en/developer/articles/tool/pytorch-prerequisites-for-intel-gpus.html
-
-# Users should update to the latest version as it becomes available
-
-function install_ubuntu() {
-    . /etc/os-release
-    if [[ ! " jammy " =~ " ${VERSION_CODENAME} " ]]; then
-        echo "Ubuntu version ${VERSION_CODENAME} not supported"
-        exit
-    fi
-
-    apt-get update -y
-    apt-get install -y gpg-agent wget
-    # To add the online network package repository for the GPU Driver
-    wget -qO - https://repositories.intel.com/gpu/intel-graphics.key \
-        | gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics.gpg
-    echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] \
-        https://repositories.intel.com/gpu/ubuntu ${VERSION_CODENAME}${XPU_DRIVER_VERSION} unified" \
-        | tee /etc/apt/sources.list.d/intel-gpu-${VERSION_CODENAME}.list
-    # To add the online network network package repository for the Intel Support Packages
-    wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
-        | gpg --dearmor > /usr/share/keyrings/intel-for-pytorch-gpu-dev-keyring.gpg
-    echo "deb [signed-by=/usr/share/keyrings/intel-for-pytorch-gpu-dev-keyring.gpg] \
-        https://apt.repos.intel.com/intel-for-pytorch-gpu-dev all main" \
-        | tee /etc/apt/sources.list.d/intel-for-pytorch-gpu-dev.list
-
-    # Update the packages list and repository index
-    apt-get update
-
-    # The xpu-smi packages
-    apt-get install -y flex bison xpu-smi
-    # Compute and Media Runtimes
-    apt-get install -y \
-        intel-opencl-icd intel-level-zero-gpu level-zero \
-        intel-media-va-driver-non-free libmfx1 libmfxgen1 libvpl2 \
-        libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \
-        libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \
-        mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo
-    # Development Packages
-    apt-get install -y libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev level-zero-dev
-    # Install Intel Support Packages
-    if [ -n "$XPU_VERSION" ]; then
-        apt-get install -y intel-for-pytorch-gpu-dev-${XPU_VERSION} intel-pti-dev
-    else
-        apt-get install -y intel-for-pytorch-gpu-dev intel-pti-dev
-    fi
-
-    # Cleanup
-    apt-get autoclean && apt-get clean
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-}
-
-function install_rhel() {
-    . /etc/os-release
-    if [[ "${ID}" == "rhel" ]]; then
-        if [[ ! " 8.6 8.8 8.9 9.0 9.2 9.3 " =~ " ${VERSION_ID} " ]]; then
-            echo "RHEL version ${VERSION_ID} not supported"
-            exit
-        fi
-    elif [[ "${ID}" == "almalinux" ]]; then
-        # Workaround for almalinux8 which used by quay.io/pypa/manylinux_2_28_x86_64
-        VERSION_ID="8.6"
-    fi
-
-    dnf install -y 'dnf-command(config-manager)'
-    # To add the online network package repository for the GPU Driver
-    dnf config-manager --add-repo \
-        https://repositories.intel.com/gpu/rhel/${VERSION_ID}${XPU_DRIVER_VERSION}/unified/intel-gpu-${VERSION_ID}.repo
-    # To add the online network network package repository for the Intel Support Packages
-    tee > /etc/yum.repos.d/intel-for-pytorch-gpu-dev.repo << EOF
-[intel-for-pytorch-gpu-dev]
-name=Intel for Pytorch GPU dev repository
-baseurl=https://yum.repos.intel.com/intel-for-pytorch-gpu-dev
-enabled=1
-gpgcheck=1
-repo_gpgcheck=1
-gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-EOF
-
-    # The xpu-smi packages
-    dnf install -y xpu-smi
-    # Compute and Media Runtimes
-    dnf install --skip-broken -y \
-        intel-opencl intel-media intel-mediasdk libmfxgen1 libvpl2\
-        level-zero intel-level-zero-gpu mesa-dri-drivers mesa-vulkan-drivers \
-        mesa-vdpau-drivers libdrm mesa-libEGL mesa-libgbm mesa-libGL \
-        mesa-libxatracker libvpl-tools intel-metrics-discovery \
-        intel-metrics-library intel-igc-core intel-igc-cm \
-        libva libva-utils intel-gmmlib libmetee intel-gsc intel-ocloc
-    # Development packages
-    dnf install -y --refresh \
-        intel-igc-opencl-devel level-zero-devel intel-gsc-devel libmetee-devel \
-        level-zero-devel
-    # Install Intel Support Packages
-    yum install -y intel-for-pytorch-gpu-dev intel-pti-dev
-
-    # Cleanup
-    dnf clean all
-    rm -rf /var/cache/yum
-    rm -rf /var/lib/yum/yumdb
-    rm -rf /var/lib/yum/history
-}
-
-function install_sles() {
-    . /etc/os-release
-    VERSION_SP=${VERSION_ID//./sp}
-    if [[ ! " 15sp4 15sp5 " =~ " ${VERSION_SP} " ]]; then
-        echo "SLES version ${VERSION_ID} not supported"
-        exit
-    fi
-
-    # To add the online network package repository for the GPU Driver
-    zypper addrepo -f -r \
-        https://repositories.intel.com/gpu/sles/${VERSION_SP}${XPU_DRIVER_VERSION}/unified/intel-gpu-${VERSION_SP}.repo
-    rpm --import https://repositories.intel.com/gpu/intel-graphics.key
-    # To add the online network network package repository for the Intel Support Packages
-    zypper addrepo https://yum.repos.intel.com/intel-for-pytorch-gpu-dev intel-for-pytorch-gpu-dev
-    rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-
-    # The xpu-smi packages
-    zypper install -y lsb-release flex bison xpu-smi
-    # Compute and Media Runtimes
-    zypper install -y intel-level-zero-gpu level-zero intel-gsc intel-opencl intel-ocloc \
-        intel-media-driver libigfxcmrt7 libvpl2 libvpl-tools libmfxgen1 libmfx1
-    # Development packages
-    zypper install -y libigdfcl-devel intel-igc-cm libigfxcmrt-devel level-zero-devel
-
-    # Install Intel Support Packages
-    zypper install -y intel-for-pytorch-gpu-dev intel-pti-dev
-
-}
-
-# Default use GPU driver LTS releases
-XPU_DRIVER_VERSION="/lts/2350"
-if [[ "${XPU_DRIVER_TYPE,,}" == "rolling" ]]; then
-    # Use GPU driver rolling releases
-    XPU_DRIVER_VERSION=""
-fi
-
-# The installation depends on the base OS
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-    ubuntu)
-        install_ubuntu
-    ;;
-    rhel|almalinux)
-        install_rhel
-    ;;
-    sles)
-        install_sles
-    ;;
-    *)
-        echo "Unable to determine OS..."
-        exit 1
-    ;;
-esac

.ci/docker/conda/Dockerfile

@@ -1,100 +0,0 @@
-ARG CUDA_VERSION=10.2
-ARG BASE_TARGET=cuda${CUDA_VERSION}
-FROM centos:7 as base
-
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
-
-ARG DEVTOOLSET_VERSION=9
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-RUN yum update -y
-RUN yum install -y wget curl perl util-linux xz bzip2 git patch which unzip
-# Just add everything as a safe.directory for git since these will be used in multiple places with git
-RUN git config --global --add safe.directory '*'
-RUN yum install -y yum-utils centos-release-scl
-RUN yum-config-manager --enable rhel-server-rhscl-7-rpms
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-RUN yum install -y devtoolset-${DEVTOOLSET_VERSION}-gcc devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran devtoolset-${DEVTOOLSET_VERSION}-binutils
-# EPEL for cmake
-RUN yum --enablerepo=extras install -y epel-release
-
-# cmake
-RUN yum install -y cmake3 && \
-    ln -s /usr/bin/cmake3 /usr/bin/cmake
-ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
-
-RUN yum install -y autoconf aclocal automake make sudo
-RUN rm -rf /usr/local/cuda-*
-
-FROM base as patchelf
-# Install patchelf
-ADD ./common/install_patchelf.sh install_patchelf.sh
-RUN bash ./install_patchelf.sh && rm install_patchelf.sh && cp $(which patchelf) /patchelf
-
-FROM base as openssl
-# Install openssl
-ADD ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh && rm install_openssl.sh
-
-FROM base as conda
-# Install Anaconda
-ADD ./common/install_conda_docker.sh install_conda.sh
-RUN bash ./install_conda.sh && rm install_conda.sh
-
-# Install CUDA
-FROM base as cuda
-ARG CUDA_VERSION=10.2
-RUN rm -rf /usr/local/cuda-*
-ADD ./common/install_cuda.sh install_cuda.sh
-ENV CUDA_HOME=/usr/local/cuda-${CUDA_VERSION}
-# Preserve CUDA_VERSION for the builds
-ENV CUDA_VERSION=${CUDA_VERSION}
-# Make things in our path by default
-ENV PATH=/usr/local/cuda-${CUDA_VERSION}/bin:$PATH
-
-FROM cuda as cuda11.8
-RUN bash ./install_cuda.sh 11.8
-ENV DESIRED_CUDA=11.8
-
-FROM cuda as cuda12.1
-RUN bash ./install_cuda.sh 12.1
-ENV DESIRED_CUDA=12.1
-
-FROM cuda as cuda12.4
-RUN bash ./install_cuda.sh 12.4
-ENV DESIRED_CUDA=12.4
-
-# Install MNIST test data
-FROM base as mnist
-ADD ./common/install_mnist.sh install_mnist.sh
-RUN bash ./install_mnist.sh
-
-FROM base as all_cuda
-COPY --from=cuda11.8  /usr/local/cuda-11.8 /usr/local/cuda-11.8
-COPY --from=cuda12.1  /usr/local/cuda-12.1 /usr/local/cuda-12.1
-COPY --from=cuda12.4  /usr/local/cuda-12.4 /usr/local/cuda-12.4
-
-# Final step
-FROM ${BASE_TARGET} as final
-COPY --from=openssl            /opt/openssl           /opt/openssl
-COPY --from=patchelf           /patchelf              /usr/local/bin/patchelf
-COPY --from=conda              /opt/conda             /opt/conda
-
-# Add jni.h for java host build.
-COPY ./common/install_jni.sh install_jni.sh
-COPY ./java/jni.h jni.h
-RUN bash ./install_jni.sh && rm install_jni.sh
-
-ENV  PATH /opt/conda/bin:$PATH
-COPY --from=mnist  /usr/local/mnist /usr/local/mnist
-RUN rm -rf /usr/local/cuda
-RUN chmod o+rw /usr/local
-RUN touch /.condarc && \
-    chmod o+rw /.condarc && \
-    chmod -R o+rw /opt/conda

.ci/docker/conda/build.sh

@@ -1,76 +0,0 @@
-#!/usr/bin/env bash
-# Script used only in CD pipeline
-
-set -eou pipefail
-
-image="$1"
-shift
-
-if [ -z "${image}" ]; then
-  echo "Usage: $0 IMAGE"
-  exit 1
-fi
-
-DOCKER_IMAGE_NAME="pytorch/${image}"
-
-
-export DOCKER_BUILDKIT=1
-TOPDIR=$(git rev-parse --show-toplevel)
-
-CUDA_VERSION=${CUDA_VERSION:-12.1}
-
-case ${CUDA_VERSION} in
-  cpu)
-    BASE_TARGET=base
-    DOCKER_TAG=cpu
-    ;;
-  all)
-    BASE_TARGET=all_cuda
-    DOCKER_TAG=latest
-    ;;
-  *)
-    BASE_TARGET=cuda${CUDA_VERSION}
-    DOCKER_TAG=cuda${CUDA_VERSION}
-    ;;
-esac
-
-
-(
-  set -x
-  docker build \
-    --target final \
-    --progress plain \
-    --build-arg "BASE_TARGET=${BASE_TARGET}" \
-    --build-arg "CUDA_VERSION=${CUDA_VERSION}" \
-    --build-arg "DEVTOOLSET_VERSION=9" \
-    -t ${DOCKER_IMAGE_NAME} \
-    $@ \
-    -f "${TOPDIR}/.ci/docker/conda/Dockerfile" \
-    ${TOPDIR}/.ci/docker/
-)
-
-if [[ "${DOCKER_TAG}" =~ ^cuda* ]]; then
-  # Test that we're using the right CUDA compiler
-  (
-    set -x
-    docker run --rm "${DOCKER_IMAGE_NAME}" nvcc --version | grep "cuda_${CUDA_VERSION}"
-  )
-fi
-
-GITHUB_REF=${GITHUB_REF:-$(git symbolic-ref -q HEAD || git describe --tags --exact-match)}
-GIT_BRANCH_NAME=${GITHUB_REF##*/}
-GIT_COMMIT_SHA=${GITHUB_SHA:-$(git rev-parse HEAD)}
-DOCKER_IMAGE_BRANCH_TAG=${DOCKER_IMAGE_NAME}-${GIT_BRANCH_NAME}
-DOCKER_IMAGE_SHA_TAG=${DOCKER_IMAGE_NAME}-${GIT_COMMIT_SHA}
-if [[ "${WITH_PUSH:-}" == true ]]; then
-  (
-    set -x
-    docker push "${DOCKER_IMAGE_NAME}"
-    if [[ -n ${GITHUB_REF} ]]; then
-        docker tag ${DOCKER_IMAGE_NAME} ${DOCKER_IMAGE_BRANCH_TAG}
-        docker tag ${DOCKER_IMAGE_NAME} ${DOCKER_IMAGE_SHA_TAG}
-        docker push "${DOCKER_IMAGE_BRANCH_TAG}"
-        docker push "${DOCKER_IMAGE_SHA_TAG}"
-    fi
-  )
-fi

.ci/docker/libtorch/Dockerfile

@@ -1,107 +0,0 @@
-ARG BASE_TARGET=base
-ARG GPU_IMAGE=ubuntu:20.04
-FROM ${GPU_IMAGE} as base
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get clean && apt-get update
-RUN apt-get install -y curl locales g++ git-all autoconf automake make cmake wget unzip sudo
-# Just add everything as a safe.directory for git since these will be used in multiple places with git
-RUN git config --global --add safe.directory '*'
-
-RUN locale-gen en_US.UTF-8
-
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
-
-# Install openssl
-FROM base as openssl
-ADD ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh && rm install_openssl.sh
-
-# Install python
-FROM base as python
-ADD common/install_cpython.sh install_cpython.sh
-RUN apt-get update -y && \
-    apt-get install build-essential gdb lcov libbz2-dev libffi-dev \
-        libgdbm-dev liblzma-dev libncurses5-dev libreadline6-dev \
-        libsqlite3-dev libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g-dev -y && \
-    bash ./install_cpython.sh && \
-    rm install_cpython.sh && \
-    apt-get clean
-
-FROM base as conda
-ADD ./common/install_conda_docker.sh install_conda.sh
-RUN bash ./install_conda.sh && rm install_conda.sh
-
-FROM base as cpu
-# Install Anaconda
-COPY --from=conda /opt/conda /opt/conda
-# Install python
-COPY --from=python /opt/python    /opt/python
-COPY --from=python /opt/_internal /opt/_internal
-ENV PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
-# Install MKL
-ADD ./common/install_mkl.sh install_mkl.sh
-RUN bash ./install_mkl.sh && rm install_mkl.sh
-
-FROM cpu as cuda
-ADD ./common/install_cuda.sh install_cuda.sh
-ADD ./common/install_magma.sh install_magma.sh
-ENV CUDA_HOME /usr/local/cuda
-
-FROM cuda as cuda11.8
-RUN bash ./install_cuda.sh 11.8
-RUN bash ./install_magma.sh 11.8
-RUN ln -sf /usr/local/cuda-11.8 /usr/local/cuda
-
-FROM cuda as cuda12.1
-RUN bash ./install_cuda.sh 12.1
-RUN bash ./install_magma.sh 12.1
-RUN ln -sf /usr/local/cuda-12.1 /usr/local/cuda
-
-FROM cuda as cuda12.4
-RUN bash ./install_cuda.sh 12.4
-RUN bash ./install_magma.sh 12.4
-RUN ln -sf /usr/local/cuda-12.4 /usr/local/cuda
-
-FROM cpu as rocm
-ARG PYTORCH_ROCM_ARCH
-ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
-ENV MKLROOT /opt/intel
-# Adding ROCM_PATH env var so that LoadHip.cmake (even with logic updated for ROCm6.0)
-# find HIP works for ROCm5.7. Not needed for ROCm6.0 and above.
-# Remove below when ROCm5.7 is not in support matrix anymore.
-ENV ROCM_PATH /opt/rocm
-# No need to install ROCm as base docker image should have full ROCm install
-#ADD ./common/install_rocm.sh install_rocm.sh
-ADD ./common/install_rocm_drm.sh install_rocm_drm.sh
-ADD ./common/install_rocm_magma.sh install_rocm_magma.sh
-# gfortran and python needed for building magma from source for ROCm
-RUN apt-get update -y && \
-    apt-get install gfortran -y && \
-    apt-get install python -y && \
-    apt-get clean
-
-RUN bash ./install_rocm_drm.sh && rm install_rocm_drm.sh
-RUN bash ./install_rocm_magma.sh && rm install_rocm_magma.sh
-
-# Install AOTriton
-COPY ./common/common_utils.sh common_utils.sh
-COPY ./aotriton_version.txt aotriton_version.txt
-COPY ./common/install_aotriton.sh install_aotriton.sh
-RUN bash ./install_aotriton.sh /opt/rocm && rm install_aotriton.sh aotriton_version.txt
-ENV AOTRITON_INSTALLED_PREFIX /opt/rocm/aotriton
-
-FROM ${BASE_TARGET} as final
-COPY --from=openssl            /opt/openssl           /opt/openssl
-# Install patchelf
-ADD ./common/install_patchelf.sh install_patchelf.sh
-RUN bash ./install_patchelf.sh && rm install_patchelf.sh
-# Install Anaconda
-COPY --from=conda /opt/conda /opt/conda
-# Install python
-COPY --from=python /opt/python    /opt/python
-COPY --from=python /opt/_internal /opt/_internal
-ENV PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH

.ci/docker/libtorch/build.sh

@@ -1,93 +0,0 @@
-#!/usr/bin/env bash
-# Script used only in CD pipeline
-
-set -eou pipefail
-
-image="$1"
-shift
-
-if [ -z "${image}" ]; then
-  echo "Usage: $0 IMAGE"
-  exit 1
-fi
-
-DOCKER_IMAGE="pytorch/${image}"
-
-TOPDIR=$(git rev-parse --show-toplevel)
-
-GPU_ARCH_TYPE=${GPU_ARCH_TYPE:-cpu}
-GPU_ARCH_VERSION=${GPU_ARCH_VERSION:-}
-
-WITH_PUSH=${WITH_PUSH:-}
-
-DOCKER=${DOCKER:-docker}
-
-case ${GPU_ARCH_TYPE} in
-    cpu)
-        BASE_TARGET=cpu
-        DOCKER_TAG=cpu
-        GPU_IMAGE=ubuntu:20.04
-        DOCKER_GPU_BUILD_ARG=""
-        ;;
-    cuda)
-        BASE_TARGET=cuda${GPU_ARCH_VERSION}
-        DOCKER_TAG=cuda${GPU_ARCH_VERSION}
-        GPU_IMAGE=ubuntu:20.04
-        DOCKER_GPU_BUILD_ARG=""
-        ;;
-    rocm)
-        BASE_TARGET=rocm
-        DOCKER_TAG=rocm${GPU_ARCH_VERSION}
-        GPU_IMAGE=rocm/dev-ubuntu-20.04:${GPU_ARCH_VERSION}-complete
-        PYTORCH_ROCM_ARCH="gfx900;gfx906;gfx908;gfx90a;gfx1030;gfx1100"
-        ROCM_REGEX="([0-9]+)\.([0-9]+)[\.]?([0-9]*)"
-        if [[ $GPU_ARCH_VERSION =~ $ROCM_REGEX ]]; then
-            ROCM_VERSION_INT=$((${BASH_REMATCH[1]}*10000 + ${BASH_REMATCH[2]}*100 + ${BASH_REMATCH[3]:-0}))
-        else
-            echo "ERROR: rocm regex failed"
-            exit 1
-        fi
-        if [[ $ROCM_VERSION_INT -ge 60000 ]]; then
-            PYTORCH_ROCM_ARCH+=";gfx942"
-        fi
-        DOCKER_GPU_BUILD_ARG="--build-arg PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}"
-        ;;
-    *)
-        echo "ERROR: Unrecognized GPU_ARCH_TYPE: ${GPU_ARCH_TYPE}"
-        exit 1
-        ;;
-esac
-
-
-(
-    set -x
-    DOCKER_BUILDKIT=1 ${DOCKER} build \
-         --target final \
-        ${DOCKER_GPU_BUILD_ARG} \
-        --build-arg "GPU_IMAGE=${GPU_IMAGE}" \
-        --build-arg "BASE_TARGET=${BASE_TARGET}" \
-        -t "${DOCKER_IMAGE}" \
-        $@ \
-        -f "${TOPDIR}/.ci/docker/libtorch/Dockerfile" \
-        "${TOPDIR}/.ci/docker/"
-
-)
-
-GITHUB_REF=${GITHUB_REF:-$(git symbolic-ref -q HEAD || git describe --tags --exact-match)}
-GIT_BRANCH_NAME=${GITHUB_REF##*/}
-GIT_COMMIT_SHA=${GITHUB_SHA:-$(git rev-parse HEAD)}
-DOCKER_IMAGE_BRANCH_TAG=${DOCKER_IMAGE}-${GIT_BRANCH_NAME}
-DOCKER_IMAGE_SHA_TAG=${DOCKER_IMAGE}-${GIT_COMMIT_SHA}
-
-if [[ "${WITH_PUSH}" == true ]]; then
-  (
-    set -x
-    ${DOCKER} push "${DOCKER_IMAGE}"
-    if [[ -n ${GITHUB_REF} ]]; then
-        ${DOCKER} tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_BRANCH_TAG}
-        ${DOCKER} tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_SHA_TAG}
-        ${DOCKER} push "${DOCKER_IMAGE_BRANCH_TAG}"
-        ${DOCKER} push "${DOCKER_IMAGE_SHA_TAG}"
-    fi
-  )
-fi

.ci/docker/linter-cuda/Dockerfile

@@ -1,44 +0,0 @@
-ARG UBUNTU_VERSION
-
-FROM ubuntu:${UBUNTU_VERSION}
-
-ARG UBUNTU_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install missing libomp-dev
-RUN apt-get update && apt-get install -y --no-install-recommends libomp-dev && apt-get autoclean && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ARG CONDA_CMAKE
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-COPY requirements-ci.txt /opt/conda/requirements-ci.txt
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
-
-# Install cuda and cudnn
-ARG CUDA_VERSION
-COPY ./common/install_cuda.sh install_cuda.sh
-RUN bash ./install_cuda.sh ${CUDA_VERSION} && rm install_cuda.sh
-ENV DESIRED_CUDA ${CUDA_VERSION}
-ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH
-
-# Note that Docker build forbids copying file outside the build context
-COPY ./common/install_linter.sh install_linter.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_linter.sh
-RUN rm install_linter.sh common_utils.sh
-
-USER jenkins
-CMD ["bash"]

.ci/docker/linter/Dockerfile

@@ -1,34 +0,0 @@
-ARG UBUNTU_VERSION
-
-FROM ubuntu:${UBUNTU_VERSION}
-
-ARG UBUNTU_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ARG CONDA_CMAKE
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-COPY requirements-ci.txt /opt/conda/requirements-ci.txt
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
-
-# Note that Docker build forbids copying file outside the build context
-COPY ./common/install_linter.sh install_linter.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_linter.sh
-RUN rm install_linter.sh common_utils.sh
-
-USER jenkins
-CMD ["bash"]

.ci/docker/manywheel/Dockerfile

@@ -1,202 +0,0 @@
-# syntax = docker/dockerfile:experimental
-ARG ROCM_VERSION=3.7
-ARG BASE_CUDA_VERSION=11.8
-
-ARG GPU_IMAGE=centos:7
-FROM centos:7 as base
-
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
-
-ARG DEVTOOLSET_VERSION=9
-# Note: This is required patch since CentOS have reached EOL
-# otherwise any yum install setp will fail
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-RUN yum install -y wget curl perl util-linux xz bzip2 git patch which perl zlib-devel
-# Just add everything as a safe.directory for git since these will be used in multiple places with git
-RUN git config --global --add safe.directory '*'
-RUN yum install -y yum-utils centos-release-scl
-RUN yum-config-manager --enable rhel-server-rhscl-7-rpms
-# Note: After running yum-config-manager --enable rhel-server-rhscl-7-rpms
-# patch is required once again. Somehow this steps adds mirror.centos.org
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-RUN yum install -y devtoolset-${DEVTOOLSET_VERSION}-gcc devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran devtoolset-${DEVTOOLSET_VERSION}-binutils
-ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
-
-RUN yum --enablerepo=extras install -y epel-release
-
-# cmake-3.18.4 from pip
-RUN yum install -y python3-pip && \
-    python3 -mpip install cmake==3.18.4 && \
-    ln -s /usr/local/bin/cmake /usr/bin/cmake
-
-RUN yum install -y autoconf aclocal automake make sudo
-
-FROM base as openssl
-# Install openssl (this must precede `build python` step)
-# (In order to have a proper SSL module, Python is compiled
-# against a recent openssl [see env vars above], which is linked
-# statically. We delete openssl afterwards.)
-ADD ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh && rm install_openssl.sh
-
-# EPEL for cmake
-FROM base as patchelf
-# Install patchelf
-ADD ./common/install_patchelf.sh install_patchelf.sh
-RUN bash ./install_patchelf.sh && rm install_patchelf.sh
-RUN cp $(which patchelf) /patchelf
-
-FROM patchelf as python
-# build python
-COPY manywheel/build_scripts /build_scripts
-ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh
-RUN bash build_scripts/build.sh && rm -r build_scripts
-
-FROM base as cuda
-ARG BASE_CUDA_VERSION=10.2
-# Install CUDA
-ADD ./common/install_cuda.sh install_cuda.sh
-RUN bash ./install_cuda.sh ${BASE_CUDA_VERSION} && rm install_cuda.sh
-
-FROM base as intel
-# MKL
-ADD ./common/install_mkl.sh install_mkl.sh
-RUN bash ./install_mkl.sh && rm install_mkl.sh
-
-FROM base as magma
-ARG BASE_CUDA_VERSION=10.2
-# Install magma
-ADD ./common/install_magma.sh install_magma.sh
-RUN bash ./install_magma.sh ${BASE_CUDA_VERSION} && rm install_magma.sh
-
-FROM base as jni
-# Install java jni header
-ADD ./common/install_jni.sh install_jni.sh
-ADD ./java/jni.h jni.h
-RUN bash ./install_jni.sh && rm install_jni.sh
-
-FROM base as libpng
-# Install libpng
-ADD ./common/install_libpng.sh install_libpng.sh
-RUN bash ./install_libpng.sh && rm install_libpng.sh
-
-FROM ${GPU_IMAGE} as common
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
-RUN yum install -y \
-        aclocal \
-        autoconf \
-        automake \
-        bison \
-        bzip2 \
-        curl \
-        diffutils \
-        file \
-        git \
-        make \
-        patch \
-        perl \
-        unzip \
-        util-linux \
-        wget \
-        which \
-        xz \
-        yasm
-RUN yum install -y \
-    https://repo.ius.io/ius-release-el7.rpm \
-    https://ossci-linux.s3.amazonaws.com/epel-release-7-14.noarch.rpm
-
-RUN yum swap -y git git236-core
-# git236+ would refuse to run git commands in repos owned by other users
-# Which causes version check to fail, as pytorch repo is bind-mounted into the image
-# Override this behaviour by treating every folder as safe
-# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
-RUN git config --global --add safe.directory "*"
-
-ENV SSL_CERT_FILE=/opt/_internal/certs.pem
-# Install LLVM version
-COPY --from=openssl            /opt/openssl                          /opt/openssl
-COPY --from=python             /opt/python                           /opt/python
-COPY --from=python             /opt/_internal                        /opt/_internal
-COPY --from=python             /opt/python/cp39-cp39/bin/auditwheel /usr/local/bin/auditwheel
-COPY --from=intel              /opt/intel                            /opt/intel
-COPY --from=patchelf           /usr/local/bin/patchelf               /usr/local/bin/patchelf
-COPY --from=jni                /usr/local/include/jni.h              /usr/local/include/jni.h
-COPY --from=libpng             /usr/local/bin/png*                   /usr/local/bin/
-COPY --from=libpng             /usr/local/bin/libpng*                /usr/local/bin/
-COPY --from=libpng             /usr/local/include/png*               /usr/local/include/
-COPY --from=libpng             /usr/local/include/libpng*            /usr/local/include/
-COPY --from=libpng             /usr/local/lib/libpng*                /usr/local/lib/
-COPY --from=libpng             /usr/local/lib/pkgconfig              /usr/local/lib/pkgconfig
-
-FROM common as cpu_final
-ARG BASE_CUDA_VERSION=10.1
-ARG DEVTOOLSET_VERSION=9
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-
-RUN yum install -y yum-utils centos-release-scl
-RUN yum-config-manager --enable rhel-server-rhscl-7-rpms
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-RUN yum install -y devtoolset-${DEVTOOLSET_VERSION}-gcc devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran devtoolset-${DEVTOOLSET_VERSION}-binutils
-ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
-
-# cmake is already installed inside the rocm base image, so remove if present
-RUN rpm -e cmake || true
-# cmake-3.18.4 from pip
-RUN yum install -y python3-pip && \
-    python3 -mpip install cmake==3.18.4 && \
-    ln -s /usr/local/bin/cmake /usr/bin/cmake
-
-# ninja
-RUN yum install -y ninja-build
-
-FROM cpu_final as cuda_final
-RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION}
-COPY --from=cuda     /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
-COPY --from=magma    /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
-RUN ln -sf /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda
-ENV PATH=/usr/local/cuda/bin:$PATH
-
-FROM cpu_final as rocm_final
-ARG ROCM_VERSION=3.7
-ARG PYTORCH_ROCM_ARCH
-ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
-# Adding ROCM_PATH env var so that LoadHip.cmake (even with logic updated for ROCm6.0)
-# find HIP works for ROCm5.7. Not needed for ROCm6.0 and above.
-# Remove below when ROCm5.7 is not in support matrix anymore.
-ENV ROCM_PATH /opt/rocm
-ENV MKLROOT /opt/intel
-# No need to install ROCm as base docker image should have full ROCm install
-#ADD ./common/install_rocm.sh install_rocm.sh
-#RUN ROCM_VERSION=${ROCM_VERSION} bash ./install_rocm.sh && rm install_rocm.sh
-ADD ./common/install_rocm_drm.sh install_rocm_drm.sh
-RUN bash ./install_rocm_drm.sh && rm install_rocm_drm.sh
-# cmake3 is needed for the MIOpen build
-RUN ln -sf /usr/local/bin/cmake /usr/bin/cmake3
-ADD ./common/install_rocm_magma.sh install_rocm_magma.sh
-RUN bash ./install_rocm_magma.sh && rm install_rocm_magma.sh
-ADD ./common/install_miopen.sh install_miopen.sh
-RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh
-
-# Install AOTriton
-COPY ./common/common_utils.sh common_utils.sh
-COPY ./aotriton_version.txt aotriton_version.txt
-COPY ./common/install_aotriton.sh install_aotriton.sh
-RUN bash ./install_aotriton.sh /opt/rocm && rm install_aotriton.sh aotriton_version.txt
-ENV AOTRITON_INSTALLED_PREFIX /opt/rocm/aotriton

.ci/docker/manywheel/Dockerfile_2014

@@ -1,153 +0,0 @@
-# syntax = docker/dockerfile:experimental
-ARG ROCM_VERSION=3.7
-ARG BASE_CUDA_VERSION=10.2
-ARG GPU_IMAGE=nvidia/cuda:${BASE_CUDA_VERSION}-devel-centos7
-FROM quay.io/pypa/manylinux2014_x86_64 as base
-
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
-
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-RUN yum install -y wget curl perl util-linux xz bzip2 git patch which perl zlib-devel
-RUN yum install -y yum-utils centos-release-scl sudo
-RUN yum-config-manager --enable rhel-server-rhscl-7-rpms
-RUN yum install -y devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran devtoolset-7-binutils
-ENV PATH=/opt/rh/devtoolset-7/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:$LD_LIBRARY_PATH
-
-# cmake
-RUN yum install -y cmake3 && \
-    ln -s /usr/bin/cmake3 /usr/bin/cmake
-FROM base as openssl
-# Install openssl (this must precede `build python` step)
-# (In order to have a proper SSL module, Python is compiled
-# against a recent openssl [see env vars above], which is linked
-# statically. We delete openssl afterwards.)
-ADD ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh && rm install_openssl.sh
-
-
-
-# remove unncessary python versions
-RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2
-RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4
-RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6
-RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6
-
-FROM base as cuda
-ARG BASE_CUDA_VERSION=10.2
-# Install CUDA
-ADD ./common/install_cuda.sh install_cuda.sh
-RUN bash ./install_cuda.sh ${BASE_CUDA_VERSION} && rm install_cuda.sh
-
-FROM base as intel
-# MKL
-ADD ./common/install_mkl.sh install_mkl.sh
-RUN bash ./install_mkl.sh && rm install_mkl.sh
-
-FROM base as magma
-ARG BASE_CUDA_VERSION=10.2
-# Install magma
-ADD ./common/install_magma.sh install_magma.sh
-RUN bash ./install_magma.sh ${BASE_CUDA_VERSION} && rm install_magma.sh
-
-FROM base as jni
-# Install java jni header
-ADD ./common/install_jni.sh install_jni.sh
-ADD ./java/jni.h jni.h
-RUN bash ./install_jni.sh && rm install_jni.sh
-
-FROM base as libpng
-# Install libpng
-ADD ./common/install_libpng.sh install_libpng.sh
-RUN bash ./install_libpng.sh && rm install_libpng.sh
-
-FROM ${GPU_IMAGE} as common
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
-RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
-RUN yum install -y \
-        aclocal \
-        autoconf \
-        automake \
-        bison \
-        bzip2 \
-        curl \
-        diffutils \
-        file \
-        git \
-        make \
-        patch \
-        perl \
-        unzip \
-        util-linux \
-        wget \
-        which \
-        xz \
-        yasm
-RUN yum install -y \
-    https://repo.ius.io/ius-release-el7.rpm \
-    https://ossci-linux.s3.amazonaws.com/epel-release-7-14.noarch.rpm
-
-RUN yum swap -y git git236-core
-# git236+ would refuse to run git commands in repos owned by other users
-# Which causes version check to fail, as pytorch repo is bind-mounted into the image
-# Override this behaviour by treating every folder as safe
-# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
-RUN git config --global --add safe.directory "*"
-
-ENV SSL_CERT_FILE=/opt/_internal/certs.pem
-# Install LLVM version
-COPY --from=openssl            /opt/openssl                          /opt/openssl
-COPY --from=base               /opt/python                           /opt/python
-COPY --from=base               /opt/_internal                        /opt/_internal
-COPY --from=base               /usr/local/bin/auditwheel             /usr/local/bin/auditwheel
-COPY --from=intel              /opt/intel                            /opt/intel
-COPY --from=base               /usr/local/bin/patchelf               /usr/local/bin/patchelf
-COPY --from=libpng             /usr/local/bin/png*                   /usr/local/bin/
-COPY --from=libpng             /usr/local/bin/libpng*                /usr/local/bin/
-COPY --from=libpng             /usr/local/include/png*               /usr/local/include/
-COPY --from=libpng             /usr/local/include/libpng*            /usr/local/include/
-COPY --from=libpng             /usr/local/lib/libpng*                /usr/local/lib/
-COPY --from=libpng             /usr/local/lib/pkgconfig              /usr/local/lib/pkgconfig
-COPY --from=jni                /usr/local/include/jni.h              /usr/local/include/jni.h
-
-FROM common as cpu_final
-ARG BASE_CUDA_VERSION=10.2
-RUN yum install -y yum-utils centos-release-scl
-RUN yum-config-manager --enable rhel-server-rhscl-7-rpms
-RUN yum install -y devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran devtoolset-7-binutils
-ENV PATH=/opt/rh/devtoolset-7/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:$LD_LIBRARY_PATH
-
-# cmake
-RUN yum install -y cmake3 && \
-    ln -s /usr/bin/cmake3 /usr/bin/cmake
-
-# ninja
-RUN yum install -y http://repo.okay.com.mx/centos/7/x86_64/release/okay-release-1-1.noarch.rpm
-RUN yum install -y ninja-build
-
-FROM cpu_final as cuda_final
-RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION}
-COPY --from=cuda     /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
-COPY --from=magma    /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
-
-FROM common as rocm_final
-ARG ROCM_VERSION=3.7
-# Install ROCm
-ADD ./common/install_rocm.sh install_rocm.sh
-RUN bash ./install_rocm.sh ${ROCM_VERSION} && rm install_rocm.sh
-# cmake is already installed inside the rocm base image, but both 2 and 3 exist
-# cmake3 is needed for the later MIOpen custom build, so that step is last.
-RUN yum install -y cmake3 && \
-    rm -f /usr/bin/cmake && \
-    ln -s /usr/bin/cmake3 /usr/bin/cmake
-ADD ./common/install_miopen.sh install_miopen.sh
-RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh

.ci/docker/manywheel/Dockerfile_2_28

@@ -1,157 +0,0 @@
-# syntax = docker/dockerfile:experimental
-ARG ROCM_VERSION=3.7
-ARG BASE_CUDA_VERSION=11.8
-ARG GPU_IMAGE=amd64/almalinux:8
-FROM quay.io/pypa/manylinux_2_28_x86_64 as base
-
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
-
-ARG DEVTOOLSET_VERSION=11
-RUN yum install -y sudo wget curl perl util-linux xz bzip2 git patch which perl zlib-devel yum-utils gcc-toolset-${DEVTOOLSET_VERSION}-toolchain
-ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
-
-# cmake-3.18.4 from pip
-RUN yum install -y python3-pip && \
-    python3 -mpip install cmake==3.18.4 && \
-    ln -s /usr/local/bin/cmake /usr/bin/cmake3
-
-FROM base as openssl
-# Install openssl (this must precede `build python` step)
-# (In order to have a proper SSL module, Python is compiled
-# against a recent openssl [see env vars above], which is linked
-# statically. We delete openssl afterwards.)
-ADD ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh && rm install_openssl.sh
-
-
-# remove unncessary python versions
-RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2
-RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4
-RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6
-RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6
-
-FROM base as cuda
-ARG BASE_CUDA_VERSION=11.8
-# Install CUDA
-ADD ./common/install_cuda.sh install_cuda.sh
-RUN bash ./install_cuda.sh ${BASE_CUDA_VERSION} && rm install_cuda.sh
-
-FROM base as intel
-# MKL
-ADD ./common/install_mkl.sh install_mkl.sh
-RUN bash ./install_mkl.sh && rm install_mkl.sh
-
-FROM base as magma
-ARG BASE_CUDA_VERSION=10.2
-# Install magma
-ADD ./common/install_magma.sh install_magma.sh
-RUN bash ./install_magma.sh ${BASE_CUDA_VERSION} && rm install_magma.sh
-
-FROM base as jni
-# Install java jni header
-ADD ./common/install_jni.sh install_jni.sh
-ADD ./java/jni.h jni.h
-RUN bash ./install_jni.sh && rm install_jni.sh
-
-FROM base as libpng
-# Install libpng
-ADD ./common/install_libpng.sh install_libpng.sh
-RUN bash ./install_libpng.sh && rm install_libpng.sh
-
-FROM ${GPU_IMAGE} as common
-ARG DEVTOOLSET_VERSION=11
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
-RUN yum -y install epel-release
-RUN yum -y update
-RUN yum install -y \
-        autoconf \
-        automake \
-        bison \
-        bzip2 \
-        curl \
-        diffutils \
-        file \
-        git \
-        make \
-        patch \
-        perl \
-        unzip \
-        util-linux \
-        wget \
-        which \
-        xz \
-        gcc-toolset-${DEVTOOLSET_VERSION}-toolchain \
-        glibc-langpack-en
-RUN yum install -y \
-    https://repo.ius.io/ius-release-el7.rpm \
-    https://ossci-linux.s3.amazonaws.com/epel-release-7-14.noarch.rpm
-
-RUN yum swap -y git git236-core
-# git236+ would refuse to run git commands in repos owned by other users
-# Which causes version check to fail, as pytorch repo is bind-mounted into the image
-# Override this behaviour by treating every folder as safe
-# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
-RUN git config --global --add safe.directory "*"
-
-ENV SSL_CERT_FILE=/opt/_internal/certs.pem
-# Install LLVM version
-COPY --from=openssl            /opt/openssl                          /opt/openssl
-COPY --from=base               /opt/python                           /opt/python
-COPY --from=base               /opt/_internal                        /opt/_internal
-COPY --from=base               /usr/local/bin/auditwheel             /usr/local/bin/auditwheel
-COPY --from=intel              /opt/intel                            /opt/intel
-COPY --from=base               /usr/local/bin/patchelf               /usr/local/bin/patchelf
-COPY --from=libpng             /usr/local/bin/png*                   /usr/local/bin/
-COPY --from=libpng             /usr/local/bin/libpng*                /usr/local/bin/
-COPY --from=libpng             /usr/local/include/png*               /usr/local/include/
-COPY --from=libpng             /usr/local/include/libpng*            /usr/local/include/
-COPY --from=libpng             /usr/local/lib/libpng*                /usr/local/lib/
-COPY --from=libpng             /usr/local/lib/pkgconfig              /usr/local/lib/pkgconfig
-COPY --from=jni                /usr/local/include/jni.h              /usr/local/include/jni.h
-
-FROM common as cpu_final
-ARG BASE_CUDA_VERSION=11.8
-ARG DEVTOOLSET_VERSION=11
-# Ensure the expected devtoolset is used
-ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
-
-# cmake-3.18.4 from pip
-RUN yum install -y python3-pip && \
-    python3 -mpip install cmake==3.18.4 && \
-    ln -s /usr/local/bin/cmake /usr/bin/cmake3
-
-FROM cpu_final as cuda_final
-RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION}
-COPY --from=cuda     /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
-COPY --from=magma    /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
-
-FROM common as rocm_final
-ARG ROCM_VERSION=3.7
-# Install ROCm
-ADD ./common/install_rocm.sh install_rocm.sh
-RUN bash ./install_rocm.sh ${ROCM_VERSION} && rm install_rocm.sh
-# cmake is already installed inside the rocm base image, but both 2 and 3 exist
-# cmake3 is needed for the later MIOpen custom build, so that step is last.
-RUN yum install -y cmake3 && \
-    rm -f /usr/bin/cmake && \
-    ln -s /usr/bin/cmake3 /usr/bin/cmake
-ADD ./common/install_miopen.sh install_miopen.sh
-RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh
-
-FROM cpu_final as xpu_final
-# XPU CD use rolling driver
-ENV XPU_DRIVER_TYPE ROLLING
-# cmake-3.28.4 from pip
-RUN python3 -m pip install --upgrade pip && \
-    python3 -mpip install cmake==3.28.4
-# Install setuptools and wheel for python 3.13
-RUN /opt/python/cp313-cp313/bin/python -m pip install setuptools wheel
-ADD ./common/install_xpu.sh install_xpu.sh
-RUN bash ./install_xpu.sh && rm install_xpu.sh
-RUN pushd /opt/_internal && tar -xJf static-libs-for-embedding-only.tar.xz && popd

.ci/docker/manywheel/Dockerfile_2_28_aarch64

@@ -1,57 +0,0 @@
-FROM quay.io/pypa/manylinux_2_28_aarch64 as base
-
-# Graviton needs GCC 10 or above for the build. GCC12 is the default version in almalinux-8.
-ARG GCCTOOLSET_VERSION=11
-
-# Language variabes
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US.UTF-8
-
-# Installed needed OS packages. This is to support all
-# the binary builds (torch, vision, audio, text, data)
-RUN yum -y install epel-release
-RUN yum -y update
-RUN yum install -y \
-  autoconf \
-  automake \
-  bison \
-  bzip2 \
-  curl \
-  diffutils \
-  file \
-  git \
-  less \
-  libffi-devel \
-  libgomp \
-  make \
-  openssl-devel \
-  patch \
-  perl \
-  unzip \
-  util-linux \
-  wget \
-  which \
-  xz \
-  yasm \
-  zstd \
-  sudo \
-  gcc-toolset-${GCCTOOLSET_VERSION}-toolchain
-
-# Ensure the expected devtoolset is used
-ENV PATH=/opt/rh/gcc-toolset-${GCCTOOLSET_VERSION}/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${GCCTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${GCCTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
-
-# git236+ would refuse to run git commands in repos owned by other users
-# Which causes version check to fail, as pytorch repo is bind-mounted into the image
-# Override this behaviour by treating every folder as safe
-# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
-RUN git config --global --add safe.directory "*"
-
-FROM base as final
-
-# remove unncessary python versions
-RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2
-RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4
-RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6
-RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6

.ci/docker/manywheel/Dockerfile_aarch64

@@ -1,94 +0,0 @@
-FROM quay.io/pypa/manylinux2014_aarch64 as base
-
-
-# Graviton needs GCC 10 for the build
-ARG DEVTOOLSET_VERSION=10
-
-# Language variabes
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US.UTF-8
-
-# Installed needed OS packages. This is to support all
-# the binary builds (torch, vision, audio, text, data)
-RUN yum -y install epel-release
-RUN yum -y update
-RUN yum install -y \
-  autoconf \
-  automake \
-  bison \
-  bzip2 \
-  curl \
-  diffutils \
-  file \
-  git \
-  make \
-  patch \
-  perl \
-  unzip \
-  util-linux \
-  wget \
-  which \
-  xz \
-  yasm \
-  less \
-  zstd \
-  libgomp \
-  sudo \
-  devtoolset-${DEVTOOLSET_VERSION}-gcc \
-  devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ \
-  devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran \
-  devtoolset-${DEVTOOLSET_VERSION}-binutils
-
-# Ensure the expected devtoolset is used
-ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
-
-
-# git236+ would refuse to run git commands in repos owned by other users
-# Which causes version check to fail, as pytorch repo is bind-mounted into the image
-# Override this behaviour by treating every folder as safe
-# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
-RUN git config --global --add safe.directory "*"
-
-
-###############################################################################
-# libglfortran.a hack
-#
-# libgfortran.a from quay.io/pypa/manylinux2014_aarch64 is not compiled with -fPIC.
-# This causes __stack_chk_guard@@GLIBC_2.17 on pytorch build. To solve, get
-# ubuntu's libgfortran.a which is compiled with -fPIC
-# NOTE: Need a better way to get this library as Ubuntu's package can be removed by the vender, or changed
-###############################################################################
-RUN cd ~/ \
-  && curl -L -o ~/libgfortran-10-dev.deb http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/gcc-10/libgfortran-10-dev_10.5.0-1ubuntu1_arm64.deb \
-  && ar x ~/libgfortran-10-dev.deb \
-  && tar --use-compress-program=unzstd -xvf data.tar.zst -C ~/ \
-  && cp -f ~/usr/lib/gcc/aarch64-linux-gnu/10/libgfortran.a /opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10/
-
-# install cmake
-RUN yum install -y cmake3 && \
-    ln -s /usr/bin/cmake3 /usr/bin/cmake
-
-FROM base as openssl
-# Install openssl (this must precede `build python` step)
-# (In order to have a proper SSL module, Python is compiled
-# against a recent openssl [see env vars above], which is linked
-# statically. We delete openssl afterwards.)
-ADD ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh && rm install_openssl.sh
-ENV SSL_CERT_FILE=/opt/_internal/certs.pem
-
-FROM base as openblas
-# Install openblas
-ADD ./common/install_openblas.sh install_openblas.sh
-RUN bash ./install_openblas.sh && rm install_openblas.sh
-
-FROM openssl as final
-# remove unncessary python versions
-RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2
-RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4
-RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6
-RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6
-COPY --from=openblas     /opt/OpenBLAS/  /opt/OpenBLAS/
-ENV LD_LIBRARY_PATH=/opt/OpenBLAS/lib:$LD_LIBRARY_PATH

.ci/docker/manywheel/Dockerfile_cuda_aarch64

@@ -1,91 +0,0 @@
-FROM quay.io/pypa/manylinux_2_28_aarch64 as base
-
-# Cuda ARM build needs gcc 11
-ARG DEVTOOLSET_VERSION=11
-
-# Language variables
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US.UTF-8
-
-# Installed needed OS packages. This is to support all
-# the binary builds (torch, vision, audio, text, data)
-RUN yum -y install epel-release
-RUN yum -y update
-RUN yum install -y \
-  autoconf \
-  automake \
-  bison \
-  bzip2 \
-  curl \
-  diffutils \
-  file \
-  git \
-  make \
-  patch \
-  perl \
-  unzip \
-  util-linux \
-  wget \
-  which \
-  xz \
-  yasm \
-  less \
-  zstd \
-  libgomp \
-  sudo \
-  gcc-toolset-${DEVTOOLSET_VERSION}-toolchain
-
-# Ensure the expected devtoolset is used
-ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
-
-# git236+ would refuse to run git commands in repos owned by other users
-# Which causes version check to fail, as pytorch repo is bind-mounted into the image
-# Override this behaviour by treating every folder as safe
-# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
-RUN git config --global --add safe.directory "*"
-
-
-FROM base as openssl
-# Install openssl (this must precede `build python` step)
-# (In order to have a proper SSL module, Python is compiled
-# against a recent openssl [see env vars above], which is linked
-# statically. We delete openssl afterwards.)
-ADD ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh && rm install_openssl.sh
-ENV SSL_CERT_FILE=/opt/_internal/certs.pem
-
-FROM openssl as final
-# remove unncessary python versions
-RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2
-RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4
-RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6
-RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6
-
-FROM base as cuda
-ARG BASE_CUDA_VERSION
-# Install CUDA
-ADD ./common/install_cuda_aarch64.sh install_cuda_aarch64.sh
-RUN bash ./install_cuda_aarch64.sh ${BASE_CUDA_VERSION} && rm install_cuda_aarch64.sh
-
-FROM base as magma
-ARG BASE_CUDA_VERSION
-# Install magma
-ADD ./common/install_magma.sh install_magma.sh
-RUN bash ./install_magma.sh ${BASE_CUDA_VERSION} && rm install_magma.sh
-
-FROM base as nvpl
-# Install nvpl
-ADD ./common/install_nvpl.sh install_nvpl.sh
-RUN bash ./install_nvpl.sh && rm install_nvpl.sh
-
-FROM final as cuda_final
-ARG BASE_CUDA_VERSION
-RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION}
-COPY --from=cuda     /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
-COPY --from=magma    /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
-COPY --from=nvpl /opt/nvpl/lib/  /usr/local/lib/
-COPY --from=nvpl /opt/nvpl/include/  /usr/local/include/
-RUN ln -sf /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda
-ENV PATH=/usr/local/cuda/bin:$PATH

.ci/docker/manywheel/Dockerfile_cxx11-abi

@@ -1,71 +0,0 @@
-FROM centos:8 as base
-
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
-ENV PATH /opt/rh/gcc-toolset-11/root/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
-# change to a valid repo
-RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-*.repo
-# enable to install ninja-build
-RUN sed -i 's|enabled=0|enabled=1|g' /etc/yum.repos.d/CentOS-Linux-PowerTools.repo
-
-RUN yum -y update
-RUN yum install -y wget curl perl util-linux xz bzip2 git patch which zlib-devel sudo
-RUN yum install -y autoconf automake make cmake gdb gcc-toolset-11-gcc-c++
-
-
-FROM base as openssl
-ADD ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh && rm install_openssl.sh
-
-# Install python
-FROM base as python
-RUN yum install -y openssl-devel zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel libpcap-devel xz-devel libffi-devel
-ADD common/install_cpython.sh install_cpython.sh
-RUN bash ./install_cpython.sh && rm install_cpython.sh
-
-FROM base as conda
-ADD ./common/install_conda_docker.sh install_conda.sh
-RUN bash ./install_conda.sh && rm install_conda.sh
-RUN /opt/conda/bin/conda install -y cmake
-
-FROM base as intel
-# Install MKL
-COPY --from=python             /opt/python                           /opt/python
-COPY --from=python             /opt/_internal                        /opt/_internal
-COPY --from=conda              /opt/conda                            /opt/conda
-ENV PATH=/opt/conda/bin:$PATH
-ADD ./common/install_mkl.sh install_mkl.sh
-RUN bash ./install_mkl.sh && rm install_mkl.sh
-
-FROM base as patchelf
-ADD ./common/install_patchelf.sh install_patchelf.sh
-RUN bash ./install_patchelf.sh && rm install_patchelf.sh
-RUN cp $(which patchelf) /patchelf
-
-FROM base as jni
-ADD ./common/install_jni.sh install_jni.sh
-ADD ./java/jni.h jni.h
-RUN bash ./install_jni.sh && rm install_jni.sh
-
-FROM base as libpng
-ADD ./common/install_libpng.sh install_libpng.sh
-RUN bash ./install_libpng.sh && rm install_libpng.sh
-
-FROM base as final
-COPY --from=openssl            /opt/openssl                          /opt/openssl
-COPY --from=python             /opt/python                           /opt/python
-COPY --from=python             /opt/_internal                        /opt/_internal
-COPY --from=intel              /opt/intel                            /opt/intel
-COPY --from=conda              /opt/conda                            /opt/conda
-COPY --from=patchelf           /usr/local/bin/patchelf               /usr/local/bin/patchelf
-COPY --from=jni                /usr/local/include/jni.h              /usr/local/include/jni.h
-COPY --from=libpng             /usr/local/bin/png*                   /usr/local/bin/
-COPY --from=libpng             /usr/local/bin/libpng*                /usr/local/bin/
-COPY --from=libpng             /usr/local/include/png*               /usr/local/include/
-COPY --from=libpng             /usr/local/include/libpng*            /usr/local/include/
-COPY --from=libpng             /usr/local/lib/libpng*                /usr/local/lib/
-COPY --from=libpng             /usr/local/lib/pkgconfig              /usr/local/lib/pkgconfig
-
-RUN yum install -y ninja-build

.ci/docker/manywheel/Dockerfile_s390x

@@ -1,73 +0,0 @@
-FROM --platform=linux/s390x docker.io/ubuntu:24.04 as base
-
-# Language variables
-ENV LC_ALL=C.UTF-8
-ENV LANG=C.UTF-8
-ENV LANGUAGE=C.UTF-8
-
-# Installed needed OS packages. This is to support all
-# the binary builds (torch, vision, audio, text, data)
-RUN apt update ; apt upgrade -y
-RUN apt install -y \
-  build-essential \
-  autoconf \
-  automake \
-  bzip2 \
-  curl \
-  diffutils \
-  file \
-  git \
-  make \
-  patch \
-  perl \
-  unzip \
-  util-linux \
-  wget \
-  which \
-  xz-utils \
-  less \
-  zstd \
-  cmake \
-  python3 \
-  python3-dev \
-  python3-setuptools \
-  python3-yaml \
-  python3-typing-extensions \
-  libblas-dev \
-  libopenblas-dev \
-  liblapack-dev \
-  libatlas-base-dev
-
-# git236+ would refuse to run git commands in repos owned by other users
-# Which causes version check to fail, as pytorch repo is bind-mounted into the image
-# Override this behaviour by treating every folder as safe
-# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
-RUN git config --global --add safe.directory "*"
-
-FROM base as openssl
-# Install openssl (this must precede `build python` step)
-# (In order to have a proper SSL module, Python is compiled
-# against a recent openssl [see env vars above], which is linked
-# statically. We delete openssl afterwards.)
-ADD ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh && rm install_openssl.sh
-ENV SSL_CERT_FILE=/opt/_internal/certs.pem
-
-# EPEL for cmake
-FROM base as patchelf
-# Install patchelf
-ADD ./common/install_patchelf.sh install_patchelf.sh
-RUN bash ./install_patchelf.sh && rm install_patchelf.sh
-RUN cp $(which patchelf) /patchelf
-
-FROM patchelf as python
-# build python
-COPY manywheel/build_scripts /build_scripts
-ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh
-RUN bash build_scripts/build.sh && rm -r build_scripts
-
-FROM openssl as final
-COPY --from=python             /opt/python                           /opt/python
-COPY --from=python             /opt/_internal                        /opt/_internal
-COPY --from=python             /opt/python/cp39-cp39/bin/auditwheel /usr/local/bin/auditwheel
-COPY --from=patchelf           /usr/local/bin/patchelf               /usr/local/bin/patchelf

.ci/docker/manywheel/build.sh

@@ -1,154 +0,0 @@
-#!/usr/bin/env bash
-# Script used only in CD pipeline
-
-set -eou pipefail
-
-TOPDIR=$(git rev-parse --show-toplevel)
-
-image="$1"
-shift
-
-if [ -z "${image}" ]; then
-  echo "Usage: $0 IMAGE"
-  exit 1
-fi
-
-DOCKER_IMAGE="pytorch/${image}"
-
-DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}"
-
-GPU_ARCH_TYPE=${GPU_ARCH_TYPE:-cpu}
-GPU_ARCH_VERSION=${GPU_ARCH_VERSION:-}
-MANY_LINUX_VERSION=${MANY_LINUX_VERSION:-}
-DOCKERFILE_SUFFIX=${DOCKERFILE_SUFFIX:-}
-WITH_PUSH=${WITH_PUSH:-}
-
-case ${GPU_ARCH_TYPE} in
-    cpu)
-        TARGET=cpu_final
-        DOCKER_TAG=cpu
-        GPU_IMAGE=centos:7
-        DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=9"
-        ;;
-    cpu-manylinux_2_28)
-        TARGET=cpu_final
-        DOCKER_TAG=cpu
-        GPU_IMAGE=amd64/almalinux:8
-        DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11"
-        MANY_LINUX_VERSION="2_28"
-        ;;
-    cpu-aarch64)
-        TARGET=final
-        DOCKER_TAG=cpu-aarch64
-        GPU_IMAGE=arm64v8/centos:7
-        DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=10"
-        MANY_LINUX_VERSION="aarch64"
-        ;;
-    cpu-aarch64-2_28)
-        TARGET=final
-        DOCKER_TAG=cpu-aarch64
-        GPU_IMAGE=arm64v8/almalinux:8
-        DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11"
-        MANY_LINUX_VERSION="2_28_aarch64"
-        ;;
-    cpu-cxx11-abi)
-        TARGET=final
-        DOCKER_TAG=cpu-cxx11-abi
-        GPU_IMAGE=""
-        DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=9"
-        MANY_LINUX_VERSION="cxx11-abi"
-        ;;
-    cpu-s390x)
-        TARGET=final
-        DOCKER_TAG=cpu-s390x
-        GPU_IMAGE=redhat/ubi9
-        DOCKER_GPU_BUILD_ARG=""
-        MANY_LINUX_VERSION="s390x"
-        ;;
-    cuda)
-        TARGET=cuda_final
-        DOCKER_TAG=cuda${GPU_ARCH_VERSION}
-        # Keep this up to date with the minimum version of CUDA we currently support
-        GPU_IMAGE=centos:7
-        DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=9"
-        ;;
-    cuda-manylinux_2_28)
-        TARGET=cuda_final
-        DOCKER_TAG=cuda${GPU_ARCH_VERSION}
-        GPU_IMAGE=amd64/almalinux:8
-        DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=11"
-        MANY_LINUX_VERSION="2_28"
-        ;;
-    cuda-aarch64)
-        TARGET=cuda_final
-        DOCKER_TAG=cuda${GPU_ARCH_VERSION}
-        GPU_IMAGE=arm64v8/centos:7
-        DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=11"
-        MANY_LINUX_VERSION="aarch64"
-        DOCKERFILE_SUFFIX="_cuda_aarch64"
-        ;;
-    rocm)
-        TARGET=rocm_final
-        DOCKER_TAG=rocm${GPU_ARCH_VERSION}
-        GPU_IMAGE=rocm/dev-centos-7:${GPU_ARCH_VERSION}-complete
-        PYTORCH_ROCM_ARCH="gfx900;gfx906;gfx908;gfx90a;gfx1030;gfx1100"
-        ROCM_REGEX="([0-9]+)\.([0-9]+)[\.]?([0-9]*)"
-        if [[ $GPU_ARCH_VERSION =~ $ROCM_REGEX ]]; then
-            ROCM_VERSION_INT=$((${BASH_REMATCH[1]}*10000 + ${BASH_REMATCH[2]}*100 + ${BASH_REMATCH[3]:-0}))
-        else
-            echo "ERROR: rocm regex failed"
-            exit 1
-        fi
-        if [[ $ROCM_VERSION_INT -ge 60000 ]]; then
-            PYTORCH_ROCM_ARCH+=";gfx942"
-        fi
-        DOCKER_GPU_BUILD_ARG="--build-arg ROCM_VERSION=${GPU_ARCH_VERSION} --build-arg PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} --build-arg DEVTOOLSET_VERSION=9"
-        ;;
-    xpu)
-        TARGET=xpu_final
-        DOCKER_TAG=xpu
-        GPU_IMAGE=amd64/almalinux:8
-        DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11"
-        MANY_LINUX_VERSION="2_28"
-        ;;
-    *)
-        echo "ERROR: Unrecognized GPU_ARCH_TYPE: ${GPU_ARCH_TYPE}"
-        exit 1
-        ;;
-esac
-
-IMAGES=''
-
-if [[ -n ${MANY_LINUX_VERSION} && -z ${DOCKERFILE_SUFFIX} ]]; then
-    DOCKERFILE_SUFFIX=_${MANY_LINUX_VERSION}
-fi
-(
-    set -x
-    DOCKER_BUILDKIT=1 docker build \
-        ${DOCKER_GPU_BUILD_ARG} \
-        --build-arg "GPU_IMAGE=${GPU_IMAGE}" \
-        --target "${TARGET}" \
-        -t "${DOCKER_IMAGE}" \
-        $@ \
-        -f "${TOPDIR}/.ci/docker/manywheel/Dockerfile${DOCKERFILE_SUFFIX}" \
-        "${TOPDIR}/.ci/docker/"
-)
-
-GITHUB_REF=${GITHUB_REF:-$(git symbolic-ref -q HEAD || git describe --tags --exact-match)}
-GIT_BRANCH_NAME=${GITHUB_REF##*/}
-GIT_COMMIT_SHA=${GITHUB_SHA:-$(git rev-parse HEAD)}
-DOCKER_IMAGE_BRANCH_TAG=${DOCKER_IMAGE}-${GIT_BRANCH_NAME}
-DOCKER_IMAGE_SHA_TAG=${DOCKER_IMAGE}-${GIT_COMMIT_SHA}
-
-if [[ "${WITH_PUSH}" == true ]]; then
-    (
-        set -x
-        docker push "${DOCKER_IMAGE}"
-        if [[ -n ${GITHUB_REF} ]]; then
-            docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_BRANCH_TAG}
-            docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_SHA_TAG}
-            docker push "${DOCKER_IMAGE_BRANCH_TAG}"
-            docker push "${DOCKER_IMAGE_SHA_TAG}"
-        fi
-    )
-fi

.ci/docker/manywheel/build_scripts/build.sh

@@ -1,131 +0,0 @@
-#!/bin/bash
-# Top-level build script called from Dockerfile
-# Script used only in CD pipeline
-
-# Stop at any error, show all commands
-set -ex
-
-# openssl version to build, with expected sha256 hash of .tar.gz
-# archive
-OPENSSL_ROOT=openssl-1.1.1l
-OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1
-DEVTOOLS_HASH=a8ebeb4bed624700f727179e6ef771dafe47651131a00a78b342251415646acc
-PATCHELF_HASH=d9afdff4baeacfbc64861454f368b7f2c15c44d245293f7587bbf726bfe722fb
-CURL_ROOT=curl-7.73.0
-CURL_HASH=cf34fe0b07b800f1c01a499a6e8b2af548f6d0e044dca4a29d88a4bee146d131
-AUTOCONF_ROOT=autoconf-2.69
-AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969
-
-# Get build utilities
-MY_DIR=$(dirname "${BASH_SOURCE[0]}")
-source $MY_DIR/build_utils.sh
-
-if [ "$(uname -m)" != "s390x" ] ; then
-    # Dependencies for compiling Python that we want to remove from
-    # the final image after compiling Python
-    PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel libffi-devel"
-
-    # Libraries that are allowed as part of the manylinux1 profile
-    MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel  mesa-libGL-devel libICE-devel libSM-devel ncurses-devel"
-
-    # Development tools and libraries
-    yum -y install bzip2 make git patch unzip bison yasm diffutils \
-        automake which file cmake28 \
-        kernel-devel-`uname -r` \
-        ${PYTHON_COMPILE_DEPS}
-else
-    # Dependencies for compiling Python that we want to remove from
-    # the final image after compiling Python
-    PYTHON_COMPILE_DEPS="zlib1g-dev libbz2-dev libncurses-dev libsqlite3-dev libdb-dev libpcap-dev liblzma-dev libffi-dev"
-
-    # Libraries that are allowed as part of the manylinux1 profile
-    MANYLINUX1_DEPS="libglib2.0-dev libX11-dev libncurses-dev"
-
-    # Development tools and libraries
-    apt install -y bzip2 make git patch unzip diffutils \
-        automake which file cmake \
-        linux-headers-virtual \
-        ${PYTHON_COMPILE_DEPS}
-fi
-
-# Install newest autoconf
-build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
-autoconf --version
-
-# Compile the latest Python releases.
-# (In order to have a proper SSL module, Python is compiled
-# against a recent openssl [see env vars above], which is linked
-# statically. We delete openssl afterwards.)
-build_openssl $OPENSSL_ROOT $OPENSSL_HASH
-/build_scripts/install_cpython.sh
-
-PY39_BIN=/opt/python/cp39-cp39/bin
-
-# Our openssl doesn't know how to find the system CA trust store
-#   (https://github.com/pypa/manylinux/issues/53)
-# And it's not clear how up-to-date that is anyway
-# So let's just use the same one pip and everyone uses
-$PY39_BIN/pip install certifi
-ln -s $($PY39_BIN/python -c 'import certifi; print(certifi.where())') \
-      /opt/_internal/certs.pem
-# If you modify this line you also have to modify the versions in the
-# Dockerfiles:
-export SSL_CERT_FILE=/opt/_internal/certs.pem
-
-# Install newest curl
-build_curl $CURL_ROOT $CURL_HASH
-rm -rf /usr/local/include/curl /usr/local/lib/libcurl* /usr/local/lib/pkgconfig/libcurl.pc
-hash -r
-curl --version
-curl-config --features
-
-# Install patchelf (latest with unreleased bug fixes)
-curl -sLOk https://nixos.org/releases/patchelf/patchelf-0.10/patchelf-0.10.tar.gz
-# check_sha256sum patchelf-0.9njs2.tar.gz $PATCHELF_HASH
-tar -xzf patchelf-0.10.tar.gz
-(cd patchelf-0.10 && ./configure && make && make install)
-rm -rf patchelf-0.10.tar.gz patchelf-0.10
-
-# Install latest pypi release of auditwheel
-$PY39_BIN/pip install auditwheel
-ln -s $PY39_BIN/auditwheel /usr/local/bin/auditwheel
-
-# Clean up development headers and other unnecessary stuff for
-# final image
-if [ "$(uname -m)" != "s390x" ] ; then
-    yum -y erase wireless-tools gtk2 libX11 hicolor-icon-theme \
-        avahi freetype bitstream-vera-fonts \
-        ${PYTHON_COMPILE_DEPS} || true > /dev/null 2>&1
-    yum -y install ${MANYLINUX1_DEPS}
-    yum -y clean all > /dev/null 2>&1
-    yum list installed
-else
-    apt purge -y ${PYTHON_COMPILE_DEPS} || true > /dev/null 2>&1
-fi
-# we don't need libpython*.a, and they're many megabytes
-find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f
-# Strip what we can -- and ignore errors, because this just attempts to strip
-# *everything*, including non-ELF files:
-find /opt/_internal -type f -print0 \
-    | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true
-# We do not need the Python test suites, or indeed the precompiled .pyc and
-# .pyo files. Partially cribbed from:
-#    https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile
-find /opt/_internal \
-     \( -type d -a -name test -o -name tests \) \
-  -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \
-  -print0 | xargs -0 rm -f
-
-for PYTHON in /opt/python/*/bin/python; do
-    # Smoke test to make sure that our Pythons work, and do indeed detect as
-    # being manylinux compatible:
-    $PYTHON $MY_DIR/manylinux1-check.py
-    # Make sure that SSL cert checking works
-    $PYTHON $MY_DIR/ssl-check.py
-done
-
-# Fix libc headers to remain compatible with C99 compilers.
-find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} +
-
-# Now we can delete our built SSL
-rm -rf /usr/local/ssl

.ci/docker/manywheel/build_scripts/build_utils.sh

@@ -1,91 +0,0 @@
-#!/bin/bash
-# Helper utilities for build
-# Script used only in CD pipeline
-
-OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/old/1.1.1/
-CURL_DOWNLOAD_URL=https://curl.askapache.com/download
-
-AUTOCONF_DOWNLOAD_URL=https://ftp.gnu.org/gnu/autoconf
-
-
-function check_var {
-    if [ -z "$1" ]; then
-        echo "required variable not defined"
-        exit 1
-    fi
-}
-
-
-function do_openssl_build {
-    ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null
-    make > /dev/null
-    make install > /dev/null
-}
-
-
-function check_sha256sum {
-    local fname=$1
-    check_var ${fname}
-    local sha256=$2
-    check_var ${sha256}
-
-    echo "${sha256}  ${fname}" > ${fname}.sha256
-    sha256sum -c ${fname}.sha256
-    rm -f ${fname}.sha256
-}
-
-
-function build_openssl {
-    local openssl_fname=$1
-    check_var ${openssl_fname}
-    local openssl_sha256=$2
-    check_var ${openssl_sha256}
-    check_var ${OPENSSL_DOWNLOAD_URL}
-    curl -sLO ${OPENSSL_DOWNLOAD_URL}/${openssl_fname}.tar.gz
-    check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256}
-    tar -xzf ${openssl_fname}.tar.gz
-    (cd ${openssl_fname} && do_openssl_build)
-    rm -rf ${openssl_fname} ${openssl_fname}.tar.gz
-}
-
-
-function do_curl_build {
-    LIBS=-ldl ./configure --with-ssl --disable-shared > /dev/null
-    make > /dev/null
-    make install > /dev/null
-}
-
-
-function build_curl {
-    local curl_fname=$1
-    check_var ${curl_fname}
-    local curl_sha256=$2
-    check_var ${curl_sha256}
-    check_var ${CURL_DOWNLOAD_URL}
-    curl -sLO ${CURL_DOWNLOAD_URL}/${curl_fname}.tar.bz2
-    check_sha256sum ${curl_fname}.tar.bz2 ${curl_sha256}
-    tar -jxf ${curl_fname}.tar.bz2
-    (cd ${curl_fname} && do_curl_build)
-    rm -rf ${curl_fname} ${curl_fname}.tar.bz2
-}
-
-
-function do_standard_install {
-    ./configure > /dev/null
-    make > /dev/null
-    make install > /dev/null
-}
-
-
-function build_autoconf {
-    local autoconf_fname=$1
-    check_var ${autoconf_fname}
-    local autoconf_sha256=$2
-    check_var ${autoconf_sha256}
-    check_var ${AUTOCONF_DOWNLOAD_URL}
-    curl -sLO ${AUTOCONF_DOWNLOAD_URL}/${autoconf_fname}.tar.gz
-    check_sha256sum ${autoconf_fname}.tar.gz ${autoconf_sha256}
-    tar -zxf ${autoconf_fname}.tar.gz
-    (cd ${autoconf_fname} && do_standard_install)
-    rm -rf ${autoconf_fname} ${autoconf_fname}.tar.gz
-}

.ci/docker/manywheel/build_scripts/manylinux1-check.py

@@ -1,60 +0,0 @@
-# Logic copied from PEP 513
-
-
-def is_manylinux1_compatible():
-    # Only Linux, and only x86-64 / i686
-    from distutils.util import get_platform
-
-    if get_platform() not in ["linux-x86_64", "linux-i686", "linux-s390x"]:
-        return False
-
-    # Check for presence of _manylinux module
-    try:
-        import _manylinux
-
-        return bool(_manylinux.manylinux1_compatible)
-    except (ImportError, AttributeError):
-        # Fall through to heuristic check below
-        pass
-
-    # Check glibc version. CentOS 5 uses glibc 2.5.
-    return have_compatible_glibc(2, 5)
-
-
-def have_compatible_glibc(major, minimum_minor):
-    import ctypes
-
-    process_namespace = ctypes.CDLL(None)
-    try:
-        gnu_get_libc_version = process_namespace.gnu_get_libc_version
-    except AttributeError:
-        # Symbol doesn't exist -> therefore, we are not linked to
-        # glibc.
-        return False
-
-    # Call gnu_get_libc_version, which returns a string like "2.5".
-    gnu_get_libc_version.restype = ctypes.c_char_p
-    version_str = gnu_get_libc_version()
-    # py2 / py3 compatibility:
-    if not isinstance(version_str, str):
-        version_str = version_str.decode("ascii")
-
-    # Parse string and check against requested version.
-    version = [int(piece) for piece in version_str.split(".")]
-    assert len(version) == 2
-    if major != version[0]:
-        return False
-    if minimum_minor > version[1]:
-        return False
-    return True
-
-
-import sys
-
-
-if is_manylinux1_compatible():
-    print(f"{sys.executable} is manylinux1 compatible")
-    sys.exit(0)
-else:
-    print(f"{sys.executable} is NOT manylinux1 compatible")
-    sys.exit(1)

.ci/docker/manywheel/build_scripts/ssl-check.py

@@ -1,35 +0,0 @@
-# cf. https://github.com/pypa/manylinux/issues/53
-
-GOOD_SSL = "https://google.com"
-BAD_SSL = "https://self-signed.badssl.com"
-
-import sys
-
-
-print("Testing SSL certificate checking for Python:", sys.version)
-
-if sys.version_info[:2] < (2, 7) or sys.version_info[:2] < (3, 4):
-    print("This version never checks SSL certs; skipping tests")
-    sys.exit(0)
-
-if sys.version_info[0] >= 3:
-    from urllib.request import urlopen
-
-    EXC = OSError
-else:
-    from urllib import urlopen
-
-    EXC = IOError
-
-print(f"Connecting to {GOOD_SSL} should work")
-urlopen(GOOD_SSL)
-print("...it did, yay.")
-
-print(f"Connecting to {BAD_SSL} should fail")
-try:
-    urlopen(BAD_SSL)
-    # If we get here then we failed:
-    print("...it DIDN'T!!!!!11!!1one!")
-    sys.exit(1)
-except EXC:
-    print("...it did, yay.")

.ci/docker/requirements-ci.txt

@@ -1,334 +0,0 @@
-# Python dependencies required for unit tests
-
-#awscli==1.6 #this breaks some platforms
-#Description: AWS command line interface
-#Pinned versions: 1.6
-#test that import:
-
-boto3==1.19.12
-#Description: AWS SDK for python
-#Pinned versions: 1.19.12, 1.16.34
-#test that import:
-
-click
-#Description: Command Line Interface Creation Kit
-#Pinned versions:
-#test that import:
-
-coremltools==5.0b5 ; python_version < "3.12"
-#Description: Apple framework for ML integration
-#Pinned versions: 5.0b5
-#test that import:
-
-#dataclasses #this breaks some platforms
-#Description: Provides decorators for auto adding special methods to user classes
-#Pinned versions:
-#test that import:
-
-dill==0.3.7
-#Description: dill extends pickle with serializing and de-serializing for most built-ins
-#Pinned versions: 0.3.7
-#test that import: dynamo/test_replay_record.py test_dataloader.py test_datapipe.py test_serialization.py
-
-expecttest==0.1.6
-#Description: method for writing tests where test framework auto populates
-# the expected output based on previous runs
-#Pinned versions: 0.1.6
-#test that import:
-
-flatbuffers==2.0
-#Description: cross platform serialization library
-#Pinned versions: 2.0
-#test that import:
-
-hypothesis==5.35.1
-# Pin hypothesis to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136
-#Description: advanced library for generating parametrized tests
-#Pinned versions: 3.44.6, 4.53.2
-#test that import: test_xnnpack_integration.py, test_pruning_op.py, test_nn.py
-
-junitparser==2.1.1
-#Description: unitparser handles JUnit/xUnit Result XML files
-#Pinned versions: 2.1.1
-#test that import:
-
-lark==0.12.0
-#Description: parser
-#Pinned versions: 0.12.0
-#test that import:
-
-librosa>=0.6.2 ; python_version < "3.11"
-#Description: A python package for music and audio analysis
-#Pinned versions: >=0.6.2
-#test that import: test_spectral_ops.py
-
-#mkl #this breaks linux-bionic-rocm4.5-py3.7
-#Description: Intel oneAPI Math Kernel Library
-#Pinned versions:
-#test that import: test_profiler.py, test_public_bindings.py, test_testing.py,
-#test_nn.py, test_mkldnn.py, test_jit.py, test_fx_experimental.py,
-#test_autograd.py
-
-#mkl-devel
-# see mkl
-
-#mock
-#Description: A testing library that allows you to replace parts of your
-#system under test with mock objects
-#Pinned versions:
-#test that import: test_modules.py, test_nn.py,
-#test_testing.py
-
-#MonkeyType # breaks pytorch-xla-linux-bionic-py3.7-clang8
-#Description: collects runtime types of function arguments and return
-#values, and can automatically generate stub files
-#Pinned versions:
-#test that import:
-
-mypy==1.10.0
-# Pin MyPy version because new errors are likely to appear with each release
-#Description: linter
-#Pinned versions: 1.10.0
-#test that import: test_typing.py, test_type_hints.py
-
-networkx==2.8.8
-#Description: creation, manipulation, and study of
-#the structure, dynamics, and functions of complex networks
-#Pinned versions: 2.8.8
-#test that import: functorch
-
-#ninja
-#Description: build system.  Note that it install from
-#here breaks things so it is commented out
-#Pinned versions: 1.10.0.post1
-#test that import: run_test.py, test_cpp_extensions_aot.py,test_determination.py
-
-numba==0.49.0 ; python_version < "3.9"
-numba==0.54.1 ; python_version == "3.9"
-numba==0.55.2 ; python_version == "3.10"
-#Description: Just-In-Time Compiler for Numerical Functions
-#Pinned versions: 0.54.1, 0.49.0, <=0.49.1
-#test that import: test_numba_integration.py
-#For numba issue see https://github.com/pytorch/pytorch/issues/51511
-
-#numpy
-#Description: Provides N-dimensional arrays and linear algebra
-#Pinned versions: 1.20
-#test that import: test_view_ops.py, test_unary_ufuncs.py, test_type_promotion.py,
-#test_type_info.py, test_torch.py, test_tensorexpr_pybind.py, test_tensorexpr.py,
-#test_tensorboard.py, test_tensor_creation_ops.py, test_static_runtime.py,
-#test_spectral_ops.py, test_sort_and_select.py, test_shape_ops.py,
-#test_segment_reductions.py, test_reductions.py, test_pruning_op.py,
-#test_overrides.py, test_numpy_interop.py, test_numba_integration.py
-#test_nn.py, test_namedtensor.py, test_linalg.py, test_jit_cuda_fuser.py,
-#test_jit.py, test_indexing.py, test_datapipe.py, test_dataloader.py,
-#test_binary_ufuncs.py
-
-#onnxruntime
-#Description: scoring engine for Open Neural Network Exchange (ONNX) models
-#Pinned versions: 1.9.0
-#test that import:
-
-opt-einsum==3.3
-#Description: Python library to optimize tensor contraction order, used in einsum
-#Pinned versions: 3.3
-#test that import: test_linalg.py
-
-optree==0.12.1
-#Description: A library for tree manipulation
-#Pinned versions: 0.12.1
-#test that import: test_vmap.py, test_aotdispatch.py, test_dynamic_shapes.py,
-#test_pytree.py, test_ops.py, test_control_flow.py, test_modules.py,
-#common_utils.py, test_eager_transforms.py, test_python_dispatch.py,
-#test_expanded_weights.py, test_decomp.py, test_overrides.py, test_masked.py,
-#test_ops.py, test_prims.py, test_subclass.py, test_functionalization.py,
-#test_schema_check.py, test_profiler_tree.py, test_meta.py, test_torchxla_num_output.py,
-#test_utils.py, test_proxy_tensor.py, test_memory_profiler.py, test_view_ops.py,
-#test_pointwise_ops.py, test_dtensor_ops.py, test_torchinductor.py, test_fx.py,
-#test_fake_tensor.py, test_mps.py
-
-pillow==10.3.0
-#Description:  Python Imaging Library fork
-#Pinned versions: 10.3.0
-#test that import:
-
-protobuf==3.20.2
-#Description:  Google’s data interchange format
-#Pinned versions: 3.20.1
-#test that import: test_tensorboard.py
-
-psutil
-#Description: information on running processes and system utilization
-#Pinned versions:
-#test that import: test_profiler.py, test_openmp.py, test_dataloader.py
-
-pytest==7.3.2
-#Description: testing framework
-#Pinned versions:
-#test that import: test_typing.py, test_cpp_extensions_aot.py, run_test.py
-
-pytest-xdist==3.3.1
-#Description: plugin for running pytest in parallel
-#Pinned versions:
-#test that import:
-
-pytest-flakefinder==1.1.0
-#Description: plugin for rerunning tests a fixed number of times in pytest
-#Pinned versions: 1.1.0
-#test that import:
-
-pytest-rerunfailures>=10.3
-#Description: plugin for rerunning failure tests in pytest
-#Pinned versions:
-#test that import:
-
-#pytest-benchmark
-#Description: fixture for benchmarking code
-#Pinned versions: 3.2.3
-#test that import:
-
-#pytest-sugar
-#Description: shows failures and errors instantly
-#Pinned versions:
-#test that import:
-
-xdoctest==1.1.0
-#Description: runs doctests in pytest
-#Pinned versions: 1.1.0
-#test that import:
-
-pygments==2.15.0
-#Description: support doctest highlighting
-#Pinned versions: 2.12.0
-#test that import: the doctests
-
-#PyYAML
-#Description: data serialization format
-#Pinned versions:
-#test that import:
-
-#requests
-#Description: HTTP library
-#Pinned versions:
-#test that import: test_type_promotion.py
-
-#rich
-#Description: rich text and beautiful formatting in the terminal
-#Pinned versions: 10.9.0
-#test that import:
-
-scikit-image==0.19.3 ; python_version < "3.10"
-scikit-image==0.22.0 ; python_version >= "3.10"
-#Description: image processing routines
-#Pinned versions:
-#test that import: test_nn.py
-
-#scikit-learn
-#Description: machine learning package
-#Pinned versions: 0.20.3
-#test that import:
-
-scipy==1.10.1 ; python_version <= "3.11"
-scipy==1.12.0 ; python_version == "3.12"
-# Pin SciPy because of failing distribution tests (see #60347)
-#Description: scientific python
-#Pinned versions: 1.10.1
-#test that import: test_unary_ufuncs.py, test_torch.py,test_tensor_creation_ops.py
-#test_spectral_ops.py, test_sparse_csr.py, test_reductions.py,test_nn.py
-#test_linalg.py, test_binary_ufuncs.py
-
-#tabulate
-#Description: Pretty-print tabular data
-#Pinned versions:
-#test that import:
-
-tb-nightly==2.13.0a20230426
-#Description: TensorBoard
-#Pinned versions:
-#test that import:
-
-# needed by torchgen utils
-typing-extensions
-#Description: type hints for python
-#Pinned versions:
-#test that import:
-
-#virtualenv
-#Description: virtual environment for python
-#Pinned versions:
-#test that import:
-
-unittest-xml-reporting<=3.2.0,>=2.0.0
-#Description: saves unit test results to xml
-#Pinned versions:
-#test that import:
-
-#lintrunner is supported on aarch64-linux only from 0.12.4 version
-lintrunner==0.12.5
-#Description: all about linters!
-#Pinned versions: 0.12.5
-#test that import:
-
-redis>=4.0.0
-#Description: redis database
-#test that import: anything that tests OSS caching/mocking (inductor/test_codecache.py, inductor/test_max_autotune.py)
-
-rockset==1.0.3
-#Description: queries Rockset
-#Pinned versions: 1.0.3
-#test that import:
-
-ghstack==0.8.0
-#Description: ghstack tool
-#Pinned versions: 0.8.0
-#test that import:
-
-jinja2==3.1.4
-#Description: jinja2 template engine
-#Pinned versions: 3.1.4
-#test that import:
-
-pytest-cpp==2.3.0
-#Description: This is used by pytest to invoke C++ tests
-#Pinned versions: 2.3.0
-#test that import:
-
-z3-solver==4.12.2.0
-#Description: The Z3 Theorem Prover Project
-#Pinned versions:
-#test that import:
-
-tensorboard==2.13.0
-#Description: Also included in .ci/docker/requirements-docs.txt
-#Pinned versions:
-#test that import: test_tensorboard
-
-pywavelets==1.4.1 ; python_version < "3.12"
-pywavelets==1.5.0 ; python_version >= "3.12"
-#Description: This is a requirement of scikit-image, we need to pin
-# it here because 1.5.0 conflicts with numpy 1.21.2 used in CI
-#Pinned versions: 1.4.1
-#test that import:
-
-lxml==5.0.0
-#Description: This is a requirement of unittest-xml-reporting
-
-# Python-3.9 binaries
-
-PyGithub==2.3.0
-
-sympy==1.12.1 ; python_version == "3.8"
-sympy==1.13.1 ; python_version >= "3.9"
-#Description: Required by coremltools, also pinned in .github/requirements/pip-requirements-macOS.txt
-#Pinned versions:
-#test that import:
-
-onnx==1.16.1
-#Description: Required by mypy and test_public_bindings.py when checking torch.onnx._internal
-#Pinned versions:
-#test that import:
-
-onnxscript==0.1.0.dev20240817
-#Description: Required by mypy and test_public_bindings.py when checking torch.onnx._internal
-#Pinned versions:
-#test that import:

.ci/docker/requirements-docs.txt

@@ -1,49 +0,0 @@
-sphinx==5.3.0
-#Description: This is used to generate PyTorch docs
-#Pinned versions: 5.3.0
--e git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
-
-# TODO: sphinxcontrib.katex 0.9.0 adds a local KaTeX server to speed up pre-rendering
-# but it doesn't seem to work and hangs around idly. The initial thought is probably
-# something related to Docker setup. We can investigate this later
-sphinxcontrib.katex==0.8.6
-#Description: This is used to generate PyTorch docs
-#Pinned versions: 0.8.6
-
-matplotlib==3.5.3
-#Description: This is used to generate PyTorch docs
-#Pinned versions: 3.5.3
-
-tensorboard==2.13.0
-#Description: This is used to generate PyTorch docs
-#Pinned versions: 2.13.0
-
-breathe==4.34.0
-#Description: This is used to generate PyTorch C++ docs
-#Pinned versions: 4.34.0
-
-exhale==0.2.3
-#Description: This is used to generate PyTorch C++ docs
-#Pinned versions: 0.2.3
-
-docutils==0.16
-#Description: This is used to generate PyTorch C++ docs
-#Pinned versions: 0.16
-
-bs4==0.0.1
-#Description: This is used to generate PyTorch C++ docs
-#Pinned versions: 0.0.1
-
-IPython==8.12.0
-#Description: This is used to generate PyTorch functorch docs
-#Pinned versions: 8.12.0
-
-myst-nb==0.17.2
-#Description: This is used to generate PyTorch functorch docs
-#Pinned versions: 0.13.2
-
-# The following are required to build torch.distributed.elastic.rendezvous.etcd* docs
-python-etcd==0.4.5
-sphinx-copybutton==0.5.0
-sphinx-panels==0.4.1
-myst-parser==0.18.1

.ci/docker/triton_version.txt

@@ -1 +0,0 @@
-3.0.0

.ci/docker/ubuntu-cuda/Dockerfile

@@ -1,172 +0,0 @@
-ARG UBUNTU_VERSION
-ARG CUDA_VERSION
-ARG IMAGE_NAME
-
-FROM ${IMAGE_NAME}
-
-ARG UBUNTU_VERSION
-ARG CUDA_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install katex
-ARG KATEX
-COPY ./common/install_docs_reqs.sh install_docs_reqs.sh
-RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-ARG CONDA_CMAKE
-COPY requirements-ci.txt /opt/conda/requirements-ci.txt
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
-
-# Install gcc
-ARG GCC_VERSION
-COPY ./common/install_gcc.sh install_gcc.sh
-RUN bash ./install_gcc.sh && rm install_gcc.sh
-
-# Install clang
-ARG CLANG_VERSION
-COPY ./common/install_clang.sh install_clang.sh
-RUN bash ./install_clang.sh && rm install_clang.sh
-
-# (optional) Install protobuf for ONNX
-ARG PROTOBUF
-COPY ./common/install_protobuf.sh install_protobuf.sh
-RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
-RUN rm install_protobuf.sh
-ENV INSTALLED_PROTOBUF ${PROTOBUF}
-
-# (optional) Install database packages like LMDB and LevelDB
-ARG DB
-COPY ./common/install_db.sh install_db.sh
-RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
-RUN rm install_db.sh
-ENV INSTALLED_DB ${DB}
-
-# (optional) Install vision packages like OpenCV
-ARG VISION
-COPY ./common/install_vision.sh ./common/cache_vision_models.sh ./common/common_utils.sh ./
-RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
-RUN rm install_vision.sh cache_vision_models.sh common_utils.sh
-ENV INSTALLED_VISION ${VISION}
-
-# (optional) Install UCC
-ARG UCX_COMMIT
-ARG UCC_COMMIT
-ENV UCX_COMMIT $UCX_COMMIT
-ENV UCC_COMMIT $UCC_COMMIT
-ENV UCX_HOME /usr
-ENV UCC_HOME /usr
-ADD ./common/install_ucc.sh install_ucc.sh
-RUN if [ -n "${UCX_COMMIT}" ] && [ -n "${UCC_COMMIT}" ]; then bash ./install_ucc.sh; fi
-RUN rm install_ucc.sh
-
-COPY ./common/install_openssl.sh install_openssl.sh
-ENV OPENSSL_ROOT_DIR /opt/openssl
-RUN bash ./install_openssl.sh
-ENV OPENSSL_DIR /opt/openssl
-
-ARG INDUCTOR_BENCHMARKS
-COPY ./common/install_inductor_benchmark_deps.sh install_inductor_benchmark_deps.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/huggingface.txt huggingface.txt
-COPY ci_commit_pins/timm.txt timm.txt
-RUN if [ -n "${INDUCTOR_BENCHMARKS}" ]; then bash ./install_inductor_benchmark_deps.sh; fi
-RUN rm install_inductor_benchmark_deps.sh common_utils.sh timm.txt huggingface.txt
-
-# (optional) Install non-default CMake version
-ARG CMAKE_VERSION
-COPY ./common/install_cmake.sh install_cmake.sh
-RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
-RUN rm install_cmake.sh
-
-ARG TRITON
-# Install triton, this needs to be done before sccache because the latter will
-# try to reach out to S3, which docker build runners don't have access
-COPY ./common/install_triton.sh install_triton.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/triton.txt triton.txt
-COPY triton_version.txt triton_version.txt
-RUN if [ -n "${TRITON}" ]; then bash ./install_triton.sh; fi
-RUN rm install_triton.sh common_utils.sh triton.txt triton_version.txt
-
-ARG HALIDE
-# Build and install halide
-COPY ./common/install_halide.sh install_halide.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/halide.txt halide.txt
-RUN if [ -n "${HALIDE}" ]; then bash ./install_halide.sh; fi
-RUN rm install_halide.sh common_utils.sh halide.txt
-
-# Install ccache/sccache (do this last, so we get priority in PATH)
-COPY ./common/install_cache.sh install_cache.sh
-ENV PATH /opt/cache/bin:$PATH
-# See https://github.com/pytorch/pytorch/issues/82174
-# TODO(sdym@fb.com):
-# check if this is needed after full off Xenial migration
-ENV CARGO_NET_GIT_FETCH_WITH_CLI true
-RUN bash ./install_cache.sh && rm install_cache.sh
-ENV CMAKE_CUDA_COMPILER_LAUNCHER=/opt/cache/bin/sccache
-
-# Add jni.h for java host build
-COPY ./common/install_jni.sh install_jni.sh
-COPY ./java/jni.h jni.h
-RUN bash ./install_jni.sh && rm install_jni.sh
-
-# Install Open MPI for CUDA
-COPY ./common/install_openmpi.sh install_openmpi.sh
-RUN if [ -n "${CUDA_VERSION}" ]; then bash install_openmpi.sh; fi
-RUN rm install_openmpi.sh
-
-# Include BUILD_ENVIRONMENT environment variable in image
-ARG BUILD_ENVIRONMENT
-ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
-
-# AWS specific CUDA build guidance
-ENV TORCH_CUDA_ARCH_LIST Maxwell
-ENV TORCH_NVCC_FLAGS "-Xfatbin -compress-all"
-ENV CUDA_PATH /usr/local/cuda
-
-# Install LLVM dev version (Defined in the pytorch/builder github repository)
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
-
-# Install CUDNN
-ARG CUDNN_VERSION
-ARG CUDA_VERSION
-COPY ./common/install_cudnn.sh install_cudnn.sh
-RUN if [ -n "${CUDNN_VERSION}" ]; then bash install_cudnn.sh; fi
-RUN rm install_cudnn.sh
-
-# Install CUSPARSELT
-ARG CUDA_VERSION
-COPY ./common/install_cusparselt.sh install_cusparselt.sh
-RUN bash install_cusparselt.sh
-RUN rm install_cusparselt.sh
-
-# Install CUDSS
-ARG CUDA_VERSION
-COPY ./common/install_cudss.sh install_cudss.sh
-RUN bash install_cudss.sh
-RUN rm install_cudss.sh
-
-# Delete /usr/local/cuda-11.X/cuda-11.X symlinks
-RUN if [ -h /usr/local/cuda-11.6/cuda-11.6 ]; then rm /usr/local/cuda-11.6/cuda-11.6; fi
-RUN if [ -h /usr/local/cuda-11.7/cuda-11.7 ]; then rm /usr/local/cuda-11.7/cuda-11.7; fi
-RUN if [ -h /usr/local/cuda-12.1/cuda-12.1 ]; then rm /usr/local/cuda-12.1/cuda-12.1; fi
-RUN if [ -h /usr/local/cuda-12.4/cuda-12.4 ]; then rm /usr/local/cuda-12.4/cuda-12.4; fi
-
-USER jenkins
-CMD ["bash"]

.ci/docker/ubuntu-rocm/Dockerfile

@@ -1,125 +0,0 @@
-ARG UBUNTU_VERSION
-
-FROM ubuntu:${UBUNTU_VERSION}
-
-ARG UBUNTU_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Set AMD gpu targets to build for
-ARG PYTORCH_ROCM_ARCH
-ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install clang
-ARG LLVMDEV
-ARG CLANG_VERSION
-COPY ./common/install_clang.sh install_clang.sh
-RUN bash ./install_clang.sh && rm install_clang.sh
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ARG CONDA_CMAKE
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-COPY requirements-ci.txt /opt/conda/requirements-ci.txt
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
-
-# Install gcc
-ARG GCC_VERSION
-COPY ./common/install_gcc.sh install_gcc.sh
-RUN bash ./install_gcc.sh && rm install_gcc.sh
-
-# (optional) Install protobuf for ONNX
-ARG PROTOBUF
-COPY ./common/install_protobuf.sh install_protobuf.sh
-RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
-RUN rm install_protobuf.sh
-ENV INSTALLED_PROTOBUF ${PROTOBUF}
-
-# (optional) Install database packages like LMDB and LevelDB
-ARG DB
-COPY ./common/install_db.sh install_db.sh
-RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
-RUN rm install_db.sh
-ENV INSTALLED_DB ${DB}
-
-# (optional) Install vision packages like OpenCV
-ARG VISION
-COPY ./common/install_vision.sh ./common/cache_vision_models.sh ./common/common_utils.sh ./
-RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
-RUN rm install_vision.sh cache_vision_models.sh common_utils.sh
-ENV INSTALLED_VISION ${VISION}
-
-# Install rocm
-ARG ROCM_VERSION
-COPY ./common/install_rocm.sh install_rocm.sh
-RUN bash ./install_rocm.sh
-RUN rm install_rocm.sh
-COPY ./common/install_rocm_magma.sh install_rocm_magma.sh
-RUN bash ./install_rocm_magma.sh
-RUN rm install_rocm_magma.sh
-ENV ROCM_PATH /opt/rocm
-ENV PATH /opt/rocm/bin:$PATH
-ENV PATH /opt/rocm/hcc/bin:$PATH
-ENV PATH /opt/rocm/hip/bin:$PATH
-ENV PATH /opt/rocm/opencl/bin:$PATH
-ENV PATH /opt/rocm/llvm/bin:$PATH
-ENV MAGMA_HOME /opt/rocm/magma
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-
-# Install amdsmi
-COPY ./common/install_amdsmi.sh install_amdsmi.sh
-RUN bash ./install_amdsmi.sh
-RUN rm install_amdsmi.sh
-
-# (optional) Install non-default CMake version
-ARG CMAKE_VERSION
-COPY ./common/install_cmake.sh install_cmake.sh
-RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
-RUN rm install_cmake.sh
-
-# (optional) Install non-default Ninja version
-ARG NINJA_VERSION
-COPY ./common/install_ninja.sh install_ninja.sh
-RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
-RUN rm install_ninja.sh
-
-ARG TRITON
-# Install triton, this needs to be done before sccache because the latter will
-# try to reach out to S3, which docker build runners don't have access
-COPY ./common/install_triton.sh install_triton.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/triton-rocm.txt triton-rocm.txt
-COPY triton_version.txt triton_version.txt
-RUN if [ -n "${TRITON}" ]; then bash ./install_triton.sh; fi
-RUN rm install_triton.sh common_utils.sh triton-rocm.txt triton_version.txt
-
-# Install AOTriton
-COPY ./aotriton_version.txt aotriton_version.txt
-COPY ./common/common_utils.sh common_utils.sh
-COPY ./common/install_aotriton.sh install_aotriton.sh
-RUN ["/bin/bash", "-c", "./install_aotriton.sh /opt/rocm && rm -rf install_aotriton.sh aotriton_version.txt common_utils.sh"]
-ENV AOTRITON_INSTALLED_PREFIX /opt/rocm/aotriton
-
-# Install ccache/sccache (do this last, so we get priority in PATH)
-COPY ./common/install_cache.sh install_cache.sh
-ENV PATH /opt/cache/bin:$PATH
-RUN bash ./install_cache.sh && rm install_cache.sh
-
-# Include BUILD_ENVIRONMENT environment variable in image
-ARG BUILD_ENVIRONMENT
-ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
-
-USER jenkins
-CMD ["bash"]

.ci/docker/ubuntu-xpu/Dockerfile

@@ -1,119 +0,0 @@
-ARG UBUNTU_VERSION
-
-FROM ubuntu:${UBUNTU_VERSION}
-
-ARG UBUNTU_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-ARG CLANG_VERSION
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install clang
-ARG LLVMDEV
-COPY ./common/install_clang.sh install_clang.sh
-RUN bash ./install_clang.sh && rm install_clang.sh
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install katex
-ARG KATEX
-COPY ./common/install_docs_reqs.sh install_docs_reqs.sh
-RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ARG CONDA_CMAKE
-ARG DOCS
-ARG BUILD_ENVIRONMENT
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-ENV DOCS=$DOCS
-COPY requirements-ci.txt requirements-docs.txt /opt/conda/
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt /opt/conda/requirements-docs.txt
-
-# Install gcc
-ARG GCC_VERSION
-COPY ./common/install_gcc.sh install_gcc.sh
-RUN bash ./install_gcc.sh && rm install_gcc.sh
-
-# Install lcov for C++ code coverage
-COPY ./common/install_lcov.sh install_lcov.sh
-RUN  bash ./install_lcov.sh && rm install_lcov.sh
-
-COPY ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh
-ENV OPENSSL_ROOT_DIR /opt/openssl
-ENV OPENSSL_DIR /opt/openssl
-RUN rm install_openssl.sh
-
-ARG INDUCTOR_BENCHMARKS
-COPY ./common/install_inductor_benchmark_deps.sh install_inductor_benchmark_deps.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/huggingface.txt huggingface.txt
-COPY ci_commit_pins/timm.txt timm.txt
-RUN if [ -n "${INDUCTOR_BENCHMARKS}" ]; then bash ./install_inductor_benchmark_deps.sh; fi
-RUN rm install_inductor_benchmark_deps.sh common_utils.sh timm.txt huggingface.txt
-
-# Install XPU Dependencies
-ARG XPU_VERSION
-COPY ./common/install_xpu.sh install_xpu.sh
-RUN bash ./install_xpu.sh && rm install_xpu.sh
-
-ARG TRITON
-# Install triton, this needs to be done before sccache because the latter will
-# try to reach out to S3, which docker build runners don't have access
-COPY ./common/install_triton.sh install_triton.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/triton-xpu.txt triton-xpu.txt
-COPY triton_version.txt triton_version.txt
-RUN if [ -n "${TRITON}" ]; then bash ./install_triton.sh; fi
-RUN rm install_triton.sh common_utils.sh triton-xpu.txt triton_version.txt
-
-# (optional) Install database packages like LMDB and LevelDB
-ARG DB
-COPY ./common/install_db.sh install_db.sh
-RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
-RUN rm install_db.sh
-ENV INSTALLED_DB ${DB}
-
-# (optional) Install vision packages like OpenCV
-ARG VISION
-COPY ./common/install_vision.sh ./common/cache_vision_models.sh ./common/common_utils.sh ./
-RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
-RUN rm install_vision.sh cache_vision_models.sh common_utils.sh
-ENV INSTALLED_VISION ${VISION}
-
-# (optional) Install non-default CMake version
-ARG CMAKE_VERSION
-COPY ./common/install_cmake.sh install_cmake.sh
-RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
-RUN rm install_cmake.sh
-
-# (optional) Install non-default Ninja version
-ARG NINJA_VERSION
-COPY ./common/install_ninja.sh install_ninja.sh
-RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
-RUN rm install_ninja.sh
-
-# Install ccache/sccache (do this last, so we get priority in PATH)
-COPY ./common/install_cache.sh install_cache.sh
-ENV PATH /opt/cache/bin:$PATH
-RUN bash ./install_cache.sh && rm install_cache.sh
-
-# Include BUILD_ENVIRONMENT environment variable in image
-ARG BUILD_ENVIRONMENT
-ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
-
-# Install LLVM dev version (Defined in the pytorch/builder github repository)
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
-
-USER jenkins
-CMD ["bash"]

.ci/docker/ubuntu/Dockerfile

@@ -1,211 +0,0 @@
-ARG UBUNTU_VERSION
-
-FROM ubuntu:${UBUNTU_VERSION}
-
-ARG UBUNTU_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-ARG CLANG_VERSION
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install clang
-ARG LLVMDEV
-COPY ./common/install_clang.sh install_clang.sh
-RUN bash ./install_clang.sh && rm install_clang.sh
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install katex
-ARG KATEX
-COPY ./common/install_docs_reqs.sh install_docs_reqs.sh
-RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ARG CONDA_CMAKE
-ARG DOCS
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-ENV DOCS=$DOCS
-COPY requirements-ci.txt requirements-docs.txt /opt/conda/
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt /opt/conda/requirements-docs.txt
-RUN if [ -n "${UNINSTALL_DILL}" ]; then pip uninstall -y dill; fi
-
-# Install gcc
-ARG GCC_VERSION
-COPY ./common/install_gcc.sh install_gcc.sh
-RUN bash ./install_gcc.sh && rm install_gcc.sh
-
-# Install lcov for C++ code coverage
-COPY ./common/install_lcov.sh install_lcov.sh
-RUN  bash ./install_lcov.sh && rm install_lcov.sh
-
-# Install cuda and cudnn
-ARG CUDA_VERSION
-COPY ./common/install_cuda.sh install_cuda.sh
-RUN bash ./install_cuda.sh ${CUDA_VERSION} && rm install_cuda.sh
-ENV DESIRED_CUDA ${CUDA_VERSION}
-ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH
-
-# (optional) Install UCC
-ARG UCX_COMMIT
-ARG UCC_COMMIT
-ENV UCX_COMMIT $UCX_COMMIT
-ENV UCC_COMMIT $UCC_COMMIT
-ENV UCX_HOME /usr
-ENV UCC_HOME /usr
-ADD ./common/install_ucc.sh install_ucc.sh
-RUN if [ -n "${UCX_COMMIT}" ] && [ -n "${UCC_COMMIT}" ]; then bash ./install_ucc.sh; fi
-RUN rm install_ucc.sh
-
-# (optional) Install protobuf for ONNX
-ARG PROTOBUF
-COPY ./common/install_protobuf.sh install_protobuf.sh
-RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
-RUN rm install_protobuf.sh
-ENV INSTALLED_PROTOBUF ${PROTOBUF}
-
-# (optional) Install database packages like LMDB and LevelDB
-ARG DB
-COPY ./common/install_db.sh install_db.sh
-RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
-RUN rm install_db.sh
-ENV INSTALLED_DB ${DB}
-
-# (optional) Install vision packages like OpenCV
-ARG VISION
-COPY ./common/install_vision.sh ./common/cache_vision_models.sh ./common/common_utils.sh ./
-RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
-RUN rm install_vision.sh cache_vision_models.sh common_utils.sh
-ENV INSTALLED_VISION ${VISION}
-
-# (optional) Install Android NDK
-ARG ANDROID
-ARG ANDROID_NDK
-ARG GRADLE_VERSION
-COPY ./common/install_android.sh ./common/cache_vision_models.sh ./common/common_utils.sh ./
-COPY ./android/AndroidManifest.xml AndroidManifest.xml
-COPY ./android/build.gradle build.gradle
-RUN if [ -n "${ANDROID}" ]; then bash ./install_android.sh; fi
-RUN rm install_android.sh cache_vision_models.sh common_utils.sh
-RUN rm AndroidManifest.xml
-RUN rm build.gradle
-ENV INSTALLED_ANDROID ${ANDROID}
-
-# (optional) Install Vulkan SDK
-ARG VULKAN_SDK_VERSION
-COPY ./common/install_vulkan_sdk.sh install_vulkan_sdk.sh
-RUN if [ -n "${VULKAN_SDK_VERSION}" ]; then bash ./install_vulkan_sdk.sh; fi
-RUN rm install_vulkan_sdk.sh
-
-# (optional) Install swiftshader
-ARG SWIFTSHADER
-COPY ./common/install_swiftshader.sh install_swiftshader.sh
-RUN if [ -n "${SWIFTSHADER}" ]; then bash ./install_swiftshader.sh; fi
-RUN rm install_swiftshader.sh
-
-# (optional) Install non-default CMake version
-ARG CMAKE_VERSION
-COPY ./common/install_cmake.sh install_cmake.sh
-RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
-RUN rm install_cmake.sh
-
-# (optional) Install non-default Ninja version
-ARG NINJA_VERSION
-COPY ./common/install_ninja.sh install_ninja.sh
-RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
-RUN rm install_ninja.sh
-
-COPY ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh
-ENV OPENSSL_ROOT_DIR /opt/openssl
-ENV OPENSSL_DIR /opt/openssl
-RUN rm install_openssl.sh
-
-ARG INDUCTOR_BENCHMARKS
-COPY ./common/install_inductor_benchmark_deps.sh install_inductor_benchmark_deps.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/huggingface.txt huggingface.txt
-COPY ci_commit_pins/timm.txt timm.txt
-RUN if [ -n "${INDUCTOR_BENCHMARKS}" ]; then bash ./install_inductor_benchmark_deps.sh; fi
-RUN rm install_inductor_benchmark_deps.sh common_utils.sh timm.txt huggingface.txt
-
-ARG TRITON
-# Install triton, this needs to be done before sccache because the latter will
-# try to reach out to S3, which docker build runners don't have access
-COPY ./common/install_triton.sh install_triton.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/triton.txt triton.txt
-RUN if [ -n "${TRITON}" ]; then bash ./install_triton.sh; fi
-RUN rm install_triton.sh common_utils.sh triton.txt
-
-ARG EXECUTORCH
-# Build and install executorch
-COPY ./common/install_executorch.sh install_executorch.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/executorch.txt executorch.txt
-RUN if [ -n "${EXECUTORCH}" ]; then bash ./install_executorch.sh; fi
-RUN rm install_executorch.sh common_utils.sh executorch.txt
-
-ARG HALIDE
-# Build and install halide
-COPY ./common/install_halide.sh install_halide.sh
-COPY ./common/common_utils.sh common_utils.sh
-COPY ci_commit_pins/halide.txt halide.txt
-RUN if [ -n "${HALIDE}" ]; then bash ./install_halide.sh; fi
-RUN rm install_halide.sh common_utils.sh halide.txt
-
-ARG ONNX
-# Install ONNX dependencies
-COPY ./common/install_onnx.sh ./common/common_utils.sh ./
-RUN if [ -n "${ONNX}" ]; then bash ./install_onnx.sh; fi
-RUN rm install_onnx.sh common_utils.sh
-
-# (optional) Build ACL
-ARG ACL
-COPY ./common/install_acl.sh install_acl.sh
-RUN if [ -n "${ACL}" ]; then bash ./install_acl.sh; fi
-RUN rm install_acl.sh
-ENV INSTALLED_ACL ${ACL}
-
-# Install ccache/sccache (do this last, so we get priority in PATH)
-ARG SKIP_SCCACHE_INSTALL
-COPY ./common/install_cache.sh install_cache.sh
-ENV PATH /opt/cache/bin:$PATH
-RUN if [ -z "${SKIP_SCCACHE_INSTALL}" ]; then bash ./install_cache.sh; fi
-RUN rm install_cache.sh
-
-# Add jni.h for java host build
-COPY ./common/install_jni.sh install_jni.sh
-COPY ./java/jni.h jni.h
-RUN bash ./install_jni.sh && rm install_jni.sh
-
-# Install Open MPI for CUDA
-COPY ./common/install_openmpi.sh install_openmpi.sh
-RUN if [ -n "${CUDA_VERSION}" ]; then bash install_openmpi.sh; fi
-RUN rm install_openmpi.sh
-
-# Include BUILD_ENVIRONMENT environment variable in image
-ARG BUILD_ENVIRONMENT
-ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
-
-# Install LLVM dev version (Defined in the pytorch/builder github repository)
-ARG SKIP_LLVM_SRC_BUILD_INSTALL
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
-RUN if [ -n "${SKIP_LLVM_SRC_BUILD_INSTALL}" ]; then set -eu; rm -rf /opt/llvm; fi
-
-# AWS specific CUDA build guidance
-ENV TORCH_CUDA_ARCH_LIST Maxwell
-ENV TORCH_NVCC_FLAGS "-Xfatbin -compress-all"
-ENV CUDA_PATH /usr/local/cuda
-
-USER jenkins
-CMD ["bash"]

.ci/onnx/README.md

@@ -1,14 +0,0 @@
-# Jenkins
-
-The scripts in this directory are the entrypoint for testing ONNX exporter.
-
-The environment variable `BUILD_ENVIRONMENT` is expected to be set to
-the build environment you intend to test. It is a hint for the build
-and test scripts to configure Caffe2 a certain way and include/exclude
-tests. Docker images, they equal the name of the image itself. For
-example: `py2-cuda9.0-cudnn7-ubuntu16.04`. The Docker images that are
-built on Jenkins and are used in triggered builds already have this
-environment variable set in their manifest. Also see
-`./docker/jenkins/*/Dockerfile` and search for `BUILD_ENVIRONMENT`.
-
-Our Jenkins installation is located at https://ci.pytorch.org/jenkins/.