Revert "[release/1.6] .circleci: Don't use SCCACHE for windows release builds (#42024)"

This reverts commit 994b37b36ef11c8e26464512e68ff5573ce81fe6.

.bazelrc

@@ -1,115 +1,3 @@
-build --cxxopt=--std=c++17
+build --copt=--std=c++14
 build --copt=-I.
-# Bazel does not support including its cc_library targets as system
-# headers. We work around this for generated code
-# (e.g. c10/macros/cmake_macros.h) by making the generated directory a
-# system include path.
 build --copt=-isystem --copt bazel-out/k8-fastbuild/bin
-build --copt=-isystem --copt bazel-out/darwin-fastbuild/bin
-build --experimental_ui_max_stdouterr_bytes=2048576
-
-# Configuration to disable tty features for environments like CI
-build:no-tty --curses no
-build:no-tty --progress_report_interval 10
-build:no-tty --show_progress_rate_limit 10
-
-# Build with GPU support by default.
-build --define=cuda=true
-# rules_cuda configuration
-build --@rules_cuda//cuda:enable_cuda
-build --@rules_cuda//cuda:cuda_targets=sm_52
-build --@rules_cuda//cuda:compiler=nvcc
-build --repo_env=CUDA_PATH=/usr/local/cuda
-
-# Configuration to build without GPU support
-build:cpu-only --define=cuda=false
-# define a separate build folder for faster switching between configs
-build:cpu-only --platform_suffix=-cpu-only
-# See the note on the config-less build for details about why we are
-# doing this. We must also do it for the "-cpu-only" platform suffix.
-build --copt=-isystem --copt=bazel-out/k8-fastbuild-cpu-only/bin
-# rules_cuda configuration
-build:cpu-only --@rules_cuda//cuda:enable_cuda=False
-
-# Definition of --config=shell
-# interactive shell immediately before execution
-build:shell --run_under="//tools/bazel_tools:shellwrap"
-
-# Disable all warnings for external repositories. We don't care about
-# their warnings.
-build --per_file_copt=^external/@-w
-
-# Set additional warnings to error level.
-#
-# Implementation notes:
-#  * we use file extensions to determine if we are using the C++
-#    compiler or the cuda compiler
-#  * we use ^// at the start of the regex to only permit matching
-#    PyTorch files. This excludes external repos.
-#
-# Note that because this is logically a command-line flag, it is
-# considered the word on what warnings are enabled. This has the
-# unfortunate consequence of preventing us from disabling an error at
-# the target level because those flags will come before these flags in
-# the action invocation. Instead we provide per-file exceptions after
-# this.
-#
-# On the bright side, this means we don't have to more broadly apply
-# the exceptions to an entire target.
-#
-# Looking for CUDA flags? We have a cu_library macro that we can edit
-# directly. Look in //tools/rules:cu.bzl for details. Editing the
-# macro over this has the following advantages:
-#  * making changes does not require discarding the Bazel analysis
-#    cache
-#  * it allows for selective overrides on individual targets since the
-#    macro-level opts will come earlier than target level overrides
-
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Werror=all
-# The following warnings come from -Wall. We downgrade them from error
-# to warnings here.
-#
-# sign-compare has a tremendous amount of violations in the
-# codebase. It will be a lot of work to fix them, just disable it for
-# now.
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-sign-compare
-# We intentionally use #pragma unroll, which is compiler specific.
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-error=unknown-pragmas
-
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Werror=extra
-# The following warnings come from -Wextra. We downgrade them from error
-# to warnings here.
-#
-# unused-parameter-compare has a tremendous amount of violations in the
-# codebase. It will be a lot of work to fix them, just disable it for
-# now.
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-unused-parameter
-# missing-field-parameters has both a large number of violations in
-# the codebase, but it also is used pervasively in the Python C
-# API. There are a couple of catches though:
-# * we use multiple versions of the Python API and hence have
-#   potentially multiple different versions of each relevant
-#   struct. They may have different numbers of fields. It will be
-#   unwieldy to support multiple versions in the same source file.
-# * Python itself for many of these structs recommends only
-#   initializing a subset of the fields. We should respect the API
-#   usage conventions of our dependencies.
-#
-# Hence, we just disable this warning altogether. We may want to clean
-# up some of the clear-cut cases that could be risky, but we still
-# likely want to have this disabled for the most part.
-build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-missing-field-initializers
-
-build --per_file_copt='//:aten/src/ATen/RegisterCompositeExplicitAutograd\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterCompositeImplicitAutograd\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterMkldnnCPU\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterNestedTensorCPU\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterQuantizedCPU\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterSparseCPU\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterSparseCsrCPU\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterNestedTensorMeta\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterSparseMeta\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterQuantizedMeta\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:aten/src/ATen/RegisterZeroTensor\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:torch/csrc/lazy/generated/RegisterAutogradLazy\.cpp$'@-Wno-error=unused-function
-build --per_file_copt='//:torch/csrc/lazy/generated/RegisterLazy\.cpp$'@-Wno-error=unused-function

.bazelversion

@@ -1 +1 @@
-4.2.1
+3.1.0

.buckconfig.oss

@@ -1,25 +0,0 @@
-[pt]
-  is_oss=1
-
-[buildfile]
-  name = BUCK.oss
-  includes = //tools/build_defs/select.bzl
-
-[repositories]
-  bazel_skylib = third_party/bazel-skylib/
-  ovr_config = .
-
-[download]
-  in_build = true
-
-[cxx]
-  cxxflags = -std=c++17
-  should_remap_host_platform = true
-  cpp = /usr/bin/clang
-  cc = /usr/bin/clang
-  cxx = /usr/bin/clang++
-  cxxpp = /usr/bin/clang++
-  ld = /usr/bin/clang++
-
-[project]
-  default_flavors_mode=all

.ci/caffe2/common.sh

@@ -1,36 +0,0 @@
-set -ex
-
-LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
-TEST_DIR="$ROOT_DIR/test"
-gtest_reports_dir="${TEST_DIR}/test-reports/cpp"
-pytest_reports_dir="${TEST_DIR}/test-reports/python"
-
-# Figure out which Python to use
-PYTHON="$(which python)"
-if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
-  PYTHON=$(which "python${BASH_REMATCH[1]}")
-fi
-
-if [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
-    # HIP_PLATFORM is auto-detected by hipcc; unset to avoid build errors
-    unset HIP_PLATFORM
-    if which sccache > /dev/null; then
-        # Save sccache logs to file
-        sccache --stop-server || true
-        rm -f ~/sccache_error.log || true
-        SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=0 sccache --start-server
-
-        # Report sccache stats for easier debugging
-        sccache --zero-stats
-    fi
-fi
-
-# /usr/local/caffe2 is where the cpp bits are installed to in cmake-only
-# builds. In +python builds the cpp tests are copied to /usr/local/caffe2 so
-# that the test code in .ci/test.sh is the same
-INSTALL_PREFIX="/usr/local/caffe2"
-
-mkdir -p "$gtest_reports_dir" || true
-mkdir -p "$pytest_reports_dir" || true
-mkdir -p "$INSTALL_PREFIX" || true

.ci/caffe2/test.sh

@@ -1,172 +0,0 @@
-#!/bin/bash
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-
-if [[ ${BUILD_ENVIRONMENT} == *onnx* ]]; then
-  pip install click mock tabulate networkx==2.0
-  pip -q install --user "file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx"
-fi
-
-# Skip tests in environments where they are not built/applicable
-if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
-  echo 'Skipping tests'
-  exit 0
-fi
-if [[ "${BUILD_ENVIRONMENT}" == *-rocm* ]]; then
-  # temporary to locate some kernel issues on the CI nodes
-  export HSAKMT_DEBUG_LEVEL=4
-fi
-# These additional packages are needed for circleci ROCm builds.
-if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
-    # Need networkx 2.0 because bellmand_ford was moved in 2.1 . Scikit-image by
-    # defaults installs the most recent networkx version, so we install this lower
-    # version explicitly before scikit-image pulls it in as a dependency
-    pip install networkx==2.0
-    # click - onnx
-    pip install --progress-bar off click protobuf tabulate virtualenv mock typing-extensions
-fi
-
-# Find where cpp tests and Caffe2 itself are installed
-if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
-  # For cmake only build we install everything into /usr/local
-  cpp_test_dir="$INSTALL_PREFIX/cpp_test"
-  ld_library_path="$INSTALL_PREFIX/lib"
-else
-  # For Python builds we install into python
-  # cd to /usr first so the python import doesn't get confused by any 'caffe2'
-  # directory in cwd
-  python_installation="$(dirname $(dirname $(cd /usr && $PYTHON -c 'import os; import caffe2; print(os.path.realpath(caffe2.__file__))')))"
-  caffe2_pypath="$python_installation/caffe2"
-  cpp_test_dir="$python_installation/torch/test"
-  ld_library_path="$python_installation/torch/lib"
-fi
-
-################################################################################
-# C++ tests #
-################################################################################
-# Only run cpp tests in the first shard, don't run cpp tests a second time in the second shard
-if [[ "${SHARD_NUMBER:-1}" == "1" ]]; then
-  echo "Running C++ tests.."
-  for test in $(find "$cpp_test_dir" -executable -type f); do
-    case "$test" in
-      # skip tests we know are hanging or bad
-      */mkl_utils_test|*/aten/integer_divider_test)
-        continue
-        ;;
-      */scalar_tensor_test|*/basic|*/native_test)
-        if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
-          continue
-        else
-          LD_LIBRARY_PATH="$ld_library_path" "$test"
-        fi
-        ;;
-      */*_benchmark)
-        LD_LIBRARY_PATH="$ld_library_path" "$test" --benchmark_color=false
-        ;;
-      *)
-        # Currently, we use a mixture of gtest (caffe2) and Catch2 (ATen). While
-        # planning to migrate to gtest as the common PyTorch c++ test suite, we
-        # currently do NOT use the xml test reporter, because Catch doesn't
-        # support multiple reporters
-        # c.f. https://github.com/catchorg/Catch2/blob/master/docs/release-notes.md#223
-        # which means that enabling XML output means you lose useful stdout
-        # output for Jenkins.  It's more important to have useful console
-        # output than it is to have XML output for Jenkins.
-        # Note: in the future, if we want to use xml test reporter once we switch
-        # to all gtest, one can simply do:
-        LD_LIBRARY_PATH="$ld_library_path" \
-            "$test" --gtest_output=xml:"$gtest_reports_dir/$(basename $test).xml"
-        ;;
-    esac
-  done
-fi
-
-################################################################################
-# Python tests #
-################################################################################
-if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
-  exit 0
-fi
-
-# If pip is installed as root, we must use sudo.
-# CircleCI docker images could install conda as jenkins user, or use the OS's python package.
-PIP=$(which pip)
-PIP_USER=$(stat --format '%U' $PIP)
-CURRENT_USER=$(id -u -n)
-if [[ "$PIP_USER" = root && "$CURRENT_USER" != root ]]; then
-  MAYBE_SUDO=sudo
-fi
-
-# Uninstall pre-installed hypothesis and coverage to use an older version as newer
-# versions remove the timeout parameter from settings which ideep/conv_transpose_test.py uses
-$MAYBE_SUDO pip -q uninstall -y hypothesis
-$MAYBE_SUDO pip -q uninstall -y coverage
-
-# "pip install hypothesis==3.44.6" from official server is unreliable on
-# CircleCI, so we host a copy on S3 instead
-$MAYBE_SUDO pip -q install attrs==18.1.0 -f https://s3.amazonaws.com/ossci-linux/wheels/attrs-18.1.0-py2.py3-none-any.whl
-$MAYBE_SUDO pip -q install coverage==4.5.1 -f https://s3.amazonaws.com/ossci-linux/wheels/coverage-4.5.1-cp36-cp36m-macosx_10_12_x86_64.whl
-$MAYBE_SUDO pip -q install hypothesis==3.44.6 -f https://s3.amazonaws.com/ossci-linux/wheels/hypothesis-3.44.6-py3-none-any.whl
-
-# Collect additional tests to run (outside caffe2/python)
-EXTRA_TESTS=()
-
-# CUDA builds always include NCCL support
-if [[ "$BUILD_ENVIRONMENT" == *-cuda* ]] || [[ "$BUILD_ENVIRONMENT" == *-rocm* ]]; then
-  EXTRA_TESTS+=("$caffe2_pypath/contrib/nccl")
-fi
-
-rocm_ignore_test=()
-if [[ $BUILD_ENVIRONMENT == *-rocm* ]]; then
-  # Currently these tests are failing on ROCM platform:
-
-  # On ROCm, RCCL (distributed) development isn't complete.
-  # https://github.com/ROCmSoftwarePlatform/rccl
-  rocm_ignore_test+=("--ignore $caffe2_pypath/python/data_parallel_model_test.py")
-
-  # This test has been flaky in ROCm CI (but note the tests are
-  # cpu-only so should be unrelated to ROCm)
-  rocm_ignore_test+=("--ignore $caffe2_pypath/python/operator_test/blobs_queue_db_test.py")
-  # This test is skipped on Jenkins(compiled without MKL) and otherwise known flaky
-  rocm_ignore_test+=("--ignore $caffe2_pypath/python/ideep/convfusion_op_test.py")
-  # This test is skipped on Jenkins(compiled without MKL) and causing segfault on Circle
-  rocm_ignore_test+=("--ignore $caffe2_pypath/python/ideep/pool_op_test.py")
-fi
-
-echo "Running Python tests.."
-# locale setting is required by click package
-for loc in "en_US.utf8" "C.UTF-8"; do
-  if locale -a | grep "$loc" >/dev/null 2>&1; then
-    export LC_ALL="$loc"
-    export LANG="$loc"
-    break;
-  fi
-done
-
-# Some Caffe2 tests fail when run using AVX512 ISA, see https://github.com/pytorch/pytorch/issues/66111
-export DNNL_MAX_CPU_ISA=AVX2
-
-# Should still run even in the absence of SHARD_NUMBER
-if [[ "${SHARD_NUMBER:-1}" == "1" ]]; then
-  # TODO(sdym@meta.com) remove this when the linked issue resolved.
-  # py is temporary until https://github.com/Teemu/pytest-sugar/issues/241 is fixed
-  pip install --user py==1.11.0
-  pip install --user pytest-sugar
-  # NB: Warnings are disabled because they make it harder to see what
-  # the actual erroring test is
-  "$PYTHON" \
-    -m pytest \
-    -x \
-    -v \
-    --disable-warnings \
-    --junit-xml="$pytest_reports_dir/result.xml" \
-    --ignore "$caffe2_pypath/python/test/executor_test.py" \
-    --ignore "$caffe2_pypath/python/operator_test/matmul_op_test.py" \
-    --ignore "$caffe2_pypath/python/operator_test/pack_ops_test.py" \
-    --ignore "$caffe2_pypath/python/mkl/mkl_sbn_speed_test.py" \
-    --ignore "$caffe2_pypath/python/trt/test_pt_onnx_trt.py" \
-    ${rocm_ignore_test[@]} \
-    "$caffe2_pypath/python" \
-    "${EXTRA_TESTS[@]}"
-fi

.ci/docker/README.md

@@ -1,31 +0,0 @@
-# Docker images for Jenkins
-
-This directory contains everything needed to build the Docker images
-that are used in our CI
-
-The Dockerfiles located in subdirectories are parameterized to
-conditionally run build stages depending on build arguments passed to
-`docker build`. This lets us use only a few Dockerfiles for many
-images. The different configurations are identified by a freeform
-string that we call a _build environment_. This string is persisted in
-each image as the `BUILD_ENVIRONMENT` environment variable.
-
-See `build.sh` for valid build environments (it's the giant switch).
-
-Docker builds are now defined with `.circleci/cimodel/data/simple/docker_definitions.py`
-
-## Contents
-
-* `build.sh` -- dispatch script to launch all builds
-* `common` -- scripts used to execute individual Docker build stages
-* `ubuntu-cuda` -- Dockerfile for Ubuntu image with CUDA support for nvidia-docker
-
-## Usage
-
-```bash
-# Build a specific image
-./build.sh pytorch-linux-bionic-py3.8-gcc9 -t myimage:latest
-
-# Set flags (see build.sh) and build image
-sudo bash -c 'PROTOBUF=1 ./build.sh pytorch-linux-bionic-py3.8-gcc9 -t myimage:latest
-```

.ci/docker/build.sh

@@ -1,392 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-image="$1"
-shift
-
-if [ -z "${image}" ]; then
-  echo "Usage: $0 IMAGE"
-  exit 1
-fi
-
-function extract_version_from_image_name() {
-  eval export $2=$(echo "${image}" | perl -n -e"/$1(\d+(\.\d+)?(\.\d+)?)/ && print \$1")
-  if [ "x${!2}" = x ]; then
-    echo "variable '$2' not correctly parsed from image='$image'"
-    exit 1
-  fi
-}
-
-function extract_all_from_image_name() {
-  # parts $image into array, splitting on '-'
-  keep_IFS="$IFS"
-  IFS="-"
-  declare -a parts=($image)
-  IFS="$keep_IFS"
-  unset keep_IFS
-
-  for part in "${parts[@]}"; do
-    name=$(echo "${part}" | perl -n -e"/([a-zA-Z]+)\d+(\.\d+)?(\.\d+)?/ && print \$1")
-    vername="${name^^}_VERSION"
-    # "py" is the odd one out, needs this special case
-    if [ "x${name}" = xpy ]; then
-      vername=ANACONDA_PYTHON_VERSION
-    fi
-    # skip non-conforming fields such as "pytorch", "linux" or "bionic" without version string
-    if [ -n "${name}" ]; then
-      extract_version_from_image_name "${name}" "${vername}"
-    fi
-  done
-}
-
-# Use the same pre-built XLA test image from PyTorch/XLA
-if [[ "$image" == *xla* ]]; then
-  echo "Using pre-built XLA test image..."
-  exit 0
-fi
-
-if [[ "$image" == *-bionic* ]]; then
-  UBUNTU_VERSION=18.04
-elif [[ "$image" == *-focal* ]]; then
-  UBUNTU_VERSION=20.04
-elif [[ "$image" == *-jammy* ]]; then
-  UBUNTU_VERSION=22.04
-elif [[ "$image" == *ubuntu* ]]; then
-  extract_version_from_image_name ubuntu UBUNTU_VERSION
-elif [[ "$image" == *centos* ]]; then
-  extract_version_from_image_name centos CENTOS_VERSION
-fi
-
-if [ -n "${UBUNTU_VERSION}" ]; then
-  OS="ubuntu"
-elif [ -n "${CENTOS_VERSION}" ]; then
-  OS="centos"
-else
-  echo "Unable to derive operating system base..."
-  exit 1
-fi
-
-DOCKERFILE="${OS}/Dockerfile"
-# When using ubuntu - 22.04, start from Ubuntu docker image, instead of nvidia/cuda docker image.
-if [[ "$image" == *cuda* && "$UBUNTU_VERSION" != "22.04" ]]; then
-  DOCKERFILE="${OS}-cuda/Dockerfile"
-elif [[ "$image" == *rocm* ]]; then
-  DOCKERFILE="${OS}-rocm/Dockerfile"
-elif [[ "$image" == *linter* ]]; then
-  # Use a separate Dockerfile for linter to keep a small image size
-  DOCKERFILE="linter/Dockerfile"
-fi
-
-# CMake 3.18 is needed to support CUDA17 language variant
-CMAKE_VERSION=3.18.5
-
-_UCX_COMMIT=31e74cac7bee0ef66bef2af72e7d86d9c282e5ab
-_UCC_COMMIT=1c7a7127186e7836f73aafbd7697bbc274a77eee
-
-# It's annoying to rename jobs every time you want to rewrite a
-# configuration, so we hardcode everything here rather than do it
-# from scratch
-case "$image" in
-  pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7)
-    CUDA_VERSION=11.6.2
-    CUDNN_VERSION=8
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=7
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7)
-    CUDA_VERSION=11.7.0
-    CUDNN_VERSION=8
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=7
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-bionic-cuda11.8-cudnn8-py3-gcc7)
-    CUDA_VERSION=11.8.0
-    CUDNN_VERSION=8
-    ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=7
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    UCX_COMMIT=${_UCX_COMMIT}
-    UCC_COMMIT=${_UCC_COMMIT}
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-focal-py3-clang7-asan)
-    ANACONDA_PYTHON_VERSION=3.9
-    CLANG_VERSION=7
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-focal-py3-clang10-onnx)
-    ANACONDA_PYTHON_VERSION=3.8
-    CLANG_VERSION=10
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-focal-py3-clang7-android-ndk-r19c)
-    ANACONDA_PYTHON_VERSION=3.7
-    CLANG_VERSION=7
-    LLVMDEV=yes
-    PROTOBUF=yes
-    ANDROID=yes
-    ANDROID_NDK_VERSION=r19c
-    GRADLE_VERSION=6.8.3
-    NINJA_VERSION=1.9.0
-    ;;
-  pytorch-linux-bionic-py3.8-clang9)
-    ANACONDA_PYTHON_VERSION=3.8
-    CLANG_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    VULKAN_SDK_VERSION=1.2.162.1
-    SWIFTSHADER=yes
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-bionic-py3.11-clang9)
-    ANACONDA_PYTHON_VERSION=3.11
-    CLANG_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    VULKAN_SDK_VERSION=1.2.162.1
-    SWIFTSHADER=yes
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-bionic-py3.8-gcc9)
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-focal-rocm-n-1-py3)
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ROCM_VERSION=5.3
-    NINJA_VERSION=1.9.0
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-focal-rocm-n-py3)
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ROCM_VERSION=5.4.2
-    NINJA_VERSION=1.9.0
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-focal-py3.8-gcc7)
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=7
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    CONDA_CMAKE=yes
-    ;;
-  pytorch-linux-jammy-cuda11.6-cudnn8-py3.8-clang12)
-    ANACONDA_PYTHON_VERSION=3.8
-    CUDA_VERSION=11.6
-    CUDNN_VERSION=8
-    CLANG_VERSION=12
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ;;
-  pytorch-linux-jammy-cuda11.7-cudnn8-py3.8-clang12)
-    ANACONDA_PYTHON_VERSION=3.8
-    CUDA_VERSION=11.7
-    CUDNN_VERSION=8
-    CLANG_VERSION=12
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ;;
-  pytorch-linux-jammy-cuda11.8-cudnn8-py3.8-clang12)
-    ANACONDA_PYTHON_VERSION=3.8
-    CUDA_VERSION=11.8
-    CUDNN_VERSION=8
-    CLANG_VERSION=12
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ;;
-  pytorch-linux-focal-linter)
-    # TODO: Use 3.9 here because of this issue https://github.com/python/mypy/issues/13627.
-    # We will need to update mypy version eventually, but that's for another day. The task
-    # would be to upgrade mypy to 1.0.0 with Python 3.11
-    ANACONDA_PYTHON_VERSION=3.9
-    CONDA_CMAKE=yes
-    ;;
-  *)
-    # Catch-all for builds that are not hardcoded.
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    echo "image '$image' did not match an existing build configuration"
-    if [[ "$image" == *py* ]]; then
-      extract_version_from_image_name py ANACONDA_PYTHON_VERSION
-    fi
-    if [[ "$image" == *cuda* ]]; then
-      extract_version_from_image_name cuda CUDA_VERSION
-      extract_version_from_image_name cudnn CUDNN_VERSION
-    fi
-    if [[ "$image" == *rocm* ]]; then
-      extract_version_from_image_name rocm ROCM_VERSION
-      NINJA_VERSION=1.9.0
-    fi
-    if [[ "$image" == *centos7* ]]; then
-      NINJA_VERSION=1.10.2
-    fi
-    if [[ "$image" == *gcc* ]]; then
-      extract_version_from_image_name gcc GCC_VERSION
-    fi
-    if [[ "$image" == *clang* ]]; then
-      extract_version_from_image_name clang CLANG_VERSION
-    fi
-    if [[ "$image" == *devtoolset* ]]; then
-      extract_version_from_image_name devtoolset DEVTOOLSET_VERSION
-    fi
-    if [[ "$image" == *glibc* ]]; then
-      extract_version_from_image_name glibc GLIBC_VERSION
-    fi
-    if [[ "$image" == *cmake* ]]; then
-      extract_version_from_image_name cmake CMAKE_VERSION
-    fi
-  ;;
-esac
-
-tmp_tag=$(basename "$(mktemp -u)" | tr '[:upper:]' '[:lower:]')
-
-#when using cudnn version 8 install it separately from cuda
-if [[ "$image" == *cuda*  && ${OS} == "ubuntu" ]]; then
-  IMAGE_NAME="nvidia/cuda:${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel-ubuntu${UBUNTU_VERSION}"
-  if [[ ${CUDNN_VERSION} == 8 ]]; then
-    IMAGE_NAME="nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}"
-  fi
-fi
-
-# Build image
-# TODO: build-arg THRIFT is not turned on for any image, remove it once we confirm
-# it's no longer needed.
-docker build \
-       --no-cache \
-       --progress=plain \
-       --build-arg "BUILD_ENVIRONMENT=${image}" \
-       --build-arg "PROTOBUF=${PROTOBUF:-}" \
-       --build-arg "THRIFT=${THRIFT:-}" \
-       --build-arg "LLVMDEV=${LLVMDEV:-}" \
-       --build-arg "DB=${DB:-}" \
-       --build-arg "VISION=${VISION:-}" \
-       --build-arg "UBUNTU_VERSION=${UBUNTU_VERSION}" \
-       --build-arg "CENTOS_VERSION=${CENTOS_VERSION}" \
-       --build-arg "DEVTOOLSET_VERSION=${DEVTOOLSET_VERSION}" \
-       --build-arg "GLIBC_VERSION=${GLIBC_VERSION}" \
-       --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
-       --build-arg "ANACONDA_PYTHON_VERSION=${ANACONDA_PYTHON_VERSION}" \
-       --build-arg "GCC_VERSION=${GCC_VERSION}" \
-       --build-arg "CUDA_VERSION=${CUDA_VERSION}" \
-       --build-arg "CUDNN_VERSION=${CUDNN_VERSION}" \
-       --build-arg "TENSORRT_VERSION=${TENSORRT_VERSION}" \
-       --build-arg "ANDROID=${ANDROID}" \
-       --build-arg "ANDROID_NDK=${ANDROID_NDK_VERSION}" \
-       --build-arg "GRADLE_VERSION=${GRADLE_VERSION}" \
-       --build-arg "VULKAN_SDK_VERSION=${VULKAN_SDK_VERSION}" \
-       --build-arg "SWIFTSHADER=${SWIFTSHADER}" \
-       --build-arg "CMAKE_VERSION=${CMAKE_VERSION:-}" \
-       --build-arg "NINJA_VERSION=${NINJA_VERSION:-}" \
-       --build-arg "KATEX=${KATEX:-}" \
-       --build-arg "ROCM_VERSION=${ROCM_VERSION:-}" \
-       --build-arg "PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH:-gfx906}" \
-       --build-arg "IMAGE_NAME=${IMAGE_NAME}" \
-       --build-arg "UCX_COMMIT=${UCX_COMMIT}" \
-       --build-arg "UCC_COMMIT=${UCC_COMMIT}" \
-       --build-arg "CONDA_CMAKE=${CONDA_CMAKE}" \
-       -f $(dirname ${DOCKERFILE})/Dockerfile \
-       -t "$tmp_tag" \
-       "$@" \
-       .
-
-# NVIDIA dockers for RC releases use tag names like `11.0-cudnn8-devel-ubuntu18.04-rc`,
-# for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
-# find the correct image. As a result, here we have to replace the
-#   "$UBUNTU_VERSION" == "18.04-rc"
-# with
-#   "$UBUNTU_VERSION" == "18.04"
-UBUNTU_VERSION=$(echo ${UBUNTU_VERSION} | sed 's/-rc$//')
-
-function drun() {
-  docker run --rm "$tmp_tag" $*
-}
-
-if [[ "$OS" == "ubuntu" ]]; then
-
-  if !(drun lsb_release -a 2>&1 | grep -qF Ubuntu); then
-    echo "OS=ubuntu, but:"
-    drun lsb_release -a
-    exit 1
-  fi
-  if !(drun lsb_release -a 2>&1 | grep -qF "$UBUNTU_VERSION"); then
-    echo "UBUNTU_VERSION=$UBUNTU_VERSION, but:"
-    drun lsb_release -a
-    exit 1
-  fi
-fi
-
-if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
-  if !(drun python --version 2>&1 | grep -qF "Python $ANACONDA_PYTHON_VERSION"); then
-    echo "ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION, but:"
-    drun python --version
-    exit 1
-  fi
-fi
-
-if [ -n "$GCC_VERSION" ]; then
-  if !(drun gcc --version 2>&1 | grep -q " $GCC_VERSION\\W"); then
-    echo "GCC_VERSION=$GCC_VERSION, but:"
-    drun gcc --version
-    exit 1
-  fi
-fi
-
-if [ -n "$CLANG_VERSION" ]; then
-  if !(drun clang --version 2>&1 | grep -qF "clang version $CLANG_VERSION"); then
-    echo "CLANG_VERSION=$CLANG_VERSION, but:"
-    drun clang --version
-    exit 1
-  fi
-fi
-
-if [ -n "$KATEX" ]; then
-  if !(drun katex --version); then
-    echo "KATEX=$KATEX, but:"
-    drun katex --version
-    exit 1
-  fi
-fi

.ci/docker/build_docker.sh

@@ -1,60 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-retry () {
-    $*  || (sleep 1 && $*) || (sleep 2 && $*)
-}
-
-# If UPSTREAM_BUILD_ID is set (see trigger job), then we can
-# use it to tag this build with the same ID used to tag all other
-# base image builds. Also, we can try and pull the previous
-# image first, to avoid rebuilding layers that haven't changed.
-
-#until we find a way to reliably reuse previous build, this last_tag is not in use
-# last_tag="$(( CIRCLE_BUILD_NUM - 1 ))"
-tag="${DOCKER_TAG}"
-
-
-registry="308535385114.dkr.ecr.us-east-1.amazonaws.com"
-image="${registry}/pytorch/${IMAGE_NAME}"
-
-login() {
-  aws ecr get-authorization-token --region us-east-1 --output text --query 'authorizationData[].authorizationToken' |
-    base64 -d |
-    cut -d: -f2 |
-    docker login -u AWS --password-stdin "$1"
-}
-
-
-# Only run these steps if not on github actions
-if [[ -z "${GITHUB_ACTIONS}" ]]; then
-  # Retry on timeouts (can happen on job stampede).
-  retry login "${registry}"
-  # Logout on exit
-  trap "docker logout ${registry}" EXIT
-fi
-
-# Try to pull the previous image (perhaps we can reuse some layers)
-# if [ -n "${last_tag}" ]; then
-#   docker pull "${image}:${last_tag}" || true
-# fi
-
-# Build new image
-./build.sh ${IMAGE_NAME} -t "${image}:${tag}"
-
-# Only push if `DOCKER_SKIP_PUSH` = false
-if [ "${DOCKER_SKIP_PUSH:-true}" = "false" ]; then
-  # Only push if docker image doesn't exist already.
-  # ECR image tags are immutable so this will avoid pushing if only just testing if the docker jobs work
-  # NOTE: The only workflow that should push these images should be the docker-builds.yml workflow
-  if ! docker manifest inspect "${image}:${tag}" >/dev/null 2>/dev/null; then
-    docker push "${image}:${tag}"
-  fi
-fi
-
-if [ -z "${DOCKER_SKIP_S3_UPLOAD:-}" ]; then
-  trap "rm -rf ${IMAGE_NAME}:${tag}.tar" EXIT
-  docker save -o "${IMAGE_NAME}:${tag}.tar" "${image}:${tag}"
-  aws s3 cp "${IMAGE_NAME}:${tag}.tar" "s3://ossci-linux-build/pytorch/base/${IMAGE_NAME}:${tag}.tar" --acl public-read
-fi

.ci/docker/centos-rocm/Dockerfile

@@ -1,111 +0,0 @@
-ARG CENTOS_VERSION
-
-FROM centos:${CENTOS_VERSION}
-
-ARG CENTOS_VERSION
-
-# Set AMD gpu targets to build for
-ARG PYTORCH_ROCM_ARCH
-ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
-
-# Install required packages to build Caffe2
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Update CentOS git version
-RUN yum -y remove git
-RUN yum -y remove git-*
-RUN yum -y install https://packages.endpoint.com/rhel/7/os/x86_64/endpoint-repo-1.9-1.x86_64.rpm || \
-    (yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo-1.9-1.x86_64.rpm && \
-    sed -i "s/packages.endpoint/packages.endpointdev/" /etc/yum.repos.d/endpoint.repo)
-RUN yum install -y git
-
-# Install devtoolset
-ARG DEVTOOLSET_VERSION
-COPY ./common/install_devtoolset.sh install_devtoolset.sh
-RUN bash ./install_devtoolset.sh && rm install_devtoolset.sh
-ENV BASH_ENV "/etc/profile"
-
-# (optional) Install non-default glibc version
-ARG GLIBC_VERSION
-COPY ./common/install_glibc.sh install_glibc.sh
-RUN if [ -n "${GLIBC_VERSION}" ]; then bash ./install_glibc.sh; fi
-RUN rm install_glibc.sh
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ARG CONDA_CMAKE
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-COPY requirements-ci.txt /opt/conda/requirements-ci.txt
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
-
-# (optional) Install protobuf for ONNX
-ARG PROTOBUF
-COPY ./common/install_protobuf.sh install_protobuf.sh
-RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
-RUN rm install_protobuf.sh
-ENV INSTALLED_PROTOBUF ${PROTOBUF}
-
-# (optional) Install database packages like LMDB and LevelDB
-ARG DB
-COPY ./common/install_db.sh install_db.sh
-RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
-RUN rm install_db.sh
-ENV INSTALLED_DB ${DB}
-
-# (optional) Install vision packages like OpenCV and ffmpeg
-ARG VISION
-COPY ./common/install_vision.sh install_vision.sh
-RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
-RUN rm install_vision.sh
-ENV INSTALLED_VISION ${VISION}
-
-# Install rocm
-ARG ROCM_VERSION
-COPY ./common/install_rocm.sh install_rocm.sh
-RUN bash ./install_rocm.sh
-RUN rm install_rocm.sh
-COPY ./common/install_rocm_magma.sh install_rocm_magma.sh
-RUN bash ./install_rocm_magma.sh
-RUN rm install_rocm_magma.sh
-ENV PATH /opt/rocm/bin:$PATH
-ENV PATH /opt/rocm/hcc/bin:$PATH
-ENV PATH /opt/rocm/hip/bin:$PATH
-ENV PATH /opt/rocm/opencl/bin:$PATH
-ENV PATH /opt/rocm/llvm/bin:$PATH
-ENV MAGMA_HOME /opt/rocm/magma
-ENV LANG en_US.utf8
-ENV LC_ALL en_US.utf8
-
-# (optional) Install non-default CMake version
-ARG CMAKE_VERSION
-COPY ./common/install_cmake.sh install_cmake.sh
-RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
-RUN rm install_cmake.sh
-
-# (optional) Install non-default Ninja version
-ARG NINJA_VERSION
-COPY ./common/install_ninja.sh install_ninja.sh
-RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
-RUN rm install_ninja.sh
-
-# Install ccache/sccache (do this last, so we get priority in PATH)
-COPY ./common/install_cache.sh install_cache.sh
-ENV PATH /opt/cache/bin:$PATH
-RUN bash ./install_cache.sh && rm install_cache.sh
-
-# Include BUILD_ENVIRONMENT environment variable in image
-ARG BUILD_ENVIRONMENT
-ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
-
-USER jenkins
-CMD ["bash"]

.ci/docker/common/common_utils.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# Work around bug where devtoolset replaces sudo and breaks it.
-if [ -n "$DEVTOOLSET_VERSION" ]; then
-  export SUDO=/bin/sudo
-else
-  export SUDO=sudo
-fi
-
-as_jenkins() {
-  # NB: unsetting the environment variables works around a conda bug
-  # https://github.com/conda/conda/issues/6576
-  # NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
-  # NB: This must be run from a directory that jenkins has access to,
-  # works around https://github.com/conda/conda-package-handling/pull/34
-  $SUDO -H -u jenkins env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" $*
-}
-
-conda_install() {
-  # Ensure that the install command don't upgrade/downgrade Python
-  # This should be called as
-  #   conda_install pkg1 pkg2 ... [-c channel]
-  as_jenkins conda install -q -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION" $*
-}
-
-conda_run() {
-  as_jenkins conda run -n py_$ANACONDA_PYTHON_VERSION --no-capture-output $*
-}
-
-pip_install() {
-  as_jenkins conda run -n py_$ANACONDA_PYTHON_VERSION pip install --progress-bar off $*
-}

.ci/docker/common/install_android.sh

@@ -1,109 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-[ -n "${ANDROID_NDK}" ]
-
-_https_amazon_aws=https://ossci-android.s3.amazonaws.com
-
-apt-get update
-apt-get install -y --no-install-recommends autotools-dev autoconf unzip
-apt-get autoclean && apt-get clean
-rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-pushd /tmp
-curl -Os --retry 3 $_https_amazon_aws/android-ndk-${ANDROID_NDK}-linux-x86_64.zip
-popd
-_ndk_dir=/opt/ndk
-mkdir -p "$_ndk_dir"
-unzip -qo /tmp/android*.zip -d "$_ndk_dir"
-_versioned_dir=$(find "$_ndk_dir/" -mindepth 1 -maxdepth 1 -type d)
-mv "$_versioned_dir"/* "$_ndk_dir"/
-rmdir "$_versioned_dir"
-rm -rf /tmp/*
-
-# Install OpenJDK
-# https://hub.docker.com/r/picoded/ubuntu-openjdk-8-jdk/dockerfile/
-
-sudo apt-get update && \
-    apt-get install -y openjdk-8-jdk && \
-    apt-get install -y ant && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* && \
-    rm -rf /var/cache/oracle-jdk8-installer;
-
-# Fix certificate issues, found as of
-# https://bugs.launchpad.net/ubuntu/+source/ca-certificates-java/+bug/983302
-
-sudo apt-get update && \
-    apt-get install -y ca-certificates-java && \
-    apt-get clean && \
-    update-ca-certificates -f && \
-    rm -rf /var/lib/apt/lists/* && \
-    rm -rf /var/cache/oracle-jdk8-installer;
-
-export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-
-# Installing android sdk
-# https://github.com/circleci/circleci-images/blob/staging/android/Dockerfile.m4
-
-_tmp_sdk_zip=/tmp/android-sdk-linux.zip
-_android_home=/opt/android/sdk
-
-rm -rf $_android_home
-sudo mkdir -p $_android_home
-curl --silent --show-error --location --fail --retry 3 --output /tmp/android-sdk-linux.zip $_https_amazon_aws/android-sdk-linux-tools3859397-build-tools2803-2902-platforms28-29.zip
-sudo unzip -q $_tmp_sdk_zip -d $_android_home
-rm $_tmp_sdk_zip
-
-sudo chmod -R 777 $_android_home
-
-export ANDROID_HOME=$_android_home
-export ADB_INSTALL_TIMEOUT=120
-
-export PATH="${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:${PATH}"
-echo "PATH:${PATH}"
-
-# Installing Gradle
-echo "GRADLE_VERSION:${GRADLE_VERSION}"
-_gradle_home=/opt/gradle
-sudo rm -rf $gradle_home
-sudo mkdir -p $_gradle_home
-
-curl --silent --output /tmp/gradle.zip --retry 3 $_https_amazon_aws/gradle-${GRADLE_VERSION}-bin.zip
-
-sudo unzip -q /tmp/gradle.zip -d $_gradle_home
-rm /tmp/gradle.zip
-
-sudo chmod -R 777 $_gradle_home
-
-export GRADLE_HOME=$_gradle_home/gradle-$GRADLE_VERSION
-alias gradle="${GRADLE_HOME}/bin/gradle"
-
-export PATH="${GRADLE_HOME}/bin/:${PATH}"
-echo "PATH:${PATH}"
-
-gradle --version
-
-mkdir /var/lib/jenkins/gradledeps
-cp build.gradle /var/lib/jenkins/gradledeps
-cp AndroidManifest.xml /var/lib/jenkins/gradledeps
-
-pushd /var/lib/jenkins
-
-export GRADLE_LOCAL_PROPERTIES=gradledeps/local.properties
-rm -f $GRADLE_LOCAL_PROPERTIES
-echo "sdk.dir=/opt/android/sdk" >> $GRADLE_LOCAL_PROPERTIES
-echo "ndk.dir=/opt/ndk" >> $GRADLE_LOCAL_PROPERTIES
-
-chown -R jenkins /var/lib/jenkins/gradledeps
-chgrp -R jenkins /var/lib/jenkins/gradledeps
-
-sudo -H -u jenkins $GRADLE_HOME/bin/gradle -Pandroid.useAndroidX=true -p /var/lib/jenkins/gradledeps -g /var/lib/jenkins/.gradle --refresh-dependencies --debug --stacktrace assemble
-
-chown -R jenkins /var/lib/jenkins/.gradle
-chgrp -R jenkins /var/lib/jenkins/.gradle
-
-popd
-
-rm -rf /var/lib/jenkins/.gradle/daemon

.ci/docker/common/install_base.sh

@@ -1,169 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-install_ubuntu() {
-  # NVIDIA dockers for RC releases use tag names like `11.0-cudnn8-devel-ubuntu18.04-rc`,
-  # for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
-  # find the correct image. As a result, here we have to check for
-  #   "$UBUNTU_VERSION" == "18.04"*
-  # instead of
-  #   "$UBUNTU_VERSION" == "18.04"
-  if [[ "$UBUNTU_VERSION" == "18.04"* ]]; then
-    cmake3="cmake=3.10*"
-    maybe_libiomp_dev="libiomp-dev"
-  elif [[ "$UBUNTU_VERSION" == "20.04"* ]]; then
-    cmake3="cmake=3.16*"
-    maybe_libiomp_dev=""
-  elif [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
-    cmake3="cmake=3.22*"
-    maybe_libiomp_dev=""
-  else
-    cmake3="cmake=3.5*"
-    maybe_libiomp_dev="libiomp-dev"
-  fi
-
-  if [[ "$CLANG_VERSION" == 12 ]]; then
-    maybe_libomp_dev="libomp-12-dev"
-  elif [[ "$CLANG_VERSION" == 10 ]]; then
-    maybe_libomp_dev="libomp-10-dev"
-  else
-    maybe_libomp_dev=""
-  fi
-
-  # TODO: Remove this once nvidia package repos are back online
-  # Comment out nvidia repositories to prevent them from getting apt-get updated, see https://github.com/pytorch/pytorch/issues/74968
-  # shellcheck disable=SC2046
-  sed -i 's/.*nvidia.*/# &/' $(find /etc/apt/ -type f -name "*.list")
-
-  # Install common dependencies
-  apt-get update
-  # TODO: Some of these may not be necessary
-  ccache_deps="asciidoc docbook-xml docbook-xsl xsltproc"
-  deploy_deps="libffi-dev libbz2-dev libreadline-dev libncurses5-dev libncursesw5-dev libgdbm-dev libsqlite3-dev uuid-dev tk-dev"
-  numpy_deps="gfortran"
-  apt-get install -y --no-install-recommends \
-    $ccache_deps \
-    $numpy_deps \
-    ${deploy_deps} \
-    ${cmake3} \
-    apt-transport-https \
-    autoconf \
-    automake \
-    build-essential \
-    ca-certificates \
-    curl \
-    git \
-    libatlas-base-dev \
-    libc6-dbg \
-    ${maybe_libiomp_dev} \
-    libyaml-dev \
-    libz-dev \
-    libjpeg-dev \
-    libasound2-dev \
-    libsndfile-dev \
-    ${maybe_libomp_dev} \
-    software-properties-common \
-    wget \
-    sudo \
-    vim \
-    jq \
-    libtool \
-    vim \
-    unzip \
-    gdb
-
-  # Should resolve issues related to various apt package repository cert issues
-  # see: https://github.com/pytorch/pytorch/issues/65931
-  apt-get install -y libgnutls30
-
-  # cuda-toolkit does not work with gcc-11.2.0 which is default in Ubunutu 22.04
-  # see: https://github.com/NVlabs/instant-ngp/issues/119
-  if [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
-    apt-get install -y g++-10
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30
-    update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 30
-
-    # https://www.spinics.net/lists/libreoffice/msg07549.html
-    sudo rm -rf /usr/lib/gcc/x86_64-linux-gnu/11
-    wget https://github.com/gcc-mirror/gcc/commit/2b2d97fc545635a0f6aa9c9ee3b017394bc494bf.patch -O noexecpt.patch
-    sudo patch  /usr/include/c++/10/bits/range_access.h noexecpt.patch
-  fi
-
-  # Cleanup package manager
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-}
-
-install_centos() {
-  # Need EPEL for many packages we depend on.
-  # See http://fedoraproject.org/wiki/EPEL
-  yum --enablerepo=extras install -y epel-release
-
-  ccache_deps="asciidoc docbook-dtds docbook-style-xsl libxslt"
-  numpy_deps="gcc-gfortran"
-  # Note: protobuf-c-{compiler,devel} on CentOS are too old to be used
-  # for Caffe2. That said, we still install them to make sure the build
-  # system opts to build/use protoc and libprotobuf from third-party.
-  yum install -y \
-    $ccache_deps \
-    $numpy_deps \
-    autoconf \
-    automake \
-    bzip2 \
-    cmake \
-    cmake3 \
-    curl \
-    gcc \
-    gcc-c++ \
-    gflags-devel \
-    git \
-    glibc-devel \
-    glibc-headers \
-    glog-devel \
-    hiredis-devel \
-    libstdc++-devel \
-    libsndfile-devel \
-    make \
-    opencv-devel \
-    sudo \
-    wget \
-    vim \
-    unzip \
-    gdb
-
-  # Cleanup
-  yum clean all
-  rm -rf /var/cache/yum
-  rm -rf /var/lib/yum/yumdb
-  rm -rf /var/lib/yum/history
-}
-
-# Install base packages depending on the base OS
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    install_ubuntu
-    ;;
-  centos)
-    install_centos
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac
-
-# Install Valgrind separately since the apt-get version is too old.
-mkdir valgrind_build && cd valgrind_build
-VALGRIND_VERSION=3.20.0
-wget https://ossci-linux.s3.amazonaws.com/valgrind-${VALGRIND_VERSION}.tar.bz2
-tar -xjf valgrind-${VALGRIND_VERSION}.tar.bz2
-cd valgrind-${VALGRIND_VERSION}
-./configure --prefix=/usr/local
-make -j6
-sudo make install
-cd ../../
-rm -rf valgrind_build
-alias valgrind="/usr/local/bin/valgrind"

.ci/docker/common/install_cache.sh

@@ -1,121 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-install_ubuntu() {
-  echo "Preparing to build sccache from source"
-  apt-get update
-  # libssl-dev will not work as it is upgraded to libssl3 in Ubuntu-22.04.
-  # Instead use lib and headers from OpenSSL1.1 installed in `install_openssl.sh``
-  apt-get install -y cargo
-  echo "Checking out sccache repo"
-  git clone https://github.com/pytorch/sccache
-  cd sccache
-  echo "Building sccache"
-  cargo build --release
-  cp target/release/sccache /opt/cache/bin
-  echo "Cleaning up"
-  cd ..
-  rm -rf sccache
-  apt-get remove -y cargo rustc
-  apt-get autoclean && apt-get clean
-}
-
-install_binary() {
-  echo "Downloading sccache binary from S3 repo"
-  curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
-}
-
-mkdir -p /opt/cache/bin
-mkdir -p /opt/cache/lib
-sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment
-export PATH="/opt/cache/bin:$PATH"
-
-# Setup compiler cache
-if [ -n "$ROCM_VERSION" ]; then
-  curl --retry 3 http://repo.radeon.com/misc/.sccache_amd/sccache -o /opt/cache/bin/sccache
-else
-  ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-  case "$ID" in
-    ubuntu)
-      install_ubuntu
-      ;;
-    *)
-      install_binary
-      ;;
-  esac
-fi
-chmod a+x /opt/cache/bin/sccache
-
-function write_sccache_stub() {
-  # Unset LD_PRELOAD for ps because of asan + ps issues
-  # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90589
-  printf "#!/bin/sh\nif [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then\n  exec sccache $(which $1) \"\$@\"\nelse\n  exec $(which $1) \"\$@\"\nfi" > "/opt/cache/bin/$1"
-  chmod a+x "/opt/cache/bin/$1"
-}
-
-write_sccache_stub cc
-write_sccache_stub c++
-write_sccache_stub gcc
-write_sccache_stub g++
-
-# NOTE: See specific ROCM_VERSION case below.
-if [ "x$ROCM_VERSION" = x ]; then
-  write_sccache_stub clang
-  write_sccache_stub clang++
-fi
-
-if [ -n "$CUDA_VERSION" ]; then
-  # TODO: This is a workaround for the fact that PyTorch's FindCUDA
-  # implementation cannot find nvcc if it is setup this way, because it
-  # appears to search for the nvcc in PATH, and use its path to infer
-  # where CUDA is installed.  Instead, we install an nvcc symlink outside
-  # of the PATH, and set CUDA_NVCC_EXECUTABLE so that we make use of it.
-
-  write_sccache_stub nvcc
-  mv /opt/cache/bin/nvcc /opt/cache/lib/
-fi
-
-if [ -n "$ROCM_VERSION" ]; then
-  # ROCm compiler is hcc or clang. However, it is commonly invoked via hipcc wrapper.
-  # hipcc will call either hcc or clang using an absolute path starting with /opt/rocm,
-  # causing the /opt/cache/bin to be skipped. We must create the sccache wrappers
-  # directly under /opt/rocm while also preserving the original compiler names.
-  # Note symlinks will chain as follows: [hcc or clang++] -> clang -> clang-??
-  # Final link in symlink chain must point back to original directory.
-
-  # Original compiler is moved one directory deeper. Wrapper replaces it.
-  function write_sccache_stub_rocm() {
-    OLDCOMP=$1
-    COMPNAME=$(basename $OLDCOMP)
-    TOPDIR=$(dirname $OLDCOMP)
-    WRAPPED="$TOPDIR/original/$COMPNAME"
-    mv "$OLDCOMP" "$WRAPPED"
-    printf "#!/bin/sh\nexec sccache $WRAPPED \"\$@\"" > "$OLDCOMP"
-    chmod a+x "$OLDCOMP"
-  }
-
-  if [[ -e "/opt/rocm/hcc/bin/hcc" ]]; then
-    # ROCm 3.3 or earlier.
-    mkdir /opt/rocm/hcc/bin/original
-    write_sccache_stub_rocm /opt/rocm/hcc/bin/hcc
-    write_sccache_stub_rocm /opt/rocm/hcc/bin/clang
-    write_sccache_stub_rocm /opt/rocm/hcc/bin/clang++
-    # Fix last link in symlink chain, clang points to versioned clang in prior dir
-    pushd /opt/rocm/hcc/bin/original
-    ln -s ../$(readlink clang)
-    popd
-  elif [[ -e "/opt/rocm/llvm/bin/clang" ]]; then
-    # ROCm 3.5 and beyond.
-    mkdir /opt/rocm/llvm/bin/original
-    write_sccache_stub_rocm /opt/rocm/llvm/bin/clang
-    write_sccache_stub_rocm /opt/rocm/llvm/bin/clang++
-    # Fix last link in symlink chain, clang points to versioned clang in prior dir
-    pushd /opt/rocm/llvm/bin/original
-    ln -s ../$(readlink clang)
-    popd
-  else
-    echo "Cannot find ROCm compiler."
-    exit 1
-  fi
-fi

.ci/docker/common/install_clang.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-if [ -n "$CLANG_VERSION" ]; then
-
-  if [[ $CLANG_VERSION == 7 && $UBUNTU_VERSION == 16.04 ]]; then
-    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-    sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main"
-  elif [[ $CLANG_VERSION == 9 && $UBUNTU_VERSION == 18.04 ]]; then
-    sudo apt-get update
-    # gpg-agent is not available by default on 18.04
-    sudo apt-get install  -y --no-install-recommends gpg-agent
-    wget --no-check-certificate -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add  -
-    apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-${CLANG_VERSION} main"
-  elif [[ $UBUNTU_VERSION == 22.04 ]]; then
-    # work around ubuntu apt-get conflicts
-    sudo apt-get -y -f install
-  fi
-
-  sudo apt-get update
-  apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
-  apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
-
-  # Install dev version of LLVM.
-  if [ -n "$LLVMDEV" ]; then
-    sudo apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"-dev
-  fi
-
-  # Use update-alternatives to make this version the default
-  # TODO: Decide if overriding gcc as well is a good idea
-  # update-alternatives --install /usr/bin/gcc gcc /usr/bin/clang-"$CLANG_VERSION" 50
-  # update-alternatives --install /usr/bin/g++ g++ /usr/bin/clang++-"$CLANG_VERSION" 50
-  update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
-  update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
-
-  # clang's packaging is a little messed up (the runtime libs aren't
-  # added into the linker path), so give it a little help
-  clang_lib=("/usr/lib/llvm-$CLANG_VERSION/lib/clang/"*"/lib/linux")
-  echo "$clang_lib" > /etc/ld.so.conf.d/clang.conf
-  ldconfig
-
-  # Cleanup package manager
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-fi

.ci/docker/common/install_cmake.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-[ -n "$CMAKE_VERSION" ]
-
-# Remove system cmake install so it won't get used instead
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    apt-get remove cmake -y
-    ;;
-  centos)
-    yum remove cmake -y
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac
-
-# Turn 3.6.3 into v3.6
-path=$(echo "${CMAKE_VERSION}" | sed -e 's/\([0-9].[0-9]\+\).*/v\1/')
-file="cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz"
-
-# Download and install specific CMake version in /usr/local
-pushd /tmp
-curl -Os --retry 3 "https://cmake.org/files/${path}/${file}"
-tar -C /usr/local --strip-components 1 --no-same-owner -zxf cmake-*.tar.gz
-rm -f cmake-*.tar.gz
-popd

.ci/docker/common/install_conda.sh

@@ -1,98 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# Optionally install conda
-if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
-  BASE_URL="https://repo.anaconda.com/miniconda"
-
-  MAJOR_PYTHON_VERSION=$(echo "$ANACONDA_PYTHON_VERSION" | cut -d . -f 1)
-
-  case "$MAJOR_PYTHON_VERSION" in
-    2)
-      CONDA_FILE="Miniconda2-latest-Linux-x86_64.sh"
-    ;;
-    3)
-      CONDA_FILE="Miniconda3-latest-Linux-x86_64.sh"
-    ;;
-    *)
-      echo "Unsupported ANACONDA_PYTHON_VERSION: $ANACONDA_PYTHON_VERSION"
-      exit 1
-      ;;
-  esac
-
-  mkdir -p /opt/conda
-  chown jenkins:jenkins /opt/conda
-
-  source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-  pushd /tmp
-  wget -q "${BASE_URL}/${CONDA_FILE}"
-  # NB: Manually invoke bash per https://github.com/conda/conda/issues/10431
-  as_jenkins bash "${CONDA_FILE}" -b -f -p "/opt/conda"
-  popd
-
-  # NB: Don't do this, rely on the rpath to get it right
-  #echo "/opt/conda/lib" > /etc/ld.so.conf.d/conda-python.conf
-  #ldconfig
-  sed -e 's|PATH="\(.*\)"|PATH="/opt/conda/bin:\1"|g' -i /etc/environment
-  export PATH="/opt/conda/bin:$PATH"
-
-  # Ensure we run conda in a directory that jenkins has write access to
-  pushd /opt/conda
-
-  # Prevent conda from updating to 4.14.0, which causes docker build failures
-  # See https://hud.pytorch.org/pytorch/pytorch/commit/754d7f05b6841e555cea5a4b2c505dd9e0baec1d
-  # Uncomment the below when resolved to track the latest conda update
-  # as_jenkins conda update -y -n base conda
-
-  # Install correct Python version
-  as_jenkins conda create -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION"
-
-  # Install PyTorch conda deps, as per https://github.com/pytorch/pytorch README
-  CONDA_COMMON_DEPS="astunparse pyyaml mkl=2021.4.0 mkl-include=2021.4.0 setuptools"
-  if [ "$ANACONDA_PYTHON_VERSION" = "3.11" ]; then
-    # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
-    # TODO: Stop using `-c malfet`
-    conda_install numpy=1.23.5 ${CONDA_COMMON_DEPS} llvmdev=8.0.0 -c malfet
-  elif [ "$ANACONDA_PYTHON_VERSION" = "3.10" ]; then
-    # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
-    conda_install numpy=1.21.2 ${CONDA_COMMON_DEPS} llvmdev=8.0.0
-  elif [ "$ANACONDA_PYTHON_VERSION" = "3.9" ]; then
-    # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
-    conda_install numpy=1.19.2 ${CONDA_COMMON_DEPS} llvmdev=8.0.0
-  elif [ "$ANACONDA_PYTHON_VERSION" = "3.8" ]; then
-    # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
-    conda_install numpy=1.18.5 ${CONDA_COMMON_DEPS} llvmdev=8.0.0
-  else
-    # Install `typing-extensions` for 3.7
-    conda_install numpy=1.18.5 ${CONDA_COMMON_DEPS} typing-extensions
-  fi
-
-  # Use conda cmake in some cases. Conda cmake will be newer than our supported
-  # min version (3.5 for xenial and 3.10 for bionic), so we only do it in those
-  # following builds that we know should use conda. Specifically, Ubuntu bionic
-  # and focal cannot find conda mkl with stock cmake, so we need a cmake from conda
-  if [ -n "${CONDA_CMAKE}" ]; then
-    conda_install cmake
-  fi
-
-  # Magma package names are concatenation of CUDA major and minor ignoring revision
-  # I.e. magma-cuda102 package corresponds to CUDA_VERSION=10.2 and CUDA_VERSION=10.2.89
-  if [ -n "$CUDA_VERSION" ]; then
-    conda_install magma-cuda$(TMP=${CUDA_VERSION/./};echo ${TMP%.*[0-9]}) -c pytorch
-  fi
-
-  # Install some other packages, including those needed for Python test reporting
-  pip_install -r /opt/conda/requirements-ci.txt
-
-  # Update scikit-learn to a python-3.8 compatible version
-  if [[ $(python -c "import sys; print(int(sys.version_info >= (3, 8)))") == "1" ]]; then
-    pip_install -U scikit-learn
-  else
-    # Pinned scikit-learn due to https://github.com/scikit-learn/scikit-learn/issues/14485 (affects gcc 5.5 only)
-    pip_install scikit-learn==0.20.3
-  fi
-
-  popd
-fi

.ci/docker/common/install_cudnn.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-if [[ ${CUDNN_VERSION} == 8 ]]; then
-    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
-    mkdir tmp_cudnn && cd tmp_cudnn
-    CUDNN_NAME="cudnn-linux-x86_64-8.3.2.44_cuda11.5-archive"
-    if [[ ${CUDA_VERSION:0:4} == "11.7" ]]; then
-        CUDNN_NAME="cudnn-linux-x86_64-8.5.0.96_cuda11-archive"
-        curl --retry 3 -OLs https://ossci-linux.s3.amazonaws.com/${CUDNN_NAME}.tar.xz
-    elif [[ ${CUDA_VERSION:0:4} == "11.8" ]]; then
-        CUDNN_NAME="cudnn-linux-x86_64-8.7.0.84_cuda11-archive"
-        curl --retry 3 -OLs https://developer.download.nvidia.com/compute/redist/cudnn/v8.7.0/local_installers/11.8/${CUDNN_NAME}.tar.xz
-    else
-        curl --retry 3 -OLs https://developer.download.nvidia.com/compute/redist/cudnn/v8.3.2/local_installers/11.5/${CUDNN_NAME}.tar.xz
-    fi
-
-    tar xf ${CUDNN_NAME}.tar.xz
-    cp -a ${CUDNN_NAME}/include/* /usr/include/
-    cp -a ${CUDNN_NAME}/include/* /usr/local/cuda/include/
-    cp -a ${CUDNN_NAME}/include/* /usr/include/x86_64-linux-gnu/
-
-    cp -a ${CUDNN_NAME}/lib/* /usr/local/cuda/lib64/
-    cp -a ${CUDNN_NAME}/lib/* /usr/lib/x86_64-linux-gnu/
-    cd ..
-    rm -rf tmp_cudnn
-    ldconfig
-fi

.ci/docker/common/install_db.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-install_ubuntu() {
-  apt-get update
-  apt-get install -y --no-install-recommends \
-          libhiredis-dev \
-          libleveldb-dev \
-          liblmdb-dev \
-          libsnappy-dev
-
-  # Cleanup
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-}
-
-install_centos() {
-  # Need EPEL for many packages we depend on.
-  # See http://fedoraproject.org/wiki/EPEL
-  yum --enablerepo=extras install -y epel-release
-
-  yum install -y \
-      hiredis-devel \
-      leveldb-devel \
-      lmdb-devel \
-      snappy-devel
-
-  # Cleanup
-  yum clean all
-  rm -rf /var/cache/yum
-  rm -rf /var/lib/yum/yumdb
-  rm -rf /var/lib/yum/history
-}
-
-# Install base packages depending on the base OS
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    install_ubuntu
-    ;;
-  centos)
-    install_centos
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac

.ci/docker/common/install_docs_reqs.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-if [ -n "$KATEX" ]; then
-  apt-get update
-  # Ignore error if gpg-agent doesn't exist (for Ubuntu 16.04)
-  apt-get install -y gpg-agent || :
-
-  curl --retry 3 -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -
-  sudo apt-get install -y nodejs
-
-  curl --retry 3 -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
-  echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
-
-  apt-get update
-  apt-get install -y --no-install-recommends yarn
-  yarn global add katex --prefix /usr/local
-
-  sudo apt-get -y install doxygen
-
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-fi

.ci/docker/common/install_gcc.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-if [ -n "$GCC_VERSION" ]; then
-
-  # Need the official toolchain repo to get alternate packages
-  add-apt-repository ppa:ubuntu-toolchain-r/test
-  apt-get update
-  if [[ "$UBUNTU_VERSION" == "16.04" && "${GCC_VERSION:0:1}" == "5" ]]; then
-    apt-get install -y g++-5=5.4.0-6ubuntu1~16.04.12
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 50
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 50
-    update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-5 50
-  else
-    apt-get install -y g++-$GCC_VERSION
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-"$GCC_VERSION" 50
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-"$GCC_VERSION" 50
-    update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-"$GCC_VERSION" 50
-  fi
-
-
-  # Cleanup package manager
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-fi

.ci/docker/common/install_lcov.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-git clone --branch v1.15 https://github.com/linux-test-project/lcov.git
-pushd lcov
-sudo make install   # will be installed in /usr/local/bin/lcov
-popd

.ci/docker/common/install_linter.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-
-if [ -n "${UBUNTU_VERSION}" ]; then
-  apt update
-  apt-get install -y clang doxygen git graphviz nodejs npm libtinfo5
-fi
-
-# Do shallow clone of PyTorch so that we can init lintrunner in Docker build context
-git clone https://github.com/pytorch/pytorch.git --depth 1
-chown -R jenkins pytorch
-
-pushd pytorch
-# Install all linter dependencies
-pip_install -r requirements.txt
-conda_run lintrunner init
-
-# Cache .lintbin directory as part of the Docker image
-cp -r .lintbin /tmp
-popd
-
-# Node dependencies required by toc linter job
-npm install -g markdown-toc
-
-# Cleaning up
-rm -rf pytorch

.ci/docker/common/install_openmpi.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-sudo apt-get update
-# also install ssh to avoid error of:
-# --------------------------------------------------------------------------
-# The value of the MCA parameter "plm_rsh_agent" was set to a path
-# that could not be found:
-#   plm_rsh_agent: ssh : rsh
-sudo apt-get install -y ssh
-sudo apt-get install -y --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev

.ci/docker/common/install_openssl.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-OPENSSL=openssl-1.1.1k
-
-wget -q -O "${OPENSSL}.tar.gz" "https://ossci-linux.s3.amazonaws.com/${OPENSSL}.tar.gz"
-tar xf "${OPENSSL}.tar.gz"
-cd "${OPENSSL}"
-./config --prefix=/opt/openssl -d '-Wl,--enable-new-dtags,-rpath,$(LIBRPATH)'
-# NOTE: openssl install errors out when built with the -j option
-make -j6; make install_sw
-# Link the ssl libraries to the /usr/lib folder.
-sudo ln -s /opt/openssl/lib/lib* /usr/lib
-cd ..
-rm -rf "${OPENSSL}"

.ci/docker/common/install_protobuf.sh

@@ -1,56 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# This function installs protobuf 3.17
-install_protobuf_317() {
-  pb_dir="/usr/temp_pb_install_dir"
-  mkdir -p $pb_dir
-
-  # On the nvidia/cuda:9-cudnn7-devel-centos7 image we need this symlink or
-  # else it will fail with
-  #   g++: error: ./../lib64/crti.o: No such file or directory
-  ln -s /usr/lib64 "$pb_dir/lib64"
-
-  curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-all-3.17.3.tar.gz" --retry 3
-  tar -xvz -C "$pb_dir" --strip-components 1 -f protobuf-all-3.17.3.tar.gz
-  # -j6 to balance memory usage and speed.
-  # naked `-j` seems to use too much memory.
-  pushd "$pb_dir" && ./configure && make -j6 && make -j6 check && sudo make -j6 install && sudo ldconfig
-  popd
-  rm -rf $pb_dir
-}
-
-install_ubuntu() {
-  # Ubuntu 14.04 has cmake 2.8.12 as the default option, so we will
-  # install cmake3 here and use cmake3.
-  apt-get update
-  if [[ "$UBUNTU_VERSION" == 14.04 ]]; then
-    apt-get install -y --no-install-recommends cmake3
-  fi
-
-  # Cleanup
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-  install_protobuf_317
-}
-
-install_centos() {
-  install_protobuf_317
-}
-
-# Install base packages depending on the base OS
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    install_ubuntu
-    ;;
-  centos)
-    install_centos
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac

.ci/docker/common/install_rocm.sh

@@ -1,146 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-ver() {
-    printf "%3d%03d%03d%03d" $(echo "$1" | tr '.' ' ');
-}
-
-# Map ROCm version to AMDGPU version
-declare -A AMDGPU_VERSIONS=( ["5.0"]="21.50" ["5.1.1"]="22.10.1" ["5.2"]="22.20" )
-
-install_ubuntu() {
-    apt-get update
-    if [[ $UBUNTU_VERSION == 18.04 ]]; then
-      # gpg-agent is not available by default on 18.04
-      apt-get install -y --no-install-recommends gpg-agent
-    fi
-    if [[ $UBUNTU_VERSION == 20.04 ]]; then
-      # gpg-agent is not available by default on 20.04
-      apt-get install -y --no-install-recommends gpg-agent
-    fi
-    apt-get install -y kmod
-    apt-get install -y wget
-
-    # Need the libc++1 and libc++abi1 libraries to allow torch._C to load at runtime
-    apt-get install -y libc++1
-    apt-get install -y libc++abi1
-
-    if [[ $(ver $ROCM_VERSION) -ge $(ver 4.5) ]]; then
-        # Add amdgpu repository
-        UBUNTU_VERSION_NAME=`cat /etc/os-release | grep UBUNTU_CODENAME | awk -F= '{print $2}'`
-        local amdgpu_baseurl
-        if [[ $(ver $ROCM_VERSION) -ge $(ver 5.3) ]]; then
-          amdgpu_baseurl="https://repo.radeon.com/amdgpu/${ROCM_VERSION}/ubuntu"
-        else
-          amdgpu_baseurl="https://repo.radeon.com/amdgpu/${AMDGPU_VERSIONS[$ROCM_VERSION]}/ubuntu"
-        fi
-        echo "deb [arch=amd64] ${amdgpu_baseurl} ${UBUNTU_VERSION_NAME} main" > /etc/apt/sources.list.d/amdgpu.list
-    fi
-
-    ROCM_REPO="ubuntu"
-    if [[ $(ver $ROCM_VERSION) -lt $(ver 4.2) ]]; then
-        ROCM_REPO="xenial"
-    fi
-
-    if [[ $(ver $ROCM_VERSION) -ge $(ver 5.3) ]]; then
-        ROCM_REPO="${UBUNTU_VERSION_NAME}"
-    fi
-
-    # Add rocm repository
-    wget -qO - http://repo.radeon.com/rocm/rocm.gpg.key | apt-key add -
-    local rocm_baseurl="http://repo.radeon.com/rocm/apt/${ROCM_VERSION}"
-    echo "deb [arch=amd64] ${rocm_baseurl} ${ROCM_REPO} main" > /etc/apt/sources.list.d/rocm.list
-    apt-get update --allow-insecure-repositories
-
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
-                   rocm-dev \
-                   rocm-utils \
-                   rocm-libs \
-                   rccl \
-                   rocprofiler-dev \
-                   roctracer-dev
-
-    # precompiled miopen kernels added in ROCm 3.5; search for all unversioned packages
-    # if search fails it will abort this script; use true to avoid case where search fails
-    MIOPENKERNELS=$(apt-cache search --names-only miopenkernels | awk '{print $1}' | grep -F -v . || true)
-    if [[ "x${MIOPENKERNELS}" = x ]]; then
-      echo "miopenkernels package not available"
-    else
-      DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated ${MIOPENKERNELS}
-    fi
-
-    # Cleanup
-    apt-get autoclean && apt-get clean
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-}
-
-install_centos() {
-
-  yum update -y
-  yum install -y kmod
-  yum install -y wget
-  yum install -y openblas-devel
-
-  yum install -y epel-release
-  yum install -y dkms kernel-headers-`uname -r` kernel-devel-`uname -r`
-
-  if [[ $(ver $ROCM_VERSION) -ge $(ver 4.5) ]]; then
-      # Add amdgpu repository
-      local amdgpu_baseurl
-      if [[ $OS_VERSION == 9 ]]; then
-          amdgpu_baseurl="https://repo.radeon.com/amdgpu/${AMDGPU_VERSIONS[$ROCM_VERSION]}/rhel/9.0/main/x86_64"
-      else
-        if [[ $(ver $ROCM_VERSION) -ge $(ver 5.3) ]]; then
-          amdgpu_baseurl="https://repo.radeon.com/amdgpu/${ROCM_VERSION}/rhel/7.9/main/x86_64"
-        else
-          amdgpu_baseurl="https://repo.radeon.com/amdgpu/${AMDGPU_VERSIONS[$ROCM_VERSION]}/rhel/7.9/main/x86_64"
-        fi
-      fi
-      echo "[AMDGPU]" > /etc/yum.repos.d/amdgpu.repo
-      echo "name=AMDGPU" >> /etc/yum.repos.d/amdgpu.repo
-      echo "baseurl=${amdgpu_baseurl}" >> /etc/yum.repos.d/amdgpu.repo
-      echo "enabled=1" >> /etc/yum.repos.d/amdgpu.repo
-      echo "gpgcheck=1" >> /etc/yum.repos.d/amdgpu.repo
-      echo "gpgkey=http://repo.radeon.com/rocm/rocm.gpg.key" >> /etc/yum.repos.d/amdgpu.repo
-  fi
-
-  local rocm_baseurl="http://repo.radeon.com/rocm/yum/${ROCM_VERSION}"
-  echo "[ROCm]" > /etc/yum.repos.d/rocm.repo
-  echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo
-  echo "baseurl=${rocm_baseurl}" >> /etc/yum.repos.d/rocm.repo
-  echo "enabled=1" >> /etc/yum.repos.d/rocm.repo
-  echo "gpgcheck=1" >> /etc/yum.repos.d/rocm.repo
-  echo "gpgkey=http://repo.radeon.com/rocm/rocm.gpg.key" >> /etc/yum.repos.d/rocm.repo
-
-  yum update -y
-
-  yum install -y \
-                   rocm-dev \
-                   rocm-utils \
-                   rocm-libs \
-                   rccl \
-                   rocprofiler-dev \
-                   roctracer-dev
-
-  # Cleanup
-  yum clean all
-  rm -rf /var/cache/yum
-  rm -rf /var/lib/yum/yumdb
-  rm -rf /var/lib/yum/history
-}
-
-# Install Python packages depending on the base OS
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    install_ubuntu
-    ;;
-  centos)
-    install_centos
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac

.ci/docker/common/install_rocm_magma.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# "install" hipMAGMA into /opt/rocm/magma by copying after build
-git clone https://bitbucket.org/icl/magma.git
-pushd magma
-# Fixes memory leaks of magma found while executing linalg UTs
-git checkout 5959b8783e45f1809812ed96ae762f38ee701972
-cp make.inc-examples/make.inc.hip-gcc-mkl make.inc
-echo 'LIBDIR += -L$(MKLROOT)/lib' >> make.inc
-echo 'LIB += -Wl,--enable-new-dtags -Wl,--rpath,/opt/rocm/lib -Wl,--rpath,$(MKLROOT)/lib -Wl,--rpath,/opt/rocm/magma/lib' >> make.inc
-echo 'DEVCCFLAGS += --gpu-max-threads-per-block=256' >> make.inc
-export PATH="${PATH}:/opt/rocm/bin"
-if [[ -n "$PYTORCH_ROCM_ARCH" ]]; then
-  amdgpu_targets=`echo $PYTORCH_ROCM_ARCH | sed 's/;/ /g'`
-else
-  amdgpu_targets=`rocm_agent_enumerator | grep -v gfx000 | sort -u | xargs`
-fi
-for arch in $amdgpu_targets; do
-  echo "DEVCCFLAGS += --amdgpu-target=$arch" >> make.inc
-done
-# hipcc with openmp flag may cause isnan() on __device__ not to be found; depending on context, compiler may attempt to match with host definition
-sed -i 's/^FOPENMP/#FOPENMP/g' make.inc
-make -f make.gen.hipMAGMA -j $(nproc)
-LANG=C.UTF-8 make lib/libmagma.so -j $(nproc) MKLROOT=/opt/conda/envs/py_$ANACONDA_PYTHON_VERSION
-make testing/testing_dgemm -j $(nproc) MKLROOT=/opt/conda/envs/py_$ANACONDA_PYTHON_VERSION
-popd
-mv magma /opt/rocm

.ci/docker/common/install_swiftshader.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-[ -n "${SWIFTSHADER}" ]
-
-retry () {
-    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
-}
-
-_https_amazon_aws=https://ossci-android.s3.amazonaws.com
-
-# SwiftShader
-_swiftshader_dir=/var/lib/jenkins/swiftshader
-_swiftshader_file_targz=swiftshader-abe07b943-prebuilt.tar.gz
-mkdir -p $_swiftshader_dir
-_tmp_swiftshader_targz="/tmp/${_swiftshader_file_targz}"
-
-curl --silent --show-error --location --fail --retry 3 \
-  --output "${_tmp_swiftshader_targz}" "$_https_amazon_aws/${_swiftshader_file_targz}"
-
-tar -C "${_swiftshader_dir}" -xzf "${_tmp_swiftshader_targz}"
-
-export VK_ICD_FILENAMES="${_swiftshader_dir}/build/Linux/vk_swiftshader_icd.json"

.ci/docker/common/install_ucc.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-if [[ -d "/usr/local/cuda/" ]];  then
-  with_cuda=/usr/local/cuda/
-else
-  with_cuda=no
-fi
-
-function install_ucx() {
-  set -ex
-  git clone --recursive https://github.com/openucx/ucx.git
-  pushd ucx
-  git checkout ${UCX_COMMIT}
-  git submodule update --init --recursive
-
-  ./autogen.sh
-  ./configure --prefix=$UCX_HOME      \
-      --enable-mt                     \
-      --with-cuda=$with_cuda          \
-      --enable-profiling              \
-      --enable-stats
-  time make -j
-  sudo make install
-
-  popd
-  rm -rf ucx
-}
-
-function install_ucc() {
-  set -ex
-  git clone --recursive https://github.com/openucx/ucc.git
-  pushd ucc
-  git checkout ${UCC_COMMIT}
-  git submodule update --init --recursive
-
-  ./autogen.sh
-  ./configure --prefix=$UCC_HOME --with-ucx=$UCX_HOME --with-cuda=$with_cuda
-  time make -j
-  sudo make install
-
-  popd
-  rm -rf ucc
-}
-
-install_ucx
-install_ucc

.ci/docker/common/install_user.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# Mirror jenkins user in container
-# jenkins user as ec2-user should have the same user-id
-echo "jenkins:x:1000:1000::/var/lib/jenkins:" >> /etc/passwd
-echo "jenkins:x:1000:" >> /etc/group
-# Needed on focal or newer
-echo "jenkins:*:19110:0:99999:7:::" >>/etc/shadow
-
-# Create $HOME
-mkdir -p /var/lib/jenkins
-chown jenkins:jenkins /var/lib/jenkins
-mkdir -p /var/lib/jenkins/.ccache
-chown jenkins:jenkins /var/lib/jenkins/.ccache
-
-# Allow writing to /usr/local (for make install)
-chown jenkins:jenkins /usr/local
-
-# Allow sudo
-# TODO: Maybe we shouldn't
-echo 'jenkins ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/jenkins
-
-# Work around bug where devtoolset replaces sudo and breaks it.
-if [ -n "$DEVTOOLSET_VERSION" ]; then
-  SUDO=/bin/sudo
-else
-  SUDO=sudo
-fi
-
-# Test that sudo works
-$SUDO -u jenkins $SUDO -v

.ci/docker/common/install_vision.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-install_ubuntu() {
-  apt-get update
-  apt-get install -y --no-install-recommends \
-          libopencv-dev \
-          libavcodec-dev
-
-  # Cleanup
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-}
-
-install_centos() {
-  # Need EPEL for many packages we depend on.
-  # See http://fedoraproject.org/wiki/EPEL
-  yum --enablerepo=extras install -y epel-release
-
-  yum install -y \
-      opencv-devel \
-      ffmpeg-devel
-
-  # Cleanup
-  yum clean all
-  rm -rf /var/cache/yum
-  rm -rf /var/lib/yum/yumdb
-  rm -rf /var/lib/yum/history
-}
-
-# Install base packages depending on the base OS
-ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
-case "$ID" in
-  ubuntu)
-    install_ubuntu
-    ;;
-  centos)
-    install_centos
-    ;;
-  *)
-    echo "Unable to determine OS..."
-    exit 1
-    ;;
-esac

.ci/docker/common/install_vulkan_sdk.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-[ -n "${VULKAN_SDK_VERSION}" ]
-
-retry () {
-    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
-}
-
-_vulkansdk_dir=/var/lib/jenkins/vulkansdk
-_tmp_vulkansdk_targz=/tmp/vulkansdk.tar.gz
-
-curl \
-  --silent \
-  --show-error \
-  --location \
-  --fail \
-  --retry 3 \
-  --output "${_tmp_vulkansdk_targz}" "https://ossci-android.s3.amazonaws.com/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz"
-
-mkdir -p "${_vulkansdk_dir}"
-tar -C "${_vulkansdk_dir}" -xzf "${_tmp_vulkansdk_targz}" --strip-components 1
-rm -rf "${_tmp_vulkansdk_targz}"

.ci/docker/linter/Dockerfile

@@ -1,34 +0,0 @@
-ARG UBUNTU_VERSION
-
-FROM ubuntu:${UBUNTU_VERSION}
-
-ARG UBUNTU_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ARG CONDA_CMAKE
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-COPY requirements-ci.txt /opt/conda/requirements-ci.txt
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
-
-# Note that Docker build forbids copying file outside the build context
-COPY ./common/install_linter.sh install_linter.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_linter.sh
-RUN rm install_linter.sh common_utils.sh
-
-USER jenkins
-CMD ["bash"]

.ci/docker/requirements-ci.txt

@@ -1,260 +0,0 @@
-# Python dependencies required for unit tests
-
-#awscli==1.6 #this breaks some platforms
-#Description: AWS command line interface
-#Pinned versions: 1.6
-#test that import:
-
-boto3==1.19.12
-#Description: AWS SDK for python
-#Pinned versions: 1.19.12, 1.16.34
-#test that import:
-
-click
-#Description: Command Line Interface Creation Kit
-#Pinned versions:
-#test that import:
-
-coremltools==5.0b5
-#Description: Apple framework for ML integration
-#Pinned versions: 5.0b5
-#test that import:
-
-#dataclasses #this breaks some platforms
-#Description: Provides decorators for auto adding special methods to user classes
-#Pinned versions:
-#test that import:
-
-expecttest==0.1.3
-#Description: method for writing tests where test framework auto populates
-# the expected output based on previous runs
-#Pinned versions: 0.1.3
-#test that import:
-
-flatbuffers==2.0
-#Description: cross platform serialization library
-#Pinned versions: 2.0
-#test that import:
-
-hypothesis==5.35.1
-# Pin hypothesis to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136
-#Description: advanced library for generating parametrized tests
-#Pinned versions: 3.44.6, 4.53.2
-#test that import: test_xnnpack_integration.py, test_pruning_op.py, test_nn.py
-
-junitparser==2.1.1
-#Description: unitparser handles JUnit/xUnit Result XML files
-#Pinned versions: 2.1.1
-#test that import:
-
-librosa>=0.6.2 ; python_version < "3.11"
-#Description: A python package for music and audio analysis
-#Pinned versions: >=0.6.2
-#test that import: test_spectral_ops.py
-
-#mkl #this breaks linux-bionic-rocm4.5-py3.7
-#Description: Intel oneAPI Math Kernel Library
-#Pinned versions:
-#test that import: test_profiler.py, test_public_bindings.py, test_testing.py,
-#test_nn.py, test_mkldnn.py, test_jit.py, test_fx_experimental.py,
-#test_autograd.py
-
-#mkl-devel
-# see mkl
-
-#mock # breaks ci/circleci: docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
-#Description: A testing library that allows you to replace parts of your
-#system under test with mock objects
-#Pinned versions:
-#test that import: test_module_init.py, test_modules.py, test_nn.py,
-#test_testing.py
-
-#MonkeyType # breaks pytorch-xla-linux-bionic-py3.7-clang8
-#Description: collects runtime types of function arguments and return
-#values, and can automatically generate stub files
-#Pinned versions:
-#test that import:
-
-mypy==0.960
-# Pin MyPy version because new errors are likely to appear with each release
-#Description: linter
-#Pinned versions: 0.960
-#test that import: test_typing.py, test_type_hints.py
-
-networkx==2.6.3
-#Description: creation, manipulation, and study of
-#the structure, dynamics, and functions of complex networks
-#Pinned versions: 2.6.3 (latest version that works with Python 3.7+)
-#test that import: functorch
-
-#ninja
-#Description: build system.  Note that it install from
-#here breaks things so it is commented out
-#Pinned versions: 1.10.0.post1
-#test that import: run_test.py, test_cpp_extensions_aot.py,test_determination.py
-
-numba==0.49.0 ; python_version < "3.9"
-numba==0.54.1 ; python_version == "3.9"
-numba==0.55.2 ; python_version == "3.10"
-#Description: Just-In-Time Compiler for Numerical Functions
-#Pinned versions: 0.54.1, 0.49.0, <=0.49.1
-#test that import: test_numba_integration.py
-#For numba issue see https://github.com/pytorch/pytorch/issues/51511
-
-#numpy
-#Description: Provides N-dimensional arrays and linear algebra
-#Pinned versions: 1.20
-#test that import: test_view_ops.py, test_unary_ufuncs.py, test_type_promotion.py,
-#test_type_info.py, test_torch.py, test_tensorexpr_pybind.py, test_tensorexpr.py,
-#test_tensorboard.py, test_tensor_creation_ops.py, test_static_runtime.py,
-#test_spectral_ops.py, test_sort_and_select.py, test_shape_ops.py,
-#test_segment_reductions.py, test_reductions.py, test_pruning_op.py,
-#test_overrides.py, test_numpy_interop.py, test_numba_integration.py
-#test_nn.py, test_namedtensor.py, test_linalg.py, test_jit_cuda_fuser.py,
-#test_jit.py, test_indexing.py, test_datapipe.py, test_dataloader.py,
-#test_binary_ufuncs.py
-
-#onnxruntime
-#Description: scoring engine for Open Neural Network Exchange (ONNX) models
-#Pinned versions: 1.9.0
-#test that import:
-
-opt-einsum==3.3
-#Description: Python library to optimize tensor contraction order, used in einsum
-#Pinned versions: 3.3
-#test that import: test_linalg.py
-
-#pillow
-#Description:  Python Imaging Library fork
-#Pinned versions:
-#test that import:
-
-protobuf==3.20.2
-#Description:  Google’s data interchange format
-#Pinned versions: 3.20.1
-#test that import: test_tensorboard.py
-
-psutil
-#Description: information on running processes and system utilization
-#Pinned versions:
-#test that import: test_profiler.py, test_openmp.py, test_dataloader.py
-
-pytest
-#Description: testing framework
-#Pinned versions:
-#test that import: test_typing.py, test_cpp_extensions_aot.py, run_test.py
-
-pytest-xdist
-#Description: plugin for running pytest in parallel
-#Pinned versions:
-#test that import:
-
-pytest-shard
-#Description: plugin spliting up tests in pytest
-#Pinned versions:
-#test that import:
-
-pytest-flakefinder==1.1.0
-#Description: plugin for rerunning tests a fixed number of times in pytest
-#Pinned versions: 1.1.0
-#test that import:
-
-pytest-rerunfailures
-#Description: plugin for rerunning failure tests in pytest
-#Pinned versions:
-#test that import:
-
-#pytest-benchmark
-#Description: fixture for benchmarking code
-#Pinned versions: 3.2.3
-#test that import:
-
-#pytest-sugar
-#Description: shows failures and errors instantly
-#Pinned versions:
-#test that import:
-
-xdoctest==1.1.0
-#Description: runs doctests in pytest
-#Pinned versions: 1.1.0
-#test that import:
-
-pygments==2.12.0
-#Description: support doctest highlighting
-#Pinned versions: 2.12.0
-#test that import: the doctests
-
-#PyYAML
-#Description: data serialization format
-#Pinned versions:
-#test that import:
-
-#requests
-#Description: HTTP library
-#Pinned versions:
-#test that import: test_type_promotion.py
-
-#rich
-#Description: rich text and beautiful formatting in the terminal
-#Pinned versions: 10.9.0
-#test that import:
-
-scikit-image
-#Description: image processing routines
-#Pinned versions:
-#test that import: test_nn.py
-
-#scikit-learn
-#Description: machine learning package
-#Pinned versions: 0.20.3
-#test that import:
-
-scipy==1.6.3 ; python_version < "3.10"
-scipy==1.8.1 ; python_version == "3.10"
-scipy==1.9.3 ; python_version == "3.11"
-# Pin SciPy because of failing distribution tests (see #60347)
-#Description: scientific python
-#Pinned versions: 1.6.3
-#test that import: test_unary_ufuncs.py, test_torch.py,test_tensor_creation_ops.py
-#test_spectral_ops.py, test_sparse_csr.py, test_reductions.py,test_nn.py
-#test_linalg.py, test_binary_ufuncs.py
-
-#tabulate
-#Description: Pretty-print tabular data
-#Pinned versions:
-#test that import:
-
-tb-nightly
-#Description: TensorBoard
-#Pinned versions:
-#test that import:
-
-#typing-extensions
-#Description: type hints for python
-#Pinned versions:
-#test that import:
-
-#virtualenv
-#Description: virtual environment for python
-#Pinned versions:
-#test that import:
-
-unittest-xml-reporting<=3.2.0,>=2.0.0
-#Description: saves unit test results to xml
-#Pinned versions:
-#test that import:
-
-lintrunner==0.9.2
-#Description: all about linters
-#Pinned versions: 0.9.2
-#test that import:
-
-rockset==1.0.3
-#Description: queries Rockset
-#Pinned versions: 1.0.3
-#test that import:
-
-ghstack==0.7.1
-#Description: ghstack tool
-#Pinned versions: 0.7.1
-#test that import:

.ci/docker/ubuntu-cuda/Dockerfile

@@ -1,132 +0,0 @@
-ARG UBUNTU_VERSION
-ARG CUDA_VERSION
-ARG IMAGE_NAME
-
-FROM ${IMAGE_NAME}
-
-ARG UBUNTU_VERSION
-ARG CUDA_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install katex
-ARG KATEX
-COPY ./common/install_docs_reqs.sh install_docs_reqs.sh
-RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-ARG CONDA_CMAKE
-COPY requirements-ci.txt /opt/conda/requirements-ci.txt
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
-
-# Install gcc
-ARG GCC_VERSION
-COPY ./common/install_gcc.sh install_gcc.sh
-RUN bash ./install_gcc.sh && rm install_gcc.sh
-
-# Install clang
-ARG CLANG_VERSION
-COPY ./common/install_clang.sh install_clang.sh
-RUN bash ./install_clang.sh && rm install_clang.sh
-
-# (optional) Install protobuf for ONNX
-ARG PROTOBUF
-COPY ./common/install_protobuf.sh install_protobuf.sh
-RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
-RUN rm install_protobuf.sh
-ENV INSTALLED_PROTOBUF ${PROTOBUF}
-
-# (optional) Install database packages like LMDB and LevelDB
-ARG DB
-COPY ./common/install_db.sh install_db.sh
-RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
-RUN rm install_db.sh
-ENV INSTALLED_DB ${DB}
-
-# (optional) Install vision packages like OpenCV and ffmpeg
-ARG VISION
-COPY ./common/install_vision.sh install_vision.sh
-RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
-RUN rm install_vision.sh
-ENV INSTALLED_VISION ${VISION}
-
-# (optional) Install UCC
-ARG UCX_COMMIT
-ARG UCC_COMMIT
-ENV UCX_COMMIT $UCX_COMMIT
-ENV UCC_COMMIT $UCC_COMMIT
-ENV UCX_HOME /usr
-ENV UCC_HOME /usr
-ADD ./common/install_ucc.sh install_ucc.sh
-RUN if [ -n "${UCX_COMMIT}" ] && [ -n "${UCC_COMMIT}" ]; then bash ./install_ucc.sh; fi
-RUN rm install_ucc.sh
-
-COPY ./common/install_openssl.sh install_openssl.sh
-ENV OPENSSL_ROOT_DIR /opt/openssl
-RUN bash ./install_openssl.sh
-ENV OPENSSL_DIR /opt/openssl
-
-# (optional) Install non-default CMake version
-ARG CMAKE_VERSION
-COPY ./common/install_cmake.sh install_cmake.sh
-RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
-RUN rm install_cmake.sh
-
-# Install ccache/sccache (do this last, so we get priority in PATH)
-COPY ./common/install_cache.sh install_cache.sh
-ENV PATH /opt/cache/bin:$PATH
-# See https://github.com/pytorch/pytorch/issues/82174
-# TODO(sdym@fb.com):
-# check if this is needed after full off Xenial migration
-ENV CARGO_NET_GIT_FETCH_WITH_CLI true
-RUN bash ./install_cache.sh && rm install_cache.sh
-ENV CMAKE_CUDA_COMPILER_LAUNCHER=/opt/cache/bin/sccache
-
-# Add jni.h for java host build
-COPY ./common/install_jni.sh install_jni.sh
-COPY ./java/jni.h jni.h
-RUN bash ./install_jni.sh && rm install_jni.sh
-
-# Install Open MPI for CUDA
-COPY ./common/install_openmpi.sh install_openmpi.sh
-RUN if [ -n "${CUDA_VERSION}" ]; then bash install_openmpi.sh; fi
-RUN rm install_openmpi.sh
-
-# Include BUILD_ENVIRONMENT environment variable in image
-ARG BUILD_ENVIRONMENT
-ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
-
-# AWS specific CUDA build guidance
-ENV TORCH_CUDA_ARCH_LIST Maxwell
-ENV TORCH_NVCC_FLAGS "-Xfatbin -compress-all"
-ENV CUDA_PATH /usr/local/cuda
-
-# Install LLVM dev version (Defined in the pytorch/builder github repository)
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
-
-# Install CUDNN
-ARG CUDNN_VERSION
-ARG CUDA_VERSION
-COPY ./common/install_cudnn.sh install_cudnn.sh
-RUN if [ "${CUDNN_VERSION}" -eq 8 ]; then bash install_cudnn.sh; fi
-RUN rm install_cudnn.sh
-
-# Delete /usr/local/cuda-11.X/cuda-11.X symlinks
-RUN if [ -h /usr/local/cuda-11.6/cuda-11.6 ]; then rm /usr/local/cuda-11.6/cuda-11.6; fi
-RUN if [ -h /usr/local/cuda-11.7/cuda-11.7 ]; then rm /usr/local/cuda-11.7/cuda-11.7; fi
-
-USER jenkins
-CMD ["bash"]

.ci/docker/ubuntu-rocm/Dockerfile

@@ -1,102 +0,0 @@
-ARG UBUNTU_VERSION
-
-FROM ubuntu:${UBUNTU_VERSION}
-
-ARG UBUNTU_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Set AMD gpu targets to build for
-ARG PYTORCH_ROCM_ARCH
-ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install clang
-ARG LLVMDEV
-ARG CLANG_VERSION
-COPY ./common/install_clang.sh install_clang.sh
-RUN bash ./install_clang.sh && rm install_clang.sh
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ARG CONDA_CMAKE
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-COPY requirements-ci.txt /opt/conda/requirements-ci.txt
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
-
-# Install gcc
-ARG GCC_VERSION
-COPY ./common/install_gcc.sh install_gcc.sh
-RUN bash ./install_gcc.sh && rm install_gcc.sh
-
-# (optional) Install protobuf for ONNX
-ARG PROTOBUF
-COPY ./common/install_protobuf.sh install_protobuf.sh
-RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
-RUN rm install_protobuf.sh
-ENV INSTALLED_PROTOBUF ${PROTOBUF}
-
-# (optional) Install database packages like LMDB and LevelDB
-ARG DB
-COPY ./common/install_db.sh install_db.sh
-RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
-RUN rm install_db.sh
-ENV INSTALLED_DB ${DB}
-
-# (optional) Install vision packages like OpenCV and ffmpeg
-ARG VISION
-COPY ./common/install_vision.sh install_vision.sh
-RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
-RUN rm install_vision.sh
-ENV INSTALLED_VISION ${VISION}
-
-# Install rocm
-ARG ROCM_VERSION
-COPY ./common/install_rocm.sh install_rocm.sh
-RUN bash ./install_rocm.sh
-RUN rm install_rocm.sh
-COPY ./common/install_rocm_magma.sh install_rocm_magma.sh
-RUN bash ./install_rocm_magma.sh
-RUN rm install_rocm_magma.sh
-ENV PATH /opt/rocm/bin:$PATH
-ENV PATH /opt/rocm/hcc/bin:$PATH
-ENV PATH /opt/rocm/hip/bin:$PATH
-ENV PATH /opt/rocm/opencl/bin:$PATH
-ENV PATH /opt/rocm/llvm/bin:$PATH
-ENV MAGMA_HOME /opt/rocm/magma
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-
-# (optional) Install non-default CMake version
-ARG CMAKE_VERSION
-COPY ./common/install_cmake.sh install_cmake.sh
-RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
-RUN rm install_cmake.sh
-
-# (optional) Install non-default Ninja version
-ARG NINJA_VERSION
-COPY ./common/install_ninja.sh install_ninja.sh
-RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
-RUN rm install_ninja.sh
-
-# Install ccache/sccache (do this last, so we get priority in PATH)
-COPY ./common/install_cache.sh install_cache.sh
-ENV PATH /opt/cache/bin:$PATH
-RUN bash ./install_cache.sh && rm install_cache.sh
-
-# Include BUILD_ENVIRONMENT environment variable in image
-ARG BUILD_ENVIRONMENT
-ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
-
-USER jenkins
-CMD ["bash"]

.ci/docker/ubuntu/Dockerfile

@@ -1,165 +0,0 @@
-ARG UBUNTU_VERSION
-
-FROM ubuntu:${UBUNTU_VERSION}
-
-ARG UBUNTU_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-ARG CLANG_VERSION
-
-# Install common dependencies (so that this step can be cached separately)
-COPY ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install clang
-ARG LLVMDEV
-COPY ./common/install_clang.sh install_clang.sh
-RUN bash ./install_clang.sh && rm install_clang.sh
-
-# (optional) Install thrift.
-ARG THRIFT
-COPY ./common/install_thrift.sh install_thrift.sh
-RUN if [ -n "${THRIFT}" ]; then bash ./install_thrift.sh; fi
-RUN rm install_thrift.sh
-ENV INSTALLED_THRIFT ${THRIFT}
-
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install katex
-ARG KATEX
-COPY ./common/install_docs_reqs.sh install_docs_reqs.sh
-RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
-
-# Install conda and other packages (e.g., numpy, pytest)
-ARG ANACONDA_PYTHON_VERSION
-ARG CONDA_CMAKE
-ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-COPY requirements-ci.txt /opt/conda/requirements-ci.txt
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
-
-# Install gcc
-ARG GCC_VERSION
-COPY ./common/install_gcc.sh install_gcc.sh
-RUN bash ./install_gcc.sh && rm install_gcc.sh
-
-# Install lcov for C++ code coverage
-COPY ./common/install_lcov.sh install_lcov.sh
-RUN  bash ./install_lcov.sh && rm install_lcov.sh
-
-# Install cuda and cudnn
-ARG CUDA_VERSION
-RUN wget -q https://raw.githubusercontent.com/pytorch/builder/main/common/install_cuda.sh -O install_cuda.sh
-RUN bash ./install_cuda.sh ${CUDA_VERSION} && rm install_cuda.sh
-ENV DESIRED_CUDA ${CUDA_VERSION}
-ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH
-
-# (optional) Install UCC
-ARG UCX_COMMIT
-ARG UCC_COMMIT
-ENV UCX_COMMIT $UCX_COMMIT
-ENV UCC_COMMIT $UCC_COMMIT
-ENV UCX_HOME /usr
-ENV UCC_HOME /usr
-ADD ./common/install_ucc.sh install_ucc.sh
-RUN if [ -n "${UCX_COMMIT}" ] && [ -n "${UCC_COMMIT}" ]; then bash ./install_ucc.sh; fi
-RUN rm install_ucc.sh
-
-# (optional) Install protobuf for ONNX
-ARG PROTOBUF
-COPY ./common/install_protobuf.sh install_protobuf.sh
-RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
-RUN rm install_protobuf.sh
-ENV INSTALLED_PROTOBUF ${PROTOBUF}
-
-# (optional) Install database packages like LMDB and LevelDB
-ARG DB
-COPY ./common/install_db.sh install_db.sh
-RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
-RUN rm install_db.sh
-ENV INSTALLED_DB ${DB}
-
-# (optional) Install vision packages like OpenCV and ffmpeg
-ARG VISION
-COPY ./common/install_vision.sh install_vision.sh
-RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
-RUN rm install_vision.sh
-ENV INSTALLED_VISION ${VISION}
-
-# (optional) Install Android NDK
-ARG ANDROID
-ARG ANDROID_NDK
-ARG GRADLE_VERSION
-COPY ./common/install_android.sh install_android.sh
-COPY ./android/AndroidManifest.xml AndroidManifest.xml
-COPY ./android/build.gradle build.gradle
-RUN if [ -n "${ANDROID}" ]; then bash ./install_android.sh; fi
-RUN rm install_android.sh
-RUN rm AndroidManifest.xml
-RUN rm build.gradle
-ENV INSTALLED_ANDROID ${ANDROID}
-
-# (optional) Install Vulkan SDK
-ARG VULKAN_SDK_VERSION
-COPY ./common/install_vulkan_sdk.sh install_vulkan_sdk.sh
-RUN if [ -n "${VULKAN_SDK_VERSION}" ]; then bash ./install_vulkan_sdk.sh; fi
-RUN rm install_vulkan_sdk.sh
-
-# (optional) Install swiftshader
-ARG SWIFTSHADER
-COPY ./common/install_swiftshader.sh install_swiftshader.sh
-RUN if [ -n "${SWIFTSHADER}" ]; then bash ./install_swiftshader.sh; fi
-RUN rm install_swiftshader.sh
-
-# (optional) Install non-default CMake version
-ARG CMAKE_VERSION
-COPY ./common/install_cmake.sh install_cmake.sh
-RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
-RUN rm install_cmake.sh
-
-# (optional) Install non-default Ninja version
-ARG NINJA_VERSION
-COPY ./common/install_ninja.sh install_ninja.sh
-RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
-RUN rm install_ninja.sh
-
-COPY ./common/install_openssl.sh install_openssl.sh
-RUN bash ./install_openssl.sh
-ENV OPENSSL_ROOT_DIR /opt/openssl
-ENV OPENSSL_DIR /opt/openssl
-RUN rm install_openssl.sh
-
-# Install ccache/sccache (do this last, so we get priority in PATH)
-COPY ./common/install_cache.sh install_cache.sh
-ENV PATH /opt/cache/bin:$PATH
-RUN bash ./install_cache.sh && rm install_cache.sh
-
-# Add jni.h for java host build
-COPY ./common/install_jni.sh install_jni.sh
-COPY ./java/jni.h jni.h
-RUN bash ./install_jni.sh && rm install_jni.sh
-
-# Install Open MPI for CUDA
-COPY ./common/install_openmpi.sh install_openmpi.sh
-RUN if [ -n "${CUDA_VERSION}" ]; then bash install_openmpi.sh; fi
-RUN rm install_openmpi.sh
-
-# Include BUILD_ENVIRONMENT environment variable in image
-ARG BUILD_ENVIRONMENT
-ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
-
-# Install LLVM dev version (Defined in the pytorch/builder github repository)
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
-
-# AWS specific CUDA build guidance
-ENV TORCH_CUDA_ARCH_LIST Maxwell
-ENV TORCH_NVCC_FLAGS "-Xfatbin -compress-all"
-ENV CUDA_PATH /usr/local/cuda
-
-USER jenkins
-CMD ["bash"]

.ci/onnx/README.md

@@ -1,14 +0,0 @@
-# Jenkins
-
-The scripts in this directory are the entrypoint for testing ONNX exporter.
-
-The environment variable `BUILD_ENVIRONMENT` is expected to be set to
-the build environment you intend to test. It is a hint for the build
-and test scripts to configure Caffe2 a certain way and include/exclude
-tests. Docker images, they equal the name of the image itself. For
-example: `py2-cuda9.0-cudnn7-ubuntu16.04`. The Docker images that are
-built on Jenkins and are used in triggered builds already have this
-environment variable set in their manifest. Also see
-`./docker/jenkins/*/Dockerfile` and search for `BUILD_ENVIRONMENT`.
-
-Our Jenkins installation is located at https://ci.pytorch.org/jenkins/.

.ci/onnx/common.sh

@@ -1,19 +0,0 @@
-set -ex
-
-LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
-TEST_DIR="$ROOT_DIR/test"
-pytest_reports_dir="${TEST_DIR}/test-reports/python"
-
-# Figure out which Python to use
-PYTHON="$(which python)"
-if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
-  PYTHON=$(which "python${BASH_REMATCH[1]}")
-fi
-
-if [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
-    # HIP_PLATFORM is auto-detected by hipcc; unset to avoid build errors
-    unset HIP_PLATFORM
-fi
-
-mkdir -p "$pytest_reports_dir" || true

.ci/onnx/test.sh

@@ -1,74 +0,0 @@
-#!/bin/bash
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-
-if [[ ${BUILD_ENVIRONMENT} == *onnx* ]]; then
-  pip install click mock tabulate networkx==2.0
-  pip -q install --user "file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx"
-fi
-
-# Skip tests in environments where they are not built/applicable
-if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
-  echo 'Skipping tests'
-  exit 0
-fi
-if [[ "${BUILD_ENVIRONMENT}" == *-rocm* ]]; then
-  # temporary to locate some kernel issues on the CI nodes
-  export HSAKMT_DEBUG_LEVEL=4
-fi
-# These additional packages are needed for circleci ROCm builds.
-if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
-    # Need networkx 2.0 because bellmand_ford was moved in 2.1 . Scikit-image by
-    # defaults installs the most recent networkx version, so we install this lower
-    # version explicitly before scikit-image pulls it in as a dependency
-    pip install networkx==2.0
-    # click - onnx
-    pip install --progress-bar off click protobuf tabulate virtualenv mock typing-extensions
-fi
-
-################################################################################
-# Python tests #
-################################################################################
-if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
-  exit 0
-fi
-
-# If pip is installed as root, we must use sudo.
-# CircleCI docker images could install conda as jenkins user, or use the OS's python package.
-PIP=$(which pip)
-PIP_USER=$(stat --format '%U' $PIP)
-CURRENT_USER=$(id -u -n)
-if [[ "$PIP_USER" = root && "$CURRENT_USER" != root ]]; then
-  MAYBE_SUDO=sudo
-fi
-
-# Uninstall pre-installed hypothesis and coverage to use an older version as newer
-# versions remove the timeout parameter from settings which ideep/conv_transpose_test.py uses
-$MAYBE_SUDO pip -q uninstall -y hypothesis
-$MAYBE_SUDO pip -q uninstall -y coverage
-
-# "pip install hypothesis==3.44.6" from official server is unreliable on
-# CircleCI, so we host a copy on S3 instead
-$MAYBE_SUDO pip -q install attrs==18.1.0 -f https://s3.amazonaws.com/ossci-linux/wheels/attrs-18.1.0-py2.py3-none-any.whl
-$MAYBE_SUDO pip -q install coverage==4.5.1 -f https://s3.amazonaws.com/ossci-linux/wheels/coverage-4.5.1-cp36-cp36m-macosx_10_12_x86_64.whl
-$MAYBE_SUDO pip -q install hypothesis==4.57.1
-
-##############
-# ONNX tests #
-##############
-if [[ "$BUILD_ENVIRONMENT" == *onnx* ]]; then
-  pip install -q --user --no-use-pep517 "git+https://github.com/pytorch/vision.git@$(cat .github/ci_commit_pins/vision.txt)"
-  pip install -q --user transformers==4.25.1
-  pip install -q --user ninja flatbuffers==2.0 numpy==1.22.4 onnxruntime==1.14.0 beartype==0.10.4
-  # TODO: change this when onnx 1.13.1 is released.
-  pip install --no-use-pep517 'onnx @ git+https://github.com/onnx/onnx@e192ba01e438d22ca2dedd7956e28e3551626c91'
-  # TODO: change this when onnx-script is on testPypi
-  pip install 'onnx-script @ git+https://github.com/microsoft/onnx-script@a71e35bcd72537bf7572536ee57250a0c0488bf6'
-  # numba requires numpy <= 1.20, onnxruntime requires numpy >= 1.21.
-  # We don't actually need it for our tests, but it's imported if it's present, so uninstall.
-  pip uninstall -q --yes numba
-  # JIT C++ extensions require ninja, so put it into PATH.
-  export PATH="/var/lib/jenkins/.local/bin:$PATH"
-  "$ROOT_DIR/scripts/onnx/test.sh"
-fi

.ci/pytorch/.shellcheckrc

@@ -1,4 +0,0 @@
-source-path=SCRIPTDIR
-
-# we'd like to enable --external-sources here but can't
-# https://github.com/koalaman/shellcheck/issues/1818

.ci/pytorch/README.md

@@ -1,42 +0,0 @@
-This directory contains scripts for our continuous integration.
-
-One important thing to keep in mind when reading the scripts here is
-that they are all based off of Docker images, which we build for each of
-the various system configurations we want to run on Jenkins.  This means
-it is very easy to run these tests yourself:
-
-1. Figure out what Docker image you want.  The general template for our
-   images look like:
-   ``registry.pytorch.org/pytorch/pytorch-$BUILD_ENVIRONMENT:$DOCKER_VERSION``,
-   where ``$BUILD_ENVIRONMENT`` is one of the build environments
-   enumerated in
-   [pytorch-dockerfiles](https://github.com/pytorch/pytorch/blob/master/.ci/docker/build.sh). The dockerfile used by jenkins can be found under the `.ci` [directory](https://github.com/pytorch/pytorch/blob/master/.ci/docker)
-
-2. Run ``docker run -it -u jenkins $DOCKER_IMAGE``, clone PyTorch and
-   run one of the scripts in this directory.
-
-The Docker images are designed so that any "reasonable" build commands
-will work; if you look in [build.sh](build.sh) you will see that it is a
-very simple script.  This is intentional.  Idiomatic build instructions
-should work inside all of our Docker images.  You can tweak the commands
-however you need (e.g., in case you want to rebuild with DEBUG, or rerun
-the build with higher verbosity, etc.).
-
-We have to do some work to make this so.  Here is a summary of the
-mechanisms we use:
-
-- We install binaries to directories like `/usr/local/bin` which
-  are automatically part of your PATH.
-
-- We add entries to the PATH using Docker ENV variables (so
-  they apply when you enter Docker) and `/etc/environment` (so they
-  continue to apply even if you sudo), instead of modifying
-  `PATH` in our build scripts.
-
-- We use `/etc/ld.so.conf.d` to register directories containing
-  shared libraries, instead of modifying `LD_LIBRARY_PATH` in our
-  build scripts.
-
-- We reroute well known paths like `/usr/bin/gcc` to alternate
-  implementations with `update-alternatives`, instead of setting
-  `CC` and `CXX` in our implementations.

.ci/pytorch/build-asan.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Required environment variable: $BUILD_ENVIRONMENT
-# (This is set by default in the Docker images we build, so you don't
-# need to set it yourself.
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-# shellcheck source=./common-build.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
-
-echo "Clang version:"
-clang --version
-
-python tools/stats/export_test_times.py
-
-# detect_leaks=0: Python is very leaky, so we need suppress it
-# symbolize=1: Gives us much better errors when things go wrong
-export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1:symbolize=1:detect_odr_violation=0
-if [ -n "$(which conda)" ]; then
-  export CMAKE_PREFIX_PATH=/opt/conda
-fi
-
-# TODO: Make the ASAN flags a centralized env var and unify with USE_ASAN option
-CC="clang" CXX="clang++" LDSHARED="clang --shared" \
-  CFLAGS="-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize-address-use-after-scope -shared-libasan" \
-  USE_ASAN=1 USE_CUDA=0 USE_MKLDNN=0 \
-  python setup.py bdist_wheel
-  pip_install_whl "$(echo dist/*.whl)"
-
-# Test building via the sdist source tarball
-python setup.py sdist
-mkdir -p /tmp/tmp
-pushd /tmp/tmp
-tar zxf "$(dirname "${BASH_SOURCE[0]}")/../../dist/"*.tar.gz
-cd torch-*
-python setup.py build --cmake-only
-popd
-
-print_sccache_stats
-
-assert_git_not_dirty

.ci/pytorch/build-mobile.sh

@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-# DO NOT ADD 'set -x' not to reveal CircleCI secret context environment variables
-set -eu -o pipefail
-
-# This script uses linux host toolchain + mobile build options in order to
-# build & test mobile libtorch without having to setup Android/iOS
-# toolchain/simulator.
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-# shellcheck source=./common-build.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
-
-# Install torch & torchvision - used to download & trace test model.
-# Ideally we should use the libtorch built on the PR so that backward
-# incompatible changes won't break this script - but it will significantly slow
-# down mobile CI jobs.
-# Here we install nightly instead of stable so that we have an option to
-# temporarily skip mobile CI jobs on BC-breaking PRs until they are in nightly.
-retry pip install --pre torch torchvision \
-  -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html \
-  --progress-bar off
-
-# Run end-to-end process of building mobile library, linking into the predictor
-# binary, and running forward pass with a real model.
-if [[ "$BUILD_ENVIRONMENT" == *-mobile-custom-build-static* ]]; then
-  TEST_CUSTOM_BUILD_STATIC=1 test/mobile/custom_build/build.sh
-elif [[ "$BUILD_ENVIRONMENT" == *-mobile-lightweight-dispatch* ]]; then
-  test/mobile/lightweight_dispatch/build.sh
-else
-  TEST_DEFAULT_BUILD=1 test/mobile/custom_build/build.sh
-fi
-
-print_sccache_stats

.ci/pytorch/build-tsan.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Required environment variable: $BUILD_ENVIRONMENT
-# (This is set by default in the Docker images we build, so you don't
-# need to set it yourself.
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-# shellcheck source=./common-build.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
-
-echo "Clang version:"
-clang --version
-
-python tools/stats/export_test_times.py
-
-if [ -n "$(which conda)" ]; then
-  export CMAKE_PREFIX_PATH=/opt/conda
-fi
-
-CC="clang" CXX="clang++" LDSHARED="clang --shared" \
-  CFLAGS="-fsanitize=thread" \
-  USE_TSAN=1 USE_CUDA=0 USE_MKLDNN=0 \
-  python setup.py bdist_wheel
-  pip_install_whl "$(echo dist/*.whl)"
-
-print_sccache_stats
-
-assert_git_not_dirty

.ci/pytorch/build.sh

@@ -1,318 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-# Required environment variable: $BUILD_ENVIRONMENT
-# (This is set by default in the Docker images we build, so you don't
-# need to set it yourself.
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-# shellcheck source=./common-build.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
-
-if [[ "$BUILD_ENVIRONMENT" == *-clang7-asan* ]]; then
-  exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" "$@"
-fi
-
-if [[ "$BUILD_ENVIRONMENT" == *-clang7-tsan* ]]; then
-  exec "$(dirname "${BASH_SOURCE[0]}")/build-tsan.sh" "$@"
-fi
-
-if [[ "$BUILD_ENVIRONMENT" == *-mobile-*build* ]]; then
-  exec "$(dirname "${BASH_SOURCE[0]}")/build-mobile.sh" "$@"
-fi
-
-echo "Python version:"
-python --version
-
-echo "GCC version:"
-gcc --version
-
-echo "CMake version:"
-cmake --version
-
-echo "Environment variables:"
-env
-
-if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
-  echo "NVCC version:"
-  nvcc --version
-fi
-
-if [[ "$BUILD_ENVIRONMENT" == *cuda11* ]]; then
-  if [[ "$BUILD_ENVIRONMENT" != *cuda11.3* && "$BUILD_ENVIRONMENT" != *clang* ]]; then
-    # TODO: there is a linking issue when building with UCC using clang,
-    # disable it for now and to be fix later.
-    export USE_UCC=1
-    export USE_SYSTEM_UCC=1
-  fi
-fi
-
-if [[ ${BUILD_ENVIRONMENT} == *"caffe2"* ]]; then
-  echo "Caffe2 build is ON"
-  export BUILD_CAFFE2=ON
-fi
-
-if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
-  export ATEN_THREADING=TBB
-  export USE_TBB=1
-elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
-  export ATEN_THREADING=NATIVE
-fi
-
-# Enable LLVM dependency for TensorExpr testing
-if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
-  export USE_LLVM=/opt/rocm/llvm
-  export LLVM_DIR=/opt/rocm/llvm/lib/cmake/llvm
-else
-  export USE_LLVM=/opt/llvm
-  export LLVM_DIR=/opt/llvm/lib/cmake/llvm
-fi
-
-if ! which conda; then
-  # In ROCm CIs, we are doing cross compilation on build machines with
-  # intel cpu and later run tests on machines with amd cpu.
-  # Also leave out two builds to make sure non-mkldnn builds still work.
-  if [[ "$BUILD_ENVIRONMENT" != *rocm* ]]; then
-    export USE_MKLDNN=1
-  else
-    export USE_MKLDNN=0
-  fi
-else
-  export CMAKE_PREFIX_PATH=/opt/conda
-fi
-
-if [[ "$BUILD_ENVIRONMENT" == *libtorch* ]]; then
-  POSSIBLE_JAVA_HOMES=()
-  POSSIBLE_JAVA_HOMES+=(/usr/local)
-  POSSIBLE_JAVA_HOMES+=(/usr/lib/jvm/java-8-openjdk-amd64)
-  POSSIBLE_JAVA_HOMES+=(/Library/Java/JavaVirtualMachines/*.jdk/Contents/Home)
-  # Add the Windows-specific JNI
-  POSSIBLE_JAVA_HOMES+=("$PWD/.circleci/windows-jni/")
-  for JH in "${POSSIBLE_JAVA_HOMES[@]}" ; do
-    if [[ -e "$JH/include/jni.h" ]] ; then
-      # Skip if we're not on Windows but haven't found a JAVA_HOME
-      if [[ "$JH" == "$PWD/.circleci/windows-jni/" && "$OSTYPE" != "msys" ]] ; then
-        break
-      fi
-      echo "Found jni.h under $JH"
-      export JAVA_HOME="$JH"
-      export BUILD_JNI=ON
-      break
-    fi
-  done
-  if [ -z "$JAVA_HOME" ]; then
-    echo "Did not find jni.h"
-  fi
-fi
-
-# Use special scripts for Android builds
-if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
-  export ANDROID_NDK=/opt/ndk
-  build_args=()
-  if [[ "${BUILD_ENVIRONMENT}" == *-arm-v7a* ]]; then
-    build_args+=("-DANDROID_ABI=armeabi-v7a")
-  elif [[ "${BUILD_ENVIRONMENT}" == *-arm-v8a* ]]; then
-    build_args+=("-DANDROID_ABI=arm64-v8a")
-  elif [[ "${BUILD_ENVIRONMENT}" == *-x86_32* ]]; then
-    build_args+=("-DANDROID_ABI=x86")
-  elif [[ "${BUILD_ENVIRONMENT}" == *-x86_64* ]]; then
-    build_args+=("-DANDROID_ABI=x86_64")
-  fi
-  if [[ "${BUILD_ENVIRONMENT}" == *vulkan* ]]; then
-    build_args+=("-DUSE_VULKAN=ON")
-  fi
-  build_args+=("-DUSE_LITE_INTERPRETER_PROFILER=OFF")
-  exec ./scripts/build_android.sh "${build_args[@]}" "$@"
-fi
-
-if [[ "$BUILD_ENVIRONMENT" != *android* && "$BUILD_ENVIRONMENT" == *vulkan* ]]; then
-  export USE_VULKAN=1
-  # shellcheck disable=SC1091
-  source /var/lib/jenkins/vulkansdk/setup-env.sh
-fi
-
-if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
-  # hcc used to run out of memory, silently exiting without stopping
-  # the build process, leaving undefined symbols in the shared lib,
-  # causing undefined symbol errors when later running tests.
-  # We used to set MAX_JOBS to 4 to avoid, but this is no longer an issue.
-  if [ -z "$MAX_JOBS" ]; then
-    export MAX_JOBS=$(($(nproc) - 1))
-  fi
-
-  if [[ -n "$CI" && -z "$PYTORCH_ROCM_ARCH" ]]; then
-      # Set ROCM_ARCH to gfx906 for CI builds, if user doesn't override.
-      echo "Limiting PYTORCH_ROCM_ARCH to gfx906 for CI builds"
-      export PYTORCH_ROCM_ARCH="gfx906"
-  fi
-
-  # hipify sources
-  python tools/amd_build/build_amd.py
-fi
-
-# sccache will fail for CUDA builds if all cores are used for compiling
-# gcc 7 with sccache seems to have intermittent OOM issue if all cores are used
-if [ -z "$MAX_JOBS" ]; then
-  if { [[ "$BUILD_ENVIRONMENT" == *cuda* ]] || [[ "$BUILD_ENVIRONMENT" == *gcc7* ]]; } && which sccache > /dev/null; then
-    export MAX_JOBS=$(($(nproc) - 1))
-  fi
-fi
-
-# TORCH_CUDA_ARCH_LIST must be passed from an environment variable
-if [[ "$BUILD_ENVIRONMENT" == *cuda* && -z "$TORCH_CUDA_ARCH_LIST" ]]; then
-  echo "TORCH_CUDA_ARCH_LIST must be defined"
-  exit 1
-fi
-
-if [[ "${BUILD_ENVIRONMENT}" == *clang* ]]; then
-  export CC=clang
-  export CXX=clang++
-fi
-
-if [[ "${BUILD_ENVIRONMENT}" == *no-ops* ]]; then
-  export USE_PER_OPERATOR_HEADERS=0
-fi
-
-if [[ "${BUILD_ENVIRONMENT}" == *-pch* ]]; then
-    export USE_PRECOMPILED_HEADERS=1
-fi
-
-if [[ "${BUILD_ENVIRONMENT}" == *linux-focal-py3.7-gcc7-build*  ]]; then
-  export USE_GLOO_WITH_OPENSSL=ON
-fi
-
-if [[ "${BUILD_ENVIRONMENT}" != *android* && "${BUILD_ENVIRONMENT}" != *cuda* ]]; then
-  export BUILD_STATIC_RUNTIME_BENCHMARK=ON
-fi
-
-if [[ "$BUILD_ENVIRONMENT" == *-bazel-* ]]; then
-  set -e
-
-  get_bazel
-
-  # Leave 1 CPU free and use only up to 80% of memory to reduce the change of crashing
-  # the runner
-  BAZEL_MEM_LIMIT="--local_ram_resources=HOST_RAM*.8"
-  BAZEL_CPU_LIMIT="--local_cpu_resources=HOST_CPUS-1"
-
-  tools/bazel build --config=no-tty "${BAZEL_MEM_LIMIT}" "${BAZEL_CPU_LIMIT}" //...
-  # Build torch, the Python module, and tests for CPU-only
-  tools/bazel build --config=no-tty "${BAZEL_MEM_LIMIT}" "${BAZEL_CPU_LIMIT}" --config=cpu-only :torch :_C.so :all_tests
-
-else
-  # check that setup.py would fail with bad arguments
-  echo "The next three invocations are expected to fail with invalid command error messages."
-  ( ! get_exit_code python setup.py bad_argument )
-  ( ! get_exit_code python setup.py clean] )
-  ( ! get_exit_code python setup.py clean bad_argument )
-
-  if [[ "$BUILD_ENVIRONMENT" != *libtorch* ]]; then
-
-    # rocm builds fail when WERROR=1
-    # XLA test build fails when WERROR=1
-    # set only when building other architectures
-    # or building non-XLA tests.
-    if [[ "$BUILD_ENVIRONMENT" != *rocm*  &&
-          "$BUILD_ENVIRONMENT" != *xla* ]]; then
-      WERROR=1 python setup.py bdist_wheel
-    else
-      python setup.py bdist_wheel
-    fi
-    pip_install_whl "$(echo dist/*.whl)"
-
-    # TODO: I'm not sure why, but somehow we lose verbose commands
-    set -x
-
-    assert_git_not_dirty
-    # Copy ninja build logs to dist folder
-    mkdir -p dist
-    if [ -f build/.ninja_log ]; then
-      cp build/.ninja_log dist
-    fi
-
-    if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
-      # remove sccache wrappers post-build; runtime compilation of MIOpen kernels does not yet fully support them
-      sudo rm -f /opt/cache/bin/cc
-      sudo rm -f /opt/cache/bin/c++
-      sudo rm -f /opt/cache/bin/gcc
-      sudo rm -f /opt/cache/bin/g++
-      pushd /opt/rocm/llvm/bin
-      if [[ -d original ]]; then
-        sudo mv original/clang .
-        sudo mv original/clang++ .
-      fi
-      sudo rm -rf original
-      popd
-    fi
-
-    CUSTOM_TEST_ARTIFACT_BUILD_DIR=${CUSTOM_TEST_ARTIFACT_BUILD_DIR:-"build/custom_test_artifacts"}
-    CUSTOM_TEST_USE_ROCM=$([[ "$BUILD_ENVIRONMENT" == *rocm* ]] && echo "ON" || echo "OFF")
-    CUSTOM_TEST_MODULE_PATH="${PWD}/cmake/public"
-    mkdir -pv "${CUSTOM_TEST_ARTIFACT_BUILD_DIR}"
-
-    # Build custom operator tests.
-    CUSTOM_OP_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-op-build"
-    CUSTOM_OP_TEST="$PWD/test/custom_operator"
-    python --version
-    SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
-    mkdir -p "$CUSTOM_OP_BUILD"
-    pushd "$CUSTOM_OP_BUILD"
-    cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)" \
-          -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
-    make VERBOSE=1
-    popd
-    assert_git_not_dirty
-
-    # Build jit hook tests
-    JIT_HOOK_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/jit-hook-build"
-    JIT_HOOK_TEST="$PWD/test/jit_hooks"
-    python --version
-    SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
-    mkdir -p "$JIT_HOOK_BUILD"
-    pushd "$JIT_HOOK_BUILD"
-    cmake "$JIT_HOOK_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)" \
-          -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
-    make VERBOSE=1
-    popd
-    assert_git_not_dirty
-
-    # Build custom backend tests.
-    CUSTOM_BACKEND_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-backend-build"
-    CUSTOM_BACKEND_TEST="$PWD/test/custom_backend"
-    python --version
-    mkdir -p "$CUSTOM_BACKEND_BUILD"
-    pushd "$CUSTOM_BACKEND_BUILD"
-    cmake "$CUSTOM_BACKEND_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)" \
-          -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
-    make VERBOSE=1
-    popd
-    assert_git_not_dirty
-  else
-    # Test no-Python build
-    echo "Building libtorch"
-
-    # This is an attempt to mitigate flaky libtorch build OOM error. By default, the build parallelization
-    # is set to be the number of CPU minus 2. So, let's try a more conservative value here. A 4xlarge has
-    # 16 CPUs
-    MAX_JOBS=$(nproc --ignore=4)
-    export MAX_JOBS
-
-    # NB: Install outside of source directory (at the same level as the root
-    # pytorch folder) so that it doesn't get cleaned away prior to docker push.
-    BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
-    mkdir -p ../cpp-build/caffe2
-    pushd ../cpp-build/caffe2
-    WERROR=1 VERBOSE=1 DEBUG=1 python "$BUILD_LIBTORCH_PY"
-    popd
-  fi
-fi
-
-if [[ "$BUILD_ENVIRONMENT" != *libtorch* && "$BUILD_ENVIRONMENT" != *bazel* ]]; then
-  # export test times so that potential sharded tests that'll branch off this build will use consistent data
-  # don't do this for libtorch as libtorch is C++ only and thus won't have python tests run on its build
-  python tools/stats/export_test_times.py
-fi
-
-print_sccache_stats

.ci/pytorch/codegen-test.sh

@@ -1,58 +0,0 @@
-#!/usr/bin/env bash
-
-# This script can also be used to test whether your diff changes any codegen output.
-#
-# Run it before and after your change:
-#   .ci/pytorch/codegen-test.sh <baseline_output_dir>
-#   .ci/pytorch/codegen-test.sh <test_output_dir>
-#
-# Then run diff to compare the generated files:
-#   diff -Naur <baseline_output_dir> <test_output_dir>
-
-set -eu -o pipefail
-
-if [ "$#" -eq 0 ]; then
-  # shellcheck source=./common.sh
-  source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-  OUT="$(dirname "${BASH_SOURCE[0]}")/../../codegen_result"
-else
-  OUT=$1
-fi
-
-set -x
-
-rm -rf "$OUT"
-
-# aten codegen
-python -m torchgen.gen \
-  -s aten/src/ATen \
-  -d "$OUT"/torch/share/ATen
-
-# torch codegen
-python -m tools.setup_helpers.generate_code \
-  --install_dir "$OUT"
-
-# pyi codegen
-mkdir -p "$OUT"/pyi/torch/_C
-mkdir -p "$OUT"/pyi/torch/nn
-python -m tools.pyi.gen_pyi \
-  --native-functions-path aten/src/ATen/native/native_functions.yaml \
-  --tags-path aten/src/ATen/native/tags.yaml \
-  --deprecated-functions-path tools/autograd/deprecated.yaml \
-  --out "$OUT"/pyi
-
-# autograd codegen (called by torch codegen but can run independently)
-python -m tools.autograd.gen_autograd \
-  "$OUT"/torch/share/ATen/Declarations.yaml \
-  aten/src/ATen/native/native_functions.yaml \
-  aten/src/ATen/native/tags.yaml \
-  "$OUT"/autograd \
-  tools/autograd
-
-# annotated_fn_args codegen (called by torch codegen but can run independently)
-mkdir -p "$OUT"/annotated_fn_args
-python -m tools.autograd.gen_annotated_fn_args \
-  aten/src/ATen/native/native_functions.yaml \
-  aten/src/ATen/native/tags.yaml \
-  "$OUT"/annotated_fn_args \
-  tools/autograd

.ci/pytorch/common-build.sh

@@ -1,58 +0,0 @@
-#!/bin/bash
-# Required environment variables:
-#   $BUILD_ENVIRONMENT (should be set by your Docker image)
-
-if [[ "$BUILD_ENVIRONMENT" != *win-* ]]; then
-    # Save the absolute path in case later we chdir (as occurs in the gpu perf test)
-    script_dir="$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit ; pwd -P )"
-
-    if which sccache > /dev/null; then
-        # Save sccache logs to file
-        sccache --stop-server > /dev/null  2>&1 || true
-        rm -f ~/sccache_error.log || true
-
-        function sccache_epilogue() {
-            echo "::group::Sccache Compilation Log"
-            echo '=================== sccache compilation log ==================='
-            python "$script_dir/print_sccache_log.py" ~/sccache_error.log 2>/dev/null || true
-            echo '=========== If your build fails, please take a look at the log above for possible reasons ==========='
-            sccache --show-stats
-            sccache --stop-server || true
-            echo "::endgroup::"
-        }
-
-        # Register the function here so that the error log can be printed even when
-        # sccache fails to start, i.e. timeout error
-        trap_add sccache_epilogue EXIT
-
-        if [[ -n "${SKIP_SCCACHE_INITIALIZATION:-}" ]]; then
-            # sccache --start-server seems to hang forever on self hosted runners for GHA
-            # so let's just go ahead and skip the --start-server altogether since it seems
-            # as though sccache still gets used even when the sscache server isn't started
-            # explicitly
-            echo "Skipping sccache server initialization, setting environment variables"
-            export SCCACHE_IDLE_TIMEOUT=1200
-            export SCCACHE_ERROR_LOG=~/sccache_error.log
-            export RUST_LOG=sccache::server=error
-        elif [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
-            SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=0 sccache --start-server
-        else
-            # increasing SCCACHE_IDLE_TIMEOUT so that extension_backend_test.cpp can build after this PR:
-            # https://github.com/pytorch/pytorch/pull/16645
-            SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=1200 RUST_LOG=sccache::server=error sccache --start-server
-        fi
-
-        # Report sccache stats for easier debugging
-        sccache --zero-stats
-    fi
-
-    if which ccache > /dev/null; then
-        # Report ccache stats for easier debugging
-        ccache --zero-stats
-        ccache --show-stats
-        function ccache_epilogue() {
-            ccache --show-stats
-        }
-        trap_add ccache_epilogue EXIT
-    fi
-fi

.ci/pytorch/common.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-# Common setup for all Jenkins scripts
-# shellcheck source=./common_utils.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
-set -ex
-
-# Required environment variables:
-#   $BUILD_ENVIRONMENT (should be set by your Docker image)
-
-# Figure out which Python to use for ROCm
-if [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
-  # HIP_PLATFORM is auto-detected by hipcc; unset to avoid build errors
-  unset HIP_PLATFORM
-  export PYTORCH_TEST_WITH_ROCM=1
-  # temporary to locate some kernel issues on the CI nodes
-  export HSAKMT_DEBUG_LEVEL=4
-  # improve rccl performance for distributed tests
-  export HSA_FORCE_FINE_GRAIN_PCIE=1
-fi
-
-# TODO: Renable libtorch testing for MacOS, see https://github.com/pytorch/pytorch/issues/62598
-# shellcheck disable=SC2034
-BUILD_TEST_LIBTORCH=0
-
-retry () {
-  "$@"  || (sleep 1 && "$@") || (sleep 2 && "$@")
-}

.ci/pytorch/common_utils.sh

@@ -1,237 +0,0 @@
-#!/bin/bash
-
-# Common util **functions** that can be sourced in other scripts.
-
-# note: printf is used instead of echo to avoid backslash
-# processing and to properly handle values that begin with a '-'.
-
-log() { printf '%s\n' "$*"; }
-error() { log "ERROR: $*" >&2; }
-fatal() { error "$@"; exit 1; }
-
-retry () {
-    "$@" || (sleep 10 && "$@") || (sleep 20 && "$@") || (sleep 40 && "$@")
-}
-
-# compositional trap taken from https://stackoverflow.com/a/7287873/23845
-# appends a command to a trap
-#
-# - 1st arg:  code to add
-# - remaining args:  names of traps to modify
-#
-trap_add() {
-    trap_add_cmd=$1; shift || fatal "${FUNCNAME[0]} usage error"
-    for trap_add_name in "$@"; do
-        trap -- "$(
-            # helper fn to get existing trap command from output
-            # of trap -p
-            extract_trap_cmd() { printf '%s\n' "$3"; }
-            # print existing trap command with newline
-            eval "extract_trap_cmd $(trap -p "${trap_add_name}")"
-            # print the new trap command
-            printf '%s\n' "${trap_add_cmd}"
-        )" "${trap_add_name}" \
-            || fatal "unable to add to trap ${trap_add_name}"
-    done
-}
-# set the trace attribute for the above function.  this is
-# required to modify DEBUG or RETURN traps because functions don't
-# inherit them unless the trace attribute is set
-declare -f -t trap_add
-
-function assert_git_not_dirty() {
-    # TODO: we should add an option to `build_amd.py` that reverts the repo to
-    #       an unmodified state.
-    if [[ "$BUILD_ENVIRONMENT" != *rocm* ]] && [[ "$BUILD_ENVIRONMENT" != *xla* ]] ; then
-        git_status=$(git status --porcelain)
-        if [[ $git_status ]]; then
-            echo "Build left local git repository checkout dirty"
-            echo "git status --porcelain:"
-            echo "${git_status}"
-            exit 1
-        fi
-    fi
-}
-
-function pip_install_whl() {
-  # This is used to install PyTorch and other build artifacts wheel locally
-  # without using any network connection
-  python3 -mpip install --no-index --no-deps "$@"
-}
-
-function pip_install() {
-  # retry 3 times
-  # old versions of pip don't have the "--progress-bar" flag
-  pip install --progress-bar off "$@" || pip install --progress-bar off "$@" || pip install --progress-bar off "$@" ||\
-  pip install "$@" || pip install "$@" || pip install "$@"
-}
-
-function pip_uninstall() {
-  # uninstall 2 times
-  pip uninstall -y "$@" || pip uninstall -y "$@"
-}
-
-function get_exit_code() {
-  set +e
-  "$@"
-  retcode=$?
-  set -e
-  return $retcode
-}
-
-function get_bazel() {
-  if [[ $(uname) == "Darwin" ]]; then
-    # download bazel version
-    retry curl https://github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-darwin-x86_64  -Lo tools/bazel
-    # verify content
-    echo '74d93848f0c9d592e341e48341c53c87e3cb304a54a2a1ee9cff3df422f0b23c  tools/bazel' | shasum -a 256 -c >/dev/null
-  else
-    # download bazel version
-    retry curl https://ossci-linux.s3.amazonaws.com/bazel-4.2.1-linux-x86_64 -o tools/bazel
-    # verify content
-    echo '1a4f3a3ce292307bceeb44f459883859c793436d564b95319aacb8af1f20557c  tools/bazel' | shasum -a 256 -c >/dev/null
-  fi
-
-  chmod +x tools/bazel
-}
-
-function install_monkeytype {
-  # Install MonkeyType
-  pip_install MonkeyType
-}
-
-
-function get_pinned_commit() {
-  cat .github/ci_commit_pins/"${1}".txt
-}
-
-function install_torchtext() {
-  local commit
-  commit=$(get_pinned_commit text)
-  pip_install --no-use-pep517 --user "git+https://github.com/pytorch/text.git@${commit}"
-}
-
-function install_torchvision() {
-  local commit
-  commit=$(get_pinned_commit vision)
-  pip_install --no-use-pep517 --user "git+https://github.com/pytorch/vision.git@${commit}"
-}
-
-function clone_pytorch_xla() {
-  if [[ ! -d ./xla ]]; then
-    git clone --recursive -b r2.0 --quiet https://github.com/pytorch/xla.git
-    pushd xla
-    # pin the xla hash so that we don't get broken by changes to xla
-    git checkout "$(cat ../.github/ci_commit_pins/xla.txt)"
-    git submodule sync
-    git submodule update --init --recursive
-    popd
-  fi
-}
-
-function install_filelock() {
-  pip_install filelock
-}
-
-function install_triton() {
-  local commit
-  if [[ "${TEST_CONFIG}" == *rocm* ]]; then
-    echo "skipping triton due to rocm"
-  else
-    commit=$(get_pinned_commit triton)
-    if [[ "${BUILD_ENVIRONMENT}" == *gcc7* ]]; then
-      # Trition needs gcc-9 to build
-      sudo apt-get install -y g++-9
-      CXX=g++-9 pip_install --user "git+https://github.com/openai/triton@${commit}#subdirectory=python"
-    elif [[ "${BUILD_ENVIRONMENT}" == *clang* ]]; then
-      # Trition needs <filesystem> which surprisingly is not available with clang-9 toolchain
-      sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-      sudo apt-get install -y g++-9
-      CXX=g++-9 pip_install --user "git+https://github.com/openai/triton@${commit}#subdirectory=python"
-    else
-      pip_install --user "git+https://github.com/openai/triton@${commit}#subdirectory=python"
-    fi
-    pip_install --user jinja2
-  fi
-}
-
-function setup_torchdeploy_deps(){
-  conda install -y -n "py_${ANACONDA_PYTHON_VERSION}" "libpython-static=${ANACONDA_PYTHON_VERSION}"
-  local CC
-  local CXX
-  CC="$(which gcc)"
-  CXX="$(which g++)"
-  export CC
-  export CXX
-  pip install --upgrade pip
-}
-
-function checkout_install_torchdeploy() {
-  local commit
-  commit=$(get_pinned_commit multipy)
-  setup_torchdeploy_deps
-  pushd ..
-  git clone --recurse-submodules https://github.com/pytorch/multipy.git
-  pushd multipy
-  git checkout "${commit}"
-  python multipy/runtime/example/generate_examples.py
-  pip install -e . --install-option="--cudatests"
-  popd
-  popd
-}
-
-function test_torch_deploy(){
- pushd ..
- pushd multipy
- ./multipy/runtime/build/test_deploy
- ./multipy/runtime/build/test_deploy_gpu
- popd
- popd
-}
-
-function install_huggingface() {
-  local commit
-  commit=$(get_pinned_commit huggingface)
-  pip_install pandas
-  pip_install scipy
-  pip_install "git+https://github.com/huggingface/transformers.git@${commit}#egg=transformers"
-}
-
-function install_timm() {
-  local commit
-  commit=$(get_pinned_commit timm)
-  pip_install pandas
-  pip_install scipy
-  pip_install "git+https://github.com/rwightman/pytorch-image-models@${commit}"
-}
-
-function checkout_install_torchbench() {
-  git clone https://github.com/pytorch/benchmark torchbench
-  pushd torchbench
-  git checkout no_torchaudio
-
-  if [ "$1" ]; then
-    python install.py --continue_on_fail models "$@"
-  else
-    # Occasionally the installation may fail on one model but it is ok to continue
-    # to install and test other models
-    python install.py --continue_on_fail
-  fi
-  popd
-}
-
-function test_functorch() {
-  python test/run_test.py --functorch --verbose
-}
-
-function print_sccache_stats() {
-  echo 'PyTorch Build Statistics'
-  sccache --show-stats
-
-  if [[ -n "${OUR_GITHUB_JOB_ID}" ]]; then
-    sccache --show-stats --stats-format json | jq .stats \
-      > "sccache-stats-${BUILD_ENVIRONMENT}-${OUR_GITHUB_JOB_ID}.json"
-  else
-    echo "env var OUR_GITHUB_JOB_ID not set, will not write sccache stats to json"
-  fi
-}

.ci/pytorch/create_test_cert.py

@@ -1,96 +0,0 @@
-from datetime import datetime, timedelta
-from tempfile import mkdtemp
-from cryptography.hazmat.primitives import serialization
-from cryptography.hazmat.primitives.asymmetric import rsa
-from cryptography import x509
-from cryptography.x509.oid import NameOID
-from cryptography.hazmat.primitives import hashes
-
-temp_dir = mkdtemp()
-print(temp_dir)
-
-
-def genrsa(path):
-    key = rsa.generate_private_key(
-        public_exponent=65537,
-        key_size=2048,
-    )
-    with open(path, "wb") as f:
-        f.write(key.private_bytes(
-            encoding=serialization.Encoding.PEM,
-            format=serialization.PrivateFormat.TraditionalOpenSSL,
-            encryption_algorithm=serialization.NoEncryption(),
-        ))
-    return key
-
-
-def create_cert(path, C, ST, L, O, key):
-    subject = issuer = x509.Name([
-        x509.NameAttribute(NameOID.COUNTRY_NAME, C),
-        x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, ST),
-        x509.NameAttribute(NameOID.LOCALITY_NAME, L),
-        x509.NameAttribute(NameOID.ORGANIZATION_NAME, O),
-    ])
-    cert = x509.CertificateBuilder().subject_name(
-        subject
-    ).issuer_name(
-        issuer
-    ).public_key(
-        key.public_key()
-    ).serial_number(
-        x509.random_serial_number()
-    ).not_valid_before(
-        datetime.utcnow()
-    ).not_valid_after(
-        # Our certificate will be valid for 10 days
-        datetime.utcnow() + timedelta(days=10)
-    ).add_extension(
-        x509.BasicConstraints(ca=True, path_length=None), critical=True,
-    ).sign(key, hashes.SHA256())
-    # Write our certificate out to disk.
-    with open(path, "wb") as f:
-        f.write(cert.public_bytes(serialization.Encoding.PEM))
-    return cert
-
-
-def create_req(path, C, ST, L, O, key):
-    csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
-        # Provide various details about who we are.
-        x509.NameAttribute(NameOID.COUNTRY_NAME, C),
-        x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, ST),
-        x509.NameAttribute(NameOID.LOCALITY_NAME, L),
-        x509.NameAttribute(NameOID.ORGANIZATION_NAME, O),
-    ])).sign(key, hashes.SHA256())
-    with open(path, "wb") as f:
-        f.write(csr.public_bytes(serialization.Encoding.PEM))
-    return csr
-
-
-def sign_certificate_request(path, csr_cert, ca_cert, private_ca_key):
-    cert = x509.CertificateBuilder().subject_name(
-        csr_cert.subject
-    ).issuer_name(
-        ca_cert.subject
-    ).public_key(
-        csr_cert.public_key()
-    ).serial_number(
-        x509.random_serial_number()
-    ).not_valid_before(
-        datetime.utcnow()
-    ).not_valid_after(
-        # Our certificate will be valid for 10 days
-        datetime.utcnow() + timedelta(days=10)
-        # Sign our certificate with our private key
-    ).sign(private_ca_key, hashes.SHA256())
-    with open(path, "wb") as f:
-        f.write(cert.public_bytes(serialization.Encoding.PEM))
-    return cert
-
-
-ca_key = genrsa(temp_dir + "/ca.key")
-ca_cert = create_cert(temp_dir + "/ca.pem", u"US", u"New York", u"New York", u"Gloo Certificate Authority", ca_key)
-
-pkey = genrsa(temp_dir + "/pkey.key")
-csr = create_req(temp_dir + "/csr.csr", u"US", u"California", u"San Francisco", u"Gloo Testing Company", pkey)
-
-cert = sign_certificate_request(temp_dir + "/cert.pem", csr, ca_cert, ca_key)

.ci/pytorch/docker-build-test.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-
-docker build -t pytorch .

.ci/pytorch/docs-test.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-
-echo "Testing pytorch docs"
-
-cd docs
-pip_install -r requirements.txt
-make doctest

.ci/pytorch/fake_numpy/numpy.py

@@ -1 +0,0 @@
-raise ModuleNotFoundError("Sorry PyTorch, but our NumPy is in the other folder")

.ci/pytorch/macos-build.sh

@@ -1,80 +0,0 @@
-#!/bin/bash
-
-# shellcheck disable=SC2034
-# shellcheck source=./macos-common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/macos-common.sh"
-# shellcheck source=./common-build.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
-
-# Build PyTorch
-if [ -z "${CI}" ]; then
-  export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
-fi
-
-# This helper function wraps calls to binaries with sccache, but only if they're not already wrapped with sccache.
-# For example, `clang` will be `sccache clang`, but `sccache clang` will not become `sccache sccache clang`.
-# The way this is done is by detecting the command of the parent pid of the current process and checking whether
-# that is sccache, and wrapping sccache around the process if its parent were not already sccache.
-function write_sccache_stub() {
-  output=$1
-  binary=$(basename "${output}")
-
-  printf "#!/bin/sh\nif [ \$(ps auxc \$(ps auxc -o ppid \$\$ | grep \$\$ | rev | cut -d' ' -f1 | rev) | tr '\\\\n' ' ' | rev | cut -d' ' -f2 | rev) != sccache ]; then\n  exec sccache %s \"\$@\"\nelse\n  exec %s \"\$@\"\nfi" "$(which "${binary}")" "$(which "${binary}")" > "${output}"
-  chmod a+x "${output}"
-}
-
-if which sccache > /dev/null; then
-  # Create temp directory for sccache shims
-  tmp_dir=$(mktemp -d)
-  trap 'rm -rfv ${tmp_dir}' EXIT
-  write_sccache_stub "${tmp_dir}/clang++"
-  write_sccache_stub "${tmp_dir}/clang"
-
-  export PATH="${tmp_dir}:$PATH"
-fi
-
-cross_compile_arm64() {
-  # Cross compilation for arm64
-  # Explicitly set USE_DISTRIBUTED=0 to align with the default build config on mac. This also serves as the sole CI config that tests
-  # that building with USE_DISTRIBUTED=0 works at all. See https://github.com/pytorch/pytorch/issues/86448
-  USE_DISTRIBUTED=0 CMAKE_OSX_ARCHITECTURES=arm64 MACOSX_DEPLOYMENT_TARGET=11.0 USE_MKLDNN=OFF USE_QNNPACK=OFF WERROR=1 BUILD_TEST=OFF USE_PYTORCH_METAL=1 python setup.py bdist_wheel
-}
-
-compile_x86_64() {
-  USE_DISTRIBUTED=0 WERROR=1 python setup.py bdist_wheel
-}
-
-build_lite_interpreter() {
-    echo "Testing libtorch (lite interpreter)."
-
-    CPP_BUILD="$(pwd)/../cpp_build"
-    # Ensure the removal of the tmp directory
-    trap 'rm -rfv ${CPP_BUILD}' EXIT
-    rm -rf "${CPP_BUILD}"
-    mkdir -p "${CPP_BUILD}/caffe2"
-
-    # It looks libtorch need to be built in "${CPP_BUILD}/caffe2 folder.
-    BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
-    pushd "${CPP_BUILD}/caffe2" || exit
-    VERBOSE=1 DEBUG=1 python "${BUILD_LIBTORCH_PY}"
-    popd || exit
-
-    "${CPP_BUILD}/caffe2/build/bin/test_lite_interpreter_runtime"
-}
-
-if [[ ${BUILD_ENVIRONMENT} = *arm64* ]]; then
-  cross_compile_arm64
-elif [[ ${BUILD_ENVIRONMENT} = *lite-interpreter* ]]; then
-  export BUILD_LITE_INTERPRETER=1
-  build_lite_interpreter
-else
-  compile_x86_64
-fi
-
-if which sccache > /dev/null; then
-  print_sccache_stats
-fi
-
-python tools/stats/export_test_times.py
-
-assert_git_not_dirty

.ci/pytorch/macos-common.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-# Common prelude for macos-build.sh and macos-test.sh
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-
-sysctl -a | grep machdep.cpu
-
-# These are required for both the build job and the test job.
-# In the latter to test cpp extensions.
-export MACOSX_DEPLOYMENT_TARGET=10.9
-export CXX=clang++
-export CC=clang

.ci/pytorch/macos-test.sh

@@ -1,186 +0,0 @@
-#!/bin/bash
-
-# shellcheck disable=SC2034
-# shellcheck source=./macos-common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/macos-common.sh"
-
-if [[ -n "$CONDA_ENV" ]]; then
-  # Use binaries under conda environment
-  export PATH="$CONDA_ENV/bin":$PATH
-fi
-
-# Test that OpenMP is enabled for non-arm64 build
-if [[ ${BUILD_ENVIRONMENT} != *arm64* ]]; then
-  pushd test
-  if [[ ! $(python -c "import torch; print(int(torch.backends.openmp.is_available()))") == "1" ]]; then
-    echo "Build should have OpenMP enabled, but torch.backends.openmp.is_available() is False"
-    exit 1
-  fi
-  popd
-fi
-
-setup_test_python() {
-  # The CircleCI worker hostname doesn't resolve to an address.
-  # This environment variable makes ProcessGroupGloo default to
-  # using the address associated with the loopback interface.
-  export GLOO_SOCKET_IFNAME=lo0
-  echo "Ninja version: $(ninja --version)"
-
-  # Increase default limit on open file handles from 256 to 1024
-  ulimit -n 1024
-}
-
-test_python_all() {
-  setup_test_python
-
-  time python test/run_test.py --verbose --exclude-jit-executor
-
-  assert_git_not_dirty
-}
-
-test_python_shard() {
-  if [[ -z "$NUM_TEST_SHARDS" ]]; then
-    echo "NUM_TEST_SHARDS must be defined to run a Python test shard"
-    exit 1
-  fi
-
-  setup_test_python
-
-  time python test/run_test.py --verbose --exclude-jit-executor --exclude-distributed-tests --shard "$1" "$NUM_TEST_SHARDS"
-
-  assert_git_not_dirty
-}
-
-test_libtorch() {
-  # C++ API
-
-  if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
-    # NB: Install outside of source directory (at the same level as the root
-    # pytorch folder) so that it doesn't get cleaned away prior to docker push.
-    # But still clean it before we perform our own build.
-
-    echo "Testing libtorch"
-
-    CPP_BUILD="$PWD/../cpp-build"
-    rm -rf "$CPP_BUILD"
-    mkdir -p "$CPP_BUILD"/caffe2
-
-    BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
-    pushd "$CPP_BUILD"/caffe2
-    VERBOSE=1 DEBUG=1 python "$BUILD_LIBTORCH_PY"
-    popd
-
-    python tools/download_mnist.py --quiet -d test/cpp/api/mnist
-
-    # Unfortunately it seems like the test can't load from miniconda3
-    # without these paths being set
-    export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/miniconda3/lib"
-    export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/miniconda3/lib"
-    TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" "$CPP_BUILD"/caffe2/bin/test_api
-
-    assert_git_not_dirty
-  fi
-}
-
-print_cmake_info() {
-  CMAKE_EXEC=$(which cmake)
-  echo "$CMAKE_EXEC"
-
-  CONDA_INSTALLATION_DIR=$(dirname "$CMAKE_EXEC")
-  # Print all libraries under cmake rpath for debugging
-  ls -la "$CONDA_INSTALLATION_DIR/../lib"
-
-  export CMAKE_EXEC
-  # Explicitly add conda env lib folder to cmake rpath to address the flaky issue
-  # where cmake dependencies couldn't be found. This seems to point to how conda
-  # links $CMAKE_EXEC to its package cache when cloning a new environment
-  install_name_tool -add_rpath @executable_path/../lib "${CMAKE_EXEC}" || true
-  # Adding the rpath will invalidate cmake signature, so signing it again here
-  # to trust the executable. EXC_BAD_ACCESS (SIGKILL (Code Signature Invalid))
-  # with an exit code 137 otherwise
-  codesign -f -s - "${CMAKE_EXEC}" || true
-}
-
-test_custom_backend() {
-  print_cmake_info
-
-  echo "Testing custom backends"
-  pushd test/custom_backend
-  rm -rf build && mkdir build
-  pushd build
-  SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
-  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" "${CMAKE_EXEC}" ..
-  make VERBOSE=1
-  popd
-
-  # Run Python tests and export a lowered module.
-  python test_custom_backend.py -v
-  python backend.py --export-module-to=model.pt
-  # Run C++ tests using the exported module.
-  build/test_custom_backend ./model.pt
-  rm -f ./model.pt
-  popd
-  assert_git_not_dirty
-}
-
-test_custom_script_ops() {
-  print_cmake_info
-
-  echo "Testing custom script operators"
-  pushd test/custom_operator
-  # Build the custom operator library.
-  rm -rf build && mkdir build
-  pushd build
-  SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
-  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" "${CMAKE_EXEC}" ..
-  make VERBOSE=1
-  popd
-
-  # Run tests Python-side and export a script module.
-  python test_custom_ops.py -v
-  python model.py --export-script-module=model.pt
-  # Run tests C++-side and load the exported script module.
-  build/test_custom_ops ./model.pt
-  popd
-  assert_git_not_dirty
-}
-
-test_jit_hooks() {
-  print_cmake_info
-
-  echo "Testing jit hooks in cpp"
-  pushd test/jit_hooks
-  # Build the custom operator library.
-  rm -rf build && mkdir build
-  pushd build
-  SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
-  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" "${CMAKE_EXEC}" ..
-  make VERBOSE=1
-  popd
-
-  # Run tests Python-side and export a script module.
-  python model.py --export-script-module=model
-  # Run tests C++-side and load the exported script module.
-  build/test_jit_hooks ./model
-  popd
-  assert_git_not_dirty
-}
-
-if [[ "${TEST_CONFIG}" == *functorch* ]]; then
-  test_functorch
-elif [[ $NUM_TEST_SHARDS -gt 1 ]]; then
-  test_python_shard "${SHARD_NUMBER}"
-  if [[ "${SHARD_NUMBER}" == 1 ]]; then
-    test_libtorch
-    test_custom_script_ops
-  elif [[ "${SHARD_NUMBER}" == 2 ]]; then
-    test_jit_hooks
-    test_custom_backend
-  fi
-else
-  test_python_all
-  test_libtorch
-  test_custom_script_ops
-  test_jit_hooks
-  test_custom_backend
-fi

.ci/pytorch/multigpu-test.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# Required environment variable: $BUILD_ENVIRONMENT
-# (This is set by default in the Docker images we build, so you don't
-# need to set it yourself.
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-
-echo "Testing pytorch"
-
-# Disabling tests to see if they solve timeout issues; see https://github.com/pytorch/pytorch/issues/70015
-# python tools/download_mnist.py --quiet -d test/cpp/api/mnist
-# OMP_NUM_THREADS=2 TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" build/bin/test_api
-time python test/run_test.py --verbose -i distributed/test_c10d_common
-time python test/run_test.py --verbose -i distributed/test_c10d_gloo
-time python test/run_test.py --verbose -i distributed/test_c10d_nccl
-time python test/run_test.py --verbose -i distributed/test_c10d_spawn_gloo
-time python test/run_test.py --verbose -i distributed/test_c10d_spawn_nccl
-time python test/run_test.py --verbose -i distributed/test_store
-time python test/run_test.py --verbose -i distributed/test_pg_wrapper
-time python test/run_test.py --verbose -i distributed/rpc/cuda/test_tensorpipe_agent
-# FSDP tests
-for f in test/distributed/fsdp/*.py ; do time python test/run_test.py --verbose -i "${f#*/}" ; done
-# ShardedTensor tests
-time python test/run_test.py --verbose -i distributed/checkpoint/test_checkpoint
-time python test/run_test.py --verbose -i distributed/checkpoint/test_file_system_checkpoint
-time python test/run_test.py --verbose -i distributed/_shard/sharding_spec/test_sharding_spec
-time python test/run_test.py --verbose -i distributed/_shard/sharding_plan/test_sharding_plan
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/test_megatron_prototype
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/test_sharded_tensor
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/test_sharded_tensor_reshard
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/ops/test_chunk
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/ops/test_elementwise_ops
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/ops/test_embedding
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/ops/test_embedding_bag
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/ops/test_binary_cmp
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/ops/test_init
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/ops/test_linear
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/ops/test_math_ops
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/ops/test_matrix_ops
-time python test/run_test.py --verbose -i distributed/_shard/sharded_tensor/ops/test_softmax
-time python test/run_test.py --verbose -i distributed/_shard/sharded_optim/test_sharded_optim
-time python test/run_test.py --verbose -i distributed/_shard/test_partial_tensor
-time python test/run_test.py --verbose -i distributed/_shard/test_replicated_tensor
-# Other tests
-time python test/run_test.py --verbose -i test_cuda_primary_ctx
-time python test/run_test.py --verbose -i test_optim -- -k optimizers_with_varying_tensors
-assert_git_not_dirty

.ci/pytorch/perf_test/common.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-set -e
-
-run_test () {
-  rm -rf test_tmp/ && mkdir test_tmp/ && cd test_tmp/
-  "$@"
-  cd .. && rm -rf test_tmp/
-}
-
-get_runtime_of_command () {
-  TIMEFORMAT=%R
-
-  # runtime=$( { time ($@ &> /dev/null); } 2>&1 1>/dev/null)
-  runtime=$( { time "$@"; } 2>&1 1>/dev/null)
-  if [[ $runtime == *"Error"* ]]; then
-    exit 1
-  fi
-  runtime=${runtime#+++ $@}
-  runtime=$(python -c "print($runtime)")
-
-  echo "$runtime"
-}

.ci/pytorch/perf_test/test_cpu_speed_mnist.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-set -e
-
-. ./common.sh
-
-test_cpu_speed_mnist () {
-  echo "Testing: MNIST, CPU"
-
-  export OMP_NUM_THREADS=4
-  export MKL_NUM_THREADS=4
-
-  git clone https://github.com/pytorch/examples.git -b perftests
-
-  cd examples/mnist
-
-  conda install -c pytorch torchvision-cpu
-
-  # Download data
-  python main.py --epochs 0
-
-  SAMPLE_ARRAY=()
-  NUM_RUNS=$1
-
-  for (( i=1; i<=NUM_RUNS; i++ )) do
-    runtime=$(get_runtime_of_command python main.py --epochs 1 --no-log)
-    echo "$runtime"
-    SAMPLE_ARRAY+=("${runtime}")
-  done
-
-  cd ../..
-
-  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
-  echo "Runtime stats in seconds:"
-  echo "$stats"
-
-  if [ "$2" == "compare_with_baseline" ]; then
-    python ../compare_with_baseline.py --test-name "${FUNCNAME[0]}" --sample-stats "${stats}"
-  elif [ "$2" == "compare_and_update" ]; then
-    python ../compare_with_baseline.py --test-name "${FUNCNAME[0]}" --sample-stats "${stats}" --update
-  fi
-}
-
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-  run_test test_cpu_speed_mnist "$@"
-fi

.ci/pytorch/perf_test/test_cpu_speed_torch.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-. ./common.sh
-
-test_cpu_speed_torch () {
-  echo "Testing: torch.*, CPU"
-
-  export OMP_NUM_THREADS=4
-  export MKL_NUM_THREADS=4
-
-  git clone https://github.com/yf225/perf-tests.git
-
-  if [ "$1" == "compare_with_baseline" ]; then
-    export ARGS=(--compare ../cpu_runtime.json)
-  elif [ "$1" == "compare_and_update" ]; then
-    export ARGS=(--compare ../cpu_runtime.json --update ../new_cpu_runtime.json)
-  elif [ "$1" == "update_only" ]; then
-    export ARGS=(--update ../new_cpu_runtime.json)
-  fi
-
-  if ! python perf-tests/modules/test_cpu_torch.py "${ARGS[@]}"; then
-    echo "To reproduce this regression, run \`cd .ci/pytorch/perf_test/ && bash ${FUNCNAME[0]}.sh\` on your local machine and compare the runtime before/after your code change."
-    exit 1
-  fi
-}
-
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-  run_test test_cpu_speed_torch "$@"
-fi

.ci/pytorch/perf_test/test_cpu_speed_torch_tensor.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-. ./common.sh
-
-test_cpu_speed_torch_tensor () {
-  echo "Testing: torch.Tensor.*, CPU"
-
-  export OMP_NUM_THREADS=4
-  export MKL_NUM_THREADS=4
-
-  git clone https://github.com/yf225/perf-tests.git
-
-  if [ "$1" == "compare_with_baseline" ]; then
-    export ARGS=(--compare ../cpu_runtime.json)
-  elif [ "$1" == "compare_and_update" ]; then
-    export ARGS=(--compare ../cpu_runtime.json --update ../new_cpu_runtime.json)
-  elif [ "$1" == "update_only" ]; then
-    export ARGS=(--update ../new_cpu_runtime.json)
-  fi
-
-  if ! python perf-tests/modules/test_cpu_torch_tensor.py "${ARGS[@]}"; then
-    echo "To reproduce this regression, run \`cd .ci/pytorch/perf_test/ && bash ${FUNCNAME[0]}.sh\` on your local machine and compare the runtime before/after your code change."
-    exit 1
-  fi
-}
-
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-  run_test test_cpu_speed_torch_tensor "$@"
-fi

.ci/pytorch/perf_test/test_gpu_speed_cudnn_lstm.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-set -e
-
-. ./common.sh
-
-test_gpu_speed_cudnn_lstm () {
-  echo "Testing: CuDNN LSTM, GPU"
-
-  export OMP_NUM_THREADS=4
-  export MKL_NUM_THREADS=4
-
-  git clone https://github.com/pytorch/benchmark.git
-
-  cd benchmark/
-
-  git checkout 43dfb2c0370e70ef37f249dc09aff9f0ccd2ddb0
-
-  cd scripts/
-
-  SAMPLE_ARRAY=()
-  NUM_RUNS=$1
-
-  for (( i=1; i<=NUM_RUNS; i++ )) do
-    runtime=$(get_runtime_of_command python cudnn_lstm.py --skip-cpu-governor-check)
-    echo "$runtime"
-    SAMPLE_ARRAY+=("${runtime}")
-  done
-
-  cd ../..
-
-  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
-  echo "Runtime stats in seconds:"
-  echo "$stats"
-
-  if [ "$2" == "compare_with_baseline" ]; then
-    python ../compare_with_baseline.py --test-name "${FUNCNAME[0]}" --sample-stats "${stats}"
-  elif [ "$2" == "compare_and_update" ]; then
-    python ../compare_with_baseline.py --test-name "${FUNCNAME[0]}" --sample-stats "${stats}" --update
-  fi
-}
-
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-  run_test test_gpu_speed_cudnn_lstm "$@"
-fi

.ci/pytorch/perf_test/test_gpu_speed_lstm.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-set -e
-
-. ./common.sh
-
-test_gpu_speed_lstm () {
-  echo "Testing: LSTM, GPU"
-
-  export OMP_NUM_THREADS=4
-  export MKL_NUM_THREADS=4
-
-  git clone https://github.com/pytorch/benchmark.git
-
-  cd benchmark/
-
-  git checkout 43dfb2c0370e70ef37f249dc09aff9f0ccd2ddb0
-
-  cd scripts/
-
-  SAMPLE_ARRAY=()
-  NUM_RUNS=$1
-
-  for (( i=1; i<=NUM_RUNS; i++ )) do
-    runtime=$(get_runtime_of_command python lstm.py --skip-cpu-governor-check)
-    echo "$runtime"
-    SAMPLE_ARRAY+=("${runtime}")
-  done
-
-  cd ../..
-
-  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
-  echo "Runtime stats in seconds:"
-  echo "$stats"
-
-  if [ "$2" == "compare_with_baseline" ]; then
-    python ../compare_with_baseline.py --test-name "${FUNCNAME[0]}" --sample-stats "${stats}"
-  elif [ "$2" == "compare_and_update" ]; then
-    python ../compare_with_baseline.py --test-name "${FUNCNAME[0]}" --sample-stats "${stats}" --update
-  fi
-}
-
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-  run_test test_gpu_speed_lstm "$@"
-fi

.ci/pytorch/perf_test/test_gpu_speed_mlstm.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-set -e
-
-. ./common.sh
-
-test_gpu_speed_mlstm () {
-  echo "Testing: MLSTM, GPU"
-
-  export OMP_NUM_THREADS=4
-  export MKL_NUM_THREADS=4
-
-  git clone https://github.com/pytorch/benchmark.git
-
-  cd benchmark/
-
-  git checkout 43dfb2c0370e70ef37f249dc09aff9f0ccd2ddb0
-
-  cd scripts/
-
-  SAMPLE_ARRAY=()
-  NUM_RUNS=$1
-
-  for (( i=1; i<=NUM_RUNS; i++ )) do
-    runtime=$(get_runtime_of_command python mlstm.py --skip-cpu-governor-check)
-    echo "$runtime"
-    SAMPLE_ARRAY+=("${runtime}")
-  done
-
-  cd ../..
-
-  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
-  echo "Runtime stats in seconds:"
-  echo "$stats"
-
-  if [ "$2" == "compare_with_baseline" ]; then
-    python ../compare_with_baseline.py --test-name "${FUNCNAME[0]}" --sample-stats "${stats}"
-  elif [ "$2" == "compare_and_update" ]; then
-    python ../compare_with_baseline.py --test-name "${FUNCNAME[0]}" --sample-stats "${stats}" --update
-  fi
-}
-
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-  run_test test_gpu_speed_mlstm "$@"
-fi

.ci/pytorch/print_sccache_log.py

@@ -1,17 +0,0 @@
-import sys
-
-log_file_path = sys.argv[1]
-
-with open(log_file_path) as f:
-    lines = f.readlines()
-
-for line in lines:
-    # Ignore errors from CPU instruction set, symbol existing testing,
-    # or compilation error formatting
-    ignored_keywords = [
-        'src.c',
-        'CheckSymbolExists.c',
-        'test_compilation_error_formatting',
-    ]
-    if all([keyword not in line for keyword in ignored_keywords]):
-        print(line)

.ci/pytorch/run_glootls_test.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-CREATE_TEST_CERT="$(dirname "${BASH_SOURCE[0]}")/create_test_cert.py"
-TMP_CERT_DIR=$(python "$CREATE_TEST_CERT")
-
-openssl verify -CAfile "${TMP_CERT_DIR}/ca.pem" "${TMP_CERT_DIR}/cert.pem"
-
-export GLOO_DEVICE_TRANSPORT=TCP_TLS
-export GLOO_DEVICE_TRANSPORT_TCP_TLS_PKEY=${TMP_CERT_DIR}/pkey.key
-export GLOO_DEVICE_TRANSPORT_TCP_TLS_CERT=${TMP_CERT_DIR}/cert.pem
-export GLOO_DEVICE_TRANSPORT_TCP_TLS_CA_FILE=${TMP_CERT_DIR}/ca.pem
-
-time python test/run_test.py --include distributed/test_c10d_gloo --verbose -- ProcessGroupGlooTest
-
-unset GLOO_DEVICE_TRANSPORT
-unset GLOO_DEVICE_TRANSPORT_TCP_TLS_PKEY
-unset GLOO_DEVICE_TRANSPORT_TCP_TLS_CERT
-unset GLOO_DEVICE_TRANSPORT_TCP_TLS_CA_FILE

.ci/pytorch/short-perf-test-cpu.sh

@@ -1,71 +0,0 @@
-#!/bin/bash
-
-SCRIPT_PARENT_DIR=$(dirname "${BASH_SOURCE[0]}")
-
-# shellcheck source=.ci/pytorch/common.sh
-source "$SCRIPT_PARENT_DIR/common.sh"
-
-cd .ci/pytorch/perf_test
-
-echo "Running CPU perf test for PyTorch..."
-
-pip install -q awscli
-
-# Set multipart_threshold to be sufficiently high, so that `aws s3 cp` is not a multipart read
-# More info at https://github.com/aws/aws-cli/issues/2321
-aws configure set default.s3.multipart_threshold 5GB
-UPSTREAM_DEFAULT_BRANCH="$(git remote show https://github.com/pytorch/pytorch.git | awk '/HEAD branch/ {print $NF}')"
-
-if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then
-    # Get current default branch commit hash
-    DEFAULT_BRANCH_COMMIT_ID=$(git log --format="%H" -n 1)
-    export DEFAULT_BRANCH_COMMIT_ID
-fi
-
-# Find the default branch commit to test against
-git remote add upstream https://github.com/pytorch/pytorch.git
-git fetch upstream
-IFS=$'\n'
-while IFS='' read -r commit_id; do
-    if aws s3 ls s3://ossci-perf-test/pytorch/cpu_runtime/"${commit_id}".json; then
-        LATEST_TESTED_COMMIT=${commit_id}
-        break
-    fi
-done < <(git rev-list upstream/"$UPSTREAM_DEFAULT_BRANCH")
-aws s3 cp s3://ossci-perf-test/pytorch/cpu_runtime/"${LATEST_TESTED_COMMIT}".json cpu_runtime.json
-
-if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then
-    # Prepare new baseline file
-    cp cpu_runtime.json new_cpu_runtime.json
-    python update_commit_hash.py new_cpu_runtime.json "${DEFAULT_BRANCH_COMMIT_ID}"
-fi
-
-# Include tests
-# shellcheck source=./perf_test/test_cpu_speed_mini_sequence_labeler.sh
-. ./test_cpu_speed_mini_sequence_labeler.sh
-# shellcheck source=./perf_test/test_cpu_speed_mnist.sh
-. ./test_cpu_speed_mnist.sh
-# shellcheck source=./perf_test/test_cpu_speed_torch.sh
-. ./test_cpu_speed_torch.sh
-# shellcheck source=./perf_test/test_cpu_speed_torch_tensor.sh
-. ./test_cpu_speed_torch_tensor.sh
-
-# Run tests
-export TEST_MODE="compare_with_baseline"
-if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then
-    export TEST_MODE="compare_and_update"
-fi
-
-# Operator tests
-run_test test_cpu_speed_torch ${TEST_MODE}
-run_test test_cpu_speed_torch_tensor ${TEST_MODE}
-
-# Sample model tests
-run_test test_cpu_speed_mini_sequence_labeler 20 ${TEST_MODE}
-run_test test_cpu_speed_mnist 20 ${TEST_MODE}
-
-if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then
-    # This could cause race condition if we are testing the same default branch commit twice,
-    # but the chance of them executing this line at the same time is low.
-    aws s3 cp new_cpu_runtime.json s3://ossci-perf-test/pytorch/cpu_runtime/"${DEFAULT_BRANCH_COMMIT_ID}".json --acl public-read
-fi

.ci/pytorch/short-perf-test-gpu.sh

@@ -1,76 +0,0 @@
-#!/bin/bash
-
-# shellcheck source=./common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-
-pushd .ci/pytorch/perf_test
-
-echo "Running GPU perf test for PyTorch..."
-
-# Trying to uninstall PyYAML can cause problem. Workaround according to:
-# https://github.com/pypa/pip/issues/5247#issuecomment-415571153
-pip install -q awscli --ignore-installed PyYAML
-
-# Set multipart_threshold to be sufficiently high, so that `aws s3 cp` is not a multipart read
-# More info at https://github.com/aws/aws-cli/issues/2321
-aws configure set default.s3.multipart_threshold 5GB
-UPSTREAM_DEFAULT_BRANCH="$(git remote show https://github.com/pytorch/pytorch.git | awk '/HEAD branch/ {print $NF}')"
-
-if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then
-    # Get current default branch commit hash
-    DEFAULT_BRANCH_COMMIT_ID=$(git log --format="%H" -n 1)
-    export DEFAULT_BRANCH_COMMIT_ID
-fi
-
-# Find the default branch commit to test against
-git remote add upstream https://github.com/pytorch/pytorch.git
-git fetch upstream
-IFS=$'\n'
-while IFS='' read -r commit_id; do
-    if aws s3 ls s3://ossci-perf-test/pytorch/gpu_runtime/"${commit_id}".json; then
-        LATEST_TESTED_COMMIT=${commit_id}
-        break
-    fi
-done < <(git rev-list upstream/"$UPSTREAM_DEFAULT_BRANCH")
-aws s3 cp s3://ossci-perf-test/pytorch/gpu_runtime/"${LATEST_TESTED_COMMIT}".json gpu_runtime.json
-
-if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then
-    # Prepare new baseline file
-    cp gpu_runtime.json new_gpu_runtime.json
-    python update_commit_hash.py new_gpu_runtime.json "${DEFAULT_BRANCH_COMMIT_ID}"
-fi
-
-# Include tests
-# shellcheck source=./perf_test/test_gpu_speed_mnist.sh
-. ./test_gpu_speed_mnist.sh
-# shellcheck source=./perf_test/test_gpu_speed_word_language_model.sh
-. ./test_gpu_speed_word_language_model.sh
-# shellcheck source=./perf_test/test_gpu_speed_cudnn_lstm.sh
-. ./test_gpu_speed_cudnn_lstm.sh
-# shellcheck source=./perf_test/test_gpu_speed_lstm.sh
-. ./test_gpu_speed_lstm.sh
-# shellcheck source=./perf_test/test_gpu_speed_mlstm.sh
-. ./test_gpu_speed_mlstm.sh
-
-# Run tests
-if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then
-    run_test test_gpu_speed_mnist 20 compare_and_update
-    run_test test_gpu_speed_word_language_model 20 compare_and_update
-    run_test test_gpu_speed_cudnn_lstm 20 compare_and_update
-    run_test test_gpu_speed_lstm 20 compare_and_update
-    run_test test_gpu_speed_mlstm 20 compare_and_update
-else
-    run_test test_gpu_speed_mnist 20 compare_with_baseline
-    run_test test_gpu_speed_word_language_model 20 compare_with_baseline
-    run_test test_gpu_speed_cudnn_lstm 20 compare_with_baseline
-    run_test test_gpu_speed_lstm 20 compare_with_baseline
-    run_test test_gpu_speed_mlstm 20 compare_with_baseline
-fi
-
-if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then
-    # This could cause race condition if we are testing the same default branch commit twice,
-    # but the chance of them executing this line at the same time is low.
-    aws s3 cp new_gpu_runtime.json s3://ossci-perf-test/pytorch/gpu_runtime/"${DEFAULT_BRANCH_COMMIT_ID}".json --acl public-read
-fi
-
-popd

.ci/pytorch/win-build.sh

@@ -1,65 +0,0 @@
-#!/bin/bash
-
-# If you want to rebuild, run this with REBUILD=1
-# If you want to build with CUDA, run this with USE_CUDA=1
-# If you want to build without CUDA, run this with USE_CUDA=0
-
-if [ ! -f setup.py ]; then
-  echo "ERROR: Please run this build script from PyTorch root directory."
-  exit 1
-fi
-
-SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-# shellcheck source=./common.sh
-source "$SCRIPT_PARENT_DIR/common.sh"
-# shellcheck source=./common-build.sh
-source "$SCRIPT_PARENT_DIR/common-build.sh"
-
-IMAGE_COMMIT_ID=$(git rev-parse HEAD)
-export IMAGE_COMMIT_ID
-export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
-if [[ ${JOB_NAME} == *"develop"* ]]; then
-  export IMAGE_COMMIT_TAG=develop-${IMAGE_COMMIT_TAG}
-fi
-
-export TMP_DIR="${PWD}/build/win_tmp"
-TMP_DIR_WIN=$(cygpath -w "${TMP_DIR}")
-export TMP_DIR_WIN
-export PYTORCH_FINAL_PACKAGE_DIR=${PYTORCH_FINAL_PACKAGE_DIR:-/c/w/build-results}
-if [[ -n "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then
-    mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true
-fi
-
-# This directory is used only to hold "pytorch_env_restore.bat", called via "setup_pytorch_env.bat"
-CI_SCRIPTS_DIR=$TMP_DIR/ci_scripts
-mkdir -p "$CI_SCRIPTS_DIR"
-
-if [ -n "$(ls "$CI_SCRIPTS_DIR"/*)" ]; then
-    rm "$CI_SCRIPTS_DIR"/*
-fi
-
-export SCRIPT_HELPERS_DIR=$SCRIPT_PARENT_DIR/win-test-helpers
-
-set +ex
-grep -E -R 'PyLong_(From|As)(Unsigned|)Long\(' --exclude=python_numbers.h --exclude=eval_frame.c torch/
-PYLONG_API_CHECK=$?
-if [[ $PYLONG_API_CHECK == 0 ]]; then
-  echo "Usage of PyLong_{From,As}{Unsigned}Long API may lead to overflow errors on Windows"
-  echo "because \`sizeof(long) == 4\` and \`sizeof(unsigned long) == 4\`."
-  echo "Please include \"torch/csrc/utils/python_numbers.h\" and use the correspoding APIs instead."
-  echo "PyLong_FromLong -> THPUtils_packInt32 / THPUtils_packInt64"
-  echo "PyLong_AsLong -> THPUtils_unpackInt (32-bit) / THPUtils_unpackLong (64-bit)"
-  echo "PyLong_FromUnsignedLong -> THPUtils_packUInt32 / THPUtils_packUInt64"
-  echo "PyLong_AsUnsignedLong -> THPUtils_unpackUInt32 / THPUtils_unpackUInt64"
-  exit 1
-fi
-set -ex
-
-"$SCRIPT_HELPERS_DIR"/build_pytorch.bat
-
-assert_git_not_dirty
-
-if [ ! -f "${TMP_DIR}"/"${IMAGE_COMMIT_TAG}".7z ] && [ ! "${BUILD_ENVIRONMENT}" == "" ]; then
-    exit 1
-fi
-echo "BUILD PASSED"

.ci/pytorch/win-test-helpers/build_pytorch.bat

@@ -1,160 +0,0 @@
-if "%DEBUG%" == "1" (
-  set BUILD_TYPE=debug
-) ELSE (
-  set BUILD_TYPE=release
-)
-
-set PATH=C:\Program Files\CMake\bin;C:\Program Files\7-Zip;C:\ProgramData\chocolatey\bin;C:\Program Files\Git\cmd;C:\Program Files\Amazon\AWSCLI;C:\Program Files\Amazon\AWSCLI\bin;%PATH%
-
-:: This inflates our log size slightly, but it is REALLY useful to be
-:: able to see what our cl.exe commands are (since you can actually
-:: just copy-paste them into a local Windows setup to just rebuild a
-:: single file.)
-:: log sizes are too long, but leaving this here incase someone wants to use it locally
-:: set CMAKE_VERBOSE_MAKEFILE=1
-
-
-set INSTALLER_DIR=%SCRIPT_HELPERS_DIR%\installation-helpers
-
-
-call %INSTALLER_DIR%\install_mkl.bat
-if errorlevel 1 exit /b
-if not errorlevel 0 exit /b
-
-call %INSTALLER_DIR%\install_magma.bat
-if errorlevel 1 exit /b
-if not errorlevel 0 exit /b
-
-call %INSTALLER_DIR%\install_sccache.bat
-if errorlevel 1 exit /b
-if not errorlevel 0 exit /b
-
-:: Miniconda has been installed as part of the Windows AMI with all the dependencies.
-:: We just need to activate it here
-call %INSTALLER_DIR%\activate_miniconda3.bat
-if errorlevel 1 exit /b
-if not errorlevel 0 exit /b
-
-:: Override VS env here
-pushd .
-if "%VC_VERSION%" == "" (
-    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64
-) else (
-    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64 -vcvars_ver=%VC_VERSION%
-)
-if errorlevel 1 exit /b
-if not errorlevel 0 exit /b
-@echo on
-popd
-
-if not "%USE_CUDA%"=="1" goto cuda_build_end
-
-set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION%
-
-if x%CUDA_VERSION:.=%==x%CUDA_VERSION% (
-    echo CUDA version %CUDA_VERSION% format isn't correct, which doesn't contain '.'
-    exit /b 1
-)
-rem version transformer, for example 10.1 to 10_1.
-if x%CUDA_VERSION:.=%==x%CUDA_VERSION% (
-    echo CUDA version %CUDA_VERSION% format isn't correct, which doesn't contain '.'
-    exit /b 1
-)
-set VERSION_SUFFIX=%CUDA_VERSION:.=_%
-set CUDA_PATH_V%VERSION_SUFFIX%=%CUDA_PATH%
-
-set CUDNN_LIB_DIR=%CUDA_PATH%\lib\x64
-set CUDA_TOOLKIT_ROOT_DIR=%CUDA_PATH%
-set CUDNN_ROOT_DIR=%CUDA_PATH%
-set NVTOOLSEXT_PATH=C:\Program Files\NVIDIA Corporation\NvToolsExt
-set PATH=%CUDA_PATH%\bin;%CUDA_PATH%\libnvvp;%PATH%
-
-set CUDNN_LIB_DIR=%CUDA_PATH%\lib\x64
-set CUDA_TOOLKIT_ROOT_DIR=%CUDA_PATH%
-set CUDNN_ROOT_DIR=%CUDA_PATH%
-set NVTOOLSEXT_PATH=C:\Program Files\NVIDIA Corporation\NvToolsExt
-set PATH=%CUDA_PATH%\bin;%CUDA_PATH%\libnvvp;%PATH%
-
-:cuda_build_end
-
-set DISTUTILS_USE_SDK=1
-set PATH=%TMP_DIR_WIN%\bin;%PATH%
-
-:: The latest Windows CUDA test is running on AWS G5 runner with A10G GPU
-if "%TORCH_CUDA_ARCH_LIST%" == "" set TORCH_CUDA_ARCH_LIST=8.6
-
-:: The default sccache idle timeout is 600, which is too short and leads to intermittent build errors.
-set SCCACHE_IDLE_TIMEOUT=0
-set SCCACHE_IGNORE_SERVER_IO_ERROR=1
-sccache --stop-server
-sccache --start-server
-sccache --zero-stats
-set CC=sccache-cl
-set CXX=sccache-cl
-
-set CMAKE_GENERATOR=Ninja
-
-if "%USE_CUDA%"=="1" (
-  :: randomtemp is used to resolve the intermittent build error related to CUDA.
-  :: code: https://github.com/peterjc123/randomtemp-rust
-  :: issue: https://github.com/pytorch/pytorch/issues/25393
-  ::
-  :: CMake requires a single command as CUDA_NVCC_EXECUTABLE, so we push the wrappers
-  :: randomtemp.exe and sccache.exe into a batch file which CMake invokes.
-  curl -kL https://github.com/peterjc123/randomtemp-rust/releases/download/v0.4/randomtemp.exe --output %TMP_DIR_WIN%\bin\randomtemp.exe
-  if errorlevel 1 exit /b
-  if not errorlevel 0 exit /b
-  echo @"%TMP_DIR_WIN%\bin\randomtemp.exe" "%TMP_DIR_WIN%\bin\sccache.exe" "%CUDA_PATH%\bin\nvcc.exe" %%* > "%TMP_DIR%/bin/nvcc.bat"
-  cat %TMP_DIR%/bin/nvcc.bat
-  set CUDA_NVCC_EXECUTABLE=%TMP_DIR%/bin/nvcc.bat
-  for /F "usebackq delims=" %%n in (`cygpath -m "%CUDA_PATH%\bin\nvcc.exe"`) do set CMAKE_CUDA_COMPILER=%%n
-  set CMAKE_CUDA_COMPILER_LAUNCHER=%TMP_DIR%/bin/randomtemp.exe;%TMP_DIR%\bin\sccache.exe
-)
-
-@echo off
-echo @echo off >> %TMP_DIR_WIN%\ci_scripts\pytorch_env_restore.bat
-for /f "usebackq tokens=*" %%i in (`set`) do echo set "%%i" >> %TMP_DIR_WIN%\ci_scripts\pytorch_env_restore.bat
-@echo on
-
-if "%REBUILD%" == "" (
-  if NOT "%BUILD_ENVIRONMENT%" == "" (
-    :: Create a shortcut to restore pytorch environment
-    echo @echo off >> %TMP_DIR_WIN%/ci_scripts/pytorch_env_restore_helper.bat
-    echo call "%TMP_DIR_WIN%/ci_scripts/pytorch_env_restore.bat" >> %TMP_DIR_WIN%/ci_scripts/pytorch_env_restore_helper.bat
-    echo cd /D "%CD%" >> %TMP_DIR_WIN%/ci_scripts/pytorch_env_restore_helper.bat
-
-    aws s3 cp "s3://ossci-windows/Restore PyTorch Environment.lnk" "C:\Users\circleci\Desktop\Restore PyTorch Environment.lnk"
-    if errorlevel 1 exit /b
-    if not errorlevel 0 exit /b
-  )
-)
-
-python setup.py bdist_wheel
-if errorlevel 1 exit /b
-if not errorlevel 0 exit /b
-sccache --show-stats
-python -c "import os, glob; os.system('python -mpip install --no-index --no-deps ' + glob.glob('dist/*.whl')[0])"
-(
-  if "%BUILD_ENVIRONMENT%"=="" (
-    echo NOTE: To run `import torch`, please make sure to activate the conda environment by running `call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Miniconda3` in Command Prompt before running Git Bash.
-  ) else (
-    if "%USE_CUDA%"=="1" (
-        7z a %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torchgen %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\functorch %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\nvfuser && copy /Y "%TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z" "%PYTORCH_FINAL_PACKAGE_DIR%\"
-    ) else (
-        7z a %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torchgen %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\functorch && copy /Y "%TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z" "%PYTORCH_FINAL_PACKAGE_DIR%\"
-    )
-
-    if errorlevel 1 exit /b
-    if not errorlevel 0 exit /b
-
-    :: export test times so that potential sharded tests that'll branch off this build will use consistent data
-    python tools/stats/export_test_times.py
-    copy /Y ".pytorch-test-times.json" "%PYTORCH_FINAL_PACKAGE_DIR%"
-
-    :: Also save build/.ninja_log as an artifact
-    copy /Y "build\.ninja_log" "%PYTORCH_FINAL_PACKAGE_DIR%\"
-  )
-)
-
-sccache --show-stats --stats-format json | jq .stats > sccache-stats-%BUILD_ENVIRONMENT%-%OUR_GITHUB_JOB_ID%.json
-sccache --stop-server

.ci/pytorch/win-test-helpers/choose_runtime_cuda_version.bat

@@ -1,4 +0,0 @@
-REM The first argument should the CUDA version
-echo %PATH%
-echo %CUDA_PATH%
-set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%1\bin;%PATH%

.ci/pytorch/win-test-helpers/install_test_functorch.bat

@@ -1,19 +0,0 @@
-call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
-:: exit the batch once there's an error
-if not errorlevel 0 (
-  echo "setup pytorch env failed"
-  echo %errorlevel%
-  exit /b
-)
-
-echo "Test functorch"
-pushd test
-python run_test.py --functorch --shard "%SHARD_NUMBER%" "%NUM_TEST_SHARDS%" --verbose
-popd
-if ERRORLEVEL 1 goto fail
-
-:eof
-exit /b 0
-
-:fail
-exit /b 1

.ci/pytorch/win-test-helpers/installation-helpers/activate_miniconda3.bat

@@ -1,26 +0,0 @@
-if "%BUILD_ENVIRONMENT%"=="" (
-  set CONDA_PARENT_DIR=%CD%
-) else (
-  set CONDA_PARENT_DIR=C:\Jenkins
-)
-
-
-:: Be conservative here when rolling out the new AMI with conda. This will try
-:: to install conda as before if it couldn't find the conda installation. This
-:: can be removed eventually after we gain enough confidence in the AMI
-if not exist %CONDA_PARENT_DIR%\Miniconda3 (
-  set INSTALL_FRESH_CONDA=1
-)
-
-if "%INSTALL_FRESH_CONDA%"=="1" (
-  curl --retry 3 --retry-all-errors -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe --output %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe
-  if errorlevel 1 exit /b
-  if not errorlevel 0 exit /b
-
-  %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /AddToPath=0 /D=%CONDA_PARENT_DIR%\Miniconda3
-  if errorlevel 1 exit /b
-  if not errorlevel 0 exit /b
-)
-
-:: Activate conda so that we can use its commands, i.e. conda, python, pip
-call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Miniconda3

.ci/pytorch/win-test-helpers/installation-helpers/install_magma.bat

@@ -1,37 +0,0 @@
-if "%CUDA_VERSION%" == "cpu" (
-  echo skip magma installation for cpu builds
-  exit /b 0
-)
-
-rem remove dot in cuda_version, fox example 11.1 to 111
-
-if not "%USE_CUDA%"=="1" (
-    exit /b 0
-)
-
-if x%CUDA_VERSION:.=%==x%CUDA_VERSION% (
-    echo CUDA version %CUDA_VERSION% format isn't correct, which doesn't contain '.'
-    exit /b 1
-)
-
-set VERSION_SUFFIX=%CUDA_VERSION:.=%
-set CUDA_SUFFIX=cuda%VERSION_SUFFIX%
-
-if "%CUDA_SUFFIX%" == "" (
-  echo unknown CUDA version, please set `CUDA_VERSION` higher than 10.2
-  exit /b 1
-)
-
-if "%REBUILD%"=="" (
-  if "%BUILD_ENVIRONMENT%"=="" (
-    curl --retry 3 --retry-all-errors -k https://s3.amazonaws.com/ossci-windows/magma_2.5.4_%CUDA_SUFFIX%_%BUILD_TYPE%.7z --output %TMP_DIR_WIN%\magma_2.5.4_%CUDA_SUFFIX%_%BUILD_TYPE%.7z
-  ) else (
-    aws s3 cp s3://ossci-windows/magma_2.5.4_%CUDA_SUFFIX%_%BUILD_TYPE%.7z %TMP_DIR_WIN%\magma_2.5.4_%CUDA_SUFFIX%_%BUILD_TYPE%.7z --quiet
-  )
-  if errorlevel 1 exit /b
-  if not errorlevel 0 exit /b
-  7z x -aoa %TMP_DIR_WIN%\magma_2.5.4_%CUDA_SUFFIX%_%BUILD_TYPE%.7z -o%TMP_DIR_WIN%\magma
-  if errorlevel 1 exit /b
-  if not errorlevel 0 exit /b
-)
-set MAGMA_HOME=%TMP_DIR_WIN%\magma

.ci/pytorch/win-test-helpers/installation-helpers/install_mkl.bat

@@ -1,14 +0,0 @@
-if "%REBUILD%"=="" (
-  if "%BUILD_ENVIRONMENT%"=="" (
-    curl --retry 3 --retry-all-errors -k https://s3.amazonaws.com/ossci-windows/mkl_2020.2.254.7z --output %TMP_DIR_WIN%\mkl.7z
-  ) else (
-    aws s3 cp s3://ossci-windows/mkl_2020.2.254.7z %TMP_DIR_WIN%\mkl.7z --quiet
-  )
-  if errorlevel 1 exit /b
-  if not errorlevel 0 exit /b
-  7z x -aoa %TMP_DIR_WIN%\mkl.7z -o%TMP_DIR_WIN%\mkl
-  if errorlevel 1 exit /b
-  if not errorlevel 0 exit /b
-)
-set CMAKE_INCLUDE_PATH=%TMP_DIR_WIN%\mkl\include
-set LIB=%TMP_DIR_WIN%\mkl\lib;%LIB%

.ci/pytorch/win-test-helpers/installation-helpers/install_sccache.bat

@@ -1,18 +0,0 @@
-mkdir %TMP_DIR_WIN%\bin
-
-if "%REBUILD%"=="" (
-  :check_sccache
-  %TMP_DIR_WIN%\bin\sccache.exe --show-stats || (
-    taskkill /im sccache.exe /f /t || ver > nul
-    del %TMP_DIR_WIN%\bin\sccache.exe || ver > nul
-    del %TMP_DIR_WIN%\bin\sccache-cl.exe || ver > nul
-    if "%BUILD_ENVIRONMENT%"=="" (
-      curl --retry 3 --retry-all-errors -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %TMP_DIR_WIN%\bin\sccache.exe
-      curl --retry 3 --retry-all-errors -k https://s3.amazonaws.com/ossci-windows/sccache-cl.exe --output %TMP_DIR_WIN%\bin\sccache-cl.exe
-    ) else (
-      aws s3 cp s3://ossci-windows/sccache.exe %TMP_DIR_WIN%\bin\sccache.exe
-      aws s3 cp s3://ossci-windows/sccache-cl.exe %TMP_DIR_WIN%\bin\sccache-cl.exe
-    )
-    goto :check_sccache
-  )
-)

.ci/pytorch/win-test-helpers/setup_pytorch_env.bat

@@ -1,73 +0,0 @@
-if exist "%TMP_DIR%/ci_scripts/pytorch_env_restore.bat" (
-    call %TMP_DIR%/ci_scripts/pytorch_env_restore.bat
-    exit /b 0
-)
-
-set PATH=C:\Program Files\CMake\bin;C:\Program Files\7-Zip;C:\ProgramData\chocolatey\bin;C:\Program Files\Git\cmd;C:\Program Files\Amazon\AWSCLI;C:\Program Files\Amazon\AWSCLI\bin;%PATH%
-
-:: Install Miniconda3
-set INSTALLER_DIR=%SCRIPT_HELPERS_DIR%\installation-helpers
-
-:: Miniconda has been installed as part of the Windows AMI with all the dependencies.
-:: We just need to activate it here
-call %INSTALLER_DIR%\activate_miniconda3.bat
-if errorlevel 1 exit /b
-if not errorlevel 0 exit /b
-
-pushd .
-if "%VC_VERSION%" == "" (
-    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64
-) else (
-    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64 -vcvars_ver=%VC_VERSION%
-)
-if errorlevel 1 exit /b
-if not errorlevel 0 exit /b
-@echo on
-popd
-
-set DISTUTILS_USE_SDK=1
-
-if not "%USE_CUDA%"=="1" goto cuda_build_end
-
-set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION%
-
-rem version transformer, for example 10.1 to 10_1.
-set VERSION_SUFFIX=%CUDA_VERSION:.=_%
-set CUDA_PATH_V%VERSION_SUFFIX%=%CUDA_PATH%
-
-set CUDNN_LIB_DIR=%CUDA_PATH%\lib\x64
-set CUDA_TOOLKIT_ROOT_DIR=%CUDA_PATH%
-set CUDNN_ROOT_DIR=%CUDA_PATH%
-set NVTOOLSEXT_PATH=C:\Program Files\NVIDIA Corporation\NvToolsExt
-set PATH=%CUDA_PATH%\bin;%CUDA_PATH%\libnvvp;%PATH%
-set NUMBAPRO_CUDALIB=%CUDA_PATH%\bin
-set NUMBAPRO_LIBDEVICE=%CUDA_PATH%\nvvm\libdevice
-set NUMBAPRO_NVVM=%CUDA_PATH%\nvvm\bin\nvvm64_32_0.dll
-
-:cuda_build_end
-
-set PYTHONPATH=%TMP_DIR_WIN%\build;%PYTHONPATH%
-
-if NOT "%BUILD_ENVIRONMENT%"=="" (
-    pushd %TMP_DIR_WIN%\build
-    copy /Y %PYTORCH_FINAL_PACKAGE_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z %TMP_DIR_WIN%\
-    :: 7z: -aos skips if exists because this .bat can be called multiple times
-    7z x %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z -aos
-    popd
-) else (
-    xcopy /s %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch %TMP_DIR_WIN%\build\torch\
-)
-
-@echo off
-echo @echo off >> %TMP_DIR_WIN%/ci_scripts/pytorch_env_restore.bat
-for /f "usebackq tokens=*" %%i in (`set`) do echo set "%%i" >> %TMP_DIR_WIN%/ci_scripts/pytorch_env_restore.bat
-@echo on
-
-if NOT "%BUILD_ENVIRONMENT%" == "" (
-  :: Create a shortcut to restore pytorch environment
-  echo @echo off >> %TMP_DIR_WIN%/ci_scripts/pytorch_env_restore_helper.bat
-  echo call "%TMP_DIR_WIN%/ci_scripts/pytorch_env_restore.bat" >> %TMP_DIR_WIN%/ci_scripts/pytorch_env_restore_helper.bat
-  echo cd /D "%CD%" >> %TMP_DIR_WIN%/ci_scripts/pytorch_env_restore_helper.bat
-
-  aws s3 cp "s3://ossci-windows/Restore PyTorch Environment.lnk" "C:\Users\circleci\Desktop\Restore PyTorch Environment.lnk"
-)

.ci/pytorch/win-test-helpers/test_custom_backend.bat

@@ -1,36 +0,0 @@
-call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
-
-git submodule update --init --recursive third_party/pybind11
-cd test\custom_backend
-
-:: Build the custom backend library.
-mkdir build
-pushd build
-
-echo "Executing CMake for custom_backend test..."
-
-:: Note: Caffe2 does not support MSVC + CUDA + Debug mode (has to be Release mode)
-cmake -DCMAKE_PREFIX_PATH=%TMP_DIR_WIN%\build\torch -DCMAKE_BUILD_TYPE=Release -GNinja ..
-if ERRORLEVEL 1 exit /b 1
-
-echo "Executing Ninja for custom_backend test..."
-
-ninja -v
-if ERRORLEVEL 1 exit /b 1
-
-echo "Ninja succeeded for custom_backend test."
-
-popd
-
-:: Run tests Python-side and export a script module.
-python test_custom_backend.py -v
-if ERRORLEVEL 1 exit /b 1
-
-python backend.py --export-module-to="build/model.pt"
-if ERRORLEVEL 1 exit /b 1
-
-:: Run tests C++-side and load the exported script module.
-cd build
-set PATH=C:\Program Files\NVIDIA Corporation\NvToolsExt\bin\x64;%TMP_DIR_WIN%\build\torch\lib;%PATH%
-test_custom_backend.exe model.pt
-if ERRORLEVEL 1 exit /b 1

.ci/pytorch/win-test-helpers/test_distributed.bat

@@ -1,24 +0,0 @@
-REM The first argument should lead to the python interpreter
-%1\python.exe test/run_test.py --verbose -i distributed/test_c10d_common
-if %errorlevel% neq 0 ( exit /b %errorlevel% )
-
-%1\python.exe test/run_test.py --verbose -i distributed/test_c10d_gloo
-if %errorlevel% neq 0 ( exit /b %errorlevel% )
-
-%1\python.exe test/run_test.py --verbose -i distributed/test_c10d_nccl
-if %errorlevel% neq 0 ( exit /b %errorlevel% )
-
-%1\python test/run_test.py --verbose -i distributed/test_c10d_spawn_gloo
-if %errorlevel% neq 0 ( exit /b %errorlevel% )
-
-%1\python test/run_test.py --verbose -i distributed/test_c10d_spawn_nccl
-if %errorlevel% neq 0 ( exit /b %errorlevel% )
-
-%1\python.exe test/run_test.py --verbose -i distributed/test_data_parallel
-if %errorlevel% neq 0 ( exit /b %errorlevel% )
-
-%1\python.exe test/run_test.py --verbose -i distributed/test_store
-if %errorlevel% neq 0 ( exit /b %errorlevel% )
-
-%1\python.exe test/run_test.py --verbose -i distributed/test_pg_wrapper
-if %errorlevel% neq 0 ( exit /b %errorlevel% )

.ci/pytorch/win-test-helpers/test_python_jit_legacy.bat

@@ -1,12 +0,0 @@
-call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
-
-echo Copying over test times file
-copy /Y "%PYTORCH_FINAL_PACKAGE_DIR_WIN%\.pytorch-test-times.json" "%PROJECT_DIR_WIN%"
-
-pushd test
-
-echo Run jit_profiling tests
-python run_test.py --include test_jit_legacy test_jit_fuser_legacy --verbose
-if ERRORLEVEL 1 exit /b 1
-
-popd

.ci/pytorch/win-test-helpers/test_python_shard.bat

@@ -1,37 +0,0 @@
-call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
-:: exit the batch once there's an error
-if not errorlevel 0 (
-  echo "setup pytorch env failed"
-  echo %errorlevel%
-  exit /b
-)
-
-pushd test
-
-set GFLAGS_EXE="C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\gflags.exe"
-if "%SHARD_NUMBER%" == "1" (
-  if exist %GFLAGS_EXE% (
-    echo Some smoke tests
-    %GFLAGS_EXE% /i python.exe +sls
-    python %SCRIPT_HELPERS_DIR%\run_python_nn_smoketests.py
-    if ERRORLEVEL 1 goto fail
-
-    %GFLAGS_EXE% /i python.exe -sls
-    if ERRORLEVEL 1 goto fail
-  )
-)
-
-echo Copying over test times file
-copy /Y "%PYTORCH_FINAL_PACKAGE_DIR_WIN%\.pytorch-test-times.json" "%PROJECT_DIR_WIN%"
-
-echo Run nn tests
-python run_test.py --exclude-jit-executor --exclude-distributed-tests --shard "%SHARD_NUMBER%" "%NUM_TEST_SHARDS%" --verbose
-if ERRORLEVEL 1 goto fail
-
-popd
-
-:eof
-exit /b 0
-
-:fail
-exit /b 1

.ci/pytorch/win-test.sh

@@ -1,86 +0,0 @@
-#!/bin/bash
-set -ex
-
-SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-# shellcheck source=./common.sh
-source "$SCRIPT_PARENT_DIR/common.sh"
-
-IMAGE_COMMIT_ID=$(git rev-parse HEAD)
-export IMAGE_COMMIT_ID
-export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
-if [[ ${JOB_NAME} == *"develop"* ]]; then
-  export IMAGE_COMMIT_TAG=develop-${IMAGE_COMMIT_TAG}
-fi
-
-export TMP_DIR="${PWD}/build/win_tmp"
-TMP_DIR_WIN=$(cygpath -w "${TMP_DIR}")
-export TMP_DIR_WIN
-export PROJECT_DIR="${PWD}"
-PROJECT_DIR_WIN=$(cygpath -w "${PROJECT_DIR}")
-export PROJECT_DIR_WIN
-export TEST_DIR="${PWD}/test"
-TEST_DIR_WIN=$(cygpath -w "${TEST_DIR}")
-export TEST_DIR_WIN
-export PYTORCH_FINAL_PACKAGE_DIR="${PYTORCH_FINAL_PACKAGE_DIR:-/c/users/circleci/workspace/build-results}"
-PYTORCH_FINAL_PACKAGE_DIR_WIN=$(cygpath -w "${PYTORCH_FINAL_PACKAGE_DIR}")
-export PYTORCH_FINAL_PACKAGE_DIR_WIN
-
-mkdir -p "$TMP_DIR"/build/torch
-
-
-# This directory is used only to hold "pytorch_env_restore.bat", called via "setup_pytorch_env.bat"
-CI_SCRIPTS_DIR=$TMP_DIR/ci_scripts
-mkdir -p "$CI_SCRIPTS_DIR"
-
-if [ -n "$(ls "$CI_SCRIPTS_DIR"/*)" ]; then
-    rm "$CI_SCRIPTS_DIR"/*
-fi
-
-
-export SCRIPT_HELPERS_DIR=$SCRIPT_PARENT_DIR/win-test-helpers
-
-if [[ "$TEST_CONFIG" = "force_on_cpu" ]]; then
-  # run the full test suite for force_on_cpu test
-  export USE_CUDA=0
-fi
-
-if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
-  # Used so that only cuda/rocm specific versions of tests are generated
-  # mainly used so that we're not spending extra cycles testing cpu
-  # devices on expensive gpu machines
-  export PYTORCH_TESTING_DEVICE_ONLY_FOR="cuda"
-fi
-
-run_tests() {
-    # Run nvidia-smi if available
-    for path in '/c/Program Files/NVIDIA Corporation/NVSMI/nvidia-smi.exe' /c/Windows/System32/nvidia-smi.exe; do
-        if [[ -x "$path" ]]; then
-            "$path" || echo "true";
-            break
-        fi
-    done
-
-    if [[ "${TEST_CONFIG}" == *functorch* ]]; then
-        "$SCRIPT_HELPERS_DIR"/install_test_functorch.bat
-    elif [[ $NUM_TEST_SHARDS -eq 1 ]]; then
-        "$SCRIPT_HELPERS_DIR"/test_python_shard.bat
-        "$SCRIPT_HELPERS_DIR"/test_custom_script_ops.bat
-        "$SCRIPT_HELPERS_DIR"/test_custom_backend.bat
-        "$SCRIPT_HELPERS_DIR"/test_libtorch.bat
-    else
-        "$SCRIPT_HELPERS_DIR"/test_python_shard.bat
-        if [[ "${SHARD_NUMBER}" == 1 && $NUM_TEST_SHARDS -gt 1 ]]; then
-            "$SCRIPT_HELPERS_DIR"/test_libtorch.bat
-            if [[ "${USE_CUDA}" == "1" ]]; then
-              "$SCRIPT_HELPERS_DIR"/test_python_jit_legacy.bat
-            fi
-        elif [[ "${SHARD_NUMBER}" == 2 && $NUM_TEST_SHARDS -gt 1 ]]; then
-            "$SCRIPT_HELPERS_DIR"/test_custom_backend.bat
-            "$SCRIPT_HELPERS_DIR"/test_custom_script_ops.bat
-        fi
-    fi
-}
-
-run_tests
-assert_git_not_dirty
-echo "TEST PASSED"

.circleci/README.md

@@ -1,8 +1,3 @@
-Warning
-=======
-
-Contents may be out of date. Our CircleCI workflows are gradually being migrated to Github actions.
-
 Structure of CI
 ===============
 
@@ -21,6 +16,8 @@ setup job:
    not, even if there isn't a Git checkout.
 
 
+
+
 CircleCI configuration generator
 ================================
 
@@ -34,7 +31,7 @@ Usage
 1. Make changes to these scripts.
 2. Run the `regenerate.sh` script in this directory and commit the script changes and the resulting change to `config.yml`.
 
-You'll see a build failure on GitHub if the scripts don't agree with the checked-in version.
+You'll see a build failure on TravisCI if the scripts don't agree with the checked-in version.
 
 
 Motivation
@@ -58,7 +55,8 @@ Future direction
 See comment [here](https://github.com/pytorch/pytorch/pull/17323#pullrequestreview-206945747):
 
 In contrast with a full recursive tree traversal of configuration dimensions,
-> in the future I think we actually want to decrease our matrix somewhat and have only a few mostly-orthogonal builds that taste as many different features as possible on PRs, plus a more complete suite on every PR and maybe an almost full suite nightly/weekly (we don't have this yet). Specifying PR jobs in the future might be easier to read with an explicit list when we come to this.
+> in the future future I think we actually want to decrease our matrix somewhat and have only a few mostly-orthogonal builds that taste as many different features as possible on PRs, plus a more complete suite on every PR and maybe an almost full suite nightly/weekly (we don't have this yet). Specifying PR jobs in the future might be easier to read with an explicit list when we come to this.
+
 ----------------
 ----------------
 
@@ -73,9 +71,9 @@ A **binary configuration** is a collection of
 * release or nightly
     * releases are stable, nightlies are beta and built every night
 * python version
-    * linux: 3.7m (mu is wide unicode or something like that. It usually doesn't matter but you should know that it exists)
-    * macos: 3.7, 3.8
-    * windows: 3.7, 3.8
+    * linux: 3.5m, 3.6m 3.7m (mu is wide unicode or something like that. It usually doesn't matter but you should know that it exists)
+    * macos: 3.6, 3.7, 3.8
+    * windows: 3.6, 3.7, 3.8
 * cpu version
     * cpu, cuda 9.0, cuda 10.0
     * The supported cuda versions occasionally change
@@ -92,7 +90,7 @@ The binaries are built in CircleCI. There are nightly binaries built every night
 
 We have 3 types of binary packages
 
-* pip packages - nightlies are stored on s3 (pip install -f \<a s3 url\>). releases are stored in a pip repo (pip install torch) (ask Soumith about this)
+* pip packages - nightlies are stored on s3 (pip install -f <a s3 url>). releases are stored in a pip repo (pip install torch) (ask Soumith about this)
 * conda packages - nightlies and releases are both stored in a conda repo. Nighty packages have a '_nightly' suffix
 * libtorch packages - these are zips of all the c++ libraries, header files, and sometimes dependencies. These are c++ only
     * shared with dependencies (the only supported option for Windows)
@@ -106,16 +104,16 @@ All binaries are built in CircleCI workflows except Windows. There are checked-i
 
 Some quick vocab:
 
-* A \**workflow** is a CircleCI concept; it is a DAG of '**jobs**'. ctrl-f 'workflows' on https://github.com/pytorch/pytorch/blob/master/.circleci/config.yml to see the workflows.
+* A\**workflow** is a CircleCI concept; it is a DAG of '**jobs**'. ctrl-f 'workflows' on\https://github.com/pytorch/pytorch/blob/master/.circleci/config.yml to see the workflows.
 * **jobs** are a sequence of '**steps**'
-* **steps** are usually just a bash script or a builtin CircleCI command. *All steps run in new environments, environment variables declared in one script DO NOT persist to following steps*
+* **steps** are usually just a bash script or a builtin CircleCI command.* All steps run in new environments, environment variables declared in one script DO NOT persist to following steps*
 * CircleCI has a **workspace**, which is essentially a cache between steps of the *same job* in which you can store artifacts between steps.
 
 ## How are the workflows structured?
 
 The nightly binaries have 3 workflows. We have one job (actually 3 jobs:  build, test, and upload) per binary configuration
 
-1. binary_builds
+1. binarybuilds
     1. every day midnight EST
     2. linux: https://github.com/pytorch/pytorch/blob/master/.circleci/verbatim-sources/linux-binary-build-defaults.yml
     3. macos: https://github.com/pytorch/pytorch/blob/master/.circleci/verbatim-sources/macos-binary-build-defaults.yml
@@ -146,7 +144,7 @@ The nightly binaries have 3 workflows. We have one job (actually 3 jobs:  build,
 
 ## How are the jobs structured?
 
-The jobs are in https://github.com/pytorch/pytorch/tree/master/.circleci/verbatim-sources. Jobs are made of multiple steps. There are some shared steps used by all the binaries/smokes. Steps of these jobs are all delegated to scripts in https://github.com/pytorch/pytorch/tree/master/.circleci/scripts .
+The jobs are in https://github.com/pytorch/pytorch/tree/master/.circleci/verbatim-sources . Jobs are made of multiple steps. There are some shared steps used by all the binaries/smokes. Steps of these jobs are all delegated to scripts in https://github.com/pytorch/pytorch/tree/master/.circleci/scripts .
 
 * Linux jobs: https://github.com/pytorch/pytorch/blob/master/.circleci/verbatim-sources/linux-binary-build-defaults.yml
     * binary_linux_build.sh
@@ -180,7 +178,8 @@ CircleCI creates a  final yaml file by inlining every <<* segment, so if we were
 So, CircleCI has several executor types: macos, machine, and docker are the ones we use. The 'machine' executor gives you two cores on some linux vm. The 'docker' executor gives you considerably more cores (nproc was 32 instead of 2 back when I tried in February). Since the dockers are faster, we try to run everything that we can in dockers. Thus
 
 * linux build jobs use the docker executor. Running them on the docker executor was at least 2x faster than running them on the machine executor
-* linux test jobs use the machine executor in order for them to properly interface with GPUs since docker executors cannot execute with attached GPUs
+* linux test jobs use the machine executor and spin up their own docker. Why this nonsense? It's cause we run nvidia-docker for our GPU tests; any code that calls into the CUDA runtime needs to be run on nvidia-docker. To run a nvidia-docker you need to install some nvidia packages on the host machine and then call docker with the '—runtime nvidia' argument. CircleCI doesn't support this, so we have to do it ourself.
+    * This is not just a mere inconvenience. **This blocks all of our linux tests from using more than 2 cores.** But there is nothing that we can do about it, but wait for a fix on circleci's side. Right now, we only run some smoke tests (some simple imports) on the binaries, but this also affects non-binary test jobs.
 * linux upload jobs use the machine executor. The upload jobs are so short that it doesn't really matter what they use
 * linux smoke test jobs use the machine executor for the same reason as the linux test jobs
 
@@ -190,11 +189,23 @@ binary_run_in_docker.sh is a way to share the docker start-up code between the b
 
 We want all the nightly binary jobs to run on the exact same git commit, so we wrote our own checkout logic to ensure that the same commit was always picked. Later circleci changed that to use a single pytorch checkout and persist it through the workspace (they did this because our config file was too big, so they wanted to take a lot of the setup code into scripts, but the scripts needed the code repo to exist to be called, so they added a prereq step called 'setup' to checkout the code and persist the needed scripts to the workspace). The changes to the binary jobs were not properly tested, so they all broke from missing pytorch code no longer existing. We hotfixed the problem by adding the pytorch checkout back to binary_checkout, so now there's two checkouts of pytorch on the binary jobs. This problem still needs to be fixed, but it takes careful tracing of which code is being called where.
 
+# Azure Pipelines structure of the binaries
+
+TODO: fill in stuff
+
+## How are the workflows structured?
+
+TODO: fill in stuff
+
+## How are the jobs structured?
+
+TODO: fill in stuff
+
 # Code structure of the binaries (circleci agnostic)
 
 ## Overview
 
-The code that runs the binaries lives in two places, in the normal [github.com/pytorch/pytorch](http://github.com/pytorch/pytorch), but also in [github.com/pytorch/builder](http://github.com/pytorch/builder), which is a repo that defines how all the binaries are built. The relevant code is
+The code that runs the binaries lives in two places, in the normal [github.com/pytorch/pytorch](http://github.com/pytorch/pytorch), but also in [github.com/pytorch/builder](http://github.com/pytorch/builder) , which is a repo that defines how all the binaries are built. The relevant code is
 
 
 ```
@@ -205,22 +216,28 @@ pytorch/pytorch
   - config.yml              # GENERATED file that actually controls all circleci behavior
   - verbatim-sources        # Used to generate job/workflow sections in ^
   - scripts/                # Code needed to prepare circleci environments for binary build scripts
+
 - setup.py                  # Builds pytorch. This is wrapped in pytorch/builder
 - cmake files               # used in normal building of pytorch
+
 # All code needed to prepare a binary build, given an environment
 # with all the right variables/packages/paths.
 pytorch/builder
+
 # Given an installed binary and a proper python env, runs some checks
 # to make sure the binary was built the proper way. Checks things like
 # the library dependencies, symbols present, etc.
 - check_binary.sh
+
 # Given an installed binary, runs python tests to make sure everything
 # is in order. These should be de-duped. Right now they both run smoke
 # tests, but are called from different places. Usually just call some
 # import statements, but also has overlap with check_binary.sh above
 - run_tests.sh
 - smoke_test.sh
+
 # Folders that govern how packages are built. See paragraphs below
+
 - conda/
   - build_pytorch.sh          # Entrypoint. Delegates to proper conda build folder
   - switch_cuda_version.sh    # Switches activate CUDA installation in Docker
@@ -244,7 +261,7 @@ Linux, MacOS and Windows use the same code flow for the conda builds.
 Conda packages are built with conda-build, see https://conda.io/projects/conda-build/en/latest/resources/commands/conda-build.html
 
 Basically, you pass `conda build` a build folder (pytorch-nightly/ above) that contains a build script and a meta.yaml. The meta.yaml specifies in what python environment to build the package in, and what dependencies the resulting package should have, and the build script gets called in the env to build the thing.
-tl;dr on conda-build is
+tldr; on conda-build is
 
 1. Creates a brand new conda environment, based off of deps in the meta.yaml
     1. Note that environment variables do not get passed into this build env unless they are specified in the meta.yaml
@@ -254,7 +271,7 @@ tl;dr on conda-build is
 4. Runs some simple import tests (if specified in the meta.yaml)
 5. Saves the finished package as a tarball
 
-The build.sh we use is essentially a wrapper around `python setup.py build`, but it also manually copies in some of our dependent libraries into the resulting tarball and messes with some rpaths.
+The build.sh we use is essentially a wrapper around ```python setup.py build``` , but it also manually copies in some of our dependent libraries into the resulting tarball and messes with some rpaths.
 
 The entrypoint file `builder/conda/build_conda.sh` is complicated because
 
@@ -327,6 +344,7 @@ All linux builds occur in docker images. The docker images are
     * Has ALL CUDA versions installed. The script pytorch/builder/conda/switch_cuda_version.sh sets /usr/local/cuda to a symlink to e.g. /usr/local/cuda-10.0 to enable different CUDA builds
     * Also used for cpu builds
 * pytorch/manylinux-cuda90
+* pytorch/manylinux-cuda92
 * pytorch/manylinux-cuda100
     * Also used for cpu builds
 
@@ -338,39 +356,47 @@ The Dockerfiles are available in pytorch/builder, but there is no circleci job o
 
 # How to manually rebuild the binaries
 
-tl;dr make a PR that looks like https://github.com/pytorch/pytorch/pull/21159
+tldr; make a PR that looks like https://github.com/pytorch/pytorch/pull/21159
 
 Sometimes we want to push a change to master and then rebuild all of today's binaries after that change. As of May 30, 2019 there isn't a way to manually run a workflow in the UI. You can manually re-run a workflow, but it will use the exact same git commits as the first run and will not include any changes. So we have to make a PR and then force circleci to run the binary workflow instead of the normal tests. The above PR is an example of how to do this; essentially you copy-paste the binarybuilds workflow steps into the default workflow steps. If you need to point the builder repo to a different commit then you'd need to change https://github.com/pytorch/pytorch/blob/master/.circleci/scripts/binary_checkout.sh#L42-L45 to checkout what you want.
 
 ## How to test changes to the binaries via .circleci
 
-Writing PRs that test the binaries is annoying, since the default circleci jobs that run on PRs are not the jobs that you want to run. Likely, changes to the binaries will touch something under .circleci/ and require that .circleci/config.yml be regenerated (.circleci/config.yml controls all .circleci behavior, and is generated using `.circleci/regenerate.sh` in python 3.7). But you also need to manually hardcode the binary jobs that you want to test into the .circleci/config.yml workflow, so you should actually make at least two commits, one for your changes and one to temporarily hardcode jobs. See https://github.com/pytorch/pytorch/pull/22928 as an example of how to do this.
+Writing PRs that test the binaries is annoying, since the default circleci jobs that run on PRs are not the jobs that you want to run. Likely, changes to the binaries will touch something under .circleci/ and require that .circleci/config.yml be regenerated (.circleci/config.yml controls all .circleci behavior, and is generated using ```.circleci/regenerate.sh``` in python 3.7). But you also need to manually hardcode the binary jobs that you want to test into the .circleci/config.yml workflow, so you should actually make at least two commits, one for your changes and one to temporarily hardcode jobs. See https://github.com/pytorch/pytorch/pull/22928 as an example of how to do this.
 
-```sh
+```
 # Make your changes
 touch .circleci/verbatim-sources/nightly-binary-build-defaults.yml
+
 # Regenerate the yaml, has to be in python 3.7
 .circleci/regenerate.sh
+
 # Make a commit
 git add .circleci *
 git commit -m "My real changes"
 git push origin my_branch
+
 # Now hardcode the jobs that you want in the .circleci/config.yml workflows section
 # Also eliminate ensure-consistency and should_run_job checks
 # e.g. https://github.com/pytorch/pytorch/commit/2b3344bfed8772fe86e5210cc4ee915dee42b32d
+
 # Make a commit you won't keep
 git add .circleci
 git commit -m "[DO NOT LAND] testing binaries for above changes"
 git push origin my_branch
+
 # Now you need to make some changes to the first commit.
 git rebase -i HEAD~2 # mark the first commit as 'edit'
+
 # Make the changes
 touch .circleci/verbatim-sources/nightly-binary-build-defaults.yml
 .circleci/regenerate.sh
+
 # Ammend the commit and recontinue
 git add .circleci
 git commit --amend
 git rebase --continue
+
 # Update the PR, need to force since the commits are different now
 git push origin my_branch --force
 ```
@@ -383,7 +409,7 @@ The advantage of this flow is that you can make new changes to the base commit a
 
 You can build Linux binaries locally easily using docker.
 
-```sh
+```
 # Run the docker
 # Use the correct docker image, pytorch/conda-cuda used here as an example
 #
@@ -393,18 +419,22 @@ You can build Linux binaries locally easily using docker.
 #    in the docker container then you will see path/to/foo/baz on your local
 #    machine. You could also clone the pytorch and builder repos in the docker.
 #
+# If you're building a CUDA binary then use `nvidia-docker run` instead, see below.
+#
 # If you know how, add ccache as a volume too and speed up everything
 docker run \
     -v your/pytorch/repo:/pytorch \
     -v your/builder/repo:/builder \
     -v where/you/want/packages/to/appear:/final_pkgs \
     -it pytorch/conda-cuda /bin/bash
+
 # Export whatever variables are important to you. All variables that you'd
 # possibly need are in .circleci/scripts/binary_populate_env.sh
 # You should probably always export at least these 3 variables
 export PACKAGE_TYPE=conda
-export DESIRED_PYTHON=3.7
+export DESIRED_PYTHON=3.6
 export DESIRED_CUDA=cpu
+
 # Call the entrypoint
 # `|& tee foo.log` just copies all stdout and stderr output to foo.log
 # The builds generate lots of output so you probably need this when
@@ -414,7 +444,9 @@ export DESIRED_CUDA=cpu
 
 **Building CUDA binaries on docker**
 
-You can build CUDA binaries on CPU only machines, but you can only run CUDA binaries on CUDA machines. This means that you can build a CUDA binary on a docker on your laptop if you so choose (though it’s gonna take a long time).
+To build a CUDA binary you need to use `nvidia-docker run` instead of just `docker run` (or you can manually pass `--runtime=nvidia`). This adds some needed libraries and things to build CUDA stuff.
+
+You can build CUDA binaries on CPU only machines, but you can only run CUDA binaries on CUDA machines. This means that you can build a CUDA binary on a docker on your laptop if you so choose (though it’s gonna take a loong time).
 
 For Facebook employees, ask about beefy machines that have docker support and use those instead of your laptop; it will be 5x as fast.
 
@@ -424,10 +456,11 @@ There’s no easy way to generate reproducible hermetic MacOS environments. If y
 
 But if you want to try, then I’d recommend
 
-```sh
+```
 # Create a new terminal
 # Clear your LD_LIBRARY_PATH and trim as much out of your PATH as you
 # know how to do
+
 # Install a new miniconda
 # First remove any other python or conda installation from your PATH
 # Always install miniconda 3, even if building for Python <3
@@ -438,17 +471,20 @@ chmod +x "$conda_sh"
 "$conda_sh" -b -p "$MINICONDA_ROOT"
 rm -f "$conda_sh"
 export PATH="~/my_new_conda/bin:$PATH"
+
 # Create a clean python env
 # All MacOS builds use conda to manage the python env and dependencies
 # that are built with, even the pip packages
 conda create -yn binary python=2.7
 conda activate binary
+
 # Export whatever variables are important to you. All variables that you'd
 # possibly need are in .circleci/scripts/binary_populate_env.sh
 # You should probably always export at least these 3 variables
 export PACKAGE_TYPE=conda
-export DESIRED_PYTHON=3.7
+export DESIRED_PYTHON=3.6
 export DESIRED_CUDA=cpu
+
 # Call the entrypoint you want
 path/to/builder/wheel/build_wheel.sh
 ```

.circleci/cimodel/data/binary_build_data.py

@@ -25,12 +25,38 @@ DEPS_INCLUSION_DIMENSIONS = [
 ]
 
 
-def get_processor_arch_name(gpu_version):
-    return "cpu" if not gpu_version else (
-        "cu" + gpu_version.strip("cuda") if gpu_version.startswith("cuda") else gpu_version
-    )
+def get_processor_arch_name(cuda_version):
+    return "cpu" if not cuda_version else "cu" + cuda_version
+
+
+LINUX_PACKAGE_VARIANTS = OrderedDict(
+    manywheel=[
+        "3.6m",
+        "3.7m",
+        "3.8m",
+    ],
+    conda=dimensions.STANDARD_PYTHON_VERSIONS,
+    libtorch=[
+        "3.7m",
+    ],
+)
 
 CONFIG_TREE_DATA = OrderedDict(
+    linux=(dimensions.CUDA_VERSIONS, LINUX_PACKAGE_VARIANTS),
+    macos=([None], OrderedDict(
+        wheel=dimensions.STANDARD_PYTHON_VERSIONS,
+        conda=dimensions.STANDARD_PYTHON_VERSIONS,
+        libtorch=[
+            "3.7",
+        ],
+    )),
+    windows=(dimensions.CUDA_VERSIONS, OrderedDict(
+        wheel=dimensions.STANDARD_PYTHON_VERSIONS,
+        conda=dimensions.STANDARD_PYTHON_VERSIONS,
+        libtorch=[
+            "3.7",
+        ],
+    )),
 )
 
 # GCC config variants:
@@ -57,7 +83,7 @@ WINDOWS_LIBTORCH_CONFIG_VARIANTS = [
 
 class TopLevelNode(ConfigNode):
     def __init__(self, node_name, config_tree_data, smoke):
-        super().__init__(None, node_name)
+        super(TopLevelNode, self).__init__(None, node_name)
 
         self.config_tree_data = config_tree_data
         self.props["smoke"] = smoke
@@ -67,12 +93,12 @@ class TopLevelNode(ConfigNode):
 
 
 class OSConfigNode(ConfigNode):
-    def __init__(self, parent, os_name, gpu_versions, py_tree):
-        super().__init__(parent, os_name)
+    def __init__(self, parent, os_name, cuda_versions, py_tree):
+        super(OSConfigNode, self).__init__(parent, os_name)
 
         self.py_tree = py_tree
         self.props["os_name"] = os_name
-        self.props["gpu_versions"] = gpu_versions
+        self.props["cuda_versions"] = cuda_versions
 
     def get_children(self):
         return [PackageFormatConfigNode(self, k, v) for k, v in self.py_tree.items()]
@@ -80,61 +106,52 @@ class OSConfigNode(ConfigNode):
 
 class PackageFormatConfigNode(ConfigNode):
     def __init__(self, parent, package_format, python_versions):
-        super().__init__(parent, package_format)
+        super(PackageFormatConfigNode, self).__init__(parent, package_format)
 
         self.props["python_versions"] = python_versions
         self.props["package_format"] = package_format
 
-
     def get_children(self):
         if self.find_prop("os_name") == "linux":
             return [LinuxGccConfigNode(self, v) for v in LINUX_GCC_CONFIG_VARIANTS[self.find_prop("package_format")]]
         elif self.find_prop("os_name") == "windows" and self.find_prop("package_format") == "libtorch":
             return [WindowsLibtorchConfigNode(self, v) for v in WINDOWS_LIBTORCH_CONFIG_VARIANTS]
         else:
-            return [ArchConfigNode(self, v) for v in self.find_prop("gpu_versions")]
+            return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
 
 
 class LinuxGccConfigNode(ConfigNode):
     def __init__(self, parent, gcc_config_variant):
-        super().__init__(parent, "GCC_CONFIG_VARIANT=" + str(gcc_config_variant))
+        super(LinuxGccConfigNode, self).__init__(parent, "GCC_CONFIG_VARIANT=" + str(gcc_config_variant))
 
         self.props["gcc_config_variant"] = gcc_config_variant
 
     def get_children(self):
-        gpu_versions = self.find_prop("gpu_versions")
+        cuda_versions = self.find_prop("cuda_versions")
 
         # XXX devtoolset7 on CUDA 9.0 is temporarily disabled
         # see https://github.com/pytorch/pytorch/issues/20066
         if self.find_prop("gcc_config_variant") == 'devtoolset7':
-            gpu_versions = filter(lambda x: x != "cuda_90", gpu_versions)
+            cuda_versions = filter(lambda x: x != "90", cuda_versions)
 
-        # XXX disabling conda rocm build since docker images are not there
-        if self.find_prop("package_format") == 'conda':
-            gpu_versions = filter(lambda x: x not in dimensions.ROCM_VERSION_LABELS, gpu_versions)
-
-        # XXX libtorch rocm build  is temporarily disabled
-        if self.find_prop("package_format") == 'libtorch':
-            gpu_versions = filter(lambda x: x not in dimensions.ROCM_VERSION_LABELS, gpu_versions)
-
-        return [ArchConfigNode(self, v) for v in gpu_versions]
+        return [ArchConfigNode(self, v) for v in cuda_versions]
 
 
 class WindowsLibtorchConfigNode(ConfigNode):
     def __init__(self, parent, libtorch_config_variant):
-        super().__init__(parent, "LIBTORCH_CONFIG_VARIANT=" + str(libtorch_config_variant))
+        super(WindowsLibtorchConfigNode, self).__init__(parent, "LIBTORCH_CONFIG_VARIANT=" + str(libtorch_config_variant))
 
         self.props["libtorch_config_variant"] = libtorch_config_variant
 
     def get_children(self):
-        return [ArchConfigNode(self, v) for v in self.find_prop("gpu_versions")]
+        return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
 
 
 class ArchConfigNode(ConfigNode):
-    def __init__(self, parent, gpu):
-        super().__init__(parent, get_processor_arch_name(gpu))
+    def __init__(self, parent, cu):
+        super(ArchConfigNode, self).__init__(parent, get_processor_arch_name(cu))
 
-        self.props["gpu"] = gpu
+        self.props["cu"] = cu
 
     def get_children(self):
         return [PyVersionConfigNode(self, v) for v in self.find_prop("python_versions")]
@@ -142,7 +159,7 @@ class ArchConfigNode(ConfigNode):
 
 class PyVersionConfigNode(ConfigNode):
     def __init__(self, parent, pyver):
-        super().__init__(parent, pyver)
+        super(PyVersionConfigNode, self).__init__(parent, pyver)
 
         self.props["pyver"] = pyver
 
@@ -158,7 +175,7 @@ class PyVersionConfigNode(ConfigNode):
 
 class LinkingVariantConfigNode(ConfigNode):
     def __init__(self, parent, linking_variant):
-        super().__init__(parent, linking_variant)
+        super(LinkingVariantConfigNode, self).__init__(parent, linking_variant)
 
     def get_children(self):
         return [DependencyInclusionConfigNode(self, v) for v in DEPS_INCLUSION_DIMENSIONS]
@@ -166,6 +183,6 @@ class LinkingVariantConfigNode(ConfigNode):
 
 class DependencyInclusionConfigNode(ConfigNode):
     def __init__(self, parent, deps_variant):
-        super().__init__(parent, deps_variant)
+        super(DependencyInclusionConfigNode, self).__init__(parent, deps_variant)
 
         self.props["libtorch_variant"] = "-".join([self.parent.get_label(), self.get_label()])

.circleci/cimodel/data/binary_build_definitions.py

@@ -6,10 +6,10 @@ import cimodel.lib.conf_tree as conf_tree
 import cimodel.lib.miniutils as miniutils
 
 class Conf(object):
-    def __init__(self, os, gpu_version, pydistro, parms, smoke, libtorch_variant, gcc_config_variant, libtorch_config_variant):
+    def __init__(self, os, cuda_version, pydistro, parms, smoke, libtorch_variant, gcc_config_variant, libtorch_config_variant):
 
         self.os = os
-        self.gpu_version = gpu_version
+        self.cuda_version = cuda_version
         self.pydistro = pydistro
         self.parms = parms
         self.smoke = smoke
@@ -18,7 +18,7 @@ class Conf(object):
         self.libtorch_config_variant = libtorch_config_variant
 
     def gen_build_env_parms(self):
-        elems = [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.gpu_version)]
+        elems = [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)]
         if self.gcc_config_variant is not None:
             elems.append(str(self.gcc_config_variant))
         if self.libtorch_config_variant is not None:
@@ -27,19 +27,7 @@ class Conf(object):
 
     def gen_docker_image(self):
         if self.gcc_config_variant == 'gcc5.4_cxx11-abi':
-            if self.gpu_version is None:
-                return miniutils.quote("pytorch/libtorch-cxx11-builder:cpu")
-            else:
-                return miniutils.quote(
-                    f"pytorch/libtorch-cxx11-builder:{self.gpu_version}"
-                )
-        if self.pydistro == "conda":
-            if self.gpu_version is None:
-                return miniutils.quote("pytorch/conda-builder:cpu")
-            else:
-                return miniutils.quote(
-                    f"pytorch/conda-builder:{self.gpu_version}"
-                )
+            return miniutils.quote("pytorch/pytorch-binary-docker-image-ubuntu16.04:latest")
 
         docker_word_substitution = {
             "manywheel": "manylinux",
@@ -49,12 +37,9 @@ class Conf(object):
         docker_distro_prefix = miniutils.override(self.pydistro, docker_word_substitution)
 
         # The cpu nightlies are built on the pytorch/manylinux-cuda102 docker image
-        # TODO cuda images should consolidate into tag-base images similar to rocm
-        alt_docker_suffix = "cuda102" if not self.gpu_version else (
-            "rocm:" + self.gpu_version.strip("rocm") if self.gpu_version.startswith("rocm") else self.gpu_version)
-        docker_distro_suffix = alt_docker_suffix if self.pydistro != "conda" else (
-            "cuda" if alt_docker_suffix.startswith("cuda") else "rocm")
-        return miniutils.quote("pytorch/" + docker_distro_prefix + "-" + docker_distro_suffix)
+        alt_docker_suffix = self.cuda_version or "102"
+        docker_distro_suffix = "" if self.pydistro == "conda" else alt_docker_suffix
+        return miniutils.quote("pytorch/" + docker_distro_prefix + "-cuda" + docker_distro_suffix)
 
     def get_name_prefix(self):
         return "smoke" if self.smoke else "binary"
@@ -84,10 +69,14 @@ class Conf(object):
                 "update_s3_htmls",
             ]
             job_def["filters"] = branch_filters.gen_filter_dict(
-                branches_list=["postnightly"],
+                branches_list=["nightly"],
+                tags_list=[branch_filters.RC_PATTERN],
             )
         else:
-            filter_branch = r"/.*/"
+            if phase in ["upload"]:
+                filter_branch = "nightly"
+            else:
+                filter_branch = r"/.*/"
             job_def["filters"] = branch_filters.gen_filter_dict(
                 branches_list=[filter_branch],
                 tags_list=[branch_filters.RC_PATTERN],
@@ -100,61 +89,28 @@ class Conf(object):
             if not (self.smoke and self.os == "macos") and self.os != "windows":
                 job_def["docker_image"] = self.gen_docker_image()
 
-            # fix this. only works on cuda not rocm
-            if self.os != "windows" and self.gpu_version:
+            if self.os != "windows" and self.cuda_version:
                 job_def["use_cuda_docker_runtime"] = miniutils.quote("1")
         else:
             if self.os == "linux" and phase != "upload":
                 job_def["docker_image"] = self.gen_docker_image()
 
         if phase == "test":
-            if self.gpu_version:
+            if self.cuda_version:
                 if self.os == "windows":
                     job_def["executor"] = "windows-with-nvidia-gpu"
                 else:
                     job_def["resource_class"] = "gpu.medium"
+        if phase == "upload":
+            job_def["context"] = "org-member"
+            job_def["requires"] = [
+                self.gen_build_name(upload_phase_dependency, nightly)
+            ]
 
         os_name = miniutils.override(self.os, {"macos": "mac"})
         job_name = "_".join([self.get_name_prefix(), os_name, phase])
         return {job_name : job_def}
 
-    def gen_upload_job(self, phase, requires_dependency):
-        """Generate binary_upload job for configuration
-
-        Output looks similar to:
-
-      - binary_upload:
-          name: binary_linux_manywheel_3_7m_cu113_devtoolset7_nightly_upload
-          context: org-member
-          requires: binary_linux_manywheel_3_7m_cu113_devtoolset7_nightly_test
-          filters:
-            branches:
-              only:
-                - nightly
-            tags:
-              only: /v[0-9]+(\\.[0-9]+)*-rc[0-9]+/
-          package_type: manywheel
-          upload_subfolder: cu113
-        """
-        return {
-            "binary_upload": OrderedDict({
-                "name": self.gen_build_name(phase, nightly=True),
-                "context": "org-member",
-                "requires": [self.gen_build_name(
-                    requires_dependency,
-                    nightly=True
-                )],
-                "filters": branch_filters.gen_filter_dict(
-                    branches_list=["nightly"],
-                    tags_list=[branch_filters.RC_PATTERN],
-                ),
-                "package_type": self.pydistro,
-                "upload_subfolder": binary_build_data.get_processor_arch_name(
-                    self.gpu_version,
-                ),
-            })
-        }
-
 def get_root(smoke, name):
 
     return binary_build_data.TopLevelNode(
@@ -173,10 +129,10 @@ def gen_build_env_list(smoke):
     for c in config_list:
         conf = Conf(
             c.find_prop("os_name"),
-            c.find_prop("gpu"),
+            c.find_prop("cu"),
             c.find_prop("package_format"),
             [c.find_prop("pyver")],
-            c.find_prop("smoke") and not (c.find_prop("os_name") == "macos_arm64"),  # don't test arm64
+            c.find_prop("smoke"),
             c.find_prop("libtorch_variant"),
             c.find_prop("gcc_config_variant"),
             c.find_prop("libtorch_config_variant"),
@@ -193,19 +149,32 @@ def get_nightly_uploads():
     mylist = []
     for conf in configs:
         phase_dependency = "test" if predicate_exclude_macos(conf) else "build"
-        mylist.append(conf.gen_upload_job("upload", phase_dependency))
+        mylist.append(conf.gen_workflow_job("upload", phase_dependency, nightly=True))
 
     return mylist
 
 def get_post_upload_jobs():
+    """Generate jobs to update HTML indices and report binary sizes"""
+    configs = gen_build_env_list(False)
+    common_job_def = {
+        "context": "org-member",
+        "filters": branch_filters.gen_filter_dict(
+            branches_list=["nightly"],
+            tags_list=[branch_filters.RC_PATTERN],
+        ),
+        "requires": [],
+    }
+    for conf in configs:
+        upload_job_name = conf.gen_build_name(
+            build_or_test="upload",
+            nightly=True
+        )
+        common_job_def["requires"].append(upload_job_name)
     return [
         {
             "update_s3_htmls": {
                 "name": "update_s3_htmls",
-                "context": "org-member",
-                "filters": branch_filters.gen_filter_dict(
-                    branches_list=["postnightly"],
-                ),
+                **common_job_def,
             },
         },
     ]
@@ -228,9 +197,7 @@ def get_jobs(toplevel_key, smoke):
     configs = gen_build_env_list(smoke)
     phase = "build" if toplevel_key == "binarybuilds" else "test"
     for build_config in configs:
-        # don't test for macos_arm64 as it's cross compiled
-        if phase != "test" or build_config.os != "macos_arm64":
-            jobs_list.append(build_config.gen_workflow_job(phase, nightly=True))
+        jobs_list.append(build_config.gen_workflow_job(phase, nightly=True))
 
     return jobs_list
 

.circleci/cimodel/data/caffe2_build_data.py

@@ -0,0 +1,91 @@
+from cimodel.lib.conf_tree import ConfigNode, XImportant
+from cimodel.lib.conf_tree import Ver
+
+
+CONFIG_TREE_DATA = [
+    (Ver("ubuntu", "16.04"), [
+        ([Ver("clang", "7")], [XImportant("onnx_main_py3.6"),
+                               XImportant("onnx_ort1_py3.6"),
+                               XImportant("onnx_ort2_py3.6")]),
+    ]),
+]
+
+
+class TreeConfigNode(ConfigNode):
+    def __init__(self, parent, node_name, subtree):
+        super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name))
+        self.subtree = subtree
+        self.init2(node_name)
+
+    # noinspection PyMethodMayBeStatic
+    def modify_label(self, label):
+        return str(label)
+
+    def init2(self, node_name):
+        pass
+
+    def get_children(self):
+        return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
+
+    def is_build_only(self):
+        if str(self.find_prop("language_version")) == "onnx_main_py3.6" or \
+                str(self.find_prop("language_version")) == "onnx_ort1_py3.6" or \
+                str(self.find_prop("language_version")) == "onnx_ort2_py3.6":
+            return False
+        return set(str(c) for c in self.find_prop("compiler_version")).intersection({
+            "clang3.8",
+            "clang3.9",
+            "clang7",
+            "android",
+        }) or self.find_prop("distro_version").name == "macos"
+
+    def is_test_only(self):
+        if str(self.find_prop("language_version")) == "onnx_ort1_py3.6" or \
+                str(self.find_prop("language_version")) == "onnx_ort2_py3.6":
+            return True
+        return False
+
+
+class TopLevelNode(TreeConfigNode):
+    def __init__(self, node_name, subtree):
+        super(TopLevelNode, self).__init__(None, node_name, subtree)
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return DistroConfigNode
+
+
+class DistroConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["distro_version"] = node_name
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return CompilerConfigNode
+
+
+class CompilerConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["compiler_version"] = node_name
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return LanguageConfigNode
+
+
+class LanguageConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["language_version"] = node_name
+        self.props["build_only"] = self.is_build_only()
+        self.props["test_only"] = self.is_test_only()
+
+    def child_constructor(self):
+        return ImportantConfigNode
+
+
+class ImportantConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["important"] = True
+
+    def get_children(self):
+        return []

.circleci/cimodel/data/caffe2_build_definitions.py

@@ -0,0 +1,174 @@
+from collections import OrderedDict
+
+import cimodel.data.dimensions as dimensions
+import cimodel.lib.conf_tree as conf_tree
+from cimodel.lib.conf_tree import Ver
+import cimodel.lib.miniutils as miniutils
+from cimodel.data.caffe2_build_data import CONFIG_TREE_DATA, TopLevelNode
+from cimodel.data.simple.util.branch_filters import gen_filter_dict
+
+from dataclasses import dataclass
+
+
+DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
+
+DOCKER_IMAGE_VERSION = "376"
+
+
+@dataclass
+class Conf:
+    language: str
+    distro: Ver
+    # There could be multiple compiler versions configured (e.g. nvcc
+    # for gpu files and host compiler (gcc/clang) for cpu files)
+    compilers: [Ver]
+    build_only: bool
+    test_only: bool
+    is_important: bool
+
+    @property
+    def compiler_names(self):
+        return [c.name for c in self.compilers]
+
+    # TODO: Eventually we can probably just remove the cudnn7 everywhere.
+    def get_cudnn_insertion(self):
+
+        omit = self.language == "onnx_main_py3.6" \
+            or self.language == "onnx_ort1_py3.6" \
+            or self.language == "onnx_ort2_py3.6" \
+            or set(self.compiler_names).intersection({"android", "mkl", "clang"}) \
+            or str(self.distro) in ["ubuntu14.04", "macos10.13"]
+
+        return [] if omit else ["cudnn7"]
+
+    def get_build_name_root_parts(self):
+        return [
+            "caffe2",
+            self.language,
+        ] + self.get_build_name_middle_parts()
+
+    def get_build_name_middle_parts(self):
+        return [str(c) for c in self.compilers] + self.get_cudnn_insertion() + [str(self.distro)]
+
+    def construct_phase_name(self, phase):
+        root_parts = self.get_build_name_root_parts()
+
+        build_name_substitutions = {
+            "onnx_ort1_py3.6": "onnx_main_py3.6",
+            "onnx_ort2_py3.6": "onnx_main_py3.6",
+        }
+        if phase == "build":
+            root_parts = [miniutils.override(r, build_name_substitutions) for r in root_parts]
+        return "_".join(root_parts + [phase]).replace(".", "_")
+
+    def get_platform(self):
+        platform = self.distro.name
+        if self.distro.name != "macos":
+            platform = "linux"
+        return platform
+
+    def gen_docker_image(self):
+
+        lang_substitutions = {
+            "onnx_main_py3.6": "py3.6",
+            "onnx_ort1_py3.6": "py3.6",
+            "onnx_ort2_py3.6": "py3.6",
+            "cmake": "py3",
+        }
+
+        lang = miniutils.override(self.language, lang_substitutions)
+        parts = [lang] + self.get_build_name_middle_parts()
+        return miniutils.quote(DOCKER_IMAGE_PATH_BASE + "-".join(parts) + ":" + str(DOCKER_IMAGE_VERSION))
+
+    def gen_workflow_params(self, phase):
+        parameters = OrderedDict()
+        lang_substitutions = {
+            "onnx_py3": "onnx-py3",
+            "onnx_main_py3.6": "onnx-main-py3.6",
+            "onnx_ort1_py3.6": "onnx-ort1-py3.6",
+            "onnx_ort2_py3.6": "onnx-ort2-py3.6",
+        }
+
+        lang = miniutils.override(self.language, lang_substitutions)
+
+        parts = [
+            "caffe2",
+            lang,
+        ] + self.get_build_name_middle_parts() + [phase]
+
+        build_env_name = "-".join(parts)
+        parameters["build_environment"] = miniutils.quote(build_env_name)
+        if "ios" in self.compiler_names:
+            parameters["build_ios"] = miniutils.quote("1")
+        if phase == "test":
+            # TODO cuda should not be considered a compiler
+            if "cuda" in self.compiler_names:
+                parameters["use_cuda_docker_runtime"] = miniutils.quote("1")
+
+        if self.distro.name != "macos":
+            parameters["docker_image"] = self.gen_docker_image()
+            if self.build_only:
+                parameters["build_only"] = miniutils.quote("1")
+        if phase == "test":
+            resource_class = "large" if "cuda" not in self.compiler_names else "gpu.medium"
+            parameters["resource_class"] = resource_class
+
+        return parameters
+
+    def gen_workflow_job(self, phase):
+        job_def = OrderedDict()
+        job_def["name"] = self.construct_phase_name(phase)
+
+        if phase == "test":
+            job_def["requires"] = [self.construct_phase_name("build")]
+            job_name = "caffe2_" + self.get_platform() + "_test"
+        else:
+            job_name = "caffe2_" + self.get_platform() + "_build"
+
+        if not self.is_important:
+            job_def["filters"] = gen_filter_dict()
+        job_def.update(self.gen_workflow_params(phase))
+        return {job_name : job_def}
+
+
+def get_root():
+    return TopLevelNode("Caffe2 Builds", CONFIG_TREE_DATA)
+
+
+def instantiate_configs():
+
+    config_list = []
+
+    root = get_root()
+    found_configs = conf_tree.dfs(root)
+    for fc in found_configs:
+        c = Conf(
+            language=fc.find_prop("language_version"),
+            distro=fc.find_prop("distro_version"),
+            compilers=fc.find_prop("compiler_version"),
+            build_only=fc.find_prop("build_only"),
+            test_only=fc.find_prop("test_only"),
+            is_important=fc.find_prop("important"),
+        )
+
+        config_list.append(c)
+
+    return config_list
+
+
+def get_workflow_jobs():
+
+    configs = instantiate_configs()
+
+    x = []
+    for conf_options in configs:
+        phases = ["build"]
+        if not conf_options.build_only:
+            phases = dimensions.PHASES
+        if conf_options.test_only:
+            phases = ["test"]
+
+        for phase in phases:
+            x.append(conf_options.gen_workflow_job(phase))
+
+    return x

.circleci/cimodel/data/dimensions.py

@@ -1,24 +1,14 @@
 PHASES = ["build", "test"]
 
 CUDA_VERSIONS = [
+    None,  # cpu build
+    "92",
+    "101",
     "102",
-    "113",
-    "116",
-    "117",
 ]
 
-ROCM_VERSIONS = [
-    "4.3.1",
-    "4.5.2",
-]
-
-ROCM_VERSION_LABELS = ["rocm" + v for v in ROCM_VERSIONS]
-
-GPU_VERSIONS = [None] + ["cuda" + v for v in CUDA_VERSIONS] + ROCM_VERSION_LABELS
-
 STANDARD_PYTHON_VERSIONS = [
+    "3.6",
     "3.7",
-    "3.8",
-    "3.9",
-    "3.10"
+    "3.8"
 ]

.circleci/cimodel/data/pytorch_build_data.py

@@ -1,7 +1,64 @@
-from cimodel.lib.conf_tree import ConfigNode
+from cimodel.lib.conf_tree import ConfigNode, X, XImportant
 
 
 CONFIG_TREE_DATA = [
+    ("xenial", [
+        (None, [
+            X("nightly"),
+        ]),
+        ("gcc", [
+            ("5.4", [  # All this subtree rebases to master and then build
+                XImportant("3.6"),
+                ("3.6", [
+                    ("parallel_tbb", [X(True)]),
+                    ("parallel_native", [X(True)]),
+                ]),
+            ]),
+            # TODO: bring back libtorch test
+            ("7", [X("3.6")]),
+        ]),
+        ("clang", [
+            ("5", [
+                XImportant("3.6"),  # This is actually the ASAN build
+            ]),
+        ]),
+        ("cuda", [
+            ("9.2", [
+                X("3.6"),
+                ("3.6", [
+                    ("cuda_gcc_override", [X("gcc5.4")])
+                ])
+            ]),
+            ("10.1", [X("3.6")]),
+            ("10.2", [
+                XImportant("3.6"),
+                ("3.6", [
+                    ("libtorch", [XImportant(True)])
+                ]),
+            ]),
+            ("11.0", [
+                X("3.8"),
+                ("3.8", [
+                    ("libtorch", [X(True)])
+                ]),
+            ]),
+        ]),
+    ]),
+    ("bionic", [
+        ("clang", [
+            ("9", [
+                XImportant("3.6"),
+            ]),
+            ("9", [
+                ("3.6", [
+                    ("xla", [XImportant(True)]),
+                ]),
+            ]),
+        ]),
+        ("gcc", [
+            ("9", [XImportant("3.8")]),
+        ]),
+    ]),
 ]
 
 
@@ -12,7 +69,7 @@ def get_major_pyver(dotted_version):
 
 class TreeConfigNode(ConfigNode):
     def __init__(self, parent, node_name, subtree):
-        super().__init__(parent, self.modify_label(node_name))
+        super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name))
         self.subtree = subtree
         self.init2(node_name)
 
@@ -28,7 +85,7 @@ class TreeConfigNode(ConfigNode):
 
 class TopLevelNode(TreeConfigNode):
     def __init__(self, node_name, subtree):
-        super().__init__(None, node_name, subtree)
+        super(TopLevelNode, self).__init__(None, node_name, subtree)
 
     # noinspection PyMethodMayBeStatic
     def child_constructor(self):
@@ -53,8 +110,6 @@ class PyVerConfigNode(TreeConfigNode):
     def init2(self, node_name):
         self.props["pyver"] = node_name
         self.props["abbreviated_pyver"] = get_major_pyver(node_name)
-        if node_name == "3.9":
-            self.props["abbreviated_pyver"] = "py3.9"
 
     # noinspection PyMethodMayBeStatic
     def child_constructor(self):
@@ -69,44 +124,17 @@ class ExperimentalFeatureConfigNode(TreeConfigNode):
         experimental_feature = self.find_prop("experimental_feature")
 
         next_nodes = {
-            "asan": AsanConfigNode,
             "xla": XlaConfigNode,
-            "mps": MPSConfigNode,
-            "vulkan": VulkanConfigNode,
             "parallel_tbb": ParallelTBBConfigNode,
-            "crossref": CrossRefConfigNode,
-            "dynamo": DynamoConfigNode,
             "parallel_native": ParallelNativeConfigNode,
-            "onnx": ONNXConfigNode,
             "libtorch": LibTorchConfigNode,
             "important": ImportantConfigNode,
             "build_only": BuildOnlyConfigNode,
-            "shard_test": ShardTestConfigNode,
-            "cuda_gcc_override": CudaGccOverrideConfigNode,
-            "pure_torch": PureTorchConfigNode,
-            "slow_gradcheck": SlowGradcheckConfigNode,
+            "cuda_gcc_override": CudaGccOverrideConfigNode
         }
         return next_nodes[experimental_feature]
 
 
-class SlowGradcheckConfigNode(TreeConfigNode):
-    def init2(self, node_name):
-        self.props["is_slow_gradcheck"] = True
-
-    def child_constructor(self):
-        return ExperimentalFeatureConfigNode
-
-class PureTorchConfigNode(TreeConfigNode):
-    def modify_label(self, label):
-        return "PURE_TORCH=" + str(label)
-
-    def init2(self, node_name):
-        self.props["is_pure_torch"] = node_name
-
-    def child_constructor(self):
-        return ImportantConfigNode
-
-
 class XlaConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "XLA=" + str(label)
@@ -117,49 +145,6 @@ class XlaConfigNode(TreeConfigNode):
     def child_constructor(self):
         return ImportantConfigNode
 
-class MPSConfigNode(TreeConfigNode):
-    def modify_label(self, label):
-        return "MPS=" + str(label)
-
-    def init2(self, node_name):
-        self.props["is_mps"] = node_name
-
-    def child_constructor(self):
-        return ImportantConfigNode
-
-
-class AsanConfigNode(TreeConfigNode):
-    def modify_label(self, label):
-        return "Asan=" + str(label)
-
-    def init2(self, node_name):
-        self.props["is_asan"] = node_name
-
-    def child_constructor(self):
-        return ExperimentalFeatureConfigNode
-
-
-class ONNXConfigNode(TreeConfigNode):
-    def modify_label(self, label):
-        return "Onnx=" + str(label)
-
-    def init2(self, node_name):
-        self.props["is_onnx"] = node_name
-
-    def child_constructor(self):
-        return ImportantConfigNode
-
-
-class VulkanConfigNode(TreeConfigNode):
-    def modify_label(self, label):
-        return "Vulkan=" + str(label)
-
-    def init2(self, node_name):
-        self.props["is_vulkan"] = node_name
-
-    def child_constructor(self):
-        return ImportantConfigNode
-
 
 class ParallelTBBConfigNode(TreeConfigNode):
     def modify_label(self, label):
@@ -172,22 +157,6 @@ class ParallelTBBConfigNode(TreeConfigNode):
         return ImportantConfigNode
 
 
-class CrossRefConfigNode(TreeConfigNode):
-    def init2(self, node_name):
-        self.props["is_crossref"] = node_name
-
-    def child_constructor(self):
-        return ImportantConfigNode
-
-
-class DynamoConfigNode(TreeConfigNode):
-    def init2(self, node_name):
-        self.props["is_dynamo"] = node_name
-
-    def child_constructor(self):
-        return ImportantConfigNode
-
-
 class ParallelNativeConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "PARALLELNATIVE=" + str(label)
@@ -207,7 +176,7 @@ class LibTorchConfigNode(TreeConfigNode):
         self.props["is_libtorch"] = node_name
 
     def child_constructor(self):
-        return ExperimentalFeatureConfigNode
+        return ImportantConfigNode
 
 
 class CudaGccOverrideConfigNode(TreeConfigNode):
@@ -215,21 +184,13 @@ class CudaGccOverrideConfigNode(TreeConfigNode):
         self.props["cuda_gcc_override"] = node_name
 
     def child_constructor(self):
-        return ExperimentalFeatureConfigNode
-
+        return ImportantConfigNode
 
 class BuildOnlyConfigNode(TreeConfigNode):
+
     def init2(self, node_name):
         self.props["build_only"] = node_name
 
-    def child_constructor(self):
-        return ExperimentalFeatureConfigNode
-
-
-class ShardTestConfigNode(TreeConfigNode):
-    def init2(self, node_name):
-        self.props["shard_test"] = node_name
-
     def child_constructor(self):
         return ImportantConfigNode
 
@@ -246,6 +207,7 @@ class ImportantConfigNode(TreeConfigNode):
 
 
 class XenialCompilerConfigNode(TreeConfigNode):
+
     def modify_label(self, label):
         return label or "<unspecified>"
 
@@ -259,6 +221,7 @@ class XenialCompilerConfigNode(TreeConfigNode):
 
 
 class BionicCompilerConfigNode(TreeConfigNode):
+
     def modify_label(self, label):
         return label or "<unspecified>"
 

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -1,13 +1,14 @@
 from collections import OrderedDict
-from dataclasses import dataclass, field
-from typing import List, Optional
 
+from cimodel.data.pytorch_build_data import TopLevelNode, CONFIG_TREE_DATA
 import cimodel.data.dimensions as dimensions
 import cimodel.lib.conf_tree as conf_tree
 import cimodel.lib.miniutils as miniutils
-from cimodel.data.pytorch_build_data import CONFIG_TREE_DATA, TopLevelNode
-from cimodel.data.simple.util.branch_filters import gen_filter_dict, RC_PATTERN
-from cimodel.data.simple.util.docker_constants import gen_docker_image
+from cimodel.data.simple.util.branch_filters import gen_filter_dict
+from cimodel.data.simple.util.docker_constants import gen_docker_image_path
+
+from dataclasses import dataclass, field
+from typing import List, Optional
 
 
 @dataclass
@@ -17,25 +18,18 @@ class Conf:
     parms_list_ignored_for_docker_image: Optional[List[str]] = None
     pyver: Optional[str] = None
     cuda_version: Optional[str] = None
-    rocm_version: Optional[str] = None
     # TODO expand this to cover all the USE_* that we want to test for
     #  tesnrorrt, leveldb, lmdb, redis, opencv, mkldnn, ideep, etc.
     # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453608)
     is_xla: bool = False
-    is_vulkan: bool = False
-    is_pure_torch: bool = False
+    vulkan: bool = False
     restrict_phases: Optional[List[str]] = None
     gpu_resource: Optional[str] = None
     dependent_tests: List = field(default_factory=list)
-    parent_build: Optional["Conf"] = None
+    parent_build: Optional['Conf'] = None
     is_libtorch: bool = False
     is_important: bool = False
     parallel_backend: Optional[str] = None
-    build_only: bool = False
-
-    @staticmethod
-    def is_test_phase(phase):
-        return "test" in phase
 
     # TODO: Eliminate the special casing for docker paths
     # In the short term, we *will* need to support special casing as docker images are merged for caffe2 and pytorch
@@ -48,12 +42,8 @@ class Conf:
         leading.append("pytorch")
         if self.is_xla and not for_docker:
             leading.append("xla")
-        if self.is_vulkan and not for_docker:
-            leading.append("vulkan")
         if self.is_libtorch and not for_docker:
             leading.append("libtorch")
-        if self.is_pure_torch and not for_docker:
-            leading.append("pure_torch")
         if self.parallel_backend is not None and not for_docker:
             leading.append(self.parallel_backend)
 
@@ -61,34 +51,23 @@ class Conf:
         if self.cuda_version:
             cudnn = "cudnn8" if self.cuda_version.startswith("11.") else "cudnn7"
             cuda_parms.extend(["cuda" + self.cuda_version, cudnn])
-        if self.rocm_version:
-            cuda_parms.extend([f"rocm{self.rocm_version}"])
         result = leading + ["linux", self.distro] + cuda_parms + self.parms
         if not for_docker and self.parms_list_ignored_for_docker_image is not None:
             result = result + self.parms_list_ignored_for_docker_image
         return result
 
     def gen_docker_image_path(self):
-        parms_source = self.parent_build or self
-        base_build_env_name = "-".join(parms_source.get_parms(True))
-        image_name, _ = gen_docker_image(base_build_env_name)
-        return miniutils.quote(image_name)
 
-    def gen_docker_image_requires(self):
         parms_source = self.parent_build or self
         base_build_env_name = "-".join(parms_source.get_parms(True))
-        _, requires = gen_docker_image(base_build_env_name)
-        return miniutils.quote(requires)
+
+        return miniutils.quote(gen_docker_image_path(base_build_env_name))
 
     def get_build_job_name_pieces(self, build_or_test):
         return self.get_parms(False) + [build_or_test]
 
     def gen_build_name(self, build_or_test):
-        return (
-            ("_".join(map(str, self.get_build_job_name_pieces(build_or_test))))
-            .replace(".", "_")
-            .replace("-", "_")
-        )
+        return ("_".join(map(str, self.get_build_job_name_pieces(build_or_test)))).replace(".", "_").replace("-", "_")
 
     def get_dependents(self):
         return self.dependent_tests or []
@@ -100,28 +79,20 @@ class Conf:
         build_env_name = "-".join(map(str, build_job_name_pieces))
         parameters["build_environment"] = miniutils.quote(build_env_name)
         parameters["docker_image"] = self.gen_docker_image_path()
-        if Conf.is_test_phase(phase) and self.gpu_resource:
+        if phase == "test" and self.gpu_resource:
             parameters["use_cuda_docker_runtime"] = miniutils.quote("1")
-        if Conf.is_test_phase(phase):
+        if phase == "test":
             resource_class = "large"
             if self.gpu_resource:
                 resource_class = "gpu." + self.gpu_resource
-            if self.rocm_version is not None:
-                resource_class = "pytorch/amd-gpu"
             parameters["resource_class"] = resource_class
-        if phase == "build" and self.rocm_version is not None:
-            parameters["resource_class"] = "xlarge"
-        if hasattr(self, 'filters'):
-            parameters['filters'] = self.filters
-        if self.build_only:
-            parameters['build_only'] = miniutils.quote(str(int(True)))
         return parameters
 
     def gen_workflow_job(self, phase):
         job_def = OrderedDict()
         job_def["name"] = self.gen_build_name(phase)
 
-        if Conf.is_test_phase(phase):
+        if phase == "test":
 
             # TODO When merging the caffe2 and pytorch jobs, it might be convenient for a while to make a
             #  caffe2 test job dependent on a pytorch build job. This way we could quickly dedup the repeated
@@ -133,85 +104,63 @@ class Conf:
             job_name = "pytorch_linux_test"
         else:
             job_name = "pytorch_linux_build"
-            job_def["requires"] = [self.gen_docker_image_requires()]
 
         if not self.is_important:
             job_def["filters"] = gen_filter_dict()
         job_def.update(self.gen_workflow_params(phase))
 
-        return {job_name: job_def}
+        return {job_name : job_def}
 
 
 # TODO This is a hack to special case some configs just for the workflow list
 class HiddenConf(object):
-    def __init__(self, name, parent_build=None, filters=None):
+    def __init__(self, name, parent_build=None):
         self.name = name
         self.parent_build = parent_build
-        self.filters = filters
 
     def gen_workflow_job(self, phase):
-        return {
-            self.gen_build_name(phase): {
-                "requires": [self.parent_build.gen_build_name("build")],
-                "filters": self.filters,
-            }
-        }
+        return {self.gen_build_name(phase): {"requires": [self.parent_build.gen_build_name("build")]}}
 
     def gen_build_name(self, _):
         return self.name
 
-class DocPushConf(object):
-    def __init__(self, name, parent_build=None, branch="master"):
-        self.name = name
-        self.parent_build = parent_build
-        self.branch = branch
 
-    def gen_workflow_job(self, phase):
-        return {
-            "pytorch_doc_push": {
-                "name": self.name,
-                "branch": self.branch,
-                "requires": [self.parent_build],
-                "context": "org-member",
-                "filters": gen_filter_dict(branches_list=["nightly"],
-                                           tags_list=RC_PATTERN)
-            }
-        }
+# TODO Convert these to graph nodes
+def gen_dependent_configs(xenial_parent_config):
+
+    extra_parms = [
+        (["multigpu"], "large"),
+        (["NO_AVX2"], "medium"),
+        (["NO_AVX", "NO_AVX2"], "medium"),
+        (["slow"], "medium"),
+        (["nogpu"], None),
+    ]
+
+    configs = []
+    for parms, gpu in extra_parms:
+
+        c = Conf(
+            xenial_parent_config.distro,
+            ["py3"] + parms,
+            pyver="3.6",
+            cuda_version=xenial_parent_config.cuda_version,
+            restrict_phases=["test"],
+            gpu_resource=gpu,
+            parent_build=xenial_parent_config,
+            is_important=xenial_parent_config.is_important,
+        )
+
+        configs.append(c)
+
+    return configs
+
 
 def gen_docs_configs(xenial_parent_config):
     configs = []
 
-    configs.append(
-        HiddenConf(
-            "pytorch_python_doc_build",
-            parent_build=xenial_parent_config,
-            filters=gen_filter_dict(branches_list=["master", "main", "nightly"],
-                                    tags_list=RC_PATTERN),
-        )
-    )
-    configs.append(
-        DocPushConf(
-            "pytorch_python_doc_push",
-            parent_build="pytorch_python_doc_build",
-            branch="site",
-        )
-    )
+    for x in ["pytorch_python_doc_push", "pytorch_cpp_doc_push", "pytorch_doc_test"]:
+        configs.append(HiddenConf(x, parent_build=xenial_parent_config))
 
-    configs.append(
-        HiddenConf(
-            "pytorch_cpp_doc_build",
-            parent_build=xenial_parent_config,
-            filters=gen_filter_dict(branches_list=["master", "main", "nightly"],
-                                    tags_list=RC_PATTERN),
-        )
-    )
-    configs.append(
-        DocPushConf(
-            "pytorch_cpp_doc_push",
-            parent_build="pytorch_cpp_doc_build",
-            branch="master",
-        )
-    )
     return configs
 
 
@@ -225,7 +174,7 @@ def gen_tree():
     return configs_list
 
 
-def instantiate_configs(only_slow_gradcheck):
+def instantiate_configs():
 
     config_list = []
 
@@ -238,17 +187,11 @@ def instantiate_configs(only_slow_gradcheck):
         compiler_name = fc.find_prop("compiler_name")
         compiler_version = fc.find_prop("compiler_version")
         is_xla = fc.find_prop("is_xla") or False
-        is_asan = fc.find_prop("is_asan") or False
-        is_crossref = fc.find_prop("is_crossref") or False
-        is_dynamo = fc.find_prop("is_dynamo") or False
-        is_onnx = fc.find_prop("is_onnx") or False
-        is_pure_torch = fc.find_prop("is_pure_torch") or False
-        is_vulkan = fc.find_prop("is_vulkan") or False
-        is_slow_gradcheck = fc.find_prop("is_slow_gradcheck") or False
         parms_list_ignored_for_docker_image = []
 
-        if only_slow_gradcheck ^ is_slow_gradcheck:
-            continue
+        vulkan = fc.find_prop("vulkan") or False
+        if vulkan:
+            parms_list_ignored_for_docker_image.append("vulkan")
 
         python_version = None
         if compiler_name == "cuda" or compiler_name == "android":
@@ -258,14 +201,9 @@ def instantiate_configs(only_slow_gradcheck):
             parms_list = ["py" + fc.find_prop("pyver")]
 
         cuda_version = None
-        rocm_version = None
         if compiler_name == "cuda":
             cuda_version = fc.find_prop("compiler_version")
 
-        elif compiler_name == "rocm":
-            rocm_version = fc.find_prop("compiler_version")
-            restrict_phases = ["build", "test1", "test2", "caffe2_test"]
-
         elif compiler_name == "android":
             android_ndk_version = fc.find_prop("compiler_version")
             # TODO: do we need clang to compile host binaries like protoc?
@@ -279,22 +217,11 @@ def instantiate_configs(only_slow_gradcheck):
             gcc_version = compiler_name + (fc.find_prop("compiler_version") or "")
             parms_list.append(gcc_version)
 
-        if is_asan:
-            parms_list.append("asan")
-            python_version = fc.find_prop("pyver")
-            parms_list[0] = fc.find_prop("abbreviated_pyver")
-
-        if is_crossref:
-            parms_list_ignored_for_docker_image.append("crossref")
-
-        if is_dynamo:
-            parms_list_ignored_for_docker_image.append("dynamo")
-
-        if is_onnx:
-            parms_list.append("onnx")
-            python_version = fc.find_prop("pyver")
-            parms_list[0] = fc.find_prop("abbreviated_pyver")
-            restrict_phases = ["build", "ort_test1", "ort_test2"]
+            # TODO: This is a nasty special case
+            if gcc_version == 'clang5' and not is_xla:
+                parms_list.append("asan")
+                python_version = fc.find_prop("pyver")
+                parms_list[0] = fc.find_prop("abbreviated_pyver")
 
         if cuda_version:
             cuda_gcc_version = fc.find_prop("cuda_gcc_override") or "gcc7"
@@ -304,18 +231,9 @@ def instantiate_configs(only_slow_gradcheck):
         is_important = fc.find_prop("is_important") or False
         parallel_backend = fc.find_prop("parallel_backend") or None
         build_only = fc.find_prop("build_only") or False
-        shard_test = fc.find_prop("shard_test") or False
-        # TODO: fix pure_torch python test packaging issue.
-        if shard_test:
-            restrict_phases = ["build"] if restrict_phases is None else restrict_phases
-            restrict_phases.extend(["test1", "test2"])
-        if build_only or is_pure_torch:
+        if build_only and restrict_phases is None:
             restrict_phases = ["build"]
 
-        if is_slow_gradcheck:
-            parms_list_ignored_for_docker_image.append("old")
-            parms_list_ignored_for_docker_image.append("gradcheck")
-
         gpu_resource = None
         if cuda_version and cuda_version != "10":
             gpu_resource = "medium"
@@ -326,43 +244,50 @@ def instantiate_configs(only_slow_gradcheck):
             parms_list_ignored_for_docker_image,
             python_version,
             cuda_version,
-            rocm_version,
             is_xla,
-            is_vulkan,
-            is_pure_torch,
+            vulkan,
             restrict_phases,
             gpu_resource,
             is_libtorch=is_libtorch,
             is_important=is_important,
             parallel_backend=parallel_backend,
-            build_only=build_only,
         )
 
-        # run docs builds on "pytorch-linux-xenial-py3.7-gcc5.4". Docs builds
+        # run docs builds on "pytorch-linux-xenial-py3.6-gcc5.4". Docs builds
         # should run on a CPU-only build that runs on all PRs.
-        # XXX should this be updated to a more modern build?
-        if (
-            distro_name == "xenial"
-            and fc.find_prop("pyver") == "3.7"
-            and cuda_version is None
-            and parallel_backend is None
-            and not is_vulkan
-            and not is_pure_torch
-            and compiler_name == "gcc"
-            and fc.find_prop("compiler_version") == "5.4"
-        ):
-            c.filters = gen_filter_dict(branches_list=r"/.*/",
-                                        tags_list=RC_PATTERN)
+        if distro_name == 'xenial' and fc.find_prop("pyver") == '3.6' \
+                and cuda_version is None \
+                and parallel_backend is None \
+                and compiler_name == 'gcc' \
+                and fc.find_prop('compiler_version') == '5.4':
             c.dependent_tests = gen_docs_configs(c)
 
+        if cuda_version == "10.1" and python_version == "3.6" and not is_libtorch:
+            c.dependent_tests = gen_dependent_configs(c)
+
+        if (compiler_name == "gcc"
+                and compiler_version == "5.4"
+                and not is_libtorch
+                and parallel_backend is None):
+            bc_breaking_check = Conf(
+                "backward-compatibility-check",
+                [],
+                is_xla=False,
+                restrict_phases=["test"],
+                is_libtorch=False,
+                is_important=True,
+                parent_build=c,
+            )
+            c.dependent_tests.append(bc_breaking_check)
+
         config_list.append(c)
 
     return config_list
 
 
-def get_workflow_jobs(only_slow_gradcheck=False):
+def get_workflow_jobs():
 
-    config_list = instantiate_configs(only_slow_gradcheck)
+    config_list = instantiate_configs()
 
     x = []
     for conf_options in config_list:
@@ -372,7 +297,7 @@ def get_workflow_jobs(only_slow_gradcheck=False):
         for phase in phases:
 
             # TODO why does this not have a test?
-            if Conf.is_test_phase(phase) and conf_options.cuda_version == "10":
+            if phase == "test" and conf_options.cuda_version == "10":
                 continue
 
             x.append(conf_options.gen_workflow_job(phase))

.circleci/cimodel/data/simple/anaconda_prune_defintions.py

@@ -1,28 +0,0 @@
-from collections import OrderedDict
-
-from cimodel.data.simple.util.branch_filters import gen_filter_dict
-from cimodel.lib.miniutils import quote
-
-
-CHANNELS_TO_PRUNE = ["pytorch-nightly", "pytorch-test"]
-PACKAGES_TO_PRUNE = "pytorch torchvision torchaudio torchtext ignite torchcsprng"
-
-
-def gen_workflow_job(channel: str):
-    return OrderedDict(
-        {
-            "anaconda_prune": OrderedDict(
-                {
-                    "name": f"anaconda-prune-{channel}",
-                    "context": quote("org-member"),
-                    "packages": quote(PACKAGES_TO_PRUNE),
-                    "channel": channel,
-                    "filters": gen_filter_dict(branches_list=["postnightly"]),
-                }
-            )
-        }
-    )
-
-
-def get_workflow_jobs():
-    return [gen_workflow_job(channel) for channel in CHANNELS_TO_PRUNE]

.circleci/cimodel/data/simple/android_definitions.py

@@ -0,0 +1,92 @@
+import cimodel.data.simple.util.branch_filters
+from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_NDK
+
+
+class AndroidJob:
+    def __init__(self,
+                 variant,
+                 template_name,
+                 is_master_only=True):
+
+        self.variant = variant
+        self.template_name = template_name
+        self.is_master_only = is_master_only
+
+    def gen_tree(self):
+
+        base_name_parts = [
+            "pytorch",
+            "linux",
+            "xenial",
+            "py3",
+            "clang5",
+            "android",
+            "ndk",
+            "r19c",
+        ] + self.variant + [
+            "build",
+        ]
+
+        full_job_name = "_".join(base_name_parts)
+        build_env_name = "-".join(base_name_parts)
+
+        props_dict = {
+            "name": full_job_name,
+            "build_environment": "\"{}\"".format(build_env_name),
+            "docker_image": "\"{}\"".format(DOCKER_IMAGE_NDK),
+        }
+
+        if self.is_master_only:
+            props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
+
+        return [{self.template_name: props_dict}]
+
+
+class AndroidGradleJob:
+    def __init__(self,
+                 job_name,
+                 template_name,
+                 dependencies,
+                 is_master_only=True):
+
+        self.job_name = job_name
+        self.template_name = template_name
+        self.dependencies = dependencies
+        self.is_master_only = is_master_only
+
+    def gen_tree(self):
+
+        props_dict = {
+            "name": self.job_name,
+            "requires": self.dependencies,
+        }
+
+        if self.is_master_only:
+            props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
+
+        return [{self.template_name: props_dict}]
+
+
+WORKFLOW_DATA = [
+    AndroidJob(["x86_32"], "pytorch_linux_build", is_master_only=False),
+    AndroidJob(["x86_64"], "pytorch_linux_build"),
+    AndroidJob(["arm", "v7a"], "pytorch_linux_build"),
+    AndroidJob(["arm", "v8a"], "pytorch_linux_build"),
+    AndroidJob(["vulkan", "x86_32"], "pytorch_linux_build", is_master_only=False),
+    AndroidGradleJob(
+        "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32",
+        "pytorch_android_gradle_build-x86_32",
+        ["pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build"],
+        is_master_only=False),
+    AndroidGradleJob(
+        "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build",
+        "pytorch_android_gradle_build",
+        ["pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build",
+         "pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build",
+         "pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build",
+         "pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build"]),
+]
+
+
+def get_workflow_jobs():
+    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/bazel_definitions.py

@@ -0,0 +1,63 @@
+from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_GCC7
+
+
+def gen_job_name(phase):
+    job_name_parts = [
+        "pytorch",
+        "bazel",
+        phase,
+    ]
+
+    return "_".join(job_name_parts)
+
+
+class BazelJob:
+    def __init__(self, phase, extra_props=None):
+        self.phase = phase
+        self.extra_props = extra_props or {}
+
+    def gen_tree(self):
+
+        template_parts = [
+            "pytorch",
+            "linux",
+            "bazel",
+            self.phase,
+        ]
+
+        build_env_parts = [
+            "pytorch",
+            "linux",
+            "xenial",
+            "py3.6",
+            "gcc7",
+            "bazel",
+            self.phase,
+        ]
+
+        full_job_name = gen_job_name(self.phase)
+        build_env_name = "-".join(build_env_parts)
+
+        extra_requires = [gen_job_name("build")] if self.phase == "test" else []
+
+        props_dict = {
+            "build_environment": build_env_name,
+            "docker_image": DOCKER_IMAGE_GCC7,
+            "name": full_job_name,
+            "requires": extra_requires,
+        }
+
+        props_dict.update(self.extra_props)
+
+        template_name = "_".join(template_parts)
+        return [{template_name: props_dict}]
+
+
+WORKFLOW_DATA = [
+    BazelJob("build", {"resource_class": "large"}),
+    BazelJob("test"),
+]
+
+
+def get_workflow_jobs():
+    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/binary_smoketest.py

@@ -0,0 +1,193 @@
+"""
+TODO: Refactor circleci/cimodel/data/binary_build_data.py to generate this file
+       instead of doing one offs here
+ Binary builds (subset, to smoke test that they'll work)
+
+ NB: If you modify this file, you need to also modify
+ the binary_and_smoke_tests_on_pr variable in
+ pytorch-ci-hud to adjust the list of whitelisted builds
+ at https://github.com/ezyang/pytorch-ci-hud/blob/master/src/BuildHistoryDisplay.js
+
+ Note:
+ This binary build is currently broken, see https://github_com/pytorch/pytorch/issues/16710
+ - binary_linux_conda_3_6_cu90_devtoolset7_build
+ - binary_linux_conda_3_6_cu90_devtoolset7_test
+
+ TODO
+ we should test a libtorch cuda build, but they take too long
+ - binary_linux_libtorch_3_6m_cu90_devtoolset7_static-without-deps_build
+"""
+
+import cimodel.lib.miniutils as miniutils
+import cimodel.data.simple.util.branch_filters
+
+
+class SmoketestJob:
+    def __init__(self,
+                 template_name,
+                 build_env_parts,
+                 docker_image,
+                 job_name,
+                 is_master_only=False,
+                 requires=None,
+                 has_libtorch_variant=False,
+                 extra_props=None):
+
+        self.template_name = template_name
+        self.build_env_parts = build_env_parts
+        self.docker_image = docker_image
+        self.job_name = job_name
+        self.is_master_only = is_master_only
+        self.requires = requires or []
+        self.has_libtorch_variant = has_libtorch_variant
+        self.extra_props = extra_props or {}
+
+    def gen_tree(self):
+
+        props_dict = {
+            "build_environment": " ".join(self.build_env_parts),
+            "name": self.job_name,
+            "requires": self.requires,
+        }
+
+        if self.docker_image:
+            props_dict["docker_image"] = self.docker_image
+
+        if self.is_master_only:
+            props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
+
+        if self.has_libtorch_variant:
+            props_dict["libtorch_variant"] = "shared-with-deps"
+
+        props_dict.update(self.extra_props)
+
+        return [{self.template_name: props_dict}]
+
+
+WORKFLOW_DATA = [
+    SmoketestJob(
+        "binary_linux_build",
+        ["manywheel", "3.7m", "cu102", "devtoolset7"],
+        "pytorch/manylinux-cuda102",
+        "binary_linux_manywheel_3_7m_cu102_devtoolset7_build",
+        is_master_only=True,
+    ),
+    SmoketestJob(
+        "binary_linux_build",
+        ["libtorch", "3.7m", "cpu", "devtoolset7"],
+        "pytorch/manylinux-cuda102",
+        "binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build",
+        is_master_only=False,
+        has_libtorch_variant=True,
+    ),
+    SmoketestJob(
+        "binary_linux_build",
+        ["libtorch", "3.7m", "cpu", "gcc5.4_cxx11-abi"],
+        "pytorch/pytorch-binary-docker-image-ubuntu16.04:latest",
+        "binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build",
+        is_master_only=False,
+        has_libtorch_variant=True,
+    ),
+    SmoketestJob(
+        "binary_mac_build",
+        ["wheel", "3.7", "cpu"],
+        None,
+        "binary_macos_wheel_3_7_cpu_build",
+        is_master_only=True,
+    ),
+    # This job has an average run time of 3 hours o.O
+    # Now only running this on master to reduce overhead
+    SmoketestJob(
+        "binary_mac_build",
+        ["libtorch", "3.7", "cpu"],
+        None,
+        "binary_macos_libtorch_3_7_cpu_build",
+        is_master_only=True,
+    ),
+    SmoketestJob(
+        "binary_windows_build",
+        ["libtorch", "3.7", "cpu", "debug"],
+        None,
+        "binary_windows_libtorch_3_7_cpu_debug_build",
+        is_master_only=False,
+    ),
+    SmoketestJob(
+        "binary_windows_build",
+        ["libtorch", "3.7", "cpu", "release"],
+        None,
+        "binary_windows_libtorch_3_7_cpu_release_build",
+        is_master_only=False,
+    ),
+    SmoketestJob(
+        "binary_windows_build",
+        ["wheel", "3.7", "cu102"],
+        None,
+        "binary_windows_wheel_3_7_cu102_build",
+        is_master_only=True,
+    ),
+
+    SmoketestJob(
+        "binary_windows_test",
+        ["libtorch", "3.7", "cpu", "debug"],
+        None,
+        "binary_windows_libtorch_3_7_cpu_debug_test",
+        is_master_only=False,
+        requires=["binary_windows_libtorch_3_7_cpu_debug_build"],
+    ),
+    SmoketestJob(
+        "binary_windows_test",
+        ["libtorch", "3.7", "cpu", "release"],
+        None,
+        "binary_windows_libtorch_3_7_cpu_release_test",
+        is_master_only=False,
+        requires=["binary_windows_libtorch_3_7_cpu_release_build"],
+    ),
+    SmoketestJob(
+        "binary_windows_test",
+        ["wheel", "3.7", "cu102"],
+        None,
+        "binary_windows_wheel_3_7_cu102_test",
+        is_master_only=True,
+        requires=["binary_windows_wheel_3_7_cu102_build"],
+        extra_props={
+            "executor": "windows-with-nvidia-gpu",
+        },
+    ),
+
+
+
+    SmoketestJob(
+        "binary_linux_test",
+        ["manywheel", "3.7m", "cu102", "devtoolset7"],
+        "pytorch/manylinux-cuda102",
+        "binary_linux_manywheel_3_7m_cu102_devtoolset7_test",
+        is_master_only=True,
+        requires=["binary_linux_manywheel_3_7m_cu102_devtoolset7_build"],
+        extra_props={
+            "resource_class": "gpu.medium",
+            "use_cuda_docker_runtime": miniutils.quote((str(1))),
+        },
+    ),
+    SmoketestJob(
+        "binary_linux_test",
+        ["libtorch", "3.7m", "cpu", "devtoolset7"],
+        "pytorch/manylinux-cuda102",
+        "binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_test",
+        is_master_only=False,
+        requires=["binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build"],
+        has_libtorch_variant=True,
+    ),
+    SmoketestJob(
+        "binary_linux_test",
+        ["libtorch", "3.7m", "cpu", "gcc5.4_cxx11-abi"],
+        "pytorch/pytorch-binary-docker-image-ubuntu16.04:latest",
+        "binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_test",
+        is_master_only=False,
+        requires=["binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build"],
+        has_libtorch_variant=True,
+    ),
+]
+
+
+def get_workflow_jobs():
+    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/docker_definitions.py

@@ -1,39 +1,44 @@
 from collections import OrderedDict
 
 from cimodel.lib.miniutils import quote
-from cimodel.data.simple.util.branch_filters import gen_filter_dict, RC_PATTERN
 
 
-# NOTE: All hardcoded docker image builds have been migrated to GHA
+# TODO: make this generated from a matrix rather than just a static list
 IMAGE_NAMES = [
+    "pytorch-linux-bionic-cuda11.0-cudnn8-py3.6-gcc9",
+    "pytorch-linux-bionic-cuda11.0-cudnn8-py3.8-gcc9",
+    "pytorch-linux-bionic-cuda10.2-cudnn7-py3.8-gcc9",
+    "pytorch-linux-bionic-py3.6-clang9",
+    "pytorch-linux-bionic-cuda10.2-cudnn7-py3.6-clang9",
+    "pytorch-linux-bionic-py3.8-gcc9",
+    "pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7",
+    "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7",
+    "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7",
+    "pytorch-linux-xenial-cuda11.0-cudnn8-py3-gcc7",
+    "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4",
+    "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7",
+    "pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
+    "pytorch-linux-xenial-py3-clang5-asan",
+    "pytorch-linux-xenial-py3.8",
+    "pytorch-linux-xenial-py3.6-clang7",
+    "pytorch-linux-xenial-py3.6-gcc4.8",
+    "pytorch-linux-xenial-py3.6-gcc5.4",
+    "pytorch-linux-xenial-py3.6-gcc7.2",
+    "pytorch-linux-xenial-py3.6-gcc7",
+    "pytorch-linux-xenial-pynightly",
+    "pytorch-linux-xenial-rocm3.3-py3.6",
 ]
 
-# This entry should be an element from the list above
-# This should contain the image matching the "slow_gradcheck" entry in
-# pytorch_build_data.py
-SLOW_GRADCHECK_IMAGE_NAME = "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
 
-def get_workflow_jobs(images=IMAGE_NAMES, only_slow_gradcheck=False):
+def get_workflow_jobs():
     """Generates a list of docker image build definitions"""
-    ret = []
-    for image_name in images:
-        if image_name.startswith('docker-'):
-            image_name = image_name.lstrip('docker-')
-        if only_slow_gradcheck and image_name is not SLOW_GRADCHECK_IMAGE_NAME:
-            continue
-
-        parameters = OrderedDict({
-            "name": quote(f"docker-{image_name}"),
-            "image_name": quote(image_name),
-        })
-        if image_name == "pytorch-linux-xenial-py3.7-gcc5.4":
-            # pushing documentation on tags requires CircleCI to also
-            # build all the dependencies on tags, including this docker image
-            parameters['filters'] = gen_filter_dict(branches_list=r"/.*/",
-                                                    tags_list=RC_PATTERN)
-        ret.append(OrderedDict(
+    return [
+        OrderedDict(
             {
-                "docker_build_job": parameters
+                "docker_build_job": OrderedDict(
+                    {"name": quote(image_name), "image_name": quote(image_name)}
+                )
             }
-        ))
-    return ret
+        )
+        for image_name in IMAGE_NAMES
+    ]

.circleci/cimodel/data/simple/ge_config_tests.py

@@ -0,0 +1,103 @@
+import cimodel.lib.miniutils as miniutils
+from cimodel.data.simple.util.versions import MultiPartVersion, CudaVersion
+from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_BASIC, DOCKER_IMAGE_CUDA_10_2
+
+
+class GeConfigTestJob:
+    def __init__(self,
+                 py_version,
+                 gcc_version,
+                 cuda_version,
+                 variant_parts,
+                 extra_requires,
+                 use_cuda_docker=False,
+                 build_env_override=None):
+
+        self.py_version = py_version
+        self.gcc_version = gcc_version
+        self.cuda_version = cuda_version
+        self.variant_parts = variant_parts
+        self.extra_requires = extra_requires
+        self.use_cuda_docker = use_cuda_docker
+        self.build_env_override = build_env_override
+
+    def get_all_parts(self, with_dots):
+
+        maybe_py_version = self.py_version.render_dots_or_parts(with_dots) if self.py_version else []
+        maybe_gcc_version = self.gcc_version.render_dots_or_parts(with_dots) if self.gcc_version else []
+        maybe_cuda_version = self.cuda_version.render_dots_or_parts(with_dots) if self.cuda_version else []
+
+        common_parts = [
+            "pytorch",
+            "linux",
+            "xenial",
+        ] + maybe_cuda_version + maybe_py_version + maybe_gcc_version
+
+        return common_parts + self.variant_parts
+
+    def gen_tree(self):
+
+        resource_class = "gpu.medium" if self.use_cuda_docker else "large"
+        docker_image = DOCKER_IMAGE_CUDA_10_2 if self.use_cuda_docker else DOCKER_IMAGE_BASIC
+        full_name = "_".join(self.get_all_parts(False))
+        build_env = self.build_env_override or "-".join(self.get_all_parts(True))
+
+        props_dict = {
+            "name": full_name,
+            "build_environment": build_env,
+            "requires": self.extra_requires,
+            "resource_class": resource_class,
+            "docker_image": docker_image,
+        }
+
+        if self.use_cuda_docker:
+            props_dict["use_cuda_docker_runtime"] = miniutils.quote(str(1))
+
+        return [{"pytorch_linux_test": props_dict}]
+
+
+WORKFLOW_DATA = [
+    GeConfigTestJob(
+        MultiPartVersion([3, 6], "py"),
+        MultiPartVersion([5, 4], "gcc"),
+        None,
+        ["ge_config_legacy", "test"],
+        ["pytorch_linux_xenial_py3_6_gcc5_4_build"]),
+    GeConfigTestJob(
+        MultiPartVersion([3, 6], "py"),
+        MultiPartVersion([5, 4], "gcc"),
+        None,
+        ["ge_config_profiling", "test"],
+        ["pytorch_linux_xenial_py3_6_gcc5_4_build"]),
+    GeConfigTestJob(
+        MultiPartVersion([3, 6], "py"),
+        MultiPartVersion([5, 4], "gcc"),
+        None,
+        ["ge_config_simple", "test"],
+        ["pytorch_linux_xenial_py3_6_gcc5_4_build"],
+    ),
+    GeConfigTestJob(
+        None,
+        None,
+        CudaVersion(10, 2),
+        ["cudnn7", "py3", "ge_config_legacy", "test"],
+        ["pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build"],
+        use_cuda_docker=True,
+        # TODO Why does the build environment specify cuda10.1, while the
+        # job name is cuda10_2?
+        build_env_override="pytorch-linux-xenial-cuda10.1-cudnn7-ge_config_legacy-test"),
+    GeConfigTestJob(
+        None,
+        None,
+        CudaVersion(10, 2),
+        ["cudnn7", "py3", "ge_config_profiling", "test"],
+        ["pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build"],
+        use_cuda_docker=True,
+        # TODO Why does the build environment specify cuda10.1, while the
+        # job name is cuda10_2?
+        build_env_override="pytorch-linux-xenial-cuda10.1-cudnn7-ge_config_profiling-test"),
+]
+
+
+def get_workflow_jobs():
+    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/ios_definitions.py

@@ -1,18 +1,17 @@
 from cimodel.data.simple.util.versions import MultiPartVersion
-from cimodel.data.simple.util.branch_filters import gen_filter_dict_exclude
-import cimodel.lib.miniutils as miniutils
 
-XCODE_VERSION = MultiPartVersion([12, 5, 1])
+
+IOS_VERSION = MultiPartVersion([11, 2, 1])
 
 
 class ArchVariant:
-    def __init__(self, name, custom_build_name=""):
+    def __init__(self, name, is_custom=False):
         self.name = name
-        self.custom_build_name = custom_build_name
+        self.is_custom = is_custom
 
     def render(self):
-        extra_parts = [self.custom_build_name] if len(self.custom_build_name) > 0 else []
-        return "-".join([self.name] + extra_parts).replace("_", "-")
+        extra_parts = ["custom"] if self.is_custom else []
+        return "_".join([self.name] + extra_parts)
 
 
 def get_platform(arch_variant_name):
@@ -20,31 +19,36 @@ def get_platform(arch_variant_name):
 
 
 class IOSJob:
-    def __init__(self, xcode_version, arch_variant, is_org_member_context=True, extra_props=None):
-        self.xcode_version = xcode_version
+    def __init__(self, ios_version, arch_variant, is_org_member_context=True, extra_props=None):
+        self.ios_version = ios_version
         self.arch_variant = arch_variant
         self.is_org_member_context = is_org_member_context
         self.extra_props = extra_props
 
-    def gen_name_parts(self):
-        version_parts = self.xcode_version.render_dots_or_parts("-")
-        build_variant_suffix = self.arch_variant.render()
+    def gen_name_parts(self, with_version_dots):
+
+        version_parts = self.ios_version.render_dots_or_parts(with_version_dots)
+        build_variant_suffix = "_".join([self.arch_variant.render(), "build"])
+
         return [
+            "pytorch",
             "ios",
         ] + version_parts + [
             build_variant_suffix,
         ]
 
     def gen_job_name(self):
-        return "-".join(self.gen_name_parts())
+        return "_".join(self.gen_name_parts(False))
 
     def gen_tree(self):
+
         platform_name = get_platform(self.arch_variant.name)
+
         props_dict = {
-            "name": self.gen_job_name(),
-            "build_environment": self.gen_job_name(),
+            "build_environment": "-".join(self.gen_name_parts(True)),
             "ios_arch": self.arch_variant.name,
             "ios_platform": platform_name,
+            "name": self.gen_job_name(),
         }
 
         if self.is_org_member_context:
@@ -53,28 +57,13 @@ class IOSJob:
         if self.extra_props:
             props_dict.update(self.extra_props)
 
-        props_dict["filters"] = gen_filter_dict_exclude()
-
         return [{"pytorch_ios_build": props_dict}]
 
 
 WORKFLOW_DATA = [
-    IOSJob(XCODE_VERSION, ArchVariant("x86_64"), is_org_member_context=False, extra_props={
-        "lite_interpreter": miniutils.quote(str(int(True)))}),
-    # IOSJob(XCODE_VERSION, ArchVariant("arm64"), extra_props={
-    #     "lite_interpreter": miniutils.quote(str(int(True)))}),
-    # IOSJob(XCODE_VERSION, ArchVariant("arm64", "metal"), extra_props={
-    #     "use_metal": miniutils.quote(str(int(True))),
-    #     "lite_interpreter": miniutils.quote(str(int(True)))}),
-    # IOSJob(XCODE_VERSION, ArchVariant("arm64", "custom-ops"), extra_props={
-    #     "op_list": "mobilenetv2.yaml",
-    #     "lite_interpreter": miniutils.quote(str(int(True)))}),
-    IOSJob(XCODE_VERSION, ArchVariant("x86_64", "coreml"), is_org_member_context=False, extra_props={
-        "use_coreml": miniutils.quote(str(int(True))),
-        "lite_interpreter": miniutils.quote(str(int(True)))}),
-    # IOSJob(XCODE_VERSION, ArchVariant("arm64", "coreml"), extra_props={
-    #     "use_coreml": miniutils.quote(str(int(True))),
-    #     "lite_interpreter": miniutils.quote(str(int(True)))}),
+    IOSJob(IOS_VERSION, ArchVariant("x86_64"), is_org_member_context=False),
+    IOSJob(IOS_VERSION, ArchVariant("arm64")),
+    IOSJob(IOS_VERSION, ArchVariant("arm64", True), extra_props={"op_list": "mobilenetv2.yaml"}),
 ]
 
 

.circleci/cimodel/data/simple/macos_definitions.py

@@ -1,26 +1,14 @@
 class MacOsJob:
-    def __init__(self, os_version, is_build=False, is_test=False, extra_props=tuple()):
-        # extra_props is tuple type, because mutable data structures for argument defaults
-        # is not recommended.
+    def __init__(self, os_version, is_test=False):
         self.os_version = os_version
-        self.is_build = is_build
         self.is_test = is_test
-        self.extra_props = dict(extra_props)
 
     def gen_tree(self):
         non_phase_parts = ["pytorch", "macos", self.os_version, "py3"]
 
-        extra_name_list = [name for name, exist in self.extra_props.items() if exist]
-        full_job_name_list = (
-            non_phase_parts
-            + extra_name_list
-            + [
-                "build" if self.is_build else None,
-                "test" if self.is_test else None,
-            ]
-        )
+        phase_name = "test" if self.is_test else "build"
 
-        full_job_name = "_".join(list(filter(None, full_job_name_list)))
+        full_job_name = "_".join(non_phase_parts + [phase_name])
 
         test_build_dependency = "_".join(non_phase_parts + ["build"])
         extra_dependencies = [test_build_dependency] if self.is_test else []
@@ -33,21 +21,7 @@ class MacOsJob:
         return [{full_job_name: props_dict}]
 
 
-WORKFLOW_DATA = [
-    MacOsJob("10_15", is_build=True),
-    MacOsJob("10_13", is_build=True),
-    MacOsJob(
-        "10_13",
-        is_build=False,
-        is_test=True,
-    ),
-    MacOsJob(
-        "10_13",
-        is_build=True,
-        is_test=True,
-        extra_props=tuple({"lite_interpreter": True}.items()),
-    ),
-]
+WORKFLOW_DATA = [MacOsJob("10_13"), MacOsJob("10_13", True)]
 
 
 def get_workflow_jobs():