Back out "Revert D10123245: Back out "codemod cuda_gpu_id to device_id"" (#12232)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/12232

Original commit changeset: fca91fea58b7

This adds proper modifications to the DeviceType <->DeviceOption conversion code added in D10033396

Reviewed By: jerryzh168

Differential Revision: D10132473

fbshipit-source-id: 801ef777e2950982cb47b48051b1471a0a91e64b

.circleci/config.yml

@@ -0,0 +1,974 @@
+docker_config_defaults: &docker_config_defaults
+  user: jenkins
+  aws_auth:
+    # This IAM user only allows read-only access to ECR
+    aws_access_key_id: AKIAJ2J6FIG5OSZTQ3IA
+    aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_ONLY}
+
+# NOTE: We only perform the merge in build step and not in test step, because
+# all source files will be shared from build to test
+merge_pull_request_onto_master: &merge_pull_request_onto_master
+  name: Merge Onto Master
+  no_output_timeout: "10h"
+  command: |
+    if [[ "${CIRCLE_BRANCH}" != "master" ]]; then
+      git config --global user.email "circleci.ossci@gmail.com"
+      git config --global user.name "CircleCI"
+
+      git config remote.origin.url https://github.com/pytorch/pytorch.git
+      git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master
+      git fetch --tags --progress https://github.com/pytorch/pytorch.git +refs/heads/master:refs/remotes/origin/master --depth=50 --quiet
+
+      export GIT_MERGE_TARGET=`git log -n 1 --pretty=format:"%H" origin/master`
+      echo "GIT_MERGE_TARGET: " ${GIT_MERGE_TARGET}
+      export GIT_COMMIT=${CIRCLE_SHA1}
+      echo "GIT_COMMIT: " ${GIT_COMMIT}
+
+      git checkout -f ${GIT_COMMIT}
+      git reset --hard ${GIT_COMMIT}
+      git merge --no-edit --no-ff ${GIT_MERGE_TARGET}
+    fi
+
+pytorch_linux_cpu_build_test_defaults: &pytorch_linux_cpu_build_test_defaults
+  resource_class: large
+  working_directory: /var/lib/jenkins/workspace
+  steps:
+  - checkout
+  - run:
+      <<: *merge_pull_request_onto_master
+  - run:
+      name: Build
+      no_output_timeout: "10h"
+      command: |
+        export IN_CIRCLECI=1
+        export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+        export SCCACHE_MAX_JOBS=`expr $(nproc) - 1`
+        export MEMORY_LIMIT_MAX_JOBS=8  # the "large" resource class on CircleCI has 32 CPU cores, if we use all of them we'll OOM
+        export MAX_JOBS=$(( ${SCCACHE_MAX_JOBS} > ${MEMORY_LIMIT_MAX_JOBS} ? ${MEMORY_LIMIT_MAX_JOBS} : ${SCCACHE_MAX_JOBS} ))
+        # This IAM user allows write access to S3 bucket for sccache
+        export AWS_ACCESS_KEY_ID=AKIAJJZUW4G2ASX5W7KA
+        export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET}
+        git submodule sync && git submodule update --init
+        .jenkins/pytorch/build.sh
+        .jenkins/pytorch/test.sh
+
+pytorch_linux_build_defaults: &pytorch_linux_build_defaults
+  resource_class: large
+  working_directory: /var/lib/jenkins/workspace
+  steps:
+  - checkout
+  - run:
+      <<: *merge_pull_request_onto_master
+  - run:
+      name: Build
+      no_output_timeout: "10h"
+      command: |
+        export IN_CIRCLECI=1
+        export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+        if [ -n "${CUDA_VERSION}" ]; then
+          export TORCH_CUDA_ARCH_LIST=5.2
+        fi
+        export SCCACHE_MAX_JOBS=`expr $(nproc) - 1`
+        export MEMORY_LIMIT_MAX_JOBS=8  # the "large" resource class on CircleCI has 32 CPU cores, if we use all of them we'll OOM
+        export MAX_JOBS=$(( ${SCCACHE_MAX_JOBS} > ${MEMORY_LIMIT_MAX_JOBS} ? ${MEMORY_LIMIT_MAX_JOBS} : ${SCCACHE_MAX_JOBS} ))
+        # This IAM user allows write access to S3 bucket for sccache
+        export AWS_ACCESS_KEY_ID=AKIAJJZUW4G2ASX5W7KA
+        export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET}
+        git submodule sync && git submodule update --init
+        .jenkins/pytorch/build.sh
+        export PYTORCH_CI_ENV_DIR=/var/lib/jenkins/pytorch-ci-env
+        mkdir -p ${PYTORCH_CI_ENV_DIR}
+        cp -r /var/lib/jenkins/workspace ${PYTORCH_CI_ENV_DIR}/build_workspace  # This copies all source files from build step to the next step
+        cp -r /opt/conda/lib/python${PYTHON_VERSION}/site-packages/torch ${PYTORCH_CI_ENV_DIR}/torch
+        cp -r build/bin ${PYTORCH_CI_ENV_DIR}/cpp_test_bin
+        if [ -d "../cpp-build" ]; then
+          cp -r ../cpp-build ${PYTORCH_CI_ENV_DIR}/cpp-build
+        fi
+  - persist_to_workspace:
+      root: /var/lib/jenkins/pytorch-ci-env
+      paths:
+        - "*"
+
+pytorch_linux_test_defaults: &pytorch_linux_test_defaults
+  machine:
+    image: default
+  steps:
+  - run:
+      name: Prepare workspace
+      command: |
+        sudo mkdir -p /opt/workspace
+        sudo chmod -R 777 /opt/workspace
+  - attach_workspace:
+      at: /opt/workspace
+  - run:
+      name: Build
+      no_output_timeout: "10h"
+      command: |
+        set -x
+        sudo pip install awscli
+        if [ -n "${CUDA_VERSION}" ]; then
+          curl -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+          echo "deb https://nvidia.github.io/libnvidia-container/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+          echo "deb https://nvidia.github.io/nvidia-container-runtime/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+          echo "deb https://nvidia.github.io/nvidia-docker/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+        fi
+        sudo apt-get update
+        sudo apt-get remove linux-image-generic linux-headers-generic linux-generic
+        sudo apt-get install linux-headers-$(uname -r)
+        sudo apt-get install linux-image-generic
+        if [ -n "${CUDA_VERSION}" ]; then
+          wget 'https://s3.amazonaws.com/ossci-linux/nvidia_driver/NVIDIA-Linux-x86_64-396.26.run'
+          sudo /bin/bash ./NVIDIA-Linux-x86_64-396.26.run -s --no-drm
+          sudo apt-get install -y nvidia-docker2
+        fi
+        sudo pkill -SIGHUP dockerd
+        if [ -n "${CUDA_VERSION}" ]; then
+          nvidia-smi
+        fi
+        # This IAM user only allows read-only access to ECR
+        export AWS_ACCESS_KEY_ID=AKIAJ2J6FIG5OSZTQ3IA
+        export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_ONLY}
+        eval $(aws ecr get-login --region us-east-1 --no-include-email)
+        docker pull ${DOCKER_IMAGE}
+        if [ -n "${CUDA_VERSION}" ]; then
+          id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+        else
+          id=$(docker run -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+        fi
+        pwd
+
+        cp -r /opt/workspace/build_workspace/. /home/circleci/project  # This copies all source files from build step to the current step
+
+        echo "declare -x IN_CIRCLECI=1" > /home/circleci/project/env
+        echo "declare -x PYTHON_VERSION=${PYTHON_VERSION}" >> /home/circleci/project/env
+        echo "declare -x SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2" >> /home/circleci/project/env
+        # This IAM user allows write access to S3 bucket for sccache
+        echo "declare -x AWS_ACCESS_KEY_ID=AKIAJJZUW4G2ASX5W7KA" >> /home/circleci/project/env
+        echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET}" >> /home/circleci/project/env
+
+        mkdir -p /home/circleci/project/build
+        cp -r /opt/workspace/cpp_test_bin /home/circleci/project/build/bin
+        docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
+        echo "mkdir -p /opt/conda/lib/python${PYTHON_VERSION}/site-packages" | docker exec -u jenkins -i "$id" bash
+        docker cp "/opt/workspace/torch" "$id:/opt/conda/lib/python${PYTHON_VERSION}/site-packages/torch"
+        if [ -d "/opt/workspace/cpp-build" ]; then
+          docker cp "/opt/workspace/cpp-build" "$id:/var/lib/jenkins/cpp-build"
+        fi
+        if [ -n "${MULTI_GPU}" ]; then
+          (echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace /opt/conda/lib/python${PYTHON_VERSION}/site-packages/torch && cd workspace && .jenkins/pytorch/multigpu-test.sh') | docker exec -u jenkins -i "$id" bash
+        else
+          (echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace /opt/conda/lib/python${PYTHON_VERSION}/site-packages/torch && cd workspace && .jenkins/pytorch/test.sh') | docker exec -u jenkins -i "$id" bash
+        fi
+
+caffe2_linux_build_defaults: &caffe2_linux_build_defaults
+  resource_class: large
+  working_directory: /var/lib/jenkins/workspace
+  steps:
+  - checkout
+  - run:
+      <<: *merge_pull_request_onto_master
+  - run:
+      name: Build
+      no_output_timeout: "10h"
+      command: |
+        export IN_CIRCLECI=1
+        export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+        # This IAM user allows write access to S3 bucket for sccache
+        export AWS_ACCESS_KEY_ID=AKIAJJZUW4G2ASX5W7KA
+        export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET}
+        export SCCACHE_MAX_JOBS=`expr $(nproc) - 1`
+        export MEMORY_LIMIT_MAX_JOBS=8  # the "large" resource class on CircleCI has 32 CPU cores, if we use all of them we'll OOM
+        export MAX_JOBS=$(( ${SCCACHE_MAX_JOBS} > ${MEMORY_LIMIT_MAX_JOBS} ? ${MEMORY_LIMIT_MAX_JOBS} : ${SCCACHE_MAX_JOBS} ))
+
+        set -ex
+
+        # Need to checkout fetch PRs for onnxbot tracking PRs
+        git submodule update --init third_party/onnx || true
+        cd third_party/onnx && git fetch --tags --progress origin +refs/pull/*:refs/remotes/origin/pr/* && cd -
+
+        # Reinitialize submodules
+        git submodule sync && git submodule update --init --recursive
+
+        # Ensure jenkins can write to the ccache root dir.
+        sudo chown jenkins:jenkins "${HOME}/.ccache"
+
+        # Make ccache log to the workspace, so we can archive it after the build
+        mkdir -p build
+        ccache -o log_file=$PWD/build/ccache.log
+
+        # Configure additional cmake arguments
+        cmake_args=()
+        cmake_args+=("$CMAKE_ARGS")
+
+        if [[ $BUILD_ENVIRONMENT == *aten* ]]; then
+          cmake_args+=("-DBUILD_ATEN=ON")
+        fi
+
+        # conda must be added to the path for Anaconda builds (this location must be
+        # the same as that in install_anaconda.sh used to build the docker image)
+        if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
+          export PATH=/opt/conda/bin:$PATH
+          sudo chown -R jenkins:jenkins '/opt/conda'
+        fi
+
+        # Build
+        if test -x ".jenkins/caffe2/build.sh"; then
+          ./.jenkins/caffe2/build.sh ${cmake_args[@]}
+        else
+          ./.jenkins/build.sh ${cmake_args[@]}
+        fi
+
+        # Show sccache stats if it is running
+        if pgrep sccache > /dev/null; then
+          sccache --show-stats
+        fi
+
+        # Copy all necessary binaries to shared workspace
+        export CAFFE2_CI_ENV_DIR=/var/lib/jenkins/caffe2-ci-env
+        mkdir -p ${CAFFE2_CI_ENV_DIR}
+        cp -r /var/lib/jenkins/workspace ${CAFFE2_CI_ENV_DIR}/build_workspace  # This copies all source files from build step to the next step
+        cp -r third_party/onnx ${CAFFE2_CI_ENV_DIR}/onnx
+        if [ -d "/usr/local/caffe2" ]; then
+          cp -r /usr/local/caffe2 ${CAFFE2_CI_ENV_DIR}/caffe2
+        fi
+        if [ -d "/opt/conda" ]; then
+          cp -r /opt/conda ${CAFFE2_CI_ENV_DIR}/conda_env
+        fi
+  - persist_to_workspace:
+      root: /var/lib/jenkins/caffe2-ci-env
+      paths:
+        - "*"
+
+caffe2_linux_test_defaults: &caffe2_linux_test_defaults
+  machine:
+    image: default
+  steps:
+  - run:
+      name: Prepare workspace
+      command: |
+        sudo mkdir -p /opt/workspace
+        sudo chmod -R 777 /opt/workspace
+  - attach_workspace:
+      at: /opt/workspace
+  - run:
+      name: Build
+      no_output_timeout: "10h"
+      command: |
+        set -x
+        sudo pip install awscli
+        if [ -n "${CUDA_VERSION}" ]; then
+          curl -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+          echo "deb https://nvidia.github.io/libnvidia-container/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+          echo "deb https://nvidia.github.io/nvidia-container-runtime/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+          echo "deb https://nvidia.github.io/nvidia-docker/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+        fi
+        sudo apt-get update
+        sudo apt-get remove linux-image-generic linux-headers-generic linux-generic
+        sudo apt-get install linux-headers-$(uname -r)
+        sudo apt-get install linux-image-generic
+        if [ -n "${CUDA_VERSION}" ]; then
+          wget 'https://s3.amazonaws.com/ossci-linux/nvidia_driver/NVIDIA-Linux-x86_64-396.26.run'
+          sudo /bin/bash ./NVIDIA-Linux-x86_64-396.26.run -s --no-drm
+          sudo apt-get install -y nvidia-docker2
+        fi
+        sudo pkill -SIGHUP dockerd
+        if [ -n "${CUDA_VERSION}" ]; then
+          nvidia-smi
+        fi
+        # This IAM user only allows read-only access to ECR
+        export AWS_ACCESS_KEY_ID=AKIAJ2J6FIG5OSZTQ3IA
+        export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_ONLY}
+        eval $(aws ecr get-login --region us-east-1 --no-include-email)
+        docker pull ${DOCKER_IMAGE}
+        if [ -n "${CUDA_VERSION}" ]; then
+          id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+        else
+          id=$(docker run -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+        fi
+        pwd
+        cp -r /opt/workspace/build_workspace/. /home/circleci/project  # This copies all source files from build step to the current step
+        echo "declare -x IN_CIRCLECI=1" > /home/circleci/project/env
+        echo "declare -x SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2" >> /home/circleci/project/env
+        # This IAM user allows write access to S3 bucket for sccache
+        echo "declare -x AWS_ACCESS_KEY_ID=AKIAJJZUW4G2ASX5W7KA" >> /home/circleci/project/env
+        echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET}" >> /home/circleci/project/env
+        echo "declare -x BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" >> /home/circleci/project/env
+
+        # TODO: merge this into Caffe2 build.sh
+        cat >/home/circleci/project/ci_build_script.sh <<EOL
+        # =================== The following code will be executed inside Docker container ===================
+        set -ex
+
+        # libdc1394 (dependency of OpenCV) expects /dev/raw1394 to exist...
+        sudo ln /dev/null /dev/raw1394
+
+        # Hotfix, use hypothesis 3.44.6 on Ubuntu 14.04
+        # See comments on https://github.com/HypothesisWorks/hypothesis-python/commit/eadd62e467d6cee6216e71b391951ec25b4f5830
+        if [[ "$BUILD_ENVIRONMENT" == *ubuntu14.04* ]]; then
+          sudo pip uninstall -y hypothesis
+          # "pip install hypothesis==3.44.6" from official server is unreliable on CircleCI, so we host a copy on S3 instead
+          sudo pip install attrs -f https://s3.amazonaws.com/ossci-linux/wheels/attrs-18.1.0-py2.py3-none-any.whl
+          sudo pip install coverage -f https://s3.amazonaws.com/ossci-linux/wheels/coverage-4.5.1-cp36-cp36m-macosx_10_12_x86_64.whl
+          sudo pip install hypothesis -f https://s3.amazonaws.com/ossci-linux/wheels/hypothesis-3.44.6-py3-none-any.whl
+        fi
+
+        # conda must be added to the path for Anaconda builds (this location must be
+        # the same as that in install_anaconda.sh used to build the docker image)
+        if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
+          export PATH=/opt/conda/bin:$PATH
+        fi
+
+        pip install --user -b /tmp/pip_install_onnx "file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx"
+        pip install --user future
+
+        # Build
+        if test -x ".jenkins/caffe2/test.sh"; then
+          ./.jenkins/caffe2/test.sh
+        else
+          ./.jenkins/test.sh
+        fi
+
+        # Remove benign core dumps.
+        # These are tests for signal handling (including SIGABRT).
+        rm -f ./crash/core.fatal_signal_as.*
+        rm -f ./crash/core.logging_test.*
+        # =================== The above code will be executed inside Docker container ===================
+        EOL
+        chmod +x /home/circleci/project/ci_build_script.sh
+        docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
+        if [ -d "/opt/workspace/caffe2" ]; then
+          echo "mkdir -p /usr/local/caffe2" | docker exec -u jenkins -i "$id" bash
+          docker cp /opt/workspace/caffe2/. "$id:/usr/local/caffe2"
+        fi
+        if [ -d "/opt/workspace/conda_env" ]; then
+          echo "sudo mkdir -p /opt/conda" | docker exec -u jenkins -i "$id" bash
+          docker cp /opt/workspace/conda_env/. "$id:/opt/conda"
+        fi
+        docker cp /opt/workspace/onnx/. "$id:/var/lib/jenkins/workspace/third_party/onnx"
+        (echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh') | docker exec -u jenkins -i "$id" bash
+
+caffe2_macos_build_defaults: &caffe2_macos_build_defaults
+  macos:
+    xcode: "9.0"
+  steps:
+    - checkout
+    - run:
+        <<: *merge_pull_request_onto_master
+    - run:
+        name: Build
+        no_output_timeout: "10h"
+        command: |
+          set -ex
+
+          export IN_CIRCLECI=1
+
+          brew install cmake
+
+          # Reinitialize submodules
+          git submodule sync && git submodule update --init --recursive
+
+          # Reinitialize path (see man page for path_helper(8))
+          eval `/usr/libexec/path_helper -s`
+
+          # Use Homebrew Python if configured to do so
+          if [ "${PYTHON_INSTALLATION}" == "homebrew" ]; then
+            export PATH=/usr/local/opt/python/libexec/bin:/usr/local/bin:$PATH
+          fi
+
+          pip install numpy
+
+          # Install Anaconda if we need to
+          if [ -n "${CAFFE2_USE_ANACONDA}" ]; then
+            rm -rf ${TMPDIR}/anaconda
+            curl -o ${TMPDIR}/anaconda.sh "https://repo.continuum.io/archive/Anaconda${ANACONDA_VERSION}-5.0.1-MacOSX-x86_64.sh"
+            /bin/bash ${TMPDIR}/anaconda.sh -b -p ${TMPDIR}/anaconda
+            rm -f ${TMPDIR}/anaconda.sh
+            export PATH="${TMPDIR}/anaconda/bin:${PATH}"
+            source ${TMPDIR}/anaconda/bin/activate
+          fi
+
+          # Install sccache
+          sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+          sudo chmod +x /usr/local/bin/sccache
+          export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+
+          # This IAM user allows write access to S3 bucket for sccache
+          export AWS_ACCESS_KEY_ID=AKIAJJZUW4G2ASX5W7KA
+          export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET}
+
+          export SCCACHE_BIN=${PWD}/sccache_bin
+          mkdir -p ${SCCACHE_BIN}
+          if which sccache > /dev/null; then
+            printf "#!/bin/sh\nexec sccache $(which clang++) \$*" > "${SCCACHE_BIN}/clang++"
+            chmod a+x "${SCCACHE_BIN}/clang++"
+
+            printf "#!/bin/sh\nexec sccache $(which clang) \$*" > "${SCCACHE_BIN}/clang"
+            chmod a+x "${SCCACHE_BIN}/clang"
+
+            export PATH="${SCCACHE_BIN}:$PATH"
+          fi
+
+          # Build
+          if [ "${BUILD_IOS:-0}" -eq 1 ]; then
+            scripts/build_ios.sh
+          elif [ -n "${CAFFE2_USE_ANACONDA}" ]; then
+            # All conda build logic should be in scripts/build_anaconda.sh
+            scripts/build_anaconda.sh
+          else
+            scripts/build_local.sh
+          fi
+
+          # Show sccache stats if it is running
+          if which sccache > /dev/null; then
+            sccache --show-stats
+          fi
+
+version: 2
+jobs:
+  pytorch_linux_trusty_py2_7_9_build_test:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-py2.7.9:238
+        <<: *docker_config_defaults
+    <<: *pytorch_linux_cpu_build_test_defaults
+
+  pytorch_linux_trusty_py2_7_build_test:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-py2.7:238
+        <<: *docker_config_defaults
+    <<: *pytorch_linux_cpu_build_test_defaults
+
+  pytorch_linux_trusty_py3_5_build_test:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-py3.5:238
+        <<: *docker_config_defaults
+    <<: *pytorch_linux_cpu_build_test_defaults
+
+  pytorch_linux_trusty_py3_6_gcc4_8_build_test:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-py3.6-gcc4.8:238
+        <<: *docker_config_defaults
+    <<: *pytorch_linux_cpu_build_test_defaults
+
+  pytorch_linux_trusty_py3_6_gcc5_4_build_test:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-py3.6-gcc5.4:238
+        <<: *docker_config_defaults
+    <<: *pytorch_linux_cpu_build_test_defaults
+
+  pytorch_linux_trusty_py3_6_gcc7_build_test:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-py3.6-gcc7:238
+        <<: *docker_config_defaults
+    <<: *pytorch_linux_cpu_build_test_defaults
+
+  pytorch_linux_trusty_pynightly_build_test:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:238
+        <<: *docker_config_defaults
+    <<: *pytorch_linux_cpu_build_test_defaults
+
+  pytorch_linux_xenial_py3_clang5_asan_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:238
+        <<: *docker_config_defaults
+    environment:
+      PYTHON_VERSION: "3.6"
+    <<: *pytorch_linux_build_defaults
+
+  pytorch_linux_xenial_py3_clang5_asan_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:238"
+      PYTHON_VERSION: "3.6"
+    resource_class: large
+    <<: *pytorch_linux_test_defaults
+
+  pytorch_linux_xenial_cuda8_cudnn6_py3_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda8-cudnn6-py3:238
+        <<: *docker_config_defaults
+    environment:
+      PYTHON_VERSION: "3.6"
+      CUDA_VERSION: "8"
+    <<: *pytorch_linux_build_defaults
+
+  pytorch_linux_xenial_cuda8_cudnn6_py3_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda8-cudnn6-py3:238"
+      PYTHON_VERSION: "3.6"
+      CUDA_VERSION: "8"
+    resource_class: gpu.medium
+    <<: *pytorch_linux_test_defaults
+
+  pytorch_linux_xenial_cuda8_cudnn6_py3_multigpu_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda8-cudnn6-py3:238"
+      PYTHON_VERSION: "3.6"
+      CUDA_VERSION: "8"
+      MULTI_GPU: "1"
+    resource_class: gpu.large
+    <<: *pytorch_linux_test_defaults
+
+  pytorch_linux_xenial_cuda9_cudnn7_py2_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py2:238
+        <<: *docker_config_defaults
+    environment:
+      PYTHON_VERSION: "2.7"
+      CUDA_VERSION: "9"
+    <<: *pytorch_linux_build_defaults
+
+  pytorch_linux_xenial_cuda9_cudnn7_py2_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py2:238"
+      PYTHON_VERSION: "2.7"
+      CUDA_VERSION: "9"
+    resource_class: gpu.medium
+    <<: *pytorch_linux_test_defaults
+
+  pytorch_linux_xenial_cuda9_cudnn7_py3_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3:238
+        <<: *docker_config_defaults
+    environment:
+      PYTHON_VERSION: "3.6"
+      CUDA_VERSION: "9"
+    <<: *pytorch_linux_build_defaults
+
+  pytorch_linux_xenial_cuda9_cudnn7_py3_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3:238"
+      PYTHON_VERSION: "3.6"
+      CUDA_VERSION: "9"
+    resource_class: gpu.medium
+    <<: *pytorch_linux_test_defaults
+
+  pytorch_linux_xenial_cuda9_2_cudnn7_py3_gcc7_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7:238
+        <<: *docker_config_defaults
+    environment:
+      PYTHON_VERSION: "3.6"
+      CUDA_VERSION: "9.2"
+    <<: *pytorch_linux_build_defaults
+
+  pytorch_linux_xenial_cuda9_2_cudnn7_py3_gcc7_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7:238"
+      PYTHON_VERSION: "3.6"
+      CUDA_VERSION: "9.2"
+    resource_class: gpu.medium
+    <<: *pytorch_linux_test_defaults
+
+  pytorch_macos_10_13_py3_build:
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          <<: *merge_pull_request_onto_master
+      - run:
+          name: Build
+          environment:
+            BUILD_ENVIRONMENT: pytorch-macos-10.13-py3
+          no_output_timeout: "10h"
+          command: |
+            set -ex
+
+            export IN_CIRCLECI=1
+
+            # Install sccache
+            sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+            sudo chmod +x /usr/local/bin/sccache
+
+            export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+            # This IAM user allows write access to S3 bucket for sccache
+            export AWS_ACCESS_KEY_ID=AKIAJJZUW4G2ASX5W7KA
+            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET}
+
+            git submodule sync && git submodule update --init
+            chmod a+x .jenkins/pytorch/macos-build.sh
+            .jenkins/pytorch/macos-build.sh
+
+            # TODO: need to share source files from build to test, when macOS builds are enabled
+
+      - persist_to_workspace:
+          root: /Users/distiller/pytorch-ci-env
+          paths:
+            - "*"
+
+  pytorch_macos_10_13_py3_test:
+    macos:
+      xcode: "9.0"
+    steps:
+      - run:
+          name: Prepare workspace
+          command: |
+            sudo mkdir -p /Users/distiller/pytorch-ci-env
+            sudo chmod -R 777 /Users/distiller/pytorch-ci-env
+      - attach_workspace:
+          at: /Users/distiller/pytorch-ci-env
+      - run:
+          name: Build
+          environment:
+            BUILD_ENVIRONMENT: pytorch-macos-10.13-py3
+          no_output_timeout: "10h"
+          command: |
+            # TODO: need to share source files from build to test, when macOS builds are enabled
+            set -ex
+            export IN_CIRCLECI=1
+            chmod a+x .jenkins/pytorch/macos-test.sh
+            .jenkins/pytorch/macos-test.sh
+
+  pytorch_macos_10_13_cuda9_2_cudnn7_py3_build:
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          <<: *merge_pull_request_onto_master
+      - run:
+          name: Build
+          environment:
+            JOB_BASE_NAME: pytorch-macos-10.13-cuda9.2-cudnn7-py3-build
+            BUILD_ENVIRONMENT: pytorch-macos-10.13-cuda9.2-cudnn7-py3
+          no_output_timeout: "10h"
+          command: |
+            set -ex
+
+            export IN_CIRCLECI=1
+
+            # Install CUDA 9.2
+            sudo rm -rf ~/cuda_9.2.64_mac_installer.app || true
+            curl https://s3.amazonaws.com/ossci-macos/cuda_9.2.64_mac_installer.zip -o ~/cuda_9.2.64_mac_installer.zip
+            unzip ~/cuda_9.2.64_mac_installer.zip -d ~/
+            sudo ~/cuda_9.2.64_mac_installer.app/Contents/MacOS/CUDAMacOSXInstaller --accept-eula --no-window
+            sudo cp /usr/local/cuda/lib/libcuda.dylib /Developer/NVIDIA/CUDA-9.2/lib/libcuda.dylib
+            sudo rm -rf /usr/local/cuda || true
+
+            # Install cuDNN 7.1 for CUDA 9.2
+            curl https://s3.amazonaws.com/ossci-macos/cudnn-9.2-osx-x64-v7.1.tgz -o ~/cudnn-9.2-osx-x64-v7.1.tgz
+            rm -rf ~/cudnn-9.2-osx-x64-v7.1 && mkdir ~/cudnn-9.2-osx-x64-v7.1
+            tar -xzvf ~/cudnn-9.2-osx-x64-v7.1.tgz -C ~/cudnn-9.2-osx-x64-v7.1
+            sudo cp ~/cudnn-9.2-osx-x64-v7.1/cuda/include/cudnn.h /Developer/NVIDIA/CUDA-9.2/include/
+            sudo cp ~/cudnn-9.2-osx-x64-v7.1/cuda/lib/libcudnn* /Developer/NVIDIA/CUDA-9.2/lib/
+            sudo chmod a+r /Developer/NVIDIA/CUDA-9.2/include/cudnn.h /Developer/NVIDIA/CUDA-9.2/lib/libcudnn*
+
+            # Install sccache
+            sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+            sudo chmod +x /usr/local/bin/sccache
+            export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+            # This IAM user allows write access to S3 bucket for sccache
+            export AWS_ACCESS_KEY_ID=AKIAJJZUW4G2ASX5W7KA
+            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET}
+
+            git submodule sync && git submodule update --init
+            chmod a+x .jenkins/pytorch/macos-build.sh
+            .jenkins/pytorch/macos-build.sh
+
+  caffe2_py2_cuda8_0_cudnn6_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda8.0-cudnn6-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      CUDA_VERSION: "8"
+      BUILD_ENVIRONMENT: "py2-cuda8.0-cudnn6-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_cuda8_0_cudnn6_ubuntu16_04_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda8.0-cudnn6-ubuntu16.04:190"
+      CUDA_VERSION: "8"
+      BUILD_ENVIRONMENT: "py2-cuda8.0-cudnn6-ubuntu16.04"
+    resource_class: gpu.medium
+    <<: *caffe2_linux_test_defaults
+
+  caffe2_py2_cuda9_0_cudnn7_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      CUDA_VERSION: "9"
+      BUILD_ENVIRONMENT: "py2-cuda9.0-cudnn7-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_cuda9_0_cudnn7_ubuntu16_04_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:190"
+      CUDA_VERSION: "9"
+      BUILD_ENVIRONMENT: "py2-cuda9.0-cudnn7-ubuntu16.04"
+    resource_class: gpu.medium
+    <<: *caffe2_linux_test_defaults
+
+  caffe2_py2_cuda9_0_cudnn7_aten_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      CUDA_VERSION: "9"
+      BUILD_ENVIRONMENT: "py2-cuda9.0-cudnn7-aten-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_cuda9_0_cudnn7_aten_ubuntu16_04_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:190"
+      CUDA_VERSION: "9"
+      BUILD_ENVIRONMENT: "py2-cuda9.0-cudnn7-aten-ubuntu16.04"
+    resource_class: gpu.medium
+    <<: *caffe2_linux_test_defaults
+
+  caffe2_py2_cuda9_1_cudnn7_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.1-cudnn7-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      CUDA_VERSION: "9.1"
+      BUILD_ENVIRONMENT: "py2-cuda9.1-cudnn7-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_cuda9_1_cudnn7_ubuntu16_04_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.1-cudnn7-ubuntu16.04:190"
+      CUDA_VERSION: "9.1"
+      BUILD_ENVIRONMENT: "py2-cuda9.1-cudnn7-ubuntu16.04"
+    resource_class: gpu.medium
+    <<: *caffe2_linux_test_defaults
+
+  caffe2_py2_mkl_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-mkl-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_mkl_ubuntu16_04_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:190"
+      BUILD_ENVIRONMENT: "py2-mkl-ubuntu16.04"
+    resource_class: large
+    <<: *caffe2_linux_test_defaults
+
+  caffe2_py2_gcc4_8_ubuntu14_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc4.8-ubuntu14.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-gcc4.8-ubuntu14.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_gcc4_8_ubuntu14_04_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc4.8-ubuntu14.04:190"
+      BUILD_ENVIRONMENT: "py2-gcc4.8-ubuntu14.04"
+    resource_class: large
+    <<: *caffe2_linux_test_defaults
+
+  caffe2_onnx_py2_gcc5_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "onnx-py2-gcc5-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_onnx_py2_gcc5_ubuntu16_04_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:190"
+      BUILD_ENVIRONMENT: "onnx-py2-gcc5-ubuntu16.04"
+    resource_class: large
+    <<: *caffe2_linux_test_defaults
+
+  caffe2_conda2_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/conda2-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "conda2-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_conda2_ubuntu16_04_test:
+    environment:
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/conda2-ubuntu16.04:190"
+      BUILD_ENVIRONMENT: "conda2-ubuntu16.04"
+    resource_class: large
+    <<: *caffe2_linux_test_defaults
+
+  caffe2_py2_cuda8_0_cudnn7_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda8.0-cudnn7-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-cuda8.0-cudnn7-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_gcc4_9_ubuntu14_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc4.9-ubuntu14.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-gcc4.9-ubuntu14.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_clang3_8_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.8-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-clang3.8-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_clang3_9_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.9-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-clang3.9-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_gcc6_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc6-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-gcc6-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_gcc7_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc7-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-gcc7-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_cuda8_0_cudnn7_aten_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda8.0-cudnn7-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-cuda8.0-cudnn7-aten-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_android_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-android-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-android-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_conda3_cuda9_0_cudnn7_ubuntu16_04_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/conda3-cuda9.0-cudnn7-ubuntu16.04:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "conda3-cuda9.0-cudnn7-ubuntu16.04"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_cuda9_0_cudnn7_centos7_build:
+    docker:
+      - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-centos7:190
+        <<: *docker_config_defaults
+    environment:
+      BUILD_ENVIRONMENT: "py2-cuda9.0-cudnn7-centos7"
+    <<: *caffe2_linux_build_defaults
+
+  caffe2_py2_ios_macos10_13_build:
+    environment:
+      BUILD_IOS: "1"
+      PYTHON_INSTALLATION: "system"
+      PYTHON_VERSION: "2"
+    <<: *caffe2_macos_build_defaults
+
+  caffe2_py2_system_macos10_13_build:
+    environment:
+      PYTHON_INSTALLATION: "system"
+      PYTHON_VERSION: "2"
+    <<: *caffe2_macos_build_defaults
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      # - pytorch_linux_trusty_py2_7_9_build_test
+      # - pytorch_linux_trusty_py2_7_build_test
+      # - pytorch_linux_trusty_py3_5_build_test
+      # - pytorch_linux_trusty_py3_6_gcc4_8_build_test
+      # - pytorch_linux_trusty_py3_6_gcc5_4_build_test
+      # - pytorch_linux_trusty_py3_6_gcc7_build_test
+      # - pytorch_linux_trusty_pynightly_build_test
+      # - pytorch_linux_xenial_py3_clang5_asan_build
+      # - pytorch_linux_xenial_py3_clang5_asan_test:
+      #     requires:
+      #       - pytorch_linux_xenial_py3_clang5_asan_build
+      # - pytorch_linux_xenial_cuda8_cudnn6_py3_build
+      # - pytorch_linux_xenial_cuda8_cudnn6_py3_test:
+      #     requires:
+      #       - pytorch_linux_xenial_cuda8_cudnn6_py3_build
+      # - pytorch_linux_xenial_cuda8_cudnn6_py3_multigpu_test:
+      #     requires:
+      #       - pytorch_linux_xenial_cuda8_cudnn6_py3_build
+      # - pytorch_linux_xenial_cuda9_cudnn7_py2_build
+      # - pytorch_linux_xenial_cuda9_cudnn7_py2_test:
+      #     requires:
+      #       - pytorch_linux_xenial_cuda9_cudnn7_py2_build
+      # - pytorch_linux_xenial_cuda9_cudnn7_py3_build
+      # - pytorch_linux_xenial_cuda9_cudnn7_py3_test:
+      #     requires:
+      #       - pytorch_linux_xenial_cuda9_cudnn7_py3_build
+      # - pytorch_linux_xenial_cuda9_2_cudnn7_py3_gcc7_build
+      # - pytorch_linux_xenial_cuda9_2_cudnn7_py3_gcc7_test:
+      #     requires:
+      #       - pytorch_linux_xenial_cuda9_2_cudnn7_py3_gcc7_build
+
+      # - pytorch_macos_10_13_py3_build
+      # - pytorch_macos_10_13_py3_test:
+      #     requires:
+      #       - pytorch_macos_10_13_py3_build
+      # - pytorch_macos_10_13_cuda9_2_cudnn7_py3_build
+
+      # - caffe2_py2_cuda8_0_cudnn6_ubuntu16_04_build
+      # - caffe2_py2_cuda8_0_cudnn6_ubuntu16_04_test:
+      #     requires:
+      #       - caffe2_py2_cuda8_0_cudnn6_ubuntu16_04_build
+      # - caffe2_py2_cuda9_0_cudnn7_ubuntu16_04_build
+      # - caffe2_py2_cuda9_0_cudnn7_ubuntu16_04_test:
+      #     requires:
+      #       - caffe2_py2_cuda9_0_cudnn7_ubuntu16_04_build
+      # - caffe2_py2_cuda9_0_cudnn7_aten_ubuntu16_04_build
+      # - caffe2_py2_cuda9_0_cudnn7_aten_ubuntu16_04_test:
+      #     requires:
+      #       - caffe2_py2_cuda9_0_cudnn7_aten_ubuntu16_04_build
+      # - caffe2_py2_mkl_ubuntu16_04_build
+      # - caffe2_py2_mkl_ubuntu16_04_test:
+      #     requires:
+      #       - caffe2_py2_mkl_ubuntu16_04_build
+      # - caffe2_py2_cuda9_1_cudnn7_ubuntu16_04_build
+      # - caffe2_py2_cuda9_1_cudnn7_ubuntu16_04_test:
+      #     requires:
+      #       - caffe2_py2_cuda9_1_cudnn7_ubuntu16_04_build
+      # - caffe2_py2_gcc4_8_ubuntu14_04_build
+      # - caffe2_py2_gcc4_8_ubuntu14_04_test:
+      #     requires:
+      #       - caffe2_py2_gcc4_8_ubuntu14_04_build
+      # - caffe2_onnx_py2_gcc5_ubuntu16_04_build
+      # - caffe2_onnx_py2_gcc5_ubuntu16_04_test:
+      #     requires:
+      #       - caffe2_onnx_py2_gcc5_ubuntu16_04_build
+      # - caffe2_conda2_ubuntu16_04_build
+      # - caffe2_conda2_ubuntu16_04_test:
+      #     requires:
+      #       - caffe2_conda2_ubuntu16_04_build
+      # - caffe2_py2_cuda8_0_cudnn7_ubuntu16_04_build
+      # - caffe2_py2_gcc4_9_ubuntu14_04_build
+      # - caffe2_py2_clang3_8_ubuntu16_04_build
+      # - caffe2_py2_clang3_9_ubuntu16_04_build
+      # - caffe2_py2_gcc6_ubuntu16_04_build
+      # - caffe2_py2_gcc7_ubuntu16_04_build
+      # - caffe2_py2_cuda8_0_cudnn7_aten_ubuntu16_04_build
+      # - caffe2_py2_android_ubuntu16_04_build
+      # - caffe2_conda3_cuda9_0_cudnn7_ubuntu16_04_build
+      # - caffe2_py2_cuda9_0_cudnn7_centos7_build
+
+      # - caffe2_py2_ios_macos10_13_build
+      # - caffe2_py2_system_macos10_13_build

.clang-format

@@ -37,7 +37,7 @@ BreakAfterJavaFieldAnnotations: false
 BreakStringLiterals: false
 ColumnLimit:     80
 CommentPragmas:  '^ IWYU pragma:'
-CompactNamespaces: true
+CompactNamespaces: false
 ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
@@ -68,7 +68,7 @@ PenaltyBreakComment: 300
 PenaltyBreakFirstLessLess: 120
 PenaltyBreakString: 1000
 PenaltyExcessCharacter: 1000000
-PenaltyReturnTypeOnItsOwnLine: 200
+PenaltyReturnTypeOnItsOwnLine: 2000000
 PointerAlignment: Left
 ReflowComments:  true
 SortIncludes:    true

.clang-tidy

@@ -0,0 +1,51 @@
+---
+# NOTE: there must be no spaces before the '-', so put the comma first.
+Checks: '
+  *
+  ,clang-analyzer-*
+  ,modernize-*
+  ,-cert-dcl21-cpp
+  ,-cert-err58-cpp
+  ,-cert-err60-cpp
+  ,-clang-diagnostic-*
+  ,-cppcoreguidelines-owning-memory
+  ,-cppcoreguidelines-pro-bounds-array-to-pointer-decay
+  ,-cppcoreguidelines-pro-bounds-constant-array-index
+  ,-cppcoreguidelines-pro-type-member-init
+  ,-cppcoreguidelines-pro-type-static-cast-downcast
+  ,-cppcoreguidelines-pro-type-union-access
+  ,-cppcoreguidelines-pro-type-vararg
+  ,-cppcoreguidelines-special-member-functions
+  ,-fuchsia-*
+  ,-google-build-using-namespace
+  ,-google-default-arguments
+  ,-google-explicit-constructor
+  ,-google-readability-braces-around-statements
+  ,-google-readability-namespace-comments
+  ,-google-readability-todo
+  ,-google-runtime-references
+  ,-google-runtime-references
+  ,-hicpp-braces-around-statements
+  ,-hicpp-explicit-conversions
+  ,-hicpp-member-init
+  ,-hicpp-no-array-decay
+  ,-hicpp-signed-bitwise
+  ,-hicpp-special-member-functions
+  ,-hicpp-vararg
+  ,-llvm-header-guard
+  ,-llvm-include-order
+  ,-llvm-namespace-comment
+  ,-misc-unused-parameters
+  ,-modernize-make-unique
+  ,-modernize-use-default-member-init
+  ,-performance-unnecessary-value-param
+  ,-readability-braces-around-statements
+  ,-readability-else-after-return
+  ,-readability-implicit-bool-conversion
+  ,-readability-named-parameter
+  '
+WarningsAsErrors: ''
+HeaderFilterRegex: 'torch/csrc/'
+AnalyzeTemporaryDtors: false
+CheckOptions:
+...

.gitattributes

@@ -0,0 +1 @@
+*.bat	text eol=crlf

.github/ISSUE_TEMPLATE.md

@@ -1,24 +1,38 @@
-PyTorch GitHub Issues Guidelines
---------------------------------
-
-We like to limit our issues to bug reports and feature requests. If you have a question or would like help and support, please visit our forums: https://discuss.pytorch.org/
+If you have a question or would like help and support, please ask at our
+[forums](https://discuss.pytorch.org/).
 
 If you are submitting a feature request, please preface the title with [feature request].
+If you are submitting a bug report, please fill in the following details.
+
+## Issue description
+
+Provide a short description.
+
+## Code example
+
+Please try to provide a minimal example to repro the bug.
+Error messages and stack traces are also helpful.
+
+## System Info
+Please copy and paste the output from our
+[environment collection script](https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py)
+(or fill out the checklist below manually).
+
+You can get the script and run it with:
+```
+wget https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py
+# For security purposes, please check the contents of collect_env.py before running it.
+python collect_env.py
+```
 
-When submitting a bug report, please include the following information (where relevant):
 - PyTorch or Caffe2:
+- How you installed PyTorch (conda, pip, source):
+- Build command you used (if compiling from source):
 - OS:
 - PyTorch version:
-- How you installed PyTorch (conda, pip, source):
 - Python version:
 - CUDA/cuDNN version:
 - GPU models and configuration:
 - GCC version (if compiling from source):
 - CMake version:
-- Build command you used (if compiling from source):
 - Versions of any other relevant libraries:
-
-In addition, including the following information will also be very helpful for us to diagnose the problem:
-- A script to reproduce the bug. Please try to provide as minimal of a test case as possible.
-- Error messages and/or stack traces of the bug
-- Context around what you are trying to do

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,49 @@
+---
+name: "\U0001F41B Bug Report"
+about: Submit a bug report to help us improve PyTorch
+
+---
+
+## 🐛 Bug
+
+<!-- A clear and concise description of what the bug is. -->
+
+## To Reproduce
+
+Steps to reproduce the behavior:
+
+1.
+1.
+1.
+
+<!-- If you have a code sample, error messages, stack traces, please provide it here as well -->
+
+## Expected behavior
+
+<!-- A clear and concise description of what you expected to happen. -->
+
+## Environment
+
+Please copy and paste the output from our
+[environment collection script](https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py)
+(or fill out the checklist below manually).
+
+You can get the script and run it with:
+```
+wget https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py
+# For security purposes, please check the contents of collect_env.py before running it.
+python collect_env.py
+```
+
+ - PyTorch Version (e.g., 1.0):
+ - OS (e.g., Linux):
+ - How you installed PyTorch (`conda`, `pip`, source):
+ - Build command you used (if compiling from source):
+ - Python version:
+ - CUDA/cuDNN version:
+ - GPU models and configuration:
+ - Any other relevant information:
+
+## Additional context
+
+<!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F4DA Documentation"
+about: Report an issue related to https://pytorch.org/docs
+
+---
+
+## 📚 Documentation
+
+<!-- A clear and concise description of what content in https://pytorch.org/docs is an issue. If this has to do with the general https://pytorch.org website, please file an issue at https://github.com/pytorch/pytorch.github.io/issues/new/choose instead. If this has to do with https://pytorch.org/tutorials, please file an issue at https://github.com/pytorch/tutorials/issues/new -->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,24 @@
+---
+name: "\U0001F680Feature Request"
+about: Submit a proposal/request for a new PyTorch feature
+
+---
+
+## 🚀 Feature
+<!-- A clear and concise description of the feature proposal -->
+
+## Motivation
+
+<!-- Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too -->
+
+## Pitch
+
+<!-- A clear and concise description of what you want to happen. -->
+
+## Alternatives
+
+<!-- A clear and concise description of any alternative solutions or features you've considered, if any. -->
+
+## Additional context
+
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/questions-help-support.md

@@ -0,0 +1,13 @@
+---
+name: "❓Questions/Help/Support"
+about: Do you need support? We have resources.
+
+---
+
+## ❓ Questions and Help
+
+### Please note that this issue tracker is not a help form and this issue will be closed.
+
+We have a set of [listed resources available on the website](https://pytorch.org/resources). Our primary means of support is our discussion forum:
+
+- [Discussion Forum](https://discuss.pytorch.org/)

.gitignore

@@ -8,53 +8,67 @@
 
 ## PyTorch
 
-build/
-dist/
-torch.egg-info/
-*/**/__pycache__
-aten/build/
-aten/src/ATen/Config.h
-third_party/build/
-torch/version.py
-torch/csrc/generic/TensorMethods.cpp
-torch/lib/*.so*
-torch/lib/*.a*
-torch/lib/*.dll*
-torch/lib/*.lib
-torch/lib/*.dylib*
-torch/lib/*.h
-torch/lib/build
-torch/lib/tmp_install
-torch/lib/include
-torch/lib/torch_shm_manager
-torch/csrc/jit/generated/*
-torch/csrc/autograd/generated/*
-torch/csrc/cudnn/cuDNN.cpp
-torch/csrc/nn/THNN.cwrap
-torch/csrc/nn/THNN.cpp
-torch/csrc/nn/THCUNN.cwrap
-torch/csrc/nn/THCUNN.cpp
-torch/csrc/nn/THNN_generic.cwrap
-torch/csrc/nn/THNN_generic.cpp
-torch/csrc/nn/THNN_generic.h
-torch/csrc/generated
-docs/src/**/*
-test/data/legacy_modules.t7
-test/data/gpu_tensors.pt
-test/htmlcov
-test/.coverage
+.mypy_cache
 */*.pyc
+*/*.so*
+*/**/__pycache__
+*/**/*.dylib*
 */**/*.pyc
+*/**/*.pyd
+*/**/*.so*
 */**/**/*.pyc
 */**/**/**/*.pyc
 */**/**/**/**/*.pyc
-*/*.so*
-*/**/*.so*
-*/**/*.dylib*
-*/**/*.pyd
+aten/build/
+aten/src/ATen/Config.h
+aten/src/ATen/cuda/CUDAConfig.h
+build/
+dist/
+docs/src/**/*
+docs/cpp/build
+docs/cpp/source/api
+test/.coverage
+test/cpp/api/mnist
+test/custom_operator/model.pt
+test/data/gpu_tensors.pt
+test/data/legacy_modules.t7
 test/data/legacy_serialized.pt
 test/data/linear.pt
-.mypy_cache
+test/htmlcov
+test/cpp_extensions/install/
+third_party/build/
+tools/shared/_utils_internal.py
+torch.egg-info/
+torch/csrc/autograd/generated/*
+torch/csrc/cudnn/cuDNN.cpp
+torch/csrc/generated
+torch/csrc/generic/TensorMethods.cpp
+torch/csrc/jit/generated/*
+torch/csrc/jit/fusers/Config.h
+torch/csrc/nn/THCUNN.cpp
+torch/csrc/nn/THCUNN.cwrap
+torch/csrc/nn/THNN_generic.cpp
+torch/csrc/nn/THNN_generic.cwrap
+torch/csrc/nn/THNN_generic.h
+torch/csrc/nn/THNN.cpp
+torch/csrc/nn/THNN.cwrap
+torch/lib/*.a*
+torch/lib/*.dll*
+torch/lib/*.exe*
+torch/lib/*.dylib*
+torch/lib/*.h
+torch/lib/*.lib
+torch/lib/*.so*
+torch/lib/build
+torch/lib/cmake
+torch/lib/include
+torch/lib/pkgconfig
+torch/lib/protoc
+torch/lib/tmp_install
+torch/lib/torch_shm_manager
+torch/lib/python*
+torch/share/
+torch/version.py
 
 # IPython notebook checkpoints
 .ipynb_checkpoints
@@ -63,6 +77,7 @@ test/data/linear.pt
 *.swn
 *.swo
 *.swp
+*.swm
 *~
 
 # macOS dir files
@@ -133,10 +148,6 @@ docs/source/scripts/activation_images/
 # PyCharm files
 .idea
 
-# Visual Studio Code files
-.vscode
-.vs
-
 # OSX dir files
 .DS_Store
 
@@ -147,7 +158,7 @@ build
 build_host_protoc
 build_android
 build_ios
-build_*
+/build_*
 .build_debug/*
 .build_release/*
 distribute/*
@@ -187,3 +198,11 @@ caffe2.egg-info
 
 # Atom/Watchman required file
 .watchmanconfig
+
+# BEGIN NOT-CLEAN-FILES (setup.py handles this marker. Do not change.)
+#
+# Below files are not deleted by "setup.py clean".
+
+# Visual Studio Code files
+.vscode
+.vs

.gitmodules

@@ -1,28 +1,15 @@
-[submodule "aten/src/ATen/cpu/cpuinfo"]
-	path = aten/src/ATen/cpu/cpuinfo
-	url = https://github.com/Maratyszcza/cpuinfo
-[submodule "aten/src/ATen/cpu/tbb/tbb_remote"]
-	path = aten/src/ATen/cpu/tbb/tbb_remote
-	url = https://github.com/01org/tbb
-	branch = tbb_2018
-[submodule "aten/src/ATen/utils/catch"]
-	path = aten/src/ATen/utils/catch
+[submodule "third_party/catch"]
+	path = third_party/catch
 	url = https://github.com/catchorg/Catch2.git
-[submodule "third_party/nanopb"]
-	path = third_party/nanopb
-	url = https://github.com/nanopb/nanopb.git
 [submodule "third_party/pybind11"]
 	path = third_party/pybind11
 	url = https://github.com/pybind/pybind11.git
-[submodule "third_party/nccl"]
-	path = third_party/nccl
-	url = https://github.com/nvidia/nccl.git
 [submodule "third_party/cub"]
 	path = third_party/cub
 	url = https://github.com/NVlabs/cub.git
 [submodule "third_party/eigen"]
 	path = third_party/eigen
-	url = https://github.com/RLovelett/eigen.git
+	url = https://github.com/eigenteam/eigen-git-mirror.git
 [submodule "third_party/googletest"]
 	path = third_party/googletest
 	url = https://github.com/google/googletest.git
@@ -77,3 +64,15 @@
 [submodule "third_party/onnx"]
 	path = third_party/onnx
 	url = https://github.com/onnx/onnx.git
+[submodule "third_party/cereal"]
+	path = third_party/cereal
+	url = https://github.com/USCiLab/cereal
+[submodule "third_party/onnx-tensorrt"]
+	path = third_party/onnx-tensorrt
+	url = https://github.com/onnx/onnx-tensorrt
+[submodule "third_party/sleef"]
+	path = third_party/sleef
+	url = https://github.com/shibatch/sleef
+[submodule "third_party/ideep"]
+	path = third_party/ideep
+	url = https://github.com/intel/ideep

.jenkins/caffe2/build.sh

@@ -2,30 +2,51 @@
 
 set -ex
 
+pip install --user --no-cache-dir hypothesis==3.59.0
+
+
+# The INSTALL_PREFIX here must match up with test.sh
+INSTALL_PREFIX="/usr/local/caffe2"
 LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
+CMAKE_ARGS=()
+SCCACHE="$(which sccache)"
 
-# Setup sccache if SCCACHE_BUCKET is set
-if [ -n "${SCCACHE_BUCKET}" ]; then
-  mkdir -p ./sccache
+if [ "$(which gcc)" != "/root/sccache/gcc" ]; then
+  # Setup SCCACHE
+  ###############################################################################
+  # Setup sccache if SCCACHE_BUCKET is set
+  if [ -n "${SCCACHE_BUCKET}" ]; then
+    mkdir -p ./sccache
 
-  SCCACHE="$(which sccache)"
-  if [ -z "${SCCACHE}" ]; then
-    echo "Unable to find sccache..."
-    exit 1
+    SCCACHE="$(which sccache)"
+    if [ -z "${SCCACHE}" ]; then
+      echo "Unable to find sccache..."
+      exit 1
+    fi
+
+    # Setup wrapper scripts
+    for compiler in cc c++ gcc g++ x86_64-linux-gnu-gcc; do
+      (
+        echo "#!/bin/sh"
+        echo "exec $SCCACHE $(which $compiler) \"\$@\""
+      ) > "./sccache/$compiler"
+      chmod +x "./sccache/$compiler"
+    done
+
+    if [[ "${BUILD_ENVIRONMENT}" == *-cuda* ]]; then
+      (
+        echo "#!/bin/sh"
+        echo "exec $SCCACHE $(which nvcc) \"\$@\""
+      ) > "./sccache/nvcc"
+      chmod +x "./sccache/nvcc"
+    fi
+
+    export CACHE_WRAPPER_DIR="$PWD/sccache"
+
+    # CMake must find these wrapper scripts
+    export PATH="$CACHE_WRAPPER_DIR:$PATH"
   fi
-
-  # Setup wrapper scripts
-  for compiler in cc c++ gcc g++ x86_64-linux-gnu-gcc; do
-    (
-      echo "#!/bin/sh"
-      echo "exec $SCCACHE $(which $compiler) \"\$@\""
-    ) > "./sccache/$compiler"
-    chmod +x "./sccache/$compiler"
-  done
-
-  # CMake must find these wrapper scripts
-  export PATH="$PWD/sccache:$PATH"
 fi
 
 # Setup ccache if configured to use it (and not sccache)
@@ -36,30 +57,55 @@ if [ -z "${SCCACHE}" ] && which ccache > /dev/null; then
   ln -sf "$(which ccache)" ./ccache/gcc
   ln -sf "$(which ccache)" ./ccache/g++
   ln -sf "$(which ccache)" ./ccache/x86_64-linux-gnu-gcc
-  export CCACHE_WRAPPER_DIR="$PWD/ccache"
-  export PATH="$CCACHE_WRAPPER_DIR:$PATH"
+  if [[ "${BUILD_ENVIRONMENT}" == *-cuda* ]]; then
+    ln -sf "$(which ccache)" ./ccache/nvcc
+  fi
+  export CACHE_WRAPPER_DIR="$PWD/ccache"
+  export PATH="$CACHE_WRAPPER_DIR:$PATH"
 fi
 
-CMAKE_ARGS=("-DBUILD_BINARY=ON")
-CMAKE_ARGS+=("-DUSE_OBSERVERS=ON")
-CMAKE_ARGS+=("-DUSE_ZSTD=ON")
+# sccache will fail for CUDA builds if all cores are used for compiling
+if [ -z "$MAX_JOBS" ]; then
+  if [[ "${BUILD_ENVIRONMENT}" == *-cuda* ]] && [ -n "${SCCACHE}" ]; then
+    MAX_JOBS=`expr $(nproc) - 1`
+  else
+    MAX_JOBS=$(nproc)
+  fi
+fi
 
-# Run build script from scripts if applicable
+report_compile_cache_stats() {
+  if [[ -n "${SCCACHE}" ]]; then
+    "$SCCACHE" --show-stats
+  elif which ccache > /dev/null; then
+    ccache -s
+  fi
+}
+
+###############################################################################
+# Explicitly set Python executable.
+###############################################################################
+# On Ubuntu 16.04 the default Python is still 2.7.
+PYTHON="$(which python)"
+if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
+  PYTHON=$(which "python${BASH_REMATCH[1]}")
+  CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=${PYTHON}")
+fi
+
+
+###############################################################################
+# Use special scripts for Android, conda, and setup builds
+###############################################################################
 if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
   export ANDROID_NDK=/opt/ndk
+  CMAKE_ARGS+=("-DBUILD_BINARY=ON")
+  CMAKE_ARGS+=("-DBUILD_TEST=ON")
+  CMAKE_ARGS+=("-DUSE_OBSERVERS=ON")
+  CMAKE_ARGS+=("-DUSE_ZSTD=ON")
   "${ROOT_DIR}/scripts/build_android.sh" ${CMAKE_ARGS[*]} "$@"
   exit 0
-fi
-if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
-
-  # click (required by onnx) wants these set
-  export LANG=C.UTF-8
-  export LC_ALL=C.UTF-8
-
-  # SKIP_CONDA_TESTS refers to only the 'test' section of the meta.yaml
-  export SKIP_CONDA_TESTS=1
-  export CONDA_INSTALL_LOCALLY=1
-  "${ROOT_DIR}/scripts/build_anaconda.sh" "$@"
+elif [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
+  "${ROOT_DIR}/scripts/build_anaconda.sh" --skip-tests --install-locally "$@"
+  report_compile_cache_stats
 
   # This build will be tested against onnx tests, which needs onnx installed.
   # At this point the visible protbuf installation will be in conda, since one
@@ -67,49 +113,63 @@ if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
   # headers are those in conda as well
   # This path comes from install_anaconda.sh which installs Anaconda into the
   # docker image
-  PROTOBUF_INCDIR=/opt/conda/include pip install "${ROOT_DIR}/third_party/onnx"
+  PROTOBUF_INCDIR=/opt/conda/include pip install -b /tmp/pip_install_onnx "file://${ROOT_DIR}/third_party/onnx#egg=onnx"
+  report_compile_cache_stats
   exit 0
 fi
 
-# Run cmake from ./build directory
-mkdir -p ./build
-cd ./build
 
-INSTALL_PREFIX="/usr/local/caffe2"
+###############################################################################
+# Set cmake args
+###############################################################################
+CMAKE_ARGS+=("-DBUILD_BINARY=ON")
+CMAKE_ARGS+=("-DBUILD_TEST=ON")
+CMAKE_ARGS+=("-DINSTALL_TEST=ON")
+CMAKE_ARGS+=("-DUSE_OBSERVERS=ON")
+CMAKE_ARGS+=("-DUSE_ZSTD=ON")
 CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}")
 
-# Explicitly set Python executable.
-# On Ubuntu 16.04 the default Python is still 2.7.
-PYTHON="$(which python)"
-if [[ "${BUILD_ENVIRONMENT}" == py3* ]]; then
-  PYTHON=/usr/bin/python3
-  CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=${PYTHON}")
+if [[ $BUILD_ENVIRONMENT == *mkl* ]]; then
+  CMAKE_ARGS+=("-DBLAS=MKL")
 fi
+if [[ $BUILD_ENVIRONMENT == *cuda* ]]; then
+  CMAKE_ARGS+=("-DUSE_CUDA=ON")
+  CMAKE_ARGS+=("-DCUDA_ARCH_NAME=Maxwell")
+  CMAKE_ARGS+=("-DUSE_NNPACK=OFF")
 
-case "${BUILD_ENVIRONMENT}" in
-  *-mkl*)
-    CMAKE_ARGS+=("-DBLAS=MKL")
-    ;;
-  *-cuda*)
-    CMAKE_ARGS+=("-DUSE_CUDA=ON")
-    CMAKE_ARGS+=("-DCUDA_ARCH_NAME=Maxwell")
-    CMAKE_ARGS+=("-DUSE_NNPACK=OFF")
+  # Explicitly set path to NVCC such that the symlink to ccache or sccache is used
+  CMAKE_ARGS+=("-DCUDA_NVCC_EXECUTABLE=${CACHE_WRAPPER_DIR}/nvcc")
 
-    # Add ccache symlink for nvcc
-    ln -sf "$(which ccache)" "${CCACHE_WRAPPER_DIR}/nvcc"
+  # Ensure FindCUDA.cmake can infer the right path to the CUDA toolkit.
+  # Setting PATH to resolve to the right nvcc alone isn't enough.
+  # See /usr/share/cmake-3.5/Modules/FindCUDA.cmake, block at line 589.
+  export CUDA_PATH="/usr/local/cuda"
 
-    # Explicitly set path to NVCC such that the symlink to ccache is used
-    CMAKE_ARGS+=("-DCUDA_NVCC_EXECUTABLE=${CCACHE_WRAPPER_DIR}/nvcc")
+  # Ensure the ccache symlink can still find the real nvcc binary.
+  export PATH="/usr/local/cuda/bin:$PATH"
+fi
+if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
+  # TODO: This is patching the official FindHip to properly handly
+  # cmake generator expression. A PR is opened in the upstream repo here:
+  # https://github.com/ROCm-Developer-Tools/HIP/pull/516
+  # remove this hack once it's merged.
+  if [[ -f /opt/rocm/hip/cmake/FindHIP.cmake ]]; then
+    sudo sed -i 's/\ -I${dir}/\ $<$<BOOL:${dir}>:-I${dir}>/' /opt/rocm/hip/cmake/FindHIP.cmake
+  fi
 
-    # Ensure FindCUDA.cmake can infer the right path to the CUDA toolkit.
-    # Setting PATH to resolve to the right nvcc alone isn't enough.
-    # See /usr/share/cmake-3.5/Modules/FindCUDA.cmake, block at line 589.
-    export CUDA_PATH="/usr/local/cuda"
+  export LANG=C.UTF-8
+  export LC_ALL=C.UTF-8
+  export HCC_AMDGPU_TARGET=gfx900
 
-    # Ensure the ccache symlink can still find the real nvcc binary.
-    export PATH="/usr/local/cuda/bin:$PATH"
-    ;;
-esac
+  # The link time of libcaffe2_hip.so takes 40 minutes, according to
+  # https://github.com/RadeonOpenCompute/hcc#thinlto-phase-1---implemented
+  # using using ThinLTO could significantly improve link-time performance.
+  export KMTHINLTO=1
+
+  ########## HIPIFY Caffe2 operators
+  ${PYTHON} "${ROOT_DIR}/tools/amd_build/build_pytorch_amd.py"
+  ${PYTHON} "${ROOT_DIR}/tools/amd_build/build_caffe2_amd.py"
+fi
 
 # Try to include Redis support for Linux builds
 if [ "$(uname)" == "Linux" ]; then
@@ -123,6 +183,9 @@ if [ "$(uname)" == "Darwin" ]; then
   CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=ON")
 fi
 
+# Use a speciallized onnx namespace in CI to catch hardcoded onnx namespace
+CMAKE_ARGS+=("-DONNX_NAMESPACE=ONNX_NAMESPACE_FOR_C2_CI")
+
 # We test the presence of cmake3 (for platforms like Centos and Ubuntu 14.04)
 # and use that if so.
 if [[ -x "$(command -v cmake3)" ]]; then
@@ -131,20 +194,57 @@ else
     CMAKE_BINARY=cmake
 fi
 
-# Configure
-${CMAKE_BINARY} "${ROOT_DIR}" ${CMAKE_ARGS[*]} "$@"
+###############################################################################
+# Configure and make
+###############################################################################
+
+if [[ -z "$INTEGRATED" ]]; then
+
+  # Run cmake from ./build_caffe2 directory so it doesn't conflict with
+  # standard PyTorch build directory. Eventually these won't need to
+  # be separate.
+  rm -rf build_caffe2
+  mkdir build_caffe2
+  cd ./build_caffe2
+
+  # Configure
+  ${CMAKE_BINARY} "${ROOT_DIR}" ${CMAKE_ARGS[*]} "$@"
+
+  # Build
+  if [ "$(uname)" == "Linux" ]; then
+    make "-j${MAX_JOBS}" install
+  else
+    echo "Don't know how to build on $(uname)"
+    exit 1
+  fi
 
-# Build
-if [ "$(uname)" == "Linux" ]; then
-  make "-j$(nproc)" install
 else
-  echo "Don't know how to build on $(uname)"
-  exit 1
+
+  # sccache will be stuck if  all cores are used for compiling
+  # see https://github.com/pytorch/pytorch/pull/7361
+  if [[ -n "${SCCACHE}" ]]; then
+    export MAX_JOBS=`expr $(nproc) - 1`
+  fi
+
+  USE_LEVELDB=1 USE_LMDB=1 USE_OPENCV=1 BUILD_BINARY=1 python setup.py install --user
+
+  # This is to save test binaries for testing
+  cp -r torch/lib/tmp_install $INSTALL_PREFIX
+
+  ls $INSTALL_PREFIX
+
+  report_compile_cache_stats
 fi
 
+
+###############################################################################
+# Install ONNX
+###############################################################################
+
 # Install ONNX into a local directory
-ONNX_INSTALL_PATH="/usr/local/onnx"
-pip install "${ROOT_DIR}/third_party/onnx" -t "${ONNX_INSTALL_PATH}"
+pip install --user -b /tmp/pip_install_onnx "file://${ROOT_DIR}/third_party/onnx#egg=onnx"
+
+report_compile_cache_stats
 
 # Symlink the caffe2 base python path into the system python path,
 # so that we can import caffe2 without having to change $PYTHONPATH.
@@ -153,30 +253,30 @@ pip install "${ROOT_DIR}/third_party/onnx" -t "${ONNX_INSTALL_PATH}"
 # This is only done when running on Jenkins!  We don't want to pollute
 # the user environment with Python symlinks and ld.so.conf.d hacks.
 #
-if [ -n "${JENKINS_URL}" ]; then
-  (
-    source /etc/os-release
+if [[ -z "$INTEGRATED" ]]; then
+  if [ -n "${JENKINS_URL}" ]; then
+    (
+      source /etc/os-release
 
-    function python_version() {
-      "$PYTHON" -c 'import sys; print("python%d.%d" % sys.version_info[0:2])'
-    }
+      function python_version() {
+        "$PYTHON" -c 'import sys; print("python%d.%d" % sys.version_info[0:2])'
+      }
 
-    # Debian/Ubuntu
-    if [[ "$ID_LIKE" == *debian* ]]; then
-      python_path="/usr/local/lib/$(python_version)/dist-packages"
-      sudo ln -sf "${INSTALL_PREFIX}/caffe2" "${python_path}"
-      sudo ln -sf "${ONNX_INSTALL_PATH}/onnx" "${python_path}"
-    fi
+      # Debian/Ubuntu
+      if [[ "$ID_LIKE" == *debian* ]]; then
+        python_path="/usr/local/lib/$(python_version)/dist-packages"
+        sudo ln -sf "${INSTALL_PREFIX}/caffe2" "${python_path}"
+      fi
 
-    # RHEL/CentOS
-    if [[ "$ID_LIKE" == *rhel* ]]; then
-      python_path="/usr/lib64/$(python_version)/site-packages/"
-      sudo ln -sf "${INSTALL_PREFIX}/caffe2" "${python_path}"
-      sudo ln -sf "${ONNX_INSTALL_PATH}/onnx" "${python_path}"
-    fi
+      # RHEL/CentOS
+      if [[ "$ID_LIKE" == *rhel* ]]; then
+        python_path="/usr/lib64/$(python_version)/site-packages/"
+        sudo ln -sf "${INSTALL_PREFIX}/caffe2" "${python_path}"
+      fi
 
-    # /etc/ld.so.conf.d is used on both Debian and RHEL
-    echo "${INSTALL_PREFIX}/lib" | sudo tee /etc/ld.so.conf.d/caffe2.conf
-    sudo ldconfig
-  )
+      # /etc/ld.so.conf.d is used on both Debian and RHEL
+      echo "${INSTALL_PREFIX}/lib" | sudo tee /etc/ld.so.conf.d/caffe2.conf
+      sudo ldconfig
+    )
+  fi
 fi

.jenkins/caffe2/dirty.sh

@@ -3,4 +3,5 @@ set -ex
 upstream="$1"
 pr="$2"
 git diff --name-only "$upstream" "$pr"
-git diff --name-only "$upstream" "$pr" | grep -Eq '^(CMakeLists.txt|Makefile|.gitmodules|.jenkins/caffe2|binaries|caffe|caffe2|cmake|conda|docker|docs/caffe2|modules|scripts|third_party)'
+# For safety, unconditionally trigger for any changes.
+#git diff --name-only "$upstream" "$pr" | grep -Eq '^(CMakeLists.txt|Makefile|.gitmodules|.jenkins/caffe2|binaries|caffe|caffe2|cmake|conda|docker|docs/caffe2|modules|scripts|third_party)'

.jenkins/caffe2/test.sh

@@ -8,12 +8,8 @@ TEST_DIR=$ROOT_DIR/caffe2_tests
 
 # Figure out which Python to use
 PYTHON="python"
-if [ -n "$BUILD_ENVIRONMENT" ]; then
-  if [[ "$BUILD_ENVIRONMENT" == py2* ]]; then
-    PYTHON="python2"
-  elif [[ "$BUILD_ENVIRONMENT" == py3* ]]; then
-    PYTHON="python3"
-  fi
+if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
+  PYTHON="python${BASH_REMATCH[1]}"
 fi
 
 # The prefix must mirror the setting from build.sh
@@ -44,8 +40,6 @@ if [[ "$BUILD_ENVIRONMENT" != conda* ]]; then
   export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${INSTALL_PREFIX}/lib"
 fi
 
-exit_code=0
-
 cd "$ROOT_DIR"
 
 if [ -d $TEST_DIR ]; then
@@ -55,30 +49,55 @@ fi
 
 mkdir -p $TEST_DIR/{cpp,python}
 
-cd ${INSTALL_PREFIX}
+if [[ $BUILD_ENVIRONMENT == *-rocm* ]]; then
+  export LANG=C.UTF-8
+  export LC_ALL=C.UTF-8
 
-# Commands below may exit with non-zero status
-set +e
+  # Pin individual runs to specific gpu so that we can schedule
+  # multiple jobs on machines that have multi-gpu.
+  NUM_AMD_GPUS=$(/opt/rocm/bin/rocminfo | grep 'Device Type.*GPU' | wc -l)
+  if (( $NUM_AMD_GPUS == 0 )); then
+      echo >&2 "No AMD GPU detected!"
+      exit 1
+  fi
+  export HIP_VISIBLE_DEVICES=$(($BUILD_NUMBER % $NUM_AMD_GPUS))
+fi
+
+cd "${WORKSPACE}"
 
 # C++ tests
 echo "Running C++ tests.."
-for test in $INSTALL_PREFIX/test/*; do
-  # Skip tests we know are hanging or bad
-  case "$(basename "$test")" in
-    mkl_utils_test)
+gtest_reports_dir="${TEST_DIR}/cpp"
+junit_reports_dir="${TEST_DIR}/junit_reports"
+mkdir -p "$gtest_reports_dir" "$junit_reports_dir"
+for test in $(find "${INSTALL_PREFIX}/test" -executable -type f); do
+  case "$test" in
+    # skip tests we know are hanging or bad
+    */mkl_utils_test|*/aten/integer_divider_test)
       continue
       ;;
-    # TODO investigate conv_op_test failures when using MKL
-    conv_op_test)
-      continue
+    */scalar_tensor_test|*/basic|*/native_test)
+	  if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+		continue
+	  else
+	    "$test"
+	  fi
+	  ;;
+	*)
+      # Currently, we use a mixture of gtest (caffe2) and Catch2 (ATen). While
+      # planning to migrate to gtest as the common PyTorch c++ test suite, we
+      # currently do NOT use the xml test reporter, because Catch doesn't
+      # support multiple reporters
+      # c.f. https://github.com/catchorg/Catch2/blob/master/docs/release-notes.md#223
+      # which means that enabling XML output means you lose useful stdout
+      # output for Jenkins.  It's more important to have useful console
+      # output than it is to have XML output for Jenkins.
+      # Note: in the future, if we want to use xml test reporter once we switch
+      # to all gtest, one can simply do:
+      # "$test" --gtest_output=xml:"$gtest_reports_dir/$(basename $test).xml"
+      "$test"
       ;;
   esac
-
-  "$test" --gtest_output=xml:"$TEST_DIR"/cpp/$(basename "$test").xml
-  tmp_exit_code="$?"
-  if [ "$exit_code" -eq 0 ]; then
-    exit_code="$tmp_exit_code"
-  fi
 done
 
 # Get the relative path to where the caffe2 python module was installed
@@ -92,9 +111,29 @@ if [[ "$BUILD_ENVIRONMENT" == *-cuda* ]]; then
   EXTRA_TESTS+=("$CAFFE2_PYPATH/contrib/nccl")
 fi
 
-# TODO find out why this breaks for conda builds
+conda_ignore_test=()
 if [[ $BUILD_ENVIRONMENT == conda* ]]; then
-  conda_ignore_test="--ignore $CAFFE2_PYPATH/python/tt_core_test.py"
+  # These tests both assume Caffe2 was built with leveldb, which is not the case
+  conda_ignore_test+=("--ignore $CAFFE2_PYPATH/python/dataio_test.py")
+  conda_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/checkpoint_test.py")
+fi
+
+rocm_ignore_test=()
+if [[ $BUILD_ENVIRONMENT == *-rocm* ]]; then
+  # Currently these tests are failing on ROCM platform:
+
+  # Unknown reasons, need to debug
+  rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/arg_ops_test.py")
+  rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/piecewise_linear_transform_test.py")
+  rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/softmax_ops_test.py")
+  rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/unique_ops_test.py")
+
+  # Need to go through roi ops to replace max(...) with fmaxf(...)
+  rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/roi_align_rotated_op_test.py")
+
+  # Our cuda top_k op has some asm code, the hipified version doesn't
+  # compile yet, so we don't have top_k operator for now
+  rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/top_k_test.py")
 fi
 
 # Python tests
@@ -108,14 +147,14 @@ echo "Running Python tests.."
   --ignore "$CAFFE2_PYPATH/python/operator_test/matmul_op_test.py" \
   --ignore "$CAFFE2_PYPATH/python/operator_test/pack_ops_test.py" \
   --ignore "$CAFFE2_PYPATH/python/mkl/mkl_sbn_speed_test.py" \
-  $conda_ignore_test \
+  ${conda_ignore_test[@]} \
+  ${rocm_ignore_test[@]} \
   "$CAFFE2_PYPATH/python" \
   "${EXTRA_TESTS[@]}"
 
-tmp_exit_code="$?"
-if [ "$exit_code" -eq 0 ]; then
-  exit_code="$tmp_exit_code"
-fi
+cd ${INSTALL_PREFIX}
 
-# Exit with the first non-zero status we got
-exit "$exit_code"
+if [[ -n "$INTEGRATED" ]]; then
+  pip install --user torchvision
+  "$ROOT_DIR/scripts/onnx/test.sh"
+fi

.jenkins/pytorch/build-asan.sh

@@ -16,7 +16,6 @@ export ASAN_OPTIONS=detect_leaks=0:symbolize=1
 
 # TODO: Make the ASAN flags a more unified env var
 CC="clang" CXX="clang++" LDSHARED="clang --shared" \
-  LDFLAGS="-stdlib=libstdc++" \
-  CFLAGS="-fsanitize=address -shared-libasan" \
+  CFLAGS="-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -shared-libasan" \
   NO_CUDA=1 \
   python setup.py install

.jenkins/pytorch/build.sh

@@ -1,12 +1,28 @@
 #!/bin/bash
 
+# For distributed, four environmental configs:
+# (1) build with only NCCL
+# (2) build with NCCL and MPI
+# (3) build with only MPI
+# (4) build with neither
+if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]]; then
+  # TODO: move this to Docker
+  sudo apt-get update
+  sudo apt-get install -y --allow-downgrades --allow-change-held-packages libnccl-dev=2.2.13-1+cuda9.0 libnccl2=2.2.13-1+cuda9.0
+fi
+
+if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda8-* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]] || [[ "$BUILD_ENVIRONMENT" == *-trusty-py2.7.9* ]]; then
+  # TODO: move this to Docker
+  sudo apt-get update
+  sudo apt-get install -y --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
+  sudo apt-get install -y --no-install-recommends openssh-client openssh-server
+  sudo mkdir -p /var/run/sshd
+fi
+
 if [[ "$BUILD_ENVIRONMENT" == "pytorch-linux-xenial-py3-clang5-asan" ]]; then
   exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" $*
 fi
 
-# Add nccl2 for distributed test.
-apt-get install libnccl-dev libnccl2
-
 # Required environment variable: $BUILD_ENVIRONMENT
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
@@ -20,48 +36,99 @@ python --version
 echo "GCC version:"
 gcc --version
 
+echo "CMake version:"
+cmake --version
+
 # TODO: Don't run this...
 pip install -r requirements.txt || true
 
+if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+  # This is necessary in order to cross compile (or else we'll have missing GPU device).
+  export HCC_AMDGPU_TARGET=gfx900
+
+  # These environment variables are not set on CI when we were running as the Jenkins user.
+  # The HIP Utility scripts require these environment variables to be set in order to run without error.
+  export LANG=C.UTF-8
+  export LC_ALL=C.UTF-8
+
+  # This environment variable enabled HCC Optimizations that speed up the linking stage.
+  # https://github.com/RadeonOpenCompute/hcc#hcc-with-thinlto-linking
+  export KMTHINLTO=1
+
+  # Need the libc++1 and libc++abi1 libraries to allow torch._C to load at runtime
+  sudo apt-get install libc++1
+  sudo apt-get install libc++abi1
+
+  python tools/amd_build/build_pytorch_amd.py
+  python tools/amd_build/build_caffe2_amd.py
+  USE_ROCM=1 python setup.py install --user
+  exit 0
+fi
+
+# TODO: Don't install this here
 if ! which conda; then
   pip install mkl mkl-devel
 fi
 
-python setup.py install
-
-# Add the ATen test binaries so that they won't be git clean'ed away
-git add -f aten/build/src/ATen/test
-
-# Testing ATen install
-if [[ "$BUILD_ENVIRONMENT" != *cuda* ]]; then
-  echo "Testing ATen install"
-  time tools/test_aten_install.sh
+# sccache will fail for CUDA builds if all cores are used for compiling
+# gcc 7 with sccache seems to have intermittent OOM issue if all cores are used
+if [ -z "$MAX_JOBS" ]; then
+  if ([[ "$BUILD_ENVIRONMENT" == *cuda* ]] || [[ "$BUILD_ENVIRONMENT" == *gcc7* ]]) && which sccache > /dev/null; then
+    export MAX_JOBS=`expr $(nproc) - 1`
+  fi
 fi
 
-# Test C FFI plugins
-# cffi install doesn't work for Python 3.7
-if [[ "$BUILD_ENVIRONMENT" != *pynightly* ]]; then
-  # TODO: Don't run this here
-  pip install cffi
-  git clone https://github.com/pytorch/extension-ffi.git
-  pushd extension-ffi/script
-  python build.py
-  popd
+# Target only our CI GPU machine's CUDA arch to speed up the build
+export TORCH_CUDA_ARCH_LIST="5.2"
+
+if [[ "$BUILD_ENVIRONMENT" == *ppc64le* ]]; then
+  export TORCH_CUDA_ARCH_LIST="6.0"
 fi
 
+if [[ "$BUILD_ENVIRONMENT" == *trusty-py3.6-gcc5.4* ]]; then
+  export DEBUG=1
+fi
+
+# ppc64le build fails when WERROR=1
+# set only when building other architectures
+# only use for "python setup.py install" line
+if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]]; then
+  WERROR=1 python setup.py install
+elif [[ "$BUILD_ENVIRONMENT" == *ppc64le* ]]; then
+  python setup.py install
+fi
+
+
+# Add the test binaries so that they won't be git clean'ed away
+git add -f build/bin
+
 # Test documentation build
 if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda8-cudnn6-py3* ]]; then
   pushd docs
   # TODO: Don't run this here
   pip install -r requirements.txt || true
-  make html
+  LC_ALL=C make html
   popd
 fi
 
 # Test no-Python build
 if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
-  echo "Building libtorch with NO_PYTHON"
+  echo "Building libtorch"
   # NB: Install outside of source directory (at the same level as the root
   # pytorch folder) so that it doesn't get cleaned away prior to docker push.
-  VERBOSE=1 tools/cpp_build/build_all.sh "$PWD/../cpp-build"
+  BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
+  mkdir -p ../cpp-build/caffe2
+  pushd ../cpp-build/caffe2
+  WERROR=1 VERBOSE=1 DEBUG=1 python $BUILD_LIBTORCH_PY
+  popd
+
+  # Build custom operator tests.
+  CUSTOM_OP_BUILD="$PWD/../custom-op-build"
+  CUSTOM_OP_TEST="$PWD/test/custom_operator"
+  SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
+  mkdir "$CUSTOM_OP_BUILD"
+  pushd "$CUSTOM_OP_BUILD"
+  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" cmake "$CUSTOM_OP_TEST"
+  make VERBOSE=1
+  popd
 fi

.jenkins/pytorch/common.sh

@@ -62,10 +62,19 @@ declare -f -t trap_add
 trap_add cleanup EXIT
 
 if which sccache > /dev/null; then
+  # Save sccache logs to file
+  sccache --stop-server || true
+  rm ~/sccache_error.log || true
+  SCCACHE_ERROR_LOG=~/sccache_error.log RUST_LOG=sccache::server=error sccache --start-server
+
   # Report sccache stats for easier debugging
-  sccache --show-stats
+  sccache --zero-stats
   function sccache_epilogue() {
-     sccache --show-stats
+    echo '=================== sccache compilation log ==================='
+    python $(dirname "${BASH_SOURCE[0]}")/print_sccache_log.py ~/sccache_error.log
+    echo '=========== If your build fails, please take a look at the log above for possible reasons ==========='
+    sccache --show-stats
+    sccache --stop-server || true
   }
   trap_add sccache_epilogue EXIT
 fi
@@ -104,8 +113,28 @@ else
 fi
 
 if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-cuda9-cudnn7-py3 ]] || \
-   [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-trusty-py3.6-gcc7.2 ]]; then
+   [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-trusty-py3.6-gcc7* ]]; then
   BUILD_TEST_LIBTORCH=1
 else
   BUILD_TEST_LIBTORCH=0
 fi
+
+# Use conda cmake in some CI build. Conda cmake will be newer than our supported
+# min version 3.5, so we only do it in two builds that we know should use conda.
+if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-cuda* ]]; then
+  if [[ "$BUILD_ENVIRONMENT" == *cuda8-cudnn6-py2* ]] || \
+     [[ "$BUILD_ENVIRONMENT" == *cuda9-cudnn7-py3* ]]; then
+    if ! which conda; then
+      echo "Expected ${BUILD_ENVIRONMENT} to use conda, but 'which conda' returns empty"
+      exit 1
+    else
+      conda install -q -y cmake
+    fi
+  else
+    if ! cmake --version | grep 'cmake version 3\.5'; then
+      echo "Expected ${BUILD_ENVIRONMENT} to have cmake version 3.5.* (min support version), but 'cmake --version' returns:"
+      cmake --version
+      exit 1
+    fi
+  fi
+fi

.jenkins/pytorch/dirty.sh

@@ -3,4 +3,8 @@ set -ex
 upstream="$1"
 pr="$2"
 git diff --name-only "$upstream" "$pr"
-git diff --name-only "$upstream" "$pr" | grep -Eq '^(aten/|.jenkins/pytorch|docs/(make.bat|Makefile|requirements.txt|source)|mypy|requirements.txt|setup.py|test/|third_party/|tools/|\.gitmodules|torch/)'
+# Now that PyTorch build depends on Caffe2, unconditionally trigger
+# for any changes.
+# TODO: Replace this with a NEGATIVE regex that allows us to blacklist
+# files (letting us skip builds when they are unnecessary)
+#git diff --name-only "$upstream" "$pr" | grep -Eq '^(aten/|caffe2/|.jenkins/pytorch|docs/(make.bat|Makefile|requirements.txt|source)|mypy|requirements.txt|setup.py|test/|third_party/|tools/|\.gitmodules|torch/)'

.jenkins/pytorch/enabled-configs.txt

@@ -12,6 +12,8 @@ pytorch-linux-xenial-cuda9-cudnn7-py2-build
 pytorch-linux-xenial-cuda9-cudnn7-py2-test
 pytorch-linux-xenial-cuda9-cudnn7-py3-build
 pytorch-linux-xenial-cuda9-cudnn7-py3-test
+pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7-build
+pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7-test
 pytorch-linux-xenial-py3-clang5-asan-build
 pytorch-linux-xenial-py3-clang5-asan-test
 pytorch-linux-trusty-py2.7.9-build
@@ -26,11 +28,21 @@ pytorch-linux-trusty-py3.6-gcc5.4-build
 pytorch-linux-trusty-py3.6-gcc5.4-test
 pytorch-linux-trusty-py3.6-gcc7.2-build
 pytorch-linux-trusty-py3.6-gcc7.2-test
+pytorch-linux-trusty-py3.6-gcc7-build
+pytorch-linux-trusty-py3.6-gcc7-test
 pytorch-linux-trusty-pynightly-build
 pytorch-linux-trusty-pynightly-test
 pytorch-win-ws2016-cuda9-cudnn7-py3-build
 pytorch-win-ws2016-cuda9-cudnn7-py3-test
-pytorch-macos-10.13-py3-build-test
+pytorch-macos-10.13-py3-build
+pytorch-macos-10.13-py3-test
+pytorch-macos-10.13-cuda9.2-cudnn7-py3-build
 pytorch-docker-build-test
 short-perf-test-cpu
 short-perf-test-gpu
+py2-clang7-rocmdeb-ubuntu16.04-build
+py2-clang7-rocmdeb-ubuntu16.04-test
+pytorch-ppc64le-cuda9.2-cudnn7-py3-build
+pytorch-ppc64le-cuda9.2-cudnn7-py3-test
+pytorch-ppc64le-cuda9.1-cudnn7-py3-build
+pytorch-ppc64le-cuda9.1-cudnn7-py3-test

.jenkins/pytorch/macos-build-test.sh

@@ -1,26 +1,9 @@
 #!/bin/bash
 
-COMPACT_JOB_NAME=pytorch-macos-10.13-py3-build-test
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-build* ]]; then
+  source "$(dirname "${BASH_SOURCE[0]}")/macos-build.sh"
+fi
 
-# Set up conda environment
-curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o $PWD/miniconda3.sh
-rm -rf $PWD/miniconda3
-bash $PWD/miniconda3.sh -b -p $PWD/miniconda3
-export PATH="$PWD/miniconda3/bin:$PATH"
-source $PWD/miniconda3/bin/activate
-conda install -y mkl mkl-include numpy pyyaml setuptools cmake cffi ninja
-
-# Build and test PyTorch
-git submodule update --init --recursive
-export CMAKE_PREFIX_PATH=$PWD/miniconda3/
-
-export MACOSX_DEPLOYMENT_TARGET=10.9
-export CXX=clang++
-export CC=clang
-# If we run too many parallel jobs, we will OOM
-export MAX_JOBS=2
-python setup.py install
-echo "Ninja version: $(ninja --version)"
-python test/run_test.py --verbose
-echo "BUILD PASSED"
+if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test* ]]; then
+  source "$(dirname "${BASH_SOURCE[0]}")/macos-test.sh"
+fi

.jenkins/pytorch/macos-build.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-build"
+export PATH="/usr/local/bin:$PATH"
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+# Set up conda environment
+export PYTORCH_ENV_DIR="${HOME}/pytorch-ci-env"
+# If a local installation of conda doesn't exist, we download and install conda
+if [ ! -d "${PYTORCH_ENV_DIR}/miniconda3" ]; then
+  mkdir -p ${PYTORCH_ENV_DIR}
+  curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ${PYTORCH_ENV_DIR}/miniconda3.sh
+  bash ${PYTORCH_ENV_DIR}/miniconda3.sh -b -p ${PYTORCH_ENV_DIR}/miniconda3
+fi
+export PATH="${PYTORCH_ENV_DIR}/miniconda3/bin:$PATH"
+source ${PYTORCH_ENV_DIR}/miniconda3/bin/activate
+conda install -y mkl mkl-include numpy pyyaml setuptools cmake cffi ninja
+rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
+
+git submodule update --init --recursive
+export CMAKE_PREFIX_PATH=${PYTORCH_ENV_DIR}/miniconda3/
+
+# Build PyTorch
+if [[ "${JOB_BASE_NAME}" == *cuda9.2* ]]; then
+  export CUDA_VERSION=9.2
+  export TORCH_CUDA_ARCH_LIST=5.2
+  export PATH=/Developer/NVIDIA/CUDA-${CUDA_VERSION}/bin${PATH:+:${PATH}}
+  export DYLD_LIBRARY_PATH=/Developer/NVIDIA/CUDA-${CUDA_VERSION}/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}}
+  export CUDA_HOME=/Developer/NVIDIA/CUDA-${CUDA_VERSION}
+  export NO_CUDA=0
+
+  if [ -z "${IN_CIRCLECI}" ]; then
+    # Eigen gives "explicit specialization of class must precede its first use" error
+    # when compiling with Xcode 9.1 toolchain, so we have to use Xcode 8.2 toolchain instead.
+    export DEVELOPER_DIR=/Library/Developer/CommandLineTools
+  fi
+else
+  if [ -z "${IN_CIRCLECI}" ]; then
+    export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
+  fi
+fi
+
+export MACOSX_DEPLOYMENT_TARGET=10.9
+export CXX=clang++
+export CC=clang
+if which sccache > /dev/null; then
+  printf "#!/bin/sh\nexec sccache $(which clang++) \$*" > "${PYTORCH_ENV_DIR}/clang++"
+  chmod a+x "${PYTORCH_ENV_DIR}/clang++"
+
+  printf "#!/bin/sh\nexec sccache $(which clang) \$*" > "${PYTORCH_ENV_DIR}/clang"
+  chmod a+x "${PYTORCH_ENV_DIR}/clang"
+
+  if [[ "${JOB_BASE_NAME}" == *cuda* ]]; then
+    printf "#!/bin/sh\nexec sccache $(which nvcc) \$*" > "${PYTORCH_ENV_DIR}/nvcc"
+    chmod a+x "${PYTORCH_ENV_DIR}/nvcc"
+    export CUDA_NVCC_EXECUTABLE="${PYTORCH_ENV_DIR}/nvcc"
+  fi
+
+  export PATH="${PYTORCH_ENV_DIR}:$PATH"
+fi
+# If we run too many parallel jobs, we will OOM
+export MAX_JOBS=2
+
+export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
+
+python setup.py install
+
+# Upload torch binaries when the build job is finished
+if [ -z "${IN_CIRCLECI}" ]; then
+  7z a ${IMAGE_COMMIT_TAG}.7z ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
+  aws s3 cp ${IMAGE_COMMIT_TAG}.7z s3://ossci-macos-build/pytorch/${IMAGE_COMMIT_TAG}.7z --acl public-read
+fi

.jenkins/pytorch/macos-test.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-test"
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+export PATH="/usr/local/bin:$PATH"
+
+# Set up conda environment
+export PYTORCH_ENV_DIR="${HOME}/pytorch-ci-env"
+# If a local installation of conda doesn't exist, we download and install conda
+if [ ! -d "${PYTORCH_ENV_DIR}/miniconda3" ]; then
+  mkdir -p ${PYTORCH_ENV_DIR}
+  curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ${PYTORCH_ENV_DIR}/miniconda3.sh
+  bash ${PYTORCH_ENV_DIR}/miniconda3.sh -b -p ${PYTORCH_ENV_DIR}/miniconda3
+fi
+export PATH="${PYTORCH_ENV_DIR}/miniconda3/bin:$PATH"
+source ${PYTORCH_ENV_DIR}/miniconda3/bin/activate
+conda install -y mkl mkl-include numpy pyyaml setuptools cmake cffi ninja future six
+if [ -z "${IN_CIRCLECI}" ]; then
+  rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
+fi
+
+git submodule update --init --recursive
+export CMAKE_PREFIX_PATH=${PYTORCH_ENV_DIR}/miniconda3/
+
+# Test PyTorch
+if [ -z "${IN_CIRCLECI}" ]; then
+  if [[ "${JOB_BASE_NAME}" == *cuda9.2* ]]; then
+    # Eigen gives "explicit specialization of class must precede its first use" error
+    # when compiling with Xcode 9.1 toolchain, so we have to use Xcode 8.2 toolchain instead.
+    export DEVELOPER_DIR=/Library/Developer/CommandLineTools
+  else
+    export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
+  fi
+fi
+export MACOSX_DEPLOYMENT_TARGET=10.9
+export CXX=clang++
+export CC=clang
+# If we run too many parallel jobs, we will OOM
+export MAX_JOBS=2
+
+export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
+
+# Download torch binaries in the test jobs
+if [ -z "${IN_CIRCLECI}" ]; then
+  rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
+  aws s3 cp s3://ossci-macos-build/pytorch/${IMAGE_COMMIT_TAG}.7z ${IMAGE_COMMIT_TAG}.7z
+  7z x ${IMAGE_COMMIT_TAG}.7z -o"${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages"
+fi
+
+test_python_all() {
+  echo "Ninja version: $(ninja --version)"
+  python test/run_test.py --verbose
+}
+
+test_cpp_api() {
+  # C++ API
+
+  # NB: Install outside of source directory (at the same level as the root
+  # pytorch folder) so that it doesn't get cleaned away prior to docker push.
+  # But still clean it before we perform our own build.
+  #
+  CPP_BUILD="$PWD/../cpp-build"
+  rm -rf $CPP_BUILD
+  mkdir -p $CPP_BUILD/caffe2
+
+  BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
+  pushd $CPP_BUILD/caffe2
+  VERBOSE=1 DEBUG=1 python $BUILD_LIBTORCH_PY
+  popd
+
+  python tools/download_mnist.py --quiet -d test/cpp/api/mnist
+
+  # Unfortunately it seems like the test can't load from miniconda3
+  # without these paths being set
+  export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/miniconda3/lib"
+  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/miniconda3/lib"
+  "$CPP_BUILD"/caffe2/bin/test_api
+}
+
+test_custom_script_ops() {
+  echo "Testing custom script operators"
+  pushd test/custom_operator
+  # Build the custom operator library.
+  rm -rf build && mkdir build
+  pushd build
+  SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
+  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" cmake ..
+  make VERBOSE=1
+  popd
+
+  # Run tests Python-side and export a script module.
+  python test_custom_ops.py -v
+  python model.py --export-script-module=model.pt
+  # Run tests C++-side and load the exported script module.
+  build/test_custom_ops ./model.pt
+  popd
+}
+
+
+if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
+  test_python_all
+  test_cpp_api
+  test_custom_script_ops
+else
+  if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
+    test_python_all
+  elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
+    test_cpp_api
+    test_custom_script_ops
+  fi
+fi

.jenkins/pytorch/multigpu-test.sh

@@ -8,4 +8,21 @@ COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-multigpu-test"
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 echo "Testing pytorch (distributed only)"
+
+if [ -n "${IN_CIRCLECI}" ]; then
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]]; then
+    # TODO: move this to Docker
+    sudo apt-get update
+    sudo apt-get install -y --allow-downgrades --allow-change-held-packages libnccl-dev=2.2.13-1+cuda9.0 libnccl2=2.2.13-1+cuda9.0
+  fi
+
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda8-* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]]; then
+    # TODO: move this to Docker
+    sudo apt-get update
+    sudo apt-get install -y --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
+    sudo apt-get install -y --no-install-recommends openssh-client openssh-server
+    sudo mkdir -p /var/run/sshd
+  fi
+fi
+
 time python test/run_test.py --verbose -i distributed

.jenkins/pytorch/perf_test/common.sh

@@ -11,7 +11,7 @@ get_runtime_of_command () {
 
   # runtime=$( { time ($@ &> /dev/null); } 2>&1 1>/dev/null)
   runtime=$( { time $@; } 2>&1 1>/dev/null)
-  if [[ $runtime == *"Warning"* ]] || [[ $runtime == *"Error"* ]]; then
+  if [[ $runtime == *"Error"* ]]; then
     exit 1
   fi
   runtime=${runtime#+++ $@}

.jenkins/pytorch/perf_test/compare_with_baseline.py

@@ -1,5 +1,6 @@
 import sys
 import json
+import math
 import numpy
 import argparse
 
@@ -35,14 +36,25 @@ else:
 print("population mean: ", mean)
 print("population sigma: ", sigma)
 
+# Let the test pass if baseline number is NaN (which happened in
+# the past when we didn't have logic for catching NaN numbers)
+if math.isnan(mean) or math.isnan(sigma):
+    mean = sys.maxsize
+    sigma = 0.001
+
 sample_stats_data = json.loads(args.sample_stats)
 
-sample_mean = sample_stats_data['mean']
-sample_sigma = sample_stats_data['sigma']
+sample_mean = float(sample_stats_data['mean'])
+sample_sigma = float(sample_stats_data['sigma'])
 
 print("sample mean: ", sample_mean)
 print("sample sigma: ", sample_sigma)
 
+if math.isnan(sample_mean):
+    raise Exception('''Error: sample mean is NaN''')
+elif math.isnan(sample_sigma):
+    raise Exception('''Error: sample sigma is NaN''')
+
 z_value = (sample_mean - mean) / sigma
 
 print("z-value: ", z_value)

.jenkins/pytorch/perf_test/test_gpu_speed_cudnn_lstm.sh

@@ -10,7 +10,11 @@ test_gpu_speed_cudnn_lstm () {
 
   git clone https://github.com/pytorch/benchmark.git
 
-  cd benchmark/scripts/
+  cd benchmark/
+
+  git checkout 43dfb2c0370e70ef37f249dc09aff9f0ccd2ddb0
+
+  cd scripts/
 
   SAMPLE_ARRAY=()
   NUM_RUNS=$1

.jenkins/pytorch/perf_test/test_gpu_speed_lstm.sh

@@ -10,7 +10,11 @@ test_gpu_speed_lstm () {
 
   git clone https://github.com/pytorch/benchmark.git
 
-  cd benchmark/scripts/
+  cd benchmark/
+
+  git checkout 43dfb2c0370e70ef37f249dc09aff9f0ccd2ddb0
+
+  cd scripts/
 
   SAMPLE_ARRAY=()
   NUM_RUNS=$1

.jenkins/pytorch/perf_test/test_gpu_speed_mlstm.sh

@@ -10,7 +10,11 @@ test_gpu_speed_mlstm () {
 
   git clone https://github.com/pytorch/benchmark.git
 
-  cd benchmark/scripts/
+  cd benchmark/
+
+  git checkout 43dfb2c0370e70ef37f249dc09aff9f0ccd2ddb0
+
+  cd scripts/
 
   SAMPLE_ARRAY=()
   NUM_RUNS=$1

.jenkins/pytorch/perf_test/test_gpu_speed_mnist.sh

@@ -20,6 +20,9 @@ test_gpu_speed_mnist () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
+  # Needs warm up to get accurate number
+  python main.py --epochs 1 --no-log
+
   for (( i=1; i<=$NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py --epochs 1 --no-log)
     echo $runtime

.jenkins/pytorch/print_sccache_log.py

@@ -0,0 +1,11 @@
+import sys
+
+log_file_path = sys.argv[1]
+
+with open(log_file_path) as f:
+    lines = f.readlines()
+
+for line in lines:
+    # Ignore errors from CPU instruction set testing
+    if 'src.c' not in line:
+        print(line)

.jenkins/pytorch/short-perf-test-gpu.sh

@@ -3,7 +3,7 @@
 COMPACT_JOB_NAME="short-perf-test-gpu"
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
-cd .jenkins/pytorch/perf_test
+pushd .jenkins/pytorch/perf_test
 
 echo "Running GPU perf test for PyTorch..."
 
@@ -64,3 +64,5 @@ if [[ "$COMMIT_SOURCE" == master ]]; then
     # but the chance of them executing this line at the same time is low.
     aws s3 cp new_gpu_runtime.json s3://ossci-perf-test/pytorch/gpu_runtime/${MASTER_COMMIT_ID}.json --acl public-read
 fi
+
+popd

.jenkins/pytorch/test.sh

@@ -9,6 +9,22 @@ source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 echo "Testing pytorch"
 
+if [ -n "${IN_CIRCLECI}" ]; then
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]]; then
+    # TODO: move this to Docker
+    sudo apt-get update
+    sudo apt-get install -y --allow-downgrades --allow-change-held-packages libnccl-dev=2.2.13-1+cuda9.0 libnccl2=2.2.13-1+cuda9.0
+  fi
+
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda8-* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]]; then
+    # TODO: move this to Docker
+    sudo apt-get update
+    sudo apt-get install -y --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
+    sudo apt-get install -y --no-install-recommends openssh-client openssh-server
+    sudo mkdir -p /var/run/sshd
+  fi
+fi
+
 # JIT C++ extensions require ninja.
 git clone https://github.com/ninja-build/ninja --quiet
 pushd ninja
@@ -16,43 +32,147 @@ python ./configure.py --bootstrap
 export PATH="$PWD:$PATH"
 popd
 
-# DANGER WILL ROBINSON.  The LD_PRELOAD here oculd cause you problems
+# DANGER WILL ROBINSON.  The LD_PRELOAD here could cause you problems
 # if you're not careful.  Check this if you made some changes and the
 # ASAN test is not working
 if [[ "$BUILD_ENVIRONMENT" == *asan* ]]; then
-    export ASAN_OPTIONS=detect_leaks=0:symbolize=1
+    export ASAN_OPTIONS=detect_leaks=0:symbolize=1:strict_init_order=true
+    # We suppress the vptr volation, since we have separate copies of
+    # libprotobuf in both libtorch.so and libcaffe2.so, and it causes
+    # the following problem:
+    #    test_cse (__main__.TestJit) ... torch/csrc/jit/export.cpp:622:38:
+    #        runtime error: member call on address ... which does not point
+    #        to an object of type 'google::protobuf::MessageLite'
+    #        ...: note: object is of type 'onnx_torch::ModelProto'
+    #
+    # This problem should be solved when libtorch.so and libcaffe2.so are
+    # merged.
+    export UBSAN_OPTIONS=print_stacktrace=1:suppressions=$PWD/ubsan.supp
     export PYTORCH_TEST_WITH_ASAN=1
+    export PYTORCH_TEST_WITH_UBSAN=1
     # TODO: Figure out how to avoid hard-coding these paths
     export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-5.0/bin/llvm-symbolizer
     export LD_PRELOAD=/usr/lib/llvm-5.0/lib/clang/5.0.0/lib/linux/libclang_rt.asan-x86_64.so
+    # Increase stack size, because ASAN red zones use more stack
+    ulimit -s 81920
+
+    function get_exit_code() {
+      set +e
+      "$@"
+      retcode=$?
+      set -e
+      return $retcode
+    }
+    (cd test && python -c "import torch")
+    echo "The next three invocations are expected to crash; if they don't that means ASAN/UBSAN is misconfigured"
+    (cd test && ! get_exit_code python -c "import torch; torch._C._crash_if_csrc_asan(3)")
+    (cd test && ! get_exit_code python -c "import torch; torch._C._crash_if_csrc_ubsan(0)")
+    (cd test && ! get_exit_code python -c "import torch; torch._C._crash_if_aten_asan(3)")
 fi
 
-time python test/run_test.py --verbose
-
-# Test ATen
-if [[ "$BUILD_ENVIRONMENT" != *asan* ]]; then
-  echo "Testing ATen"
-  TORCH_LIB_PATH=$(python -c "import site; print(site.getsitepackages()[0])")/torch/lib
-  ln -s "$TORCH_LIB_PATH"/libATen.so aten/build/src/ATen/libATen.so
-  aten/tools/run_tests.sh aten/build
+if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+  export PYTORCH_TEST_WITH_ROCM=1
 fi
 
-rm -rf ninja
-
-echo "Installing torchvision at branch master"
-rm -rf vision
-# TODO: This git clone is bad
-git clone https://github.com/pytorch/vision --quiet
-pushd vision
-time python setup.py install
-popd
-
-if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
-   echo "Testing libtorch with NO_PYTHON"
-   CPP_BUILD="$PWD/../cpp-build"
-   if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
-     "$CPP_BUILD"/libtorch/bin/test_jit
-   else
-     "$CPP_BUILD"/libtorch/bin/test_jit "[cpu]"
-   fi
+if [[ "${JOB_BASE_NAME}" == *-NO_AVX-* ]]; then
+  export ATEN_CPU_CAPABILITY=default
+elif [[ "${JOB_BASE_NAME}" == *-NO_AVX2-* ]]; then
+  export ATEN_CPU_CAPABILITY=avx
+fi
+
+test_python_nn() {
+  time python test/run_test.py --include nn --verbose
+}
+
+test_python_all_except_nn() {
+  time python test/run_test.py --exclude nn --verbose
+}
+
+test_aten() {
+  # Test ATen
+  # The following test(s) of ATen have already been skipped by caffe2 in rocm environment:
+  # scalar_tensor_test, basic, native_test
+  if ([[ "$BUILD_ENVIRONMENT" != *asan* ]] && [[ "$BUILD_ENVIRONMENT" != *rocm* ]]); then
+    echo "Running ATen tests with pytorch lib"
+    TORCH_LIB_PATH=$(python -c "import site; print(site.getsitepackages()[0])")/torch/lib
+    # NB: the ATen test binaries don't have RPATH set, so it's necessary to
+    # put the dynamic libraries somewhere were the dynamic linker can find them.
+    # This is a bit of a hack.
+    if [[ "$BUILD_ENVIRONMENT" == *ppc64le* ]]; then
+      SUDO=sudo
+    fi
+
+    ${SUDO} ln -s "$TORCH_LIB_PATH"/libc10* build/bin
+    ${SUDO} ln -s "$TORCH_LIB_PATH"/libcaffe2* build/bin
+    ${SUDO} ln -s "$TORCH_LIB_PATH"/libnccl* build/bin
+
+    ls build/bin
+    aten/tools/run_tests.sh build/bin
+  fi
+}
+
+test_torchvision() {
+  rm -rf ninja
+
+  echo "Installing torchvision at branch master"
+  rm -rf vision
+  # TODO: This git clone is bad, it means pushes to torchvision can break
+  # PyTorch CI
+  git clone https://github.com/pytorch/vision --quiet
+  pushd vision
+  # python setup.py install with a tqdm dependency is broken in the
+  # Travis Python nightly (but not in latest Python nightlies, so
+  # this should be a transient requirement...)
+  # See https://github.com/pytorch/pytorch/issues/7525
+  #time python setup.py install
+  pip install --user .
+  popd
+}
+
+test_libtorch() {
+  if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
+     echo "Testing libtorch"
+     CPP_BUILD="$PWD/../cpp-build"
+     if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
+       "$CPP_BUILD"/caffe2/bin/test_jit
+     else
+       "$CPP_BUILD"/caffe2/bin/test_jit "[cpu]"
+     fi
+     python tools/download_mnist.py --quiet -d test/cpp/api/mnist
+     OMP_NUM_THREADS=2 "$CPP_BUILD"/caffe2/bin/test_api
+  fi
+}
+
+test_custom_script_ops() {
+  if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
+    echo "Testing custom script operators"
+    CUSTOM_OP_BUILD="$PWD/../custom-op-build"
+    pushd test/custom_operator
+    cp -r "$CUSTOM_OP_BUILD" build
+    # Run tests Python-side and export a script module.
+    python test_custom_ops.py -v
+    python model.py --export-script-module=model.pt
+    # Run tests C++-side and load the exported script module.
+    build/test_custom_ops ./model.pt
+    popd
+  fi
+}
+
+if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
+  test_python_nn
+  test_python_all_except_nn
+  test_aten
+  test_torchvision
+  test_libtorch
+  test_custom_script_ops
+else
+  if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
+    test_python_nn
+  elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
+    test_python_all_except_nn
+    test_aten
+    test_torchvision
+    test_libtorch
+    test_custom_script_ops
+  fi
 fi

.jenkins/pytorch/win-build.sh

@@ -1,5 +1,14 @@
 #!/bin/bash
 
+# If you want to rebuild, run this with REBUILD=1
+# If you want to build with CUDA, run this with USE_CUDA=1
+# If you want to build without CUDA, run this with USE_CUDA=0
+
+if [ ! -f setup.py ]; then
+  echo "ERROR: Please run this build script from PyTorch root directory."
+  exit 1
+fi
+
 COMPACT_JOB_NAME=pytorch-win-ws2016-cuda9-cudnn7-py3-build
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
@@ -29,35 +38,64 @@ EOL
 
 cat >ci_scripts/build_pytorch.bat <<EOL
 
-set PATH=C:\\Program Files\\CMake\\bin;C:\\Program Files\\7-Zip;C:\\curl-7.57.0-win64-mingw\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Amazon\\AWSCLI;%PATH%
+set PATH=C:\\Program Files\\CMake\\bin;C:\\Program Files\\7-Zip;C:\\ProgramData\\chocolatey\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Amazon\\AWSCLI;%PATH%
 
 :: Install MKL
-aws s3 cp s3://ossci-windows/mkl_2018.2.185.7z mkl.7z --quiet && 7z x -aoa mkl.7z -omkl
+if "%REBUILD%"=="" (
+  if "%BUILD_ENVIRONMENT%"=="" (
+    curl -k https://s3.amazonaws.com/ossci-windows/mkl_2018.2.185.7z --output mkl.7z
+  ) else (
+    aws s3 cp s3://ossci-windows/mkl_2018.2.185.7z mkl.7z --quiet
+  )
+  7z x -aoa mkl.7z -omkl
+)
 set CMAKE_INCLUDE_PATH=%cd%\\mkl\\include
 set LIB=%cd%\\mkl\\lib;%LIB
 
 :: Install MAGMA
-aws s3 cp s3://ossci-windows/magma_cuda90_release_mkl_2018.2.185.7z magma_cuda90_release_mkl_2018.2.185.7z --quiet && 7z x -aoa magma_cuda90_release_mkl_2018.2.185.7z -omagma
+if "%REBUILD%"=="" (
+  if "%BUILD_ENVIRONMENT%"=="" (
+    curl -k https://s3.amazonaws.com/ossci-windows/magma_cuda90_release_mkl_2018.2.185.7z --output magma_cuda90_release_mkl_2018.2.185.7z
+  ) else (
+    aws s3 cp s3://ossci-windows/magma_cuda90_release_mkl_2018.2.185.7z magma_cuda90_release_mkl_2018.2.185.7z --quiet
+  )
+  7z x -aoa magma_cuda90_release_mkl_2018.2.185.7z -omagma
+)
 set MAGMA_HOME=%cd%\\magma
 
-:: Install clcache
-aws s3 cp s3://ossci-windows/clcache.7z clcache.7z --quiet && 7z x -aoa clcache.7z -oclcache
+:: Install sccache
+mkdir %CD%\\tmp_bin
+if "%REBUILD%"=="" (
+  :check_sccache
+  %CD%\\tmp_bin\\sccache.exe --show-stats || (
+    taskkill /im sccache.exe /f /t || ver > nul
+    del %CD%\\tmp_bin\\sccache.exe
+    if "%BUILD_ENVIRONMENT%"=="" (
+      curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %CD%\\tmp_bin\\sccache.exe
+    ) else (
+      aws s3 cp s3://ossci-windows/sccache.exe %CD%\\tmp_bin\\sccache.exe
+    )
+    goto :check_sccache
+  )
+)
 
 :: Install Miniconda3
-IF EXIST C:\\Jenkins\\Miniconda3 ( rd /s /q C:\\Jenkins\\Miniconda3 )
-curl https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -O
-.\Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /AddToPath=0 /D=C:\\Jenkins\\Miniconda3
+if "%REBUILD%"=="" (
+  IF EXIST C:\\Jenkins\\Miniconda3 ( rd /s /q C:\\Jenkins\\Miniconda3 )
+  curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -O
+  .\Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /AddToPath=0 /D=C:\\Jenkins\\Miniconda3
+)
 call C:\\Jenkins\\Miniconda3\\Scripts\\activate.bat C:\\Jenkins\\Miniconda3
-call conda install -y -q numpy cffi pyyaml boto3
+if "%REBUILD%"=="" ( call conda install -y -q numpy cffi pyyaml boto3 )
 
 :: Install ninja
-pip install ninja
+if "%REBUILD%"=="" ( pip install ninja )
 
 call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x86_amd64
 
 git submodule update --init --recursive
 
-set PATH=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\bin;C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\libnvvp;%PATH%
+set PATH=%CD%\\tmp_bin;C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\bin;C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\libnvvp;%PATH%
 set CUDA_PATH=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
 set CUDA_PATH_V9_0=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
 set NVTOOLSEXT_PATH=C:\\Program Files\\NVIDIA Corporation\\NvToolsExt
@@ -65,29 +103,53 @@ set CUDNN_LIB_DIR=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\l
 set CUDA_TOOLKIT_ROOT_DIR=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
 set CUDNN_ROOT_DIR=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
 
+:: Target only our CI GPU machine's CUDA arch to speed up the build
 set TORCH_CUDA_ARCH_LIST=5.2
 
-set USE_CLCACHE=1
-set CLCACHE_DIR=%cd%\\clcache_tmp
-set CC=%cd%\\clcache\\clcache_main.exe
-set CXX=%cd%\\clcache\\clcache_main.exe
+sccache --stop-server
+sccache --start-server
+sccache --zero-stats
+set CC=sccache cl
+set CXX=sccache cl
 
 set DISTUTILS_USE_SDK=1
 
 set CMAKE_GENERATOR=Ninja
 
-set NO_CUDA=1
+if not "%USE_CUDA%"=="1" (
+  if "%REBUILD%"=="" (
+    set NO_CUDA=1
+    python setup.py install
+  )
+  if errorlevel 1 exit /b 1
+  if not errorlevel 0 exit /b 1
+)
 
-python setup.py install
+if not "%USE_CUDA%"=="0" (
+  if "%REBUILD%"=="" (
+    sccache --show-stats
+    sccache --zero-stats
+    rd /s /q C:\\Jenkins\\Miniconda3\\Lib\\site-packages\\torch
+    copy %CD%\\tmp_bin\\sccache.exe tmp_bin\\nvcc.exe
+  )
 
-if %errorlevel% neq 0 exit /b %errorlevel%
+  set CUDA_NVCC_EXECUTABLE=%CD%\\tmp_bin\\nvcc
 
-rd /s /q C:\\Jenkins\\Miniconda3\\Lib\\site-packages\\torch
+  if "%REBUILD%"=="" set NO_CUDA=0
 
-set NO_CUDA=
-
-python setup.py install && 7z a %IMAGE_COMMIT_TAG%.7z C:\\Jenkins\\Miniconda3\\Lib\\site-packages\\torch && python ci_scripts\\upload_image.py %IMAGE_COMMIT_TAG%.7z
+  python setup.py install && sccache --show-stats && (
+    if "%BUILD_ENVIRONMENT%"=="" (
+      echo "NOTE: To run \`import torch\`, please make sure to activate the conda environment by running \`call C:\\Jenkins\\Miniconda3\\Scripts\\activate.bat C:\\Jenkins\\Miniconda3\` in Command Prompt before running Git Bash."
+    ) else (
+      7z a %IMAGE_COMMIT_TAG%.7z C:\\Jenkins\\Miniconda3\\Lib\\site-packages\\torch && python ci_scripts\\upload_image.py %IMAGE_COMMIT_TAG%.7z
+    )
+  )
+)
 
 EOL
 
-ci_scripts/build_pytorch.bat && echo "BUILD PASSED"
+ci_scripts/build_pytorch.bat
+if [ ! -f $IMAGE_COMMIT_TAG.7z ] && [ ! ${BUILD_ENVIRONMENT} == "" ]; then
+    exit 1
+fi
+echo "BUILD PASSED"

.jenkins/pytorch/win-test.sh

@@ -34,24 +34,9 @@ except botocore.exceptions.ClientError as e:
 
 EOL
 
-cat >ci_scripts/delete_image.py << EOL
+cat >ci_scripts/setup_pytorch_env.bat <<EOL
 
-import os
-import boto3
-
-IMAGE_COMMIT_TAG = os.getenv('IMAGE_COMMIT_TAG')
-
-session = boto3.session.Session()
-s3 = session.resource('s3')
-BUCKET_NAME = 'ossci-windows-build'
-KEY = 'pytorch/'+IMAGE_COMMIT_TAG+'.7z'
-s3.Object(BUCKET_NAME, KEY).delete()
-
-EOL
-
-cat >ci_scripts/test_pytorch.bat <<EOL
-
-set PATH=C:\\Program Files\\CMake\\bin;C:\\Program Files\\7-Zip;C:\\curl-7.57.0-win64-mingw\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Amazon\\AWSCLI;%PATH%
+set PATH=C:\\Program Files\\CMake\\bin;C:\\Program Files\\7-Zip;C:\\ProgramData\\chocolatey\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Amazon\\AWSCLI;%PATH%
 
 :: Install Miniconda3
 IF EXIST C:\\Jenkins\\Miniconda3 ( rd /s /q C:\\Jenkins\\Miniconda3 )
@@ -60,7 +45,7 @@ curl https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -O
 call C:\\Jenkins\\Miniconda3\\Scripts\\activate.bat C:\\Jenkins\\Miniconda3
 call conda install -y -q numpy mkl cffi pyyaml boto3
 
-pip install ninja
+pip install ninja future
 
 call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x86_amd64
 
@@ -78,8 +63,31 @@ cd test/
 python ..\\ci_scripts\\download_image.py %IMAGE_COMMIT_TAG%.7z
 
 7z x %IMAGE_COMMIT_TAG%.7z
-python run_test.py --verbose && python ..\\ci_scripts\\delete_image.py
+
+cd ..
 
 EOL
 
-ci_scripts/test_pytorch.bat && echo "TEST PASSED"
+cat >ci_scripts/test_python_nn.bat <<EOL
+call ci_scripts/setup_pytorch_env.bat
+cd test/ && python run_test.py --include nn --verbose && cd ..
+EOL
+
+cat >ci_scripts/test_python_all_except_nn.bat <<EOL
+call ci_scripts/setup_pytorch_env.bat
+cd test/ && python run_test.py --exclude nn --verbose && cd ..
+EOL
+
+run_tests() {
+    if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
+        ci_scripts/test_python_nn.bat && ci_scripts/test_python_all_except_nn.bat
+    else
+        if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
+            ci_scripts/test_python_nn.bat
+        elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
+            ci_scripts/test_python_all_except_nn.bat
+        fi
+    fi
+}
+
+run_tests && echo "TEST PASSED"

.travis.aten.yml

@@ -13,9 +13,10 @@ install:
     - travis_retry pip install pyyaml typing
 
 script:
+    - cd aten
     - mkdir build install
     - cd build
-    - cmake .. -DNO_CUDA=1 -DCMAKE_INSTALL_PREFIX=../install
+    - cmake .. -DUSE_CUDA=OFF -DCMAKE_INSTALL_PREFIX=../install
     - make install
     - ../tools/run_tests.sh .
     - cd ..

.travis.yml

@@ -16,7 +16,16 @@ matrix:
         python: "2.7"
         install: pip install flake8
         script: flake8
+      - env: LINT_CHECK
+        python: "3.7"
+        dist: xenial    # required for Python 3.7 (travis-ci/travis-ci#9069)
+        sudo: required  # required for Python 3.7 (travis-ci/travis-ci#9069)
+        install: pip install flake8
+        script: flake8
       - env: MYPY_TYPE_CHECK
         python: "3.6"
         install: pip install mypy mypy-extensions
         script: mypy @mypy-files.txt
+      - env: CPP_DOC_CHECK
+        install: sudo apt-get install -y doxygen
+        script: cd docs/cpp/source && ./check-doxygen.sh

CMakeLists.txt

@@ -1,15 +1,14 @@
-cmake_minimum_required(VERSION 3.2 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
 #cmake_policy(SET CMP0022 NEW)
 #cmake_policy(SET CMP0023 NEW)
 
 # ---[ Project and semantic versioning.
 project(Caffe2 CXX C)
 
-set(CAFFE2_VERSION_MAJOR 0)
-set(CAFFE2_VERSION_MINOR 8)
-set(CAFFE2_VERSION_PATCH 2)
-set(CAFFE2_VERSION
-    "${CAFFE2_VERSION_MAJOR}.${CAFFE2_VERSION_MINOR}.${CAFFE2_VERSION_PATCH}")
+set(CMAKE_CXX_STANDARD 11)
+if (NOT MSVC)
+  set(CMAKE_C_STANDARD 11)
+endif()
 
 # One variable that determines whether the current cmake process is being run
 # with the main Caffe2 library. This is useful for building modules - if
@@ -20,14 +19,47 @@ set(CAFFE2_VERSION
 #    endif()
 set(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO ON)
 
+if(NOT DEFINED BLAS_SET_BY_USER)
+  if(DEFINED BLAS)
+    set(BLAS_SET_BY_USER TRUE)
+  else()
+    message(STATUS "Not forcing any particular BLAS to be found")
+    set(BLAS_SET_BY_USER FALSE)
+  endif()
+  set(BLAS_SET_BY_USER ${BLAS_SET_BY_USER} CACHE STRING "Marks whether BLAS was manually set by user or auto-detected")
+endif()
+
+# Apple specific
+if(APPLE)
+  # These lines are an attempt to make find_package(cuda) pick up
+  # libcuda.dylib, and not cuda.framework.  It doesn't work all
+  # the time, but it seems to help for some users.
+  # TODO: replace this with a more robust fix
+  set(CMAKE_FIND_FRAMEWORK LAST)
+  set(CMAKE_FIND_APPBUNDLE LAST)
+
+  # Get clang version on macOS
+  EXECUTE_PROCESS( COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE clang_full_version_string )
+  string(REGEX REPLACE "Apple LLVM version ([0-9]+\\.[0-9]+).*" "\\1" CLANG_VERSION_STRING ${clang_full_version_string})
+  MESSAGE( STATUS "CLANG_VERSION_STRING:         " ${CLANG_VERSION_STRING} )
+
+
+  # RPATH stuff
+  set(CMAKE_MACOSX_RPATH ON)
+endif()
+
 # ---[ Options.
 # Note to developers: if you add an option below, make sure you also add it to
 # cmake/Summary.cmake so that the summary prints out the option values.
 include(CMakeDependentOption)
-option(BUILD_BINARY "Build C++ binaries" ON)
-option(BUILD_DOCS "Build documentation" OFF)
+option(BUILD_TORCH "Build Torch" OFF)
+option(ATEN_NO_TEST "Do not build ATen test binaries" OFF)
+option(BUILD_ATEN_MOBILE "Build ATen for Android and iOS" OFF)
+option(BUILD_BINARY "Build C++ binaries" OFF)
+option(BUILD_DOCS "Build Caffe2 documentation" OFF)
 option(BUILD_CUSTOM_PROTOBUF "Build and use Caffe2's own protobuf under third_party" ON)
 option(BUILD_PYTHON "Build Python binaries" ON)
+option(BUILD_CAFFE2_OPS "Build Caffe2 operators" ON)
 option(BUILD_SHARED_LIBS "Build libcaffe2.so" ON)
 cmake_dependent_option(
     CAFFE2_LINK_LOCAL_PROTOBUF "If set, build protobuf inside libcaffe2.so." ON
@@ -35,45 +67,91 @@ cmake_dependent_option(
 cmake_dependent_option(
     CAFFE2_USE_MSVC_STATIC_RUNTIME "Using MSVC static runtime libraries" ON
     "NOT BUILD_SHARED_LIBS" OFF)
-option(BUILD_TEST "Build C++ test binaries (need gtest and gbenchmark)" ON)
+option(BUILD_TEST "Build C++ test binaries (need gtest and gbenchmark)" OFF)
+cmake_dependent_option(
+    INSTALL_TEST "Install test binaries if BUILD_TEST is on" OFF
+    "BUILD_TEST" OFF)
 option(USE_ACL "Use ARM Compute Library" OFF)
 option(USE_ASAN "Use Address Sanitizer" OFF)
-option(USE_ATEN "Use ATen" OFF)
-option(USE_CUDA "Use Cuda" ON)
+option(USE_CUDA "Use CUDA" ON)
+option(USE_ROCM "Use ROCm" OFF)
+option(CAFFE2_STATIC_LINK_CUDA "Statically link CUDA libraries" OFF)
+cmake_dependent_option(
+    USE_CUDNN "Use cuDNN" ON
+    "USE_CUDA" OFF)
 option(USE_FFMPEG "Use ffmpeg" OFF)
 option(USE_GFLAGS "Use GFLAGS" ON)
 option(USE_GLOG "Use GLOG" ON)
-option(USE_GLOO "Use Gloo" ON)
 option(USE_LEVELDB "Use LEVELDB" ON)
 option(USE_LITE_PROTO "Use lite protobuf instead of full." OFF)
 option(USE_LMDB "Use LMDB" ON)
 option(USE_METAL "Use Metal for iOS build" ON)
 option(USE_MOBILE_OPENGL "Use OpenGL for mobile code" ON)
-option(USE_MPI "Use MPI" ON)
 option(USE_NATIVE_ARCH "Use -march=native" OFF)
 option(USE_NCCL "Use NCCL" ON)
+option(USE_SYSTEM_NCCL "Use system-wide NCCL" OFF)
 option(USE_NERVANA_GPU "Use Nervana GPU backend" OFF)
 option(USE_NNAPI "Use NNAPI" OFF)
 option(USE_NNPACK "Use NNPACK" ON)
 option(USE_NUMA "Use NUMA (only available on Linux)" ON)
+cmake_dependent_option(
+    USE_NVRTC "Use NVRTC. Only available if USE_CUDA is on." OFF
+    "USE_CUDA" OFF)
 option(USE_OBSERVERS "Use observers module." OFF)
-option(USE_OPENCV "Use openCV" ON)
+option(USE_OPENCL "Use OpenCL" OFF)
+option(USE_OPENCV "Use OpenCV" ON)
 option(USE_OPENMP "Use OpenMP for parallel code" OFF)
 option(USE_PROF "Use profiling" OFF)
 option(USE_REDIS "Use Redis" OFF)
 option(USE_ROCKSDB "Use RocksDB" OFF)
 option(USE_SNPE "Use Qualcomm's SNPE library" OFF)
+option(USE_SYSTEM_EIGEN_INSTALL
+    "Use system Eigen instead of the one under third_party" OFF)
 option(USE_TENSORRT "Using Nvidia TensorRT library" OFF)
 option(USE_ZMQ "Use ZMQ" OFF)
 option(USE_ZSTD "Use ZSTD" OFF)
+option(USE_MKLDNN "Use MKLDNN" OFF)
+option(USE_IDEEP "Use IDEEP interface in MKL BLAS" ON)
+option(USE_MKLML "Use MKLML interface in MKL BLAS" ON)
+option(USE_DISTRIBUTED "Use distributed" ON)
+cmake_dependent_option(
+    USE_MPI "Use MPI for Caffe2. Only available if USE_DISTRIBUTED is on." ON
+    "USE_DISTRIBUTED" OFF)
+cmake_dependent_option(
+    USE_GLOO "Use Gloo. Only available if USE_DISTRIBUTED is on." ON
+    "USE_DISTRIBUTED" OFF)
+cmake_dependent_option(
+    USE_GLOO_IBVERBS "Use Gloo IB verbs for distributed. Only available if USE_GLOO is on." OFF
+    "USE_GLOO" OFF)
+
+# Used when building Caffe2 through setup.py
+option(BUILDING_WITH_TORCH_LIBS "Tell cmake if Caffe2 is being built alongside torch libs" OFF)
+
+SET(ONNX_NAMESPACE "onnx_c2" CACHE STRING "onnx namespace")
+
+if (ANDROID OR IOS)
+  set(BUILD_ATEN_MOBILE ON)
+endif()
+
+# ---[ Utils
+# TODO: merge the following 3 files into cmake/public/utils.cmake.
+include(cmake/Utils.cmake)
+include(cmake/public/utils.cmake)
+
+# ---[ Version numbers for generated libraries
+set(TORCH_DEFAULT_VERSION "1.0.0")
+set(TORCH_BUILD_VERSION "${TORCH_DEFAULT_VERSION}" CACHE STRING "Torch build version")
+if (NOT TORCH_BUILD_VERSION)
+  # An empty string was specified so force version to the default
+  set(TORCH_BUILD_VERSION "${TORCH_DEFAULT_VERSION}"
+    CACHE STRING "Torch build version" FORCE)
+endif()
+caffe2_parse_version_str(TORCH ${TORCH_BUILD_VERSION})
+caffe2_parse_version_str(CAFFE2 ${TORCH_BUILD_VERSION})
 
 # ---[ CMake scripts + modules
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)
 
-if (MSVC AND ${BUILD_SHARED_LIBS})
-  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
-endif()
-
 # ---[ CMake build directories
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
@@ -81,17 +159,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
 enable_testing()
 
-# ---[ Misc checks to cope with various compiler modes
-include(cmake/MiscCheck.cmake)
+# ---[ Build variables set within the cmake tree
 include(cmake/BuildVariables.cmake)
-
-# External projects
-include(ExternalProject)
-
-# TODO: merge the following 3 files into cmake/public/utils.cmake.
-include(cmake/Utils.cmake)
-include(cmake/public/utils.cmake)
-
 set(CAFFE2_WHITELIST "" CACHE STRING "A whitelist file of files that one should build.")
 
 # Set default build type
@@ -100,6 +169,12 @@ if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build from: Debug Release RelWithDebInfo MinSizeRel Coverage." FORCE)
 endif()
 
+# ---[ Misc checks to cope with various compiler modes
+include(cmake/MiscCheck.cmake)
+
+# External projects
+include(ExternalProject)
+
 # ---[ Dependencies
 include(cmake/Dependencies.cmake)
 
@@ -120,7 +195,52 @@ if(NOT MSVC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing")
   # Eigen fails to build with some versions, so convert this to a warning
   # Details at http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1459
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-invalid-partial-specialization")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-type-limits")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-array-bounds")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-pragmas")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-function")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-result")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-overflow")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-aliasing")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations")
+  if (CMAKE_COMPILER_IS_GNUCXX AND NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0.0))
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-stringop-overflow")
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=pedantic")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=redundant-decls")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=old-style-cast")
+  # These flags are not available in GCC-4.8.5. Set only when using clang.
+  # Compared against https://gcc.gnu.org/onlinedocs/gcc-4.8.5/gcc/Option-Summary.html
+  if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-invalid-partial-specialization")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-typedef-redefinition")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-private-field")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-inconsistent-missing-override")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-aligned-allocation-unavailable")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++14-extensions")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-constexpr-not-const")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-braces")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments")
+  endif()
+  if ((APPLE AND (NOT ("${CLANG_VERSION_STRING}" VERSION_LESS "9.0")))
+    OR (CMAKE_COMPILER_IS_GNUCXX
+    AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0 AND NOT APPLE)))
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -faligned-new")
+  endif()
+  if ($ENV{WERROR})
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
+  endif($ENV{WERROR})
+  if (NOT APPLE)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-variable")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized")
+  endif()
 else()
   foreach(flag_var
       CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
@@ -134,10 +254,30 @@ else()
         string(REGEX REPLACE "/MT" "/MD" ${flag_var} "${${flag_var}}")
       endif()
     endif()
+    # /bigobj increases number of sections in .obj file, which is needed to link
+    # against libaries in Python 2.7 under Windows
     set(${flag_var} "${${flag_var}} /MP /bigobj")
   endforeach(flag_var)
 endif()
 
+set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -O0")
+set (CMAKE_LINKER_FLAGS_DEBUG "${CMAKE_STATIC_LINKER_FLAGS_DEBUG} -fno-omit-frame-pointer -O0")
+if (USE_ASAN)
+    set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address")
+    set (CMAKE_LINKER_FLAGS_DEBUG "${CMAKE_STATIC_LINKER_FLAGS_DEBUG} -fsanitize=address")
+endif()
+
+if (APPLE)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-private-field")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-braces")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++14-extensions")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-constexpr-not-const")
+endif()
+
+if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0.0)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-stringop-overflow")
+endif()
+
 if(ANDROID)
   if(CMAKE_COMPILER_IS_GNUCXX)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -s")
@@ -161,13 +301,13 @@ include_directories(BEFORE ${PROJECT_SOURCE_DIR})
 # in PROJECT_SOURCE_DIR.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})
 
-# ---[ Old caffe protobuf.
-add_subdirectory(caffe/proto)
+include_directories(BEFORE ${PROJECT_SOURCE_DIR}/aten/src/)
 
 # ---[ Main build
+add_subdirectory(c10)
 add_subdirectory(caffe2)
 
-# Documentation Option
+# --[ Documentation
 if(BUILD_DOCS)
   # check if Doxygen is installed
   find_package(Doxygen)
@@ -181,7 +321,7 @@ if(BUILD_DOCS)
 
     if(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/docs)
       file(REMOVE_RECURSE ${CMAKE_CURRENT_BINARY_DIR}/docs)
-    endif (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/docs)
+    endif()
 
     file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/docs)
     configure_file(${DOXYGEN_C_IN} ${DOXYGEN_C_OUT} @ONLY)
@@ -198,10 +338,10 @@ if(BUILD_DOCS)
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
         COMMENT "Generating Python API documentation with Doxygen"
         VERBATIM)
-  else (DOXYGEN_FOUND)
+  else()
     message(FATAL_ERROR "Doxygen needs to be installed to generate the documentation")
-  endif (DOXYGEN_FOUND)
-endif (BUILD_DOCS)
+  endif()
+endif()
 
 # ---[ CMake related files
 # Uninistall option.
@@ -226,7 +366,7 @@ if ((NOT USE_GLOG) OR (NOT USE_GFLAGS) OR BUILD_CUSTOM_PROTOBUF)
       "generate files that are not well tested.")
 endif()
 
-if (USE_CUDA)
+if (USE_CUDA OR USE_ROCM)
   # TODO: check if we should include other cuda dependency libraries
   # to the interface as well.
 
@@ -256,11 +396,16 @@ if (BUILD_SHARED_LIBS)
       ${PROJECT_SOURCE_DIR}/cmake/public/cuda.cmake
       ${PROJECT_SOURCE_DIR}/cmake/public/glog.cmake
       ${PROJECT_SOURCE_DIR}/cmake/public/gflags.cmake
+      ${PROJECT_SOURCE_DIR}/cmake/public/mkl.cmake
       ${PROJECT_SOURCE_DIR}/cmake/public/protobuf.cmake
       ${PROJECT_SOURCE_DIR}/cmake/public/threads.cmake
       ${PROJECT_SOURCE_DIR}/cmake/public/utils.cmake
       DESTINATION share/cmake/Caffe2/public
       COMPONENT dev)
+  install(DIRECTORY
+      ${PROJECT_SOURCE_DIR}/cmake/Modules_CUDA_fix
+      DESTINATION share/cmake/Caffe2/
+      COMPONENT dev)
   install(EXPORT Caffe2Targets DESTINATION share/cmake/Caffe2
       FILE Caffe2Targets.cmake
       COMPONENT dev)
@@ -278,7 +423,6 @@ add_subdirectory(modules)
 # are built. For the binaries, they will be linked to the Caffe2 main
 # libraries, as well as all the modules that are built with Caffe2 (the ones
 # built in the previous Modules section above).
-
 if (BUILD_BINARY)
   add_subdirectory(binaries)
 endif()

CODEOWNERS

@@ -2,11 +2,24 @@
 # Each line is a file pattern followed by one or more owners.
 
 /aten/ @apaszke @soumith @colesbury @gchanan @zdevito @ezyang
-/torch/onnx/ @anderspapitto @bddppq @dzhulgakov @ezyang @houseroad @jamesr66a @smessmer
+/aten/src/ATen/core/
 /torch/ @apaszke @soumith @colesbury @gchanan @zdevito @ezyang
-/docs/source @apaszke @soumith @colesbury @gchanan @zdevito @ezyang
+/docs/source @apaszke @soumith @colesbury @gchanan @zdevito @ezyang @ssnl @zou3519
+/docs/cpp @goldsborough @ebetica @apaszke @soumith @colesbury @gchanan @zdevito @ezyang
 /test @apaszke @soumith @colesbury @gchanan @zdevito @ezyang
 /tools @apaszke @soumith @colesbury @gchanan @zdevito @ezyang
 /README.md @apaszke @soumith @colesbury @gchanan @zdevito @ezyang
 /setup.py @apaszke @soumith @colesbury @gchanan @zdevito @ezyang
 /requirements.txt @apaszke @soumith @colesbury @gchanan @zdevito @ezyang
+/torch/csrc/api/ @apaszke @soumith @colesbury @gchanan @zdevito @ezyang @ebetica @goldsborough
+/test/cpp/api/ @apaszke @soumith @colesbury @gchanan @zdevito @ezyang @ebetica @goldsborough
+/torch/onnx/ @anderspapitto @bddppq @dzhulgakov @ezyang @houseroad @jamesr66a @smessmer @Yangqing
+/torch/csrc/onnx/ @anderspapitto @bddppq @dzhulgakov @ezyang @houseroad @jamesr66a @smessmer @Yangqing
+/torch/csrc/jit/passes/onnx/ @anderspapitto @bddppq @dzhulgakov @ezyang @houseroad @jamesr66a @smessmer @Yangqing
+/test/onnx/ @anderspapitto @bddppq @dzhulgakov @ezyang @houseroad @jamesr66a @smessmer @Yangqing
+/scripts/onnx/ @anderspapitto @bddppq @dzhulgakov @ezyang @houseroad @jamesr66a @smessmer @Yangqing
+/torch/lib/c10d/ @apaszke @pietern @teng-li
+/torch/csrc/distributed/ @apaszke @pietern @teng-li
+/torch/distributed/ @apaszke @pietern @teng-li
+/test/test_c10d.py @apaszke @pietern @teng-li
+/torch/utils/cpp_extension.py @goldsborough @fmassa @apaszke @soumith @ezyang

CONTRIBUTING.md

@@ -19,18 +19,18 @@ If you are not familiar with creating a Pull Request, here are some guides:
 - https://help.github.com/articles/creating-a-pull-request/
 
 
-## Developing locally with PyTorch
+## Developing PyTorch
 
-To locally develop with PyTorch, here are some tips:
+To develop PyTorch on your machine, here are some tips:
 
-1. Uninstall all existing pytorch installs
+1. Uninstall all existing PyTorch installs:
 ```
 conda uninstall pytorch
 pip uninstall torch
 pip uninstall torch # run this command twice
 ```
 
-2. Locally clone a copy of PyTorch from source:
+2. Clone a copy of PyTorch from source:
 
 ```
 git clone https://github.com/pytorch/pytorch
@@ -72,6 +72,9 @@ For example:
 
 You do not need to repeatedly install after modifying python files.
 
+In case you want to reinstall, make sure that you uninstall pytorch first by running `pip uninstall torch`
+and `python setup.py clean`. Then you can install in `build develop` mode again.
+
 ## Unit testing
 
 PyTorch's testing is located under `test/`. Run the entire test suite with
@@ -101,6 +104,18 @@ PyTorch uses [Google style](http://sphinxcontrib-napoleon.readthedocs.io/en/late
 for formatting docstrings. Length of line inside docstrings block must be limited to 80 characters to
 fit into Jupyter documentation popups.
 
+For C++ documentation (https://pytorch.org/cppdocs), we use
+[Doxygen](http://www.doxygen.nl/) and then convert it to
+[Sphinx](http://www.sphinx-doc.org/) via
+[Breathe](https://github.com/michaeljones/breathe) and
+[Exhale](https://github.com/svenevs/exhale). Check the [Doxygen
+reference](http://www.stack.nl/~dimitri/doxygen/manual/index.html) for more
+information on the documentation syntax. To build the documentation locally,
+`cd` into `docs/cpp` and then `make html`.
+
+We run Doxygen in CI (Travis) to verify that you do not use invalid Doxygen
+commands. To run this check locally, run `./check-doxygen.sh` from inside
+`docs/cpp`.
 
 ## Managing multiple build trees
 
@@ -136,19 +151,20 @@ not very optimized for incremental rebuilds, this will actually be very slow.
 Far better is to only request rebuilds of the parts of the project you are
 working on:
 
-- Working on `torch/csrc`?  Run `python setup.py develop` to rebuild
+- Working on the Python bindings?  Run `python setup.py develop` to rebuild
   (NB: no `build` here!)
 
-- Working on `torch/lib/TH`, did not make any cmake changes, and just want to
-  see if it compiles?  Run `(cd torch/lib/build/TH && make install -j$(getconf _NPROCESSORS_ONLN))`.  This
-  applies for any other subdirectory of `torch/lib`.  **Warning: Changes you
-  make here will not be visible from Python.**  See below.
+- Working on `torch/csrc` or `aten`?  Run `python setup.py rebuild_libtorch` to
+  rebuild and avoid having to rebuild other dependent libraries we
+  depend on.
 
-- Working on `torch/lib` and want to run your changes / rerun cmake?  Run
-  `python setup.py build_deps`.  Note that this will rerun cmake for
-  every subdirectory in TH; if you are only working on one project,
-  consider editing `torch/lib/build_all.sh` and commenting out the
-  `build` lines of libraries you are not working on.
+- Working on one of the other dependent libraries? The other valid
+  targets are listed in `dep_libs` in `setup.py`. prepend `build_` to
+  get a target, and run as e.g. `python setup.py build_gloo`.
+
+- Working on a test binary?  Run `(cd build && ninja bin/test_binary_name)` to
+  rebuild only that test binary (without rerunning cmake).  (Replace `ninja` with
+  `make` if you don't have ninja installed).
 
 On the initial build, you can also speed things up with the environment
 variables `DEBUG` and `NO_CUDA`.
@@ -165,7 +181,7 @@ Make sure you continue to pass these flags on subsequent builds.
 
 ### Code completion and IDE support
 
-When using `python setup.py develop`, PyTorch will generate 
+When using `python setup.py develop`, PyTorch will generate
 a `compile_commands.json` file that can be used by many editors
 to provide command completion and error highlighting for PyTorch's
 C++ code. You need to `pip install ninja` to generate accurate
@@ -176,12 +192,14 @@ information for the code in `torch/csrc`. More information at:
 
 #### Use Ninja
 Python `setuptools` is pretty dumb, and always rebuilds every C file in a
-project.  If you install the ninja build system with `pip install ninja`, 
+project.  If you install the ninja build system with `pip install ninja`,
 then PyTorch will use it to track dependencies correctly.
+If pytorch was already built, you will need to run `python setup.py clean` once
+after installing ninja for builds to succeed.
 
 #### Use CCache
 
-Even when dependencies are tracked with file modification, 
+Even when dependencies are tracked with file modification,
 there are many situations where files get rebuilt when a previous
 compilation was exactly the same.
 
@@ -226,15 +244,116 @@ export CUDA_NVCC_EXECUTABLE=~/ccache/cuda/nvcc
 
 If you are working on the CUDA code, here are some useful CUDA debugging tips:
 
-1. `CUDA_DEBUG=1` will enable CUDA debugging symbols (-g -G). This is particularly
-    helpful in debugging device code. However, it will slow down the build process,
-    so use wisely.
+1. `CUDA_DEVICE_DEBUG=1` will enable CUDA device function debug symbols (`-g -G`).
+    This will be particularly helpful in debugging device code. However, it will
+    slow down the build process for about 50% (compared to only `DEBUG=1`), so use wisely.
 2. `cuda-gdb` and `cuda-memcheck` are your best CUDA debugging friends. Unlike`gdb`,
    `cuda-gdb` can display actual values in a CUDA tensor (rather than all zeros).
 
 
 Hope this helps, and thanks for considering to contribute.
 
+## Windows development tips
+
+Occasionally, you will write a patch which works on Linux, but fails CI on Windows.
+There are a few aspects in which MSVC (the Windows compiler toolchain we use) is stricter
+than Linux, which are worth keeping in mind when fixing these problems.
+
+1. Symbols are NOT exported by default on Windows; instead, you have to explicitly
+   mark a symbol as exported/imported in a header file with `__declspec(dllexport)` /
+   `__declspec(dllimport)`.  We have codified this pattern into a set of macros
+   which follow the convention `*_API`, e.g., `CAFFE2_API` inside Caffe2 and ATen.
+   (Every separate shared library needs a unique macro name, because symbol visibility
+   is on a per shared library basis. See c10/macros/Macros.h for more details.)
+
+   The upshot is if you see an "unresolved external" error in your Windows build, this
+   is probably because you forgot to mark a function with `*_API`.  However, there is
+   one important counterexample to this principle: if you want a *templated* function
+   to be instantiated at the call site, do NOT mark it with `*_API` (if you do mark it,
+   you'll have to explicitly instantiate all of the specializations used by the call
+   sites.)
+
+2. If you link against a library, this does not make its dependencies transitively
+   visible. You must explicitly specify a link dependency against every library whose
+   symbols you use.  (This is different from Linux where in most environments,
+   transitive dependencies can be used to fulfill unresolved symbols.)
+
+3. If you have a Windows box (we have a few on EC2 which you can request access to) and
+   you want to run the build, the easiest way is to just run `.jenkins/pytorch/win-build.sh`.
+   If you need to rebuild, run `REBUILD=1 .jenkins/pytorch/win-build.sh` (this will avoid
+   blowing away your Conda environment.)
+
+Even if you don't know anything about MSVC, you can use cmake to build simple programs on
+Windows; this can be helpful if you want to learn more about some peculiar linking behavior
+by reproducing it on a small example.  Here's a simple example cmake file that defines
+two dynamic libraries, one linking with the other:
+
+```
+project(myproject CXX)
+set(CMAKE_CXX_STANDARD 11)
+add_library(foo SHARED foo.cpp)
+add_library(bar SHARED bar.cpp)
+# NB: don't forget to __declspec(dllexport) at least one symbol from foo,
+# otherwise foo.lib will not be created.
+target_link_libraries(bar PUBLIC foo)
+```
+
+You can build it with:
+
+```
+mkdir build
+cd build
+cmake ..
+cmake --build .
+```
+
+### Known MSVC (and MSVC with NVCC) bugs
+
+The PyTorch codebase sometimes likes to use exciting C++ features, and
+these exciting features lead to exciting bugs in Windows compilers.
+To add insult to injury, the error messages will often not tell you
+which line of code actually induced the erroring template instantiation.
+
+I've found the most effective way to debug these problems is to
+carefully read over diffs, keeping in mind known bugs in MSVC/NVCC.
+Here are a few well known pitfalls and workarounds:
+
+* This is not actually a bug per se, but in general, code generated by MSVC
+  is more sensitive to memory errors; you may have written some code
+  that does a use-after-free or stack overflows; on Linux the code
+  might work, but on Windows your program will crash.  ASAN may not
+  catch all of these problems: stay vigilant to the possibility that
+  your crash is due to a real memory problem.
+
+* (NVCC) `at::optional` does not work when used from device code.  Don't use
+  it from kernels.  Upstream issue: https://github.com/akrzemi1/Optional/issues/58
+  and our local issue #10329.
+
+* `constexpr` generally works less well on MSVC.
+
+  * The idiom `static_assert(f() == f())` to test if `f` is constexpr
+    does not work; you'll get "error C2131: expression did not evaluate
+    to a constant".  Don't use these asserts on Windows.
+    (Example: `aten/src/ATen/core/intrusive_ptr.h`)
+
+* (NVCC) Code you access inside a `static_assert` will eagerly be
+  evaluated as if it were device code, and so you might get an error
+  that the code is "not accessible".
+
+```
+class A {
+  static A singleton_;
+  static constexpr inline A* singleton() {
+    return &singleton_;
+  }
+};
+static_assert(std::is_same(A*, decltype(A::singelton()))::value, "hmm");
+```
+
+* The compiler will run out of heap if you attempt to compile files that
+  are too large.  Splitting such files into separate files helps.
+  (Example: `THTensorMath`, `THTensorMoreMath`, `THTensorEvenMoreMath`.)
+
 ## Caffe2 notes
 
 In 2018, we merged Caffe2 into the PyTorch source repository.  While the

LICENSE

@@ -1,3 +1,45 @@
+From PyTorch:
+
+Copyright (c) 2016-     Facebook, Inc            (Adam Paszke)
+Copyright (c) 2014-     Facebook, Inc            (Soumith Chintala)
+Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
+Copyright (c) 2012-2014 Deepmind Technologies    (Koray Kavukcuoglu)
+Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
+Copyright (c) 2011-2013 NYU                      (Clement Farabet)
+Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston)
+Copyright (c) 2006      Idiap Research Institute (Samy Bengio)
+Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz)
+
+From Caffe2:
+
+Copyright (c) 2016-present, Facebook Inc. All rights reserved.
+
+All contributions by Facebook:
+Copyright (c) 2016 Facebook Inc.
+ 
+All contributions by Google:
+Copyright (c) 2015 Google Inc.
+All rights reserved.
+ 
+All contributions by Yangqing Jia:
+Copyright (c) 2015 Yangqing Jia
+All rights reserved.
+ 
+All contributions from Caffe:
+Copyright(c) 2013, 2014, 2015, the respective contributors
+All rights reserved.
+ 
+All other contributions:
+Copyright(c) 2015, 2016 the respective contributors
+All rights reserved.
+ 
+Caffe2 uses a copyright model similar to Caffe: each contributor holds
+copyright over their contributions to Caffe2. The project versioning records
+all such contribution and copyright details. If a contributor wants to further
+mark their specific copyright on a particular contribution, they should
+indicate their copyright solely in the commit message of the change when it is
+committed.
+
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

NOTICE

@@ -1,45 +1,3 @@
-From PyTorch:
-
-Copyright (c) 2016-     Facebook, Inc            (Adam Paszke)
-Copyright (c) 2014-     Facebook, Inc            (Soumith Chintala)
-Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
-Copyright (c) 2012-2014 Deepmind Technologies    (Koray Kavukcuoglu)
-Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
-Copyright (c) 2011-2013 NYU                      (Clement Farabet)
-Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston)
-Copyright (c) 2006      Idiap Research Institute (Samy Bengio)
-Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz)
-
-From Caffe2:
-
-Copyright (c) 2016-present, Facebook Inc. All rights reserved.
-
-All contributions by Facebook:
-Copyright (c) 2016 Facebook Inc.
- 
-All contributions by Google:
-Copyright (c) 2015 Google Inc.
-All rights reserved.
- 
-All contributions by Yangqing Jia:
-Copyright (c) 2015 Yangqing Jia
-All rights reserved.
- 
-All contributions from Caffe:
-Copyright(c) 2013, 2014, 2015, the respective contributors
-All rights reserved.
- 
-All other contributions:
-Copyright(c) 2015, 2016 the respective contributors
-All rights reserved.
- 
-Caffe2 uses a copyright model similar to Caffe: each contributor holds
-copyright over their contributions to Caffe2. The project versioning records
-all such contribution and copyright details. If a contributor wants to further
-mark their specific copyright on a particular contribution, they should
-indicate their copyright solely in the commit message of the change when it is
-committed.
-
 =======================================================================
 Software under third_party
 =======================================================================

README.md

@@ -1,4 +1,4 @@
-<p align="center"><img width="40%" src="docs/source/_static/img/pytorch-logo-dark.png" /></p>
+![PyTorch Logo](https://github.com/pytorch/pytorch/blob/master/docs/source/_static/img/pytorch-logo-dark.png)
 
 --------------------------------------------------------------------------------
 
@@ -15,6 +15,7 @@ We are in an early-release beta. Expect some adventures and rough edges.
   - [Binaries](#binaries)
   - [From Source](#from-source)
   - [Docker Image](#docker-image)
+  - [Building the Documentation](#building-the-documentation)
   - [Previous Versions](#previous-versions)
 - [Getting Started](#getting-started)
 - [Communication](#communication)
@@ -27,37 +28,21 @@ We are in an early-release beta. Expect some adventures and rough edges.
 | Linux GPU | [![Build Status](https://ci.pytorch.org/jenkins/job/pytorch-master/badge/icon)](https://ci.pytorch.org/jenkins/job/pytorch-master/) | [![Build Status](https://ci.pytorch.org/jenkins/job/pytorch-master/badge/icon)](https://ci.pytorch.org/jenkins/job/pytorch-master/) |
 | Windows GPU | <center>—</center> | [![Build Status](https://ci.pytorch.org/jenkins/job/pytorch-builds/job/pytorch-win-ws2016-cuda9-cudnn7-py3-trigger/badge/icon)](https://ci.pytorch.org/jenkins/job/pytorch-builds/job/pytorch-win-ws2016-cuda9-cudnn7-py3-trigger/)
 
+See also the [ci.pytorch.org HUD](https://ezyang.github.io/pytorch-ci-hud/build/pytorch-master).
+
 
 ## More about PyTorch
 
 At a granular level, PyTorch is a library that consists of the following components:
 
-<table>
-<tr>
-    <td><b> torch </b></td>
-    <td> a Tensor library like NumPy, with strong GPU support </td>
-</tr>
-<tr>
-    <td><b> torch.autograd </b></td>
-    <td> a tape-based automatic differentiation library that supports all differentiable Tensor operations in torch </td>
-</tr>
-<tr>
-    <td><b> torch.nn </b></td>
-    <td> a neural networks library deeply integrated with autograd designed for maximum flexibility </td>
-</tr>
-<tr>
-    <td><b> torch.multiprocessing  </b></td>
-    <td> Python multiprocessing, but with magical memory sharing of torch Tensors across processes. Useful for data loading and Hogwild training. </td>
-</tr>
-<tr>
-    <td><b> torch.utils </b></td>
-    <td> DataLoader, Trainer and other utility functions for convenience </td>
-</tr>
-<tr>
-    <td><b> torch.legacy(.nn/.optim) </b></td>
-    <td> legacy code that has been ported over from torch for backward compatibility reasons </td>
-</tr>
-</table>
+| Component | Description |
+| ---- | --- |
+| **torch** | a Tensor library like NumPy, with strong GPU support |
+| **torch.autograd** | a tape-based automatic differentiation library that supports all differentiable Tensor operations in torch |
+| **torch.nn** | a neural networks library deeply integrated with autograd designed for maximum flexibility |
+| **torch.multiprocessing** | Python multiprocessing, but with magical memory sharing of torch Tensors across processes. Useful for data loading and Hogwild training |
+| **torch.utils** | DataLoader, Trainer and other utility functions for convenience |
+| **torch.legacy(.nn/.optim)** | legacy code that has been ported over from torch for backward compatibility reasons |
 
 Usually one uses PyTorch either as:
 
@@ -70,10 +55,10 @@ Elaborating further:
 
 If you use NumPy, then you have used Tensors (a.k.a ndarray).
 
-<p align=center><img width="30%" src="docs/source/_static/img/tensor_illustration.png" /></p>
+![Tensor illustration](https://github.com/pytorch/pytorch/blob/master/docs/source/_static/img/tensor_illustration.png)
 
-PyTorch provides Tensors that can live either on the CPU or the GPU, and accelerate
-compute by a huge amount.
+PyTorch provides Tensors that can live either on the CPU or the GPU, and accelerates the
+computation by a huge amount.
 
 We provide a wide variety of tensor routines to accelerate and fit your scientific computation needs
 such as slicing, indexing, math operations, linear algebra, reductions.
@@ -97,7 +82,7 @@ from several research papers on this topic, as well as current and past work suc
 While this technique is not unique to PyTorch, it's one of the fastest implementations of it to date.
 You get the best of speed and flexibility for your crazy research.
 
-<p align=center><img width="80%" src="docs/source/_static/img/dynamic_graph.gif" /></p>
+![Dynamic graph](https://github.com/pytorch/pytorch/blob/master/docs/source/_static/img/dynamic_graph.gif)
 
 ### Python First
 
@@ -121,8 +106,7 @@ We hope you never spend hours debugging your code because of bad stack traces or
 PyTorch has minimal framework overhead. We integrate acceleration libraries
 such as Intel MKL and NVIDIA (cuDNN, NCCL) to maximize speed.
 At the core, its CPU and GPU Tensor and neural network backends
-(TH, THC, THNN, THCUNN) are written as independent libraries with a C99 API.
-They are mature and have been tested for years.
+(TH, THC, THNN, THCUNN) are mature and have been tested for years.
 
 Hence, PyTorch is quite fast – whether you run small or large neural networks.
 
@@ -139,9 +123,8 @@ and with minimal abstractions.
 You can write new neural network layers in Python using the torch API
 [or your favorite NumPy-based libraries such as SciPy](http://pytorch.org/tutorials/advanced/numpy_extensions_tutorial.html).
 
-If you want to write your layers in C/C++, we provide an extension API based on
-[cffi](http://cffi.readthedocs.io/en/latest/) that is efficient and with minimal boilerplate.
-There is no wrapper code that needs to be written. You can see [a tutorial here](http://pytorch.org/tutorials/advanced/c_extension.html) and [an example here](https://github.com/pytorch/extension-ffi).
+If you want to write your layers in C/C++, we provide a convenient extension API that is efficient and with minimal boilerplate.
+There is no wrapper code that needs to be written. You can see [a tutorial here](http://pytorch.org/tutorials/advanced/cpp_extension.html) and [an example here](https://github.com/pytorch/extension-cpp).
 
 
 ## Installation
@@ -165,7 +148,9 @@ If you want to compile with CUDA support, install
 If you want to disable CUDA support, export environment variable `NO_CUDA=1`.
 Other potentially useful environment variables may be found in `setup.py`.
 
-If you want to build on Windows, Visual Studio 2017 and NVTX are also needed.
+If you want to build on Windows, Visual Studio 2017 14.11 toolset and NVTX are also needed.
+Especially, for CUDA 8 build on Windows, there will be an additional requirement for VS 2015 Update 3 and a patch for it.
+The details of the patch can be found out [here](https://support.microsoft.com/en-gb/help/4020481/fix-link-exe-crashes-with-a-fatal-lnk1000-error-when-you-use-wholearch).
 
 #### Install optional dependencies
 
@@ -175,9 +160,10 @@ export CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" # [anaconda root direct
 
 # Install basic dependencies
 conda install numpy pyyaml mkl mkl-include setuptools cmake cffi typing
+conda install -c mingfeima mkldnn
 
 # Add LAPACK support for the GPU
-conda install -c pytorch magma-cuda80 # or magma-cuda90 if CUDA 9
+conda install -c pytorch magma-cuda92 # or [magma-cuda80 | magma-cuda91] depending on your cuda version
 ```
 
 On macOS
@@ -212,8 +198,11 @@ On Windows
 set "VS150COMNTOOLS=C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build"
 set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
 set DISTUTILS_USE_SDK=1
-REM The following line is needed for Python 2.7, but the support for it is very experimental.
+REM The following two lines are needed for Python 2.7, but the support for it is very experimental.
 set MSSdk=1
+set FORCE_PY27_BUILD=1
+REM As for CUDA 8, VS2015 Update 3 is also required to build PyTorch. Use the following line.
+set "CUDAHOSTCXX=%VS140COMNTOOLS%\..\..\VC\bin\amd64\cl.exe"
 
 call "%VS150COMNTOOLS%\vcvarsall.bat" x64 -vcvars_ver=14.11
 python setup.py install
@@ -221,7 +210,7 @@ python setup.py install
 
 ### Docker image
 
-Dockerfile is supplied to build images with cuda support and cudnn v7. Build as usual
+Dockerfile is supplied to build images with cuda support and cudnn v7. You can pass -e PYTHON_VERSION=x.y flag to specificy which python to be used by Miniconda, or leave it unset to use the default. Build as usual
 ```
 docker build -t pytorch -f docker/pytorch/Dockerfile .
 ```
@@ -235,23 +224,35 @@ Please note that PyTorch uses shared memory to share data between processes, so
 for multithreaded data loaders) the default shared memory segment size that container runs with is not enough, and you
 should increase shared memory size either with `--ipc=host` or `--shm-size` command line options to `nvidia-docker run`.
 
+### Building the Documentation
+
+To build documentation in various formats, you will need Sphinx and the
+readthedocs theme.
+
+```
+cd docs/
+pip install -r requirements.txt
+```
+You can then build the documentation by running ``make <format>`` from the
+``docs/`` folder. Run ``make`` to get a list of all available output formats.
+
 ### Previous Versions
 
 Installation instructions and binaries for previous PyTorch versions may be found
-on [our website](http://pytorch.org/previous-versions/).
+on [our website](http://pytorch.org/previous-versions).
 
 
 ## Getting Started
 
 Three pointers to get you started:
-- [Tutorials: get you started with understanding and using PyTorch](http://pytorch.org/tutorials/)
+- [Tutorials: get you started with understanding and using PyTorch](https://pytorch.org/tutorials/)
 - [Examples: easy to understand pytorch code across all domains](https://github.com/pytorch/examples)
 - [The API Reference](http://pytorch.org/docs/)
 
 ## Communication
 * forums: discuss implementations, research, etc. http://discuss.pytorch.org
 * GitHub issues: bug reports, feature requests, install issues, RFCs, thoughts, etc.
-* Slack: general chat, online discussions, collaboration etc. https://pytorch.slack.com/ . Our slack channel is invite-only to promote a healthy balance between power-users and beginners. If you need a slack invite, ping us at soumith@pytorch.org
+* Slack: general chat, online discussions, collaboration etc. https://pytorch.slack.com/ . Our slack channel is invite-only to promote a healthy balance between power-users and beginners. If you need a slack invite, ping us at slack@pytorch.org
 * newsletter: no-noise, one-way email newsletter with important announcements about pytorch. You can sign-up here: http://eepurl.com/cbG0rv
 
 ## Releases and Contributing

aten/.gitmodules

@@ -1,10 +0,0 @@
-[submodule "src/ATen/cpu/cpuinfo"]
-	path = src/ATen/cpu/cpuinfo
-	url = https://github.com/Maratyszcza/cpuinfo
-[submodule "src/ATen/cpu/tbb/tbb_remote"]
-	path = src/ATen/cpu/tbb/tbb_remote
-	url = https://github.com/01org/tbb
-	branch = tbb_2018
-[submodule "src/ATen/utils/catch"]
-	path = src/ATen/utils/catch
-	url = https://github.com/catchorg/Catch2.git

aten/CMakeLists.txt

@@ -1,521 +1,105 @@
-cmake_minimum_required(VERSION 3.0)
-set(CMAKE_MODULE_PATH
-  ${CMAKE_CURRENT_SOURCE_DIR}/cmake
-  ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/Modules_CUDA_fix
+if (BUILD_ATEN_MOBILE)
+  return()
+endif()
+
+# Find modules
+list(APPEND CMAKE_MODULE_PATH
   /usr/lib/x86_64-linux-gnu/
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/TH/cmake
-  ${CMAKE_MODULE_PATH})
-SET(CMAKE_LIBRARY_PATH /usr/lib/x86_64-linux-gnu/ ${CMAKE_LIBRARY_PATH})
-project(ATen)
+  ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/Modules
+  ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/public
+  ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/Modules_CUDA_fix)
+list(APPEND CMAKE_LIBRARY_PATH /usr/lib/x86_64-linux-gnu/)
 
 cmake_policy(SET CMP0012 NEW)
 
-# Polyfill for upstream FindCUDA
-include(CMakeInitializeConfigs)
+#############################################
 
-# RPATH stuff
-# see https://cmake.org/Wiki/CMake_RPATH_handling
-if(APPLE)
-  set(CMAKE_MACOSX_RPATH ON)
-endif()
-set(CMAKE_SKIP_BUILD_RPATH  FALSE)
-set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
-list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir)
-if("${isSystemDir}" STREQUAL "-1")
-  set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
+set(ATen_CPU_SRCS)
+set(ATen_CPU_TEST_SRCS)
+set(ATen_CPU_INCLUDE)
+set(ATen_THIRD_PARTY_INCLUDE)
+set(ATen_CUDA_SRCS)
+set(ATen_CUDA_TEST_SRCS)
+set(ATen_CUDA_INCLUDE)
+set(ATen_CPU_DEPENDENCY_LIBS)
+set(ATen_CUDA_DEPENDENCY_LIBS)
+set(ATen_PUBLIC_CUDA_DEPENDENCY_LIBS)
+SET(ATEN_INSTALL_BIN_SUBDIR "bin" CACHE PATH "ATen install binary subdirectory")
+SET(ATEN_INSTALL_LIB_SUBDIR "lib" CACHE PATH "ATen install library subdirectory")
+SET(ATEN_INSTALL_INCLUDE_SUBDIR "include" CACHE PATH "ATen install include subdirectory")
+
+if(USE_CUDA)
+  list(APPEND ATen_CUDA_INCLUDE ${CUDA_INCLUDE_DIRS})
 endif()
 
-IF(NOT MSVC)
-  set(CMAKE_CXX_FLAGS "--std=c++11 -Wall -Wextra -Wno-missing-field-initializers -Wno-type-limits -Wno-unused-parameter -Wno-unknown-pragmas -Wno-vla -fexceptions ${CMAKE_CXX_FLAGS}")
-  set(CMAKE_C_FLAGS "-fexceptions ${CMAKE_C_FLAGS}")
-  if ($ENV{WERROR})
-    set(CMAKE_CXX_FLAGS "-Werror ${CMAKE_CXX_FLAGS}")
+set(TH_LINK_STYLE STATIC)
+add_subdirectory(src/TH)
+set(TH_CPU_INCLUDE
+  # dense
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/TH
+  ${CMAKE_CURRENT_BINARY_DIR}/src/TH
+  ${CMAKE_CURRENT_SOURCE_DIR}/src
+  ${CMAKE_CURRENT_BINARY_DIR}/src
+  ${CMAKE_BINARY_DIR}/aten/src)
+list(APPEND ATen_CPU_INCLUDE ${TH_CPU_INCLUDE})
+
+if(USE_CUDA OR USE_ROCM)
+  set(TH_CUDA_INCLUDE
+    # dense
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/THC
+    ${CMAKE_CURRENT_BINARY_DIR}/src/THC)
+  list(APPEND ATen_CUDA_INCLUDE ${TH_CUDA_INCLUDE})
+endif()
+
+add_subdirectory(src/THNN)
+
+# Find the HIP package, set the HIP paths, load the HIP CMake.
+IF(USE_ROCM)
+  include(LoadHIP)
+  if (NOT PYTORCH_FOUND_HIP)
+    MESSAGE(FATAL_ERROR
+      "Could not find HIP installation")
   endif()
-ENDIF(NOT MSVC)
-
-INCLUDE(CheckCXXSourceCompiles)
-
-# disable some verbose warnings
-IF (MSVC)
-  set(CMAKE_CXX_FLAGS "/wd4267 /wd4251 /wd4522 /wd4522 /wd4838 /wd4305 /wd4244 /wd4190 /wd4101 /wd4996 /wd4275 ${CMAKE_CXX_FLAGS}")
-ENDIF(MSVC)
-
-# windef.h will define max/min macros if NOMINMAX is not defined
-IF(MSVC)
-  add_definitions(/DNOMINMAX)
-ENDIF(MSVC)
-
-#Check if certain std functions are supported. Sometimes
-#_GLIBCXX_USE_C99 macro is not defined and some functions are missing.
-CHECK_CXX_SOURCE_COMPILES("
-#include <cmath>
-#include <string>
-
-int main() {
-  int a = std::isinf(3.0);
-  int b = std::isnan(0.0);
-  std::string s = std::to_string(1);
-
-  return 0;
-  }" SUPPORT_GLIBCXX_USE_C99)
-
-if(NOT SUPPORT_GLIBCXX_USE_C99)
-  message(FATAL_ERROR
-          "The C++ compiler does not support required functions. "
-          "This is very likely due to a known bug in GCC 5 "
-          "(and maybe other versions) on Ubuntu 17.10 and newer. "
-          "For more information, see: "
-          "https://github.com/pytorch/pytorch/issues/5229"
-         )
-endif()
-
-# Top-level build config
-############################################
-# Flags
-# When using MSVC
-
-# Detect CUDA architecture and get best NVCC flags
-# finding cuda must be first because other things depend on the result
-#
-# NB: We MUST NOT run this find_package if NO_CUDA is set, because upstream
-# FindCUDA has a bug where it will still attempt to make use of NOTFOUND
-# compiler variables to run various probe tests.  We could try to fix
-# this, but since FindCUDA upstream is subsumed by first-class support
-# for CUDA language, it seemed not worth fixing.
-IF(NOT CUDA_FOUND AND NOT NO_CUDA)
-  FIND_PACKAGE(CUDA 5.5)
 ENDIF()
 
 IF(MSVC)
   # we want to respect the standard, and we are bored of those **** .
   ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE=1)
   LIST(APPEND CUDA_NVCC_FLAGS "-Xcompiler /wd4819 -Xcompiler /wd4503 -Xcompiler /wd4190 -Xcompiler /wd4244 -Xcompiler /wd4251 -Xcompiler /wd4275 -Xcompiler /wd4522")
-  ADD_DEFINITIONS(-DTH_EXPORTS)
-  IF (NOT NO_CUDA)
-    ADD_DEFINITIONS(-DTHC_EXPORTS)
-  ENDIF()
 ENDIF(MSVC)
 
-IF (NOT MSVC)
-  IF (CMAKE_VERSION VERSION_LESS "3.1")
-    SET(CMAKE_C_FLAGS "-std=c11 ${CMAKE_C_FLAGS}")
-  ELSE ()
-    SET(CMAKE_C_STANDARD 11)
-  ENDIF ()
-ENDIF(NOT MSVC)
-
-if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "4.9")
-    if(CUDA_VERSION VERSION_LESS "8.0")
-      MESSAGE(STATUS "Found gcc >=5 and CUDA <= 7.5, adding workaround C++ flags")
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_FORCE_INLINES -D_MWAITXINTRIN_H_INCLUDED -D__STRICT_ANSI__")
-    endif(CUDA_VERSION VERSION_LESS "8.0")
-  endif(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "4.9")
-endif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-
-LIST(APPEND CUDA_NVCC_FLAGS -Wno-deprecated-gpu-targets)
-LIST(APPEND CUDA_NVCC_FLAGS --expt-extended-lambda)
-
-if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  SET(CMAKE_CXX_STANDARD 11)
-endif()
-
-IF(NOT COMMAND CUDA_SELECT_NVCC_ARCH_FLAGS)
-  INCLUDE(${CMAKE_CURRENT_SOURCE_DIR}/cmake/select_compute_arch.cmake)
-ENDIF()
-LIST(APPEND CUDA_NVCC_FLAGS $ENV{TORCH_NVCC_FLAGS})
-CUDA_SELECT_NVCC_ARCH_FLAGS(NVCC_FLAGS_EXTRA $ENV{TORCH_CUDA_ARCH_LIST})
-LIST(APPEND CUDA_NVCC_FLAGS ${NVCC_FLAGS_EXTRA})
-IF(CMAKE_POSITION_INDEPENDENT_CODE AND NOT MSVC)
-  LIST(APPEND CUDA_NVCC_FLAGS "-Xcompiler -fPIC")
-ENDIF()
-
-IF(CUDA_HAS_FP16 OR NOT ${CUDA_VERSION} LESS 7.5)
-  MESSAGE(STATUS "Found CUDA with FP16 support, compiling with torch.CudaHalfTensor")
-  LIST(APPEND CUDA_NVCC_FLAGS "-DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__")
-  SET(CMAKE_C_FLAGS "-DCUDA_HAS_FP16=1 ${CMAKE_C_FLAGS}")
-ELSE(CUDA_HAS_FP16 OR NOT ${CUDA_VERSION} LESS 7.5)
-  MESSAGE(STATUS "Could not find CUDA with FP16 support, compiling without torch.CudaHalfTensor")
-ENDIF(CUDA_HAS_FP16 OR NOT ${CUDA_VERSION} LESS 7.5)
-
-OPTION(NDEBUG "disable asserts (WARNING: this may result in silent UB e.g. with out-of-bound indices)")
-IF(NOT NDEBUG)
-  MESSAGE(STATUS "Removing -DNDEBUG from compile flags")
-  STRING(REPLACE "-DNDEBUG" "" CMAKE_C_FLAGS "" ${CMAKE_C_FLAGS})
-  STRING(REPLACE "-DNDEBUG" "" CMAKE_C_FLAGS_DEBUG "" ${CMAKE_C_FLAGS_DEBUG})
-  STRING(REPLACE "-DNDEBUG" "" CMAKE_C_FLAGS_RELEASE "" ${CMAKE_C_FLAGS_RELEASE})
-  STRING(REPLACE "-DNDEBUG" "" CMAKE_CXX_FLAGS "" ${CMAKE_CXX_FLAGS})
-  STRING(REPLACE "-DNDEBUG" "" CMAKE_CXX_FLAGS_DEBUG "" ${CMAKE_CXX_FLAGS_DEBUG})
-  STRING(REPLACE "-DNDEBUG" "" CMAKE_CXX_FLAGS_RELEASE "" ${CMAKE_CXX_FLAGS_RELEASE})
-ENDIF()
-
-# OpenMP support?
-SET(WITH_OPENMP ON CACHE BOOL "OpenMP support if available?")
-IF (APPLE AND CMAKE_COMPILER_IS_GNUCC)
-  EXEC_PROGRAM (uname ARGS -v  OUTPUT_VARIABLE DARWIN_VERSION)
-  STRING (REGEX MATCH "[0-9]+" DARWIN_VERSION ${DARWIN_VERSION})
-  MESSAGE (STATUS "MAC OS Darwin Version: ${DARWIN_VERSION}")
-  IF (DARWIN_VERSION GREATER 9)
-    SET(APPLE_OPENMP_SUCKS 1)
-  ENDIF (DARWIN_VERSION GREATER 9)
-  EXECUTE_PROCESS (COMMAND ${CMAKE_C_COMPILER} -dumpversion
-    OUTPUT_VARIABLE GCC_VERSION)
-  IF (APPLE_OPENMP_SUCKS AND GCC_VERSION VERSION_LESS 4.6.2)
-    MESSAGE(STATUS "Warning: Disabling OpenMP (unstable with this version of GCC)")
-    MESSAGE(STATUS " Install GCC >= 4.6.2 or change your OS to enable OpenMP")
-    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unknown-pragmas")
-    SET(WITH_OPENMP OFF CACHE BOOL "OpenMP support if available?" FORCE)
-  ENDIF ()
-ENDIF ()
-
-IF (WITH_OPENMP AND NOT CHECKED_OPENMP)
-  FIND_PACKAGE(OpenMP)
-  SET(CHECKED_OPENMP ON CACHE BOOL "already checked for OpenMP")
-
-  # OPENMP_FOUND is not cached in FindOpenMP.cmake (all other variables are cached)
-  # see https://github.com/Kitware/CMake/blob/master/Modules/FindOpenMP.cmake
-  SET(OPENMP_FOUND ${OPENMP_FOUND} CACHE BOOL "OpenMP Support found")
-ENDIF (WITH_OPENMP AND NOT CHECKED_OPENMP)
-
-IF(OPENMP_FOUND)
-  MESSAGE(STATUS "Compiling with OpenMP support")
-  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
-  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-ENDIF(OPENMP_FOUND)
-
-
-SET(CUDA_ATTACH_VS_BUILD_RULE_TO_CUDA_FILE OFF)
-
-FIND_PACKAGE(MAGMA)
-IF(CUDA_FOUND AND MAGMA_FOUND)
-  INCLUDE_DIRECTORIES("${MAGMA_INCLUDE_DIR}")
-  SET(CMAKE_REQUIRED_INCLUDES "${MAGMA_INCLUDE_DIR};${CUDA_INCLUDE_DIRS}")
-  INCLUDE(CheckPrototypeDefinition)
-  check_prototype_definition(magma_get_sgeqrf_nb
-   "magma_int_t magma_get_sgeqrf_nb( magma_int_t m, magma_int_t n );"
-   "0"
-   "magma.h"
-    MAGMA_V2)
-  IF (MAGMA_V2)
-    add_definitions(-DMAGMA_V2)
-  ENDIF (MAGMA_V2)
-
-  SET(USE_MAGMA 1)
-  MESSAGE(STATUS "Compiling with MAGMA support")
-  MESSAGE(STATUS "MAGMA INCLUDE DIRECTORIES: ${MAGMA_INCLUDE_DIR}")
-  MESSAGE(STATUS "MAGMA LIBRARIES: ${MAGMA_LIBRARIES}")
-  MESSAGE(STATUS "MAGMA V2 check: ${MAGMA_V2}")
-ELSE()
-  MESSAGE(STATUS "MAGMA not found. Compiling without MAGMA support")
-ENDIF()
-
-# ARM specific flags
-FIND_PACKAGE(ARM)
-IF (ASIMD_FOUND)
-  MESSAGE(STATUS "asimd/Neon found with compiler flag : -D__NEON__")
-  SET(CMAKE_C_FLAGS "-D__NEON__ ${CMAKE_C_FLAGS}")
-ELSEIF (NEON_FOUND)
-  MESSAGE(STATUS "Neon found with compiler flag : -mfpu=neon -D__NEON__")
-  SET(CMAKE_C_FLAGS "-mfpu=neon -D__NEON__ ${CMAKE_C_FLAGS}")
-ENDIF (ASIMD_FOUND)
-IF (CORTEXA8_FOUND)
-  MESSAGE(STATUS "Cortex-A8 Found with compiler flag : -mcpu=cortex-a8")
-  SET(CMAKE_C_FLAGS "-mcpu=cortex-a8 -fprefetch-loop-arrays ${CMAKE_C_FLAGS}")
-ENDIF (CORTEXA8_FOUND)
-IF (CORTEXA9_FOUND)
-  MESSAGE(STATUS "Cortex-A9 Found with compiler flag : -mcpu=cortex-a9")
-  SET(CMAKE_C_FLAGS "-mcpu=cortex-a9 ${CMAKE_C_FLAGS}")
-ENDIF (CORTEXA9_FOUND)
-
-IF(UNIX)
-  # prevent Unknown CMake command "check_function_exists".
-  INCLUDE(CheckFunctionExists)
-ENDIF(UNIX)
-
-INCLUDE (CheckIncludeFile)
-INCLUDE (CheckCSourceCompiles)
-INCLUDE (CheckCSourceRuns)
-
-# Check that our programs run.  This is different from the native CMake compiler
-# check, which just tests if the program compiles and links.  This is important
-# because with ASAN you might need to help the compiled library find some
-# dynamic libraries.
-CHECK_C_SOURCE_RUNS("
-int main() { return 0; }
-" COMPILER_WORKS)
-IF(NOT COMPILER_WORKS)
-  # Force cmake to retest next time around
-  unset(COMPILER_WORKS CACHE)
-  MESSAGE(FATAL_ERROR
-      "Could not run a simple program built with your compiler. "
-      "If you are trying to use -fsanitize=address, make sure "
-      "libasan is properly installed on your system (you can confirm "
-      "if the problem is this by attempting to build and run a "
-      "small program.)")
-ENDIF()
-
-CHECK_INCLUDE_FILE(cpuid.h HAVE_CPUID_H)
-# Check for a cpuid intrinsic
-IF(HAVE_CPUID_H)
-    CHECK_C_SOURCE_COMPILES("#include <cpuid.h>
-        int main()
-        {
-            unsigned int eax, ebx, ecx, edx;
-            return __get_cpuid(0, &eax, &ebx, &ecx, &edx);
-        }" HAVE_GCC_GET_CPUID)
-ENDIF()
-IF(HAVE_GCC_GET_CPUID)
-  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_GCC_GET_CPUID")
-ENDIF(HAVE_GCC_GET_CPUID)
-
-CHECK_C_SOURCE_COMPILES("#include <stdint.h>
-    static inline void cpuid(uint32_t *eax, uint32_t *ebx,
-    			 uint32_t *ecx, uint32_t *edx)
-    {
-      uint32_t a = *eax, b, c = *ecx, d;
-      asm volatile ( \"cpuid\" : \"+a\"(a), \"=b\"(b), \"+c\"(c), \"=d\"(d) );
-      *eax = a; *ebx = b; *ecx = c; *edx = d;
-    }
-    int main() {
-      uint32_t a,b,c,d;
-      cpuid(&a, &b, &c, &d);
-      return 0;
-    }" NO_GCC_EBX_FPIC_BUG)
-
-IF(NOT NO_GCC_EBX_FPIC_BUG)
-  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_GCC_GET_CPUID")
-ENDIF(NOT NO_GCC_EBX_FPIC_BUG)
-
-FIND_PACKAGE(SSE) # checks SSE, AVX and AVX2
-IF(C_SSE2_FOUND)
-  MESSAGE(STATUS "SSE2 Found")
-  SET(CMAKE_C_FLAGS "${C_SSE2_FLAGS} -DUSE_SSE2 ${CMAKE_C_FLAGS}")
-ENDIF(C_SSE2_FOUND)
-IF(C_SSE4_1_FOUND AND C_SSE4_2_FOUND)
-  SET(CMAKE_C_FLAGS "${C_SSE4_1_FLAGS} -DUSE_SSE4_1 ${C_SSE4_2_FLAGS} -DUSE_SSE4_2 ${CMAKE_C_FLAGS}")
-ENDIF()
-IF(C_SSE3_FOUND)
-  MESSAGE(STATUS "SSE3 Found")
-  SET(CMAKE_C_FLAGS "${C_SSE3_FLAGS} -DUSE_SSE3 ${CMAKE_C_FLAGS}")
-  SET(CMAKE_CXX_FLAGS "${C_SSE3_FLAGS} -DUSE_SSE3 ${CMAKE_CXX_FLAGS}")
-ENDIF(C_SSE3_FOUND)
-
-# we don't set -mavx and -mavx2 flags globally, but only for specific files
-# however, we want to enable the AVX codepaths, so we still need to
-# add USE_AVX and USE_AVX2 macro defines
-IF(C_AVX_FOUND)
-  MESSAGE(STATUS "AVX Found")
-  SET(CMAKE_C_FLAGS "-DUSE_AVX ${CMAKE_C_FLAGS}")
-ENDIF(C_AVX_FOUND)
-IF(C_AVX2_FOUND)
-  MESSAGE(STATUS "AVX2 Found")
-  SET(CMAKE_C_FLAGS "-DUSE_AVX2 ${CMAKE_C_FLAGS}")
-  SET(CMAKE_CXX_FLAGS "-DUSE_AVX2 ${CMAKE_CXX_FLAGS}")
-ENDIF(C_AVX2_FOUND)
-
-CHECK_C_SOURCE_RUNS("
-#include <stdatomic.h>
-// ATOMIC_INT_LOCK_FREE is flaky on some older gcc versions
-// so if this define is not usable a preprocessor definition
-// we fail this check and fall back to GCC atomics
-#if ATOMIC_INT_LOCK_FREE == 2
-#define TH_ATOMIC_IPC_REFCOUNT 1
-#endif
-int main()
-{
-  int a;
-  int oa;
-  atomic_store(&a, 1);
-  atomic_fetch_add(&a, 1);
-  oa = atomic_load(&a);
-  if(!atomic_compare_exchange_strong(&a, &oa, 3))
-    return -1;
-  return 0;
-}
-" HAS_C11_ATOMICS)
-
-IF(NOT HAS_C11_ATOMICS)
-  CHECK_C_SOURCE_RUNS("
-#include <intrin.h>
-int main()
-{
-  long a;
-  _InterlockedExchange(&a, 1);
-  _InterlockedExchangeAdd(&a, 1);
-  if(_InterlockedCompareExchange(&a, 3, 2) != 2)
-    return -1;
-  return 0;
-}
-" HAS_MSC_ATOMICS)
-
-  CHECK_C_SOURCE_RUNS("
-int main()
-{
-  int a;
-  __sync_lock_test_and_set(&a, 1);
-  __sync_fetch_and_add(&a, 1);
-  if(!__sync_bool_compare_and_swap(&a, 2, 3))
-    return -1;
-  return 0;
-}
-" HAS_GCC_ATOMICS)
-ENDIF()
-
-IF(HAS_C11_ATOMICS)
-  ADD_DEFINITIONS(-DUSE_C11_ATOMICS=1)
-  MESSAGE(STATUS "Atomics: using C11 intrinsics")
-ELSEIF(HAS_MSC_ATOMICS)
-  ADD_DEFINITIONS(-DUSE_MSC_ATOMICS=1)
-  MESSAGE(STATUS "Atomics: using MSVC intrinsics")
-ELSEIF(HAS_GCC_ATOMICS)
-  ADD_DEFINITIONS(-DUSE_GCC_ATOMICS=1)
-    MESSAGE(STATUS "Atomics: using GCC intrinsics")
-ELSE()
-  SET(CMAKE_THREAD_PREFER_PTHREAD TRUE)
-  FIND_PACKAGE(Threads)
-  IF(THREADS_FOUND)
-    ADD_DEFINITIONS(-DUSE_PTHREAD_ATOMICS=1)
-    TARGET_LINK_LIBRARIES(TH ${CMAKE_THREAD_LIBS_INIT})
-    MESSAGE(STATUS "Atomics: using pthread")
-  ENDIF()
-ENDIF()
-
-IF (WIN32 AND NOT CYGWIN)
-  SET(BLAS_INSTALL_LIBRARIES "OFF"
-    CACHE BOOL "Copy the required BLAS DLLs into the TH install dirs")
-ENDIF (WIN32 AND NOT CYGWIN)
-
-MACRO(Install_Required_Library ln)
-    get_filename_component(libpath ${ln} PATH)
-    get_filename_component(libname ${ln} NAME_WE)
-    file(GLOB libdlls "${libpath}/${libname}*.dll")
-    install(PROGRAMS ${libdlls}
-      DESTINATION "${TH_INSTALL_BIN_SUBDIR}")
-ENDMACRO(Install_Required_Library libname)
-
-FIND_PACKAGE(BLAS)
-SET(AT_MKL_ENABLED 0)
-SET(AT_MKL_MT 0)
-IF(BLAS_FOUND)
-  SET(USE_BLAS 1)
-  IF(BLAS_INFO STREQUAL "mkl")
-    ADD_DEFINITIONS(-DTH_BLAS_MKL)
-    IF(NOT BLAS_INCLUDE_DIR)
-      MESSAGE(FATAL_ERROR "MKL is used, but MKL header files are not found. \
-        You can get them by `conda install mkl-include` if using conda (if \
-        it is missing, run `conda upgrade -n root conda` first), and \
-        `pip install mkl-devel` if using pip. If build fails with header files \
-        available in the system, please make sure that CMake will search the \
-        directory containing them, e.g., by setting CMAKE_INCLUDE_PATH.")
-    ENDIF()
-    IF(MSVC AND MKL_LIBRARIES MATCHES ".*libiomp5md\\.lib.*")
-      ADD_DEFINITIONS(-D_OPENMP_NOFORCE_MANIFEST)
-      SET(AT_MKL_MT 1)
-    ENDIF()
-    INCLUDE_DIRECTORIES(${BLAS_INCLUDE_DIR})  # include MKL headers
-    SET(AT_MKL_ENABLED 1)
-  ENDIF()
-ENDIF(BLAS_FOUND)
-
-FIND_PACKAGE(LAPACK)
-IF(LAPACK_FOUND)
-  SET(USE_LAPACK 1)
-ENDIF(LAPACK_FOUND)
-
-#############################################
-
-set(ATen_CPU_SRCS)
-set(ATen_CPU_INCLUDE)
-set(ATen_CUDA_SRCS)
-set(ATen_CUDA_INCLUDE)
-SET(ATEN_INSTALL_BIN_SUBDIR "bin" CACHE PATH "ATen install binary subdirectory")
-SET(ATEN_INSTALL_LIB_SUBDIR "lib" CACHE PATH "ATen install library subdirectory")
-SET(ATEN_INSTALL_INCLUDE_SUBDIR "include" CACHE PATH "ATen install include subdirectory")
-
-add_definitions(-DTH_INDEX_BASE=0)
-set(TH_LINK_STYLE STATIC)
-add_subdirectory(src/TH)
-include_directories(
-  # dense
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/TH
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/THC
-  ${CMAKE_CURRENT_BINARY_DIR}/src/TH
-  ${CMAKE_CURRENT_BINARY_DIR}/src/THC
-  # sparse
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/THS
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/THCS
-  ${CMAKE_CURRENT_BINARY_DIR}/src/THS
-  ${CMAKE_CURRENT_BINARY_DIR}/src/THCS
-
-  ${CMAKE_CURRENT_SOURCE_DIR}/src
-  ${CMAKE_CURRENT_BINARY_DIR}/src)
-add_subdirectory(src/THNN)
-add_subdirectory(src/THS)
-
-if(NO_CUDA)
-  message("disabling CUDA because NO_CUDA is set")
-  SET(CUDA_FLAG -n)
-  SET(AT_CUDA_ENABLED 0)
-else()
+if(USE_ROCM)
+  SET(AT_CUDA_ENABLED 1)
+  add_subdirectory(src/THC)
+  add_subdirectory(src/THCUNN)
+  message("ROCm is enabled.")
+elseif(USE_CUDA)
   SET(AT_CUDA_ENABLED 1)
-  INCLUDE_DIRECTORIES(${CUDA_INCLUDE_DIRS})
-  find_package(CUDA 5.5 REQUIRED)
   add_subdirectory(src/THC)
   add_subdirectory(src/THCUNN)
-  add_subdirectory(src/THCS)
-endif()
-
-find_package(CuDNN)
-IF(NOT AT_CUDA_ENABLED OR NOT CUDNN_FOUND)
-  MESSAGE(STATUS "CuDNN not found. Compiling without CuDNN support")
-  set(AT_CUDNN_ENABLED 0)
-ELSE()
-  INCLUDE_DIRECTORIES(BEFORE ${CUDNN_INCLUDE_DIRS})
-  set(AT_CUDNN_ENABLED 1)
-ENDIF()
-
-if(NO_MKLDNN)
-  message("disabling MKLDNN because NO_MKLDNN is set")
-  set(AT_MKLDNN_ENABLED 0)
 else()
-  find_package(MKLDNN)
-  if(NOT MKLDNN_FOUND)
-    message(STATUS "MKLDNN not found. Compiling without MKLDNN support")
-    set(AT_MKLDNN_ENABLED 0)
-  else()
-    INCLUDE_DIRECTORIES(${MKLDNN_INCLUDE_DIRS})
-    set(AT_MKLDNN_ENABLED 1)
-  endif()
+  message("disabling CUDA because USE_CUDA is set false")
+  SET(AT_CUDA_ENABLED 0)
 endif()
 
-set(cwrap_files
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/ATen/Declarations.cwrap
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/THNN/generic/THNN.h
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/THCUNN/generic/THCUNN.h
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/ATen/nn.yaml
-  ${CMAKE_CURRENT_SOURCE_DIR}/src/ATen/native/native_functions.yaml
-)
-
-include_directories(
-${CMAKE_CURRENT_SOURCE_DIR}/src/THNN
-${CMAKE_CURRENT_SOURCE_DIR}/src/THCUNN)
+list(APPEND ATen_CPU_INCLUDE
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/THNN
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/THCUNN)
 
+list(APPEND ATen_CPU_INCLUDE
+  ${CMAKE_CURRENT_SOURCE_DIR}/src
+  ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/catch/single_include
+  ${CMAKE_CURRENT_BINARY_DIR}/src/ATen)
 add_subdirectory(src/ATen)
-include_directories(
-${CMAKE_CURRENT_SOURCE_DIR}/src
-${CMAKE_CURRENT_SOURCE_DIR}/src/ATen/utils/catch/single_include
-${CMAKE_CURRENT_BINARY_DIR}/src/ATen)
-if(NOT NO_CUDA)
-  include_directories(${CUDA_INCLUDE_DIRS})
-endif()
-add_subdirectory(src/ATen/test)
 
-if(ATEN_NO_CONTRIB)
-  message("disable contrib because ATEN_NO_CONTRIB is set")
-else()
-  add_subdirectory(contrib/data)
-  add_subdirectory(contrib/meter)
-endif()
+# Pass source, includes, and libs to parent
+set(ATen_CPU_SRCS ${ATen_CPU_SRCS} PARENT_SCOPE)
+set(ATen_CUDA_SRCS ${ATen_CUDA_SRCS} PARENT_SCOPE)
+set(ATen_CPU_TEST_SRCS ${ATen_CPU_TEST_SRCS} PARENT_SCOPE)
+set(ATen_CUDA_TEST_SRCS ${ATen_CUDA_TEST_SRCS} PARENT_SCOPE)
+set(ATen_CPU_INCLUDE ${ATen_CPU_INCLUDE} PARENT_SCOPE)
+set(ATen_CUDA_INCLUDE ${ATen_CUDA_INCLUDE} PARENT_SCOPE)
+set(ATen_THIRD_PARTY_INCLUDE ${ATen_THIRD_PARTY_INCLUDE} PARENT_SCOPE)
+set(ATen_CPU_DEPENDENCY_LIBS ${ATen_CPU_DEPENDENCY_LIBS} PARENT_SCOPE)
+set(ATen_CUDA_DEPENDENCY_LIBS ${ATen_CUDA_DEPENDENCY_LIBS} PARENT_SCOPE)
+set(ATen_CORE_TEST_SRCS ${ATen_CORE_TEST_SRCS} PARENT_SCOPE)

aten/README.md

@@ -2,7 +2,7 @@
 
 ATen is a simple tensor library thats exposes the Tensor operations in Torch
 and PyTorch directly in C++11. The wrapper respects the semantics of operators
-in PyTorch, except minor details due to differences between C++ in Python in
+in PyTorch, except minor details due to differences between C++ and Python in
 the way default arguments are handled. See the [documentation for tensors](http://pytorch.org/docs/tensors.html) in PyTorch for what these operations do.
 ATen's API is auto-generated from the same declarations PyTorch uses so the
 two APIs will track each other over time.
@@ -12,7 +12,7 @@ does not include templates. That is, there is one `Tensor` type. It can hold a
 CPU or CUDA Tensor, and the tensor may have Doubles, Float, Ints, etc. This design
 makes it easy to write generic code without templating everything.
 
-See the _generated_ [`Tensor.h` file](doc/Tensor.h) and [`Functions.h` file](doc/Functions.h) for the provided API. Excerpt:
+See https://pytorch.org/cppdocs for the provided API. Excerpt:
 ```c++
 Tensor atan2(const Tensor & other) const;
 Tensor & atan2_(const Tensor & other);
@@ -48,7 +48,8 @@ sudo pip install pyyaml
 mkdir build
 cd build
 cmake .. -DCMAKE_INSTALL_PREFIX=/where/you/want # specify your dest directory
-# cmake .. -DNO_CUDA=true  # for CPU only machines
+# cmake .. -DUSE_NVRTC=ON -DUSE_TENSORRT=OFF -DCMAKE_INSTALL_PREFIX=../install -DCAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO=OFF -DUSE_CUDA=ON # for CUDA
+# cmake .. -DUSE_CUDA=OFF  # for CPU only machines
 make install
 ```
 
@@ -87,7 +88,7 @@ for(auto i = 0; i < 100000; i++) {
 
 Expressions like `CUDA(kFloat)` are first-class `at::Type` objects that represent
 the type of a Tensor and are used to create Tensors when their type cannot be
-inferred. See the _generated_ [Type header](doc/Type.h) for its API.
+inferred.
 
 See more in [sample files](src/ATen/test).
 
@@ -164,7 +165,7 @@ behave as normal tensors.
 ### Scalars and zero-dimensional tensors
 
 In addition to the `Tensor` objects, ATen also includes `Scalar`s that represent a single number.
-Like a Tensor, Scalars are dynamically typed and can hold any one of ATen's [number types](doc/Type.h).
+Like a Tensor, Scalars are dynamically typed and can hold any one of ATen's number types.
 Scalars can be implicitly constructed from C++ number types. Scalars are needed because some functions like `addmm` take numbers along with Tensors and expect these
 numbers to be the same dynamic type as the tensor. They are also used in the API to indicate places where
 a function will _always_ return a Scalar value, like `sum`.

aten/cmake/FindCuDNN.cmake

@@ -1,70 +0,0 @@
-# - Try to find cuDNN
-#
-# The following variables are optionally searched for defaults
-#  CUDNN_ROOT_DIR:            Base directory where all cuDNN components are found
-#
-# The following are set after configuration is done:
-#  CUDNN_FOUND
-#  CUDNN_INCLUDE_DIRS
-#  CUDNN_LIBRARIES
-#  CUDNN_LIBRARY_DIRS
-#
-# Borrowed from https://github.com/caffe2/caffe2/blob/master/cmake/Modules/FindCuDNN.cmake
-
-include(FindPackageHandleStandardArgs)
-
-set(CUDNN_ROOT_DIR "" CACHE PATH "Folder contains NVIDIA cuDNN")
-
-if($ENV{CUDNN_INCLUDE_DIR})
-  SET(CUDNN_INCLUDE_DIR $ENV{CUDNN_INCLUDE_DIR})
-else($ENV{CUDNN_INCLUDE_DIR})
-  find_path(CUDNN_INCLUDE_DIR cudnn.h
-    HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}
-    PATH_SUFFIXES cuda/include include)
-endif($ENV{CUDNN_INCLUDE_DIR})
-
-IF ($ENV{USE_STATIC_CUDNN})
-  MESSAGE(STATUS "USE_STATIC_CUDNN detected. Linking against static CUDNN library")
-  SET(CUDNN_LIBNAME "libcudnn_static.a")
-ELSE()
-  SET(CUDNN_LIBNAME "cudnn")
-ENDIF()
-
-if($ENV{CUDNN_LIBRARY})
-  SET(CUDNN_LIBRARY $ENV{CUDNN_LIBRARY})
-else($ENV{CUDNN_LIBRARY})
-  find_library(CUDNN_LIBRARY ${CUDNN_LIBNAME}
-    HINTS ${CUDNN_LIB_DIR} ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}
-    PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64)
-endif($ENV{CUDNN_LIBRARY})
-
-find_package_handle_standard_args(
-    CUDNN DEFAULT_MSG CUDNN_INCLUDE_DIR CUDNN_LIBRARY)
-
-if(CUDNN_FOUND)
-	# get cuDNN version
-  file(READ ${CUDNN_INCLUDE_DIR}/cudnn.h CUDNN_HEADER_CONTENTS)
-	string(REGEX MATCH "define CUDNN_MAJOR * +([0-9]+)"
-				 CUDNN_VERSION_MAJOR "${CUDNN_HEADER_CONTENTS}")
-	string(REGEX REPLACE "define CUDNN_MAJOR * +([0-9]+)" "\\1"
-				 CUDNN_VERSION_MAJOR "${CUDNN_VERSION_MAJOR}")
-	string(REGEX MATCH "define CUDNN_MINOR * +([0-9]+)"
-				 CUDNN_VERSION_MINOR "${CUDNN_HEADER_CONTENTS}")
-	string(REGEX REPLACE "define CUDNN_MINOR * +([0-9]+)" "\\1"
-				 CUDNN_VERSION_MINOR "${CUDNN_VERSION_MINOR}")
-	string(REGEX MATCH "define CUDNN_PATCHLEVEL * +([0-9]+)"
-				 CUDNN_VERSION_PATCH "${CUDNN_HEADER_CONTENTS}")
-	string(REGEX REPLACE "define CUDNN_PATCHLEVEL * +([0-9]+)" "\\1"
-				 CUDNN_VERSION_PATCH "${CUDNN_VERSION_PATCH}")
-  # Assemble cuDNN version
-  if(NOT CUDNN_VERSION_MAJOR)
-    set(CUDNN_VERSION "?")
-  else()
-    set(CUDNN_VERSION "${CUDNN_VERSION_MAJOR}.${CUDNN_VERSION_MINOR}.${CUDNN_VERSION_PATCH}")
-  endif()
-
-  set(CUDNN_INCLUDE_DIRS ${CUDNN_INCLUDE_DIR})
-  set(CUDNN_LIBRARIES ${CUDNN_LIBRARY})
-  message(STATUS "Found cuDNN: v${CUDNN_VERSION}  (include: ${CUDNN_INCLUDE_DIR}, library: ${CUDNN_LIBRARY})")
-  mark_as_advanced(CUDNN_ROOT_DIR CUDNN_LIBRARY CUDNN_INCLUDE_DIR)
-endif()

aten/cmake/select_compute_arch.cmake

@@ -1,196 +0,0 @@
-# Synopsis:
-#   CUDA_SELECT_NVCC_ARCH_FLAGS(out_variable [target_CUDA_architectures])
-#   -- Selects GPU arch flags for nvcc based on target_CUDA_architectures
-#      target_CUDA_architectures : Auto | Common | All | LIST(ARCH_AND_PTX ...)
-#       - "Auto" detects local machine GPU compute arch at runtime.
-#       - "Common" and "All" cover common and entire subsets of architectures
-#      ARCH_AND_PTX : NAME | NUM.NUM | NUM.NUM(NUM.NUM) | NUM.NUM+PTX
-#      NAME: Fermi Kepler Maxwell Kepler+Tegra Kepler+Tesla Maxwell+Tegra Pascal
-#      NUM: Any number. Only those pairs are currently accepted by NVCC though:
-#            2.0 2.1 3.0 3.2 3.5 3.7 5.0 5.2 5.3 6.0 6.2
-#      Returns LIST of flags to be added to CUDA_NVCC_FLAGS in ${out_variable}
-#      Additionally, sets ${out_variable}_readable to the resulting numeric list
-#      Example:
-#       CUDA_SELECT_NVCC_ARCH_FLAGS(ARCH_FLAGS 3.0 3.5+PTX 5.2(5.0) Maxwell)
-#        LIST(APPEND CUDA_NVCC_FLAGS ${ARCH_FLAGS})
-#
-#      More info on CUDA architectures: https://en.wikipedia.org/wiki/CUDA
-#
-
-# This list will be used for CUDA_ARCH_NAME = All option
-set(CUDA_KNOWN_GPU_ARCHITECTURES  "Fermi" "Kepler" "Maxwell")
-
-# This list will be used for CUDA_ARCH_NAME = Common option (enabled by default)
-set(CUDA_COMMON_GPU_ARCHITECTURES "3.0" "3.5" "5.0")
-
-if (CUDA_VERSION VERSION_GREATER "6.5")
-  list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Kepler+Tegra" "Kepler+Tesla" "Maxwell+Tegra")
-  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "5.2")
-endif ()
-
-if (CUDA_VERSION VERSION_GREATER "7.5")
-  list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Pascal")
-  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "6.0" "6.1" "6.1+PTX")
-else()
-  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "5.2+PTX")
-endif ()
-
-
-
-################################################################################################
-# A function for automatic detection of GPUs installed  (if autodetection is enabled)
-# Usage:
-#   CUDA_DETECT_INSTALLED_GPUS(OUT_VARIABLE)
-#
-function(CUDA_DETECT_INSTALLED_GPUS OUT_VARIABLE)
-  if(NOT CUDA_GPU_DETECT_OUTPUT)
-    set(cufile ${PROJECT_BINARY_DIR}/detect_cuda_archs.cu)
-
-    file(WRITE ${cufile} ""
-      "#include <cstdio>\n"
-      "int main()\n"
-      "{\n"
-      "  int count = 0;\n"
-      "  if (cudaSuccess != cudaGetDeviceCount(&count)) return -1;\n"
-      "  if (count == 0) return -1;\n"
-      "  for (int device = 0; device < count; ++device)\n"
-      "  {\n"
-      "    cudaDeviceProp prop;\n"
-      "    if (cudaSuccess == cudaGetDeviceProperties(&prop, device))\n"
-      "      std::printf(\"%d.%d \", prop.major, prop.minor);\n"
-      "  }\n"
-      "  return 0;\n"
-      "}\n")
-
-    execute_process(COMMAND "${CUDA_NVCC_EXECUTABLE}" "--run" "${cufile}"
-                    "-ccbin" ${CMAKE_CXX_COMPILER}
-                    WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/CMakeFiles/"
-                    RESULT_VARIABLE nvcc_res OUTPUT_VARIABLE nvcc_out
-                    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-    if(nvcc_res EQUAL 0)
-      string(REPLACE "2.1" "2.1(2.0)" nvcc_out "${nvcc_out}")
-      set(CUDA_GPU_DETECT_OUTPUT ${nvcc_out} CACHE INTERNAL "Returned GPU architetures from detect_gpus tool" FORCE)
-    endif()
-  endif()
-
-  if(NOT CUDA_GPU_DETECT_OUTPUT)
-    message(STATUS "Automatic GPU detection failed. Building for common architectures.")
-    set(${OUT_VARIABLE} ${CUDA_COMMON_GPU_ARCHITECTURES} PARENT_SCOPE)
-  else()
-    set(${OUT_VARIABLE} ${CUDA_GPU_DETECT_OUTPUT} PARENT_SCOPE)
-  endif()
-endfunction()
-
-
-################################################################################################
-# Function for selecting GPU arch flags for nvcc based on CUDA architectures from parameter list
-# Usage:
-#   SELECT_NVCC_ARCH_FLAGS(out_variable [list of CUDA compute archs])
-function(CUDA_SELECT_NVCC_ARCH_FLAGS out_variable)
-  set(CUDA_ARCH_LIST "${ARGN}")
-
-  if("X${CUDA_ARCH_LIST}" STREQUAL "X" )
-    set(CUDA_ARCH_LIST "Auto")
-  endif()
-
-  set(cuda_arch_bin)
-  set(cuda_arch_ptx)
-
-  if("${CUDA_ARCH_LIST}" STREQUAL "All")
-    set(CUDA_ARCH_LIST ${CUDA_KNOWN_GPU_ARCHITECTURES})
-  elseif("${CUDA_ARCH_LIST}" STREQUAL "Common")
-    set(CUDA_ARCH_LIST ${CUDA_COMMON_GPU_ARCHITECTURES})
-  elseif("${CUDA_ARCH_LIST}" STREQUAL "Auto")
-    CUDA_DETECT_INSTALLED_GPUS(CUDA_ARCH_LIST)
-    message(STATUS "Autodetected CUDA architecture(s): ${CUDA_ARCH_LIST}")
-  endif()
-
-  # Now process the list and look for names
-  string(REGEX REPLACE "[ \t]+" ";" CUDA_ARCH_LIST "${CUDA_ARCH_LIST}")
-  list(REMOVE_DUPLICATES CUDA_ARCH_LIST)
-  foreach(arch_name ${CUDA_ARCH_LIST})
-    set(arch_bin)
-    set(add_ptx FALSE)
-    # Check to see if we are compiling PTX
-    if(arch_name MATCHES "(.*)\\+PTX$")
-      set(add_ptx TRUE)
-      set(arch_name ${CMAKE_MATCH_1})
-    endif()
-    if(arch_name MATCHES "(^[0-9]\\.[0-9](\\([0-9]\\.[0-9]\\))?)$")
-      set(arch_bin ${CMAKE_MATCH_1})
-      set(arch_ptx ${arch_bin})
-    else()
-      # Look for it in our list of known architectures
-      if(${arch_name} STREQUAL "Fermi")
-        set(arch_bin "2.0 2.1(2.0)")
-      elseif(${arch_name} STREQUAL "Kepler+Tegra")
-        set(arch_bin 3.2)
-      elseif(${arch_name} STREQUAL "Kepler+Tesla")
-        set(arch_bin 3.7)
-      elseif(${arch_name} STREQUAL "Kepler")
-        set(arch_bin 3.0 3.5)
-        set(arch_ptx 3.5)
-      elseif(${arch_name} STREQUAL "Maxwell+Tegra")
-        set(arch_bin 5.3)
-      elseif(${arch_name} STREQUAL "Maxwell")
-        set(arch_bin 5.0 5.2)
-        set(arch_ptx 5.2)
-      elseif(${arch_name} STREQUAL "Pascal")
-        set(arch_bin 6.0 6.1)
-        set(arch_ptx 6.1)
-      else()
-        message(SEND_ERROR "Unknown CUDA Architecture Name ${arch_name} in CUDA_SELECT_NVCC_ARCH_FLAGS")
-      endif()
-    endif()
-    if(NOT arch_bin)
-      message(SEND_ERROR "arch_bin wasn't set for some reason")
-    endif()
-    list(APPEND cuda_arch_bin ${arch_bin})
-    if(add_ptx)
-      if (NOT arch_ptx)
-        set(arch_ptx ${arch_bin})
-      endif()
-      list(APPEND cuda_arch_ptx ${arch_ptx})
-    endif()
-  endforeach()
-
-  # remove dots and convert to lists
-  string(REGEX REPLACE "\\." "" cuda_arch_bin "${cuda_arch_bin}")
-  string(REGEX REPLACE "\\." "" cuda_arch_ptx "${cuda_arch_ptx}")
-  string(REGEX MATCHALL "[0-9()]+" cuda_arch_bin "${cuda_arch_bin}")
-  string(REGEX MATCHALL "[0-9]+"   cuda_arch_ptx "${cuda_arch_ptx}")
-
-  if(cuda_arch_bin)
-    list(REMOVE_DUPLICATES cuda_arch_bin)
-  endif()
-  if(cuda_arch_ptx)
-    list(REMOVE_DUPLICATES cuda_arch_ptx)
-  endif()
-
-  set(nvcc_flags "")
-  set(nvcc_archs_readable "")
-
-  # Tell NVCC to add binaries for the specified GPUs
-  foreach(arch ${cuda_arch_bin})
-    if(arch MATCHES "([0-9]+)\\(([0-9]+)\\)")
-      # User explicitly specified ARCH for the concrete CODE
-      list(APPEND nvcc_flags -gencode arch=compute_${CMAKE_MATCH_2},code=sm_${CMAKE_MATCH_1})
-      list(APPEND nvcc_archs_readable sm_${CMAKE_MATCH_1})
-    else()
-      # User didn't explicitly specify ARCH for the concrete CODE, we assume ARCH=CODE
-      list(APPEND nvcc_flags -gencode arch=compute_${arch},code=sm_${arch})
-      list(APPEND nvcc_archs_readable sm_${arch})
-    endif()
-  endforeach()
-
-  # Tell NVCC to add PTX intermediate code for the specified architectures
-  foreach(arch ${cuda_arch_ptx})
-    list(APPEND nvcc_flags -gencode arch=compute_${arch},code=compute_${arch})
-    list(APPEND nvcc_archs_readable compute_${arch})
-  endforeach()
-
-  string(REPLACE ";" " " nvcc_archs_readable "${nvcc_archs_readable}")
-  set(${out_variable}          ${nvcc_flags}          PARENT_SCOPE)
-  set(${out_variable}_readable ${nvcc_archs_readable} PARENT_SCOPE)
-endfunction()

aten/contrib/data/BatchDataset.cc

@@ -1,61 +0,0 @@
-#include "BatchDataset.h"
-#include "Dataset.h"
-#include "ATen/ATen.h"
-#include <vector>
-#include <cassert>
-#include <math.h>
-
-using namespace at;
-
-BatchDataset::BatchDataset(Dataset& dataset, uint64_t batchsize) {
-   BatchDataset(dataset, batchsize, true);
-}
-
-BatchDataset::BatchDataset(Dataset& dataset, uint64_t batchsize, bool fullbatches) {
-   dataset_ = &dataset;
-   size_ = dataset_->size();
-   batchsize_ = batchsize;
-   fullbatches_ = fullbatches;
-}
-
-void BatchDataset::getField(uint64_t idx, std::string& fieldkey, at::Tensor& field) {
-
-   // assertions:
-   assert(idx < size());
-   assert(hasField(fieldkey));
-
-   // loop over samples:
-   Tensor singlefield, buffer;
-   uint64_t maxsize = std::min(batchsize_, size_ - idx * batchsize_);
-   for(int n = 0; n < maxsize; n++) {
-
-      // get sample:
-      uint64_t batchidx = idx * batchsize_ + n;
-      dataset_->getField(batchidx, fieldkey, singlefield);
-
-      // allocate memory for batch:
-      if(n == 0) {
-
-         // determine size of batch:
-         std::vector<int64_t> fieldsize;
-         fieldsize.push_back(maxsize);
-         for(uint64_t d = 0; d < singlefield.dim(); ++d) {
-            fieldsize.push_back(singlefield.size(d));
-         }
-
-         // resize buffer:
-         field.resize_(fieldsize);
-      }
-
-      // copy sample into batch:
-      buffer = select(field, 0, n);
-      buffer.copy_(singlefield);
-   }
-}
-
-uint64_t BatchDataset::size() {
-   if(fullbatches_)
-      return floor(size_ / batchsize_);
-   else
-      return ceil(size_ / batchsize_);
-}

aten/contrib/data/BatchDataset.h

@@ -1,21 +0,0 @@
-#ifndef AT_BATCH_DATASET_H
-#define AT_BATCH_DATASET_H
-
-#include "Dataset.h"
-#include "ATen/ATen.h"
-
-class BatchDataset : public Dataset
-{
-public:
-   BatchDataset(Dataset& dataset, uint64_t batchsize);
-   BatchDataset(Dataset& dataset, uint64_t batchsize, bool fullbatches);
-   virtual void getField(uint64_t idx, std::string& fieldkey, at::Tensor& field);
-   virtual uint64_t size();
-private:
-   Dataset* dataset_;
-   uint64_t batchsize_;
-   uint64_t size_;
-   bool fullbatches_;
-};
-
-#endif

aten/contrib/data/CMakeLists.txt

@@ -1,30 +0,0 @@
-cmake_minimum_required(VERSION 3.0)
-project(ATen)
-
-if(APPLE)
-  set(CMAKE_MACOSX_RPATH ON)
-endif()
-
-if(CMAKE_VERSION VERSION_LESS "3.1")
-  set(CMAKE_CXX_FLAGS "--std=c++11 ${CMAKE_CXX_FLAGS}")
-else()
-  set(CMAKE_CXX_STANDARD 11)
-endif()
-
-set(src
-  BatchDataset.cc
-  ConcatDataset.cc
-  Dataset.cc
-  MergeDataset.cc
-  ResampleDataset.cc
-  ShuffleDataset.cc
-  TensorDataset.cc
-  TransformDataset.cc
-)
-
-add_library(xtdata ${TH_LINK_STYLE} ${src})
-target_link_libraries(xtdata ATen)
-
-include_directories(.)
-# add_executable(test-data test/basic.cc)
-# target_link_libraries(test-data xtdata)

aten/contrib/data/ConcatDataset.cc

@@ -1,63 +0,0 @@
-#include "ConcatDataset.h"
-#include "Dataset.h"
-#include <vector>
-#include <cassert>
-
-using namespace at;
-
-ConcatDataset::ConcatDataset(std::vector<Dataset*>& datasets) {
-   datasets_ = &datasets;
-   size_ = 0;
-   beginindices_  = std::vector<uint64_t>();
-   endindices_    = std::vector<uint64_t>();
-   beginindices_.push_back(0);
-   for(Dataset* dataset : datasets) {
-      size_ += dataset->size();
-      uint64_t curidx = endindices_.back();
-      endindices_.push_back(beginindices_.back() + dataset->size());
-      if(endindices_.size() > 1)
-         beginindices_.push_back(curidx + 1);
-   }
-}
-
-void ConcatDataset::getField(uint64_t idx, std::string& fieldkey, Tensor &field) {
-
-   // assertions:
-   assert(idx < size());
-   assert(hasField(fieldkey));
-
-   // get sample from correct dataset:
-   uint64_t datasetidx = binarySearch(idx);
-   Dataset* curdataset = (*datasets_)[datasetidx];
-   curdataset->getField(idx - beginindices_[datasetidx], fieldkey, field);
-}
-
-uint64_t ConcatDataset::binarySearch(uint64_t idx) {
-   assert(idx < size());  // TODO: Add caching to this method.
-   uint64_t left = 0;
-   uint64_t right = size_ - 1;
-   while(left != right) {
-      uint64_t middle = (right - left) / 2;
-      if(left == middle) {
-         if(idx > endindices_[left])
-            left = right;
-         else
-            right = left;
-      }
-      else {
-         if(idx > endindices_[middle])
-            left = middle;
-         else if(idx < beginindices_[middle])
-            right = middle;
-         else {
-            left  = middle;
-            right = middle;
-         }
-      }
-   }
-   return left;
-}
-
-uint64_t ConcatDataset::size() {
-   return size_;
-}

aten/contrib/data/ConcatDataset.h

@@ -1,20 +0,0 @@
-#ifndef AT_CONCAT_DATASET_H
-#define AT_CONCAT_DATASET_H
-
-#include "Dataset.h"
-
-class ConcatDataset : public Dataset
-{
-public:
-   ConcatDataset(std::vector<Dataset*>& datasets);
-   virtual void getField(uint64_t idx, std::string& fieldkey, at::Tensor &field);
-   virtual uint64_t size();
-private:
-   uint64_t binarySearch(uint64_t idx);
-   std::vector<Dataset*>* datasets_;
-   std::vector<uint64_t> beginindices_;
-   std::vector<uint64_t> endindices_;
-   uint64_t size_;
-};
-
-#endif

aten/contrib/data/Dataset.cc

@@ -1,28 +0,0 @@
-#include "Dataset.h"
-#include <cassert>
-
-typedef std::map<std::string, at::Tensor> Fields;
-
-void Dataset::get(int64_t idx, Fields& fields) {
-   for(auto& field : fields) {
-      std::string fieldname = field.first;
-      assert(hasField(fieldname));
-      getField(idx, fieldname, field.second);
-   }
-}
-
-bool Dataset::hasField(std::string& fieldkey) {
-   auto search = fieldkeys_.find(fieldkey);
-   return (search != fieldkeys_.end());
-}
-
-std::set<std::string>& Dataset::fieldKeys() {
-   return fieldkeys_;
-}
-
-void Dataset::addFieldKey(std::string& fieldkey) {
-   fieldkeys_.insert(fieldkey);
-}
-
-Dataset::~Dataset() {
-}

aten/contrib/data/Dataset.h

@@ -1,23 +0,0 @@
-#ifndef AT_DATASET_H
-#define AT_DATASET_H
-
-#include "ATen/ATen.h"
-#include <string>
-#include <map>
-#include <set>
-
-typedef std::map<std::string, at::Tensor> Fields;
-
-class Dataset {
-   std::set<std::string> fieldkeys_;
-public:
-   virtual uint64_t size() = 0;  // pure virtual function
-   virtual void getField(uint64_t idx, std::string& fieldkey, at::Tensor& field) = 0;
-   virtual bool hasField(std::string& fieldkey);
-   virtual std::set<std::string>& fieldKeys();
-   virtual void addFieldKey(std::string& fieldkey);
-   virtual void get(int64_t idx, Fields& fields);
-   virtual ~Dataset();
-};
-
-#endif

aten/contrib/data/DatasetIterator.h

@@ -1,93 +0,0 @@
-#ifndef AT_DATASET_ITERATOR_H
-#define AT_DATASET_ITERATOR_H
-
-#include "Dataset.h"
-#include <iterator>
-
-class DatasetIterator : public std::iterator<std::forward_iterator_tag, Fields> {
-private:
-   uint64_t idx_ = 0;
-   Dataset* dataset_;
-public:
-
-   DatasetIterator(Dataset& dataset)  {
-      dataset_ = &dataset;
-      idx_ = 0;
-   }
-
-   DatasetIterator(DatasetIterator& rhs) {
-      DatasetIterator(*rhs.dataset_);
-   }
-
-   DatasetIterator& operator ++() {
-      ++idx_;
-      return *this;
-   }
-
-   DatasetIterator operator ++ (int) {
-      DatasetIterator tmp(*this);
-      ++idx_;
-      return tmp;
-   }
-
-   friend bool operator == (const DatasetIterator& lhs, const DatasetIterator& rhs);
-   friend bool operator != (const DatasetIterator& lhs, const DatasetIterator& rhs);
-
-   Fields operator* () const {
-      Fields sample;
-      dataset_->get(idx_, sample);
-      return sample;
-   }
-
-   Fields* operator-> () const {
-      Fields sample;
-      dataset_->get(idx_, sample);
-      return &sample;
-   }
-};
-
-bool operator == (const DatasetIterator& lhs, const DatasetIterator& rhs) {
-   return lhs.dataset_ == rhs.dataset_;
-}
-
-bool operator != (const DatasetIterator& lhs, const DatasetIterator& rhs) {
-   return lhs.dataset_ != rhs.dataset_;
-}
-
-typedef DatasetIterator iterator;
-//typedef DatasetIterator<const Fields> const_iterator;
-
-#endif
-
-/**
-iterator {
-    iterator(const iterator&);
-    ~iterator();
-    iterator& operator=(const iterator&);
-    iterator& operator++(); //prefix increment
-    reference operator*() const;
-    friend void swap(iterator& lhs, iterator& rhs); //C++11 I think
-};
-
-input_iterator : public virtual iterator {
-    iterator operator++(int); //postfix increment
-    value_type operator*() const;
-    pointer operator->() const;
-    friend bool operator==(const iterator&, const iterator&);
-    friend bool operator!=(const iterator&, const iterator&);
-};
-//once an input iterator has been dereferenced, it is
-//undefined to dereference one before that.
-
-output_iterator : public virtual iterator {
-    reference operator*() const;
-    iterator operator++(int); //postfix increment
-};
-//dereferences may only be on the left side of an assignment
-//once an input iterator has been dereferenced, it is
-//undefined to dereference one before that.
-
-forward_iterator : input_iterator, output_iterator {
-    forward_iterator();
-};
-**/

aten/contrib/data/MergeDataset.cc

@@ -1,30 +0,0 @@
-#include "MergeDataset.h"
-#include <cassert>
-
-using namespace at;
-
-MergeDataset::MergeDataset(std::vector<Dataset*>& datasets) {
-   datasets_ = &datasets;
-   uint64_t idx = 0;
-   for(Dataset* dataset : *datasets_) {
-      for(auto& fieldkey : dataset->fieldKeys()) {
-         std::string fieldkeyc = fieldkey;
-         addFieldKey(fieldkeyc);
-         datasetidx_[fieldkeyc] = idx;
-      }
-   }
-}
-
-void MergeDataset::getField(uint64_t idx, std::string& fieldkey, Tensor& field) {
-   assert(idx < size());
-   assert(hasField(fieldkey));
-   Dataset* curdataset = (*datasets_)[datasetidx_[fieldkey]];
-   return curdataset->getField(idx, fieldkey, field);
-}
-
-uint64_t MergeDataset::size() {
-   uint64_t size = 0;
-   for(Dataset* dataset : *datasets_)
-      size += dataset->size();
-   return size;
-}

aten/contrib/data/MergeDataset.h

@@ -1,19 +0,0 @@
-#ifndef AT_MERGE_DATASET_H
-#define AT_MERGE_DATASET_H
-
-#include "Dataset.h"
-#include <vector>
-#include <string>
-
-class MergeDataset : public Dataset
-{
-public:
-   MergeDataset(std::vector<Dataset*>& datasets);
-   virtual void getField(uint64_t idx, std::string& fieldkey, at::Tensor& field);
-   virtual uint64_t size();
-private:
-   std::vector<Dataset*>* datasets_;
-   std::map<std::string, int> datasetidx_;
-};
-
-#endif

aten/contrib/data/ResampleDataset.cc

@@ -1,47 +0,0 @@
-#include "ResampleDataset.h"
-#include "Dataset.h"
-#include <vector>
-#include <cassert>
-
-using namespace at;
-
-ResampleDataset::ResampleDataset(Dataset& dataset) {
-   dataset_ = &dataset;
-   size_ = dataset.size();
-   perm_ = std::vector<uint64_t>();
-   perm_.reserve(size_);
-   for(int n = 0; n < size_; ++n)
-      perm_[n] = n;
-}
-
-ResampleDataset::ResampleDataset(Dataset& dataset, std::vector<uint64_t>& perm) {
-   dataset_ = &dataset;
-   size_ = dataset.size();
-   perm_ = perm;
-   assert(perm_.size() == size_);
-}
-
-ResampleDataset::ResampleDataset(Dataset& dataset, std::function<uint64_t(uint64_t)> perm) {
-   dataset_ = &dataset;
-   size_ = dataset.size();
-   permfunc_ = perm;
-   resample();
-}
-
-void ResampleDataset::getField(uint64_t idx, std::string& fieldkey, at::Tensor& field) {
-   assert(idx < size());
-   assert(hasField(fieldkey));
-   dataset_->getField(perm_[idx], fieldkey, field);
-}
-
-void ResampleDataset::resample() {
-   if(permfunc_) {
-      perm_.reserve(size_);
-      for(int n = 0; n < size_; ++n)
-         perm_[n] = permfunc_(n);
-   }
-}
-
-uint64_t ResampleDataset::size() {
-   return size_;
-}

aten/contrib/data/ResampleDataset.h

@@ -1,28 +0,0 @@
-#ifndef AT_RESAMPLE_DATASET_H
-#define AT_RESAMPLE_DATASET_H
-
-#include <string>
-#include <vector>
-#include <functional>
-#include "ATen/ATen.h"
-#include "Dataset.h"
-
-class ResampleDataset : public Dataset
-{
-public:
-   ResampleDataset(Dataset& dataset);
-   ResampleDataset(Dataset& dataset, std::vector<uint64_t>& perm);
-   ResampleDataset(Dataset& dataset, std::function<uint64_t(uint64_t)> perm);
-   virtual void getField(uint64_t idx, std::string& fieldkey, at::Tensor& field);
-   virtual uint64_t size();
-   virtual void resample();
-protected:
-   std::vector<uint64_t> perm_;
-   uint64_t size_;
-private:
-   Dataset* dataset_;
-   std::function<uint64_t(uint64_t)> permfunc_;
-   std::vector<at::Tensor> fields_;
-};
-
-#endif

aten/contrib/data/ShuffleDataset.cc

@@ -1,16 +0,0 @@
-#include "ShuffleDataset.h"
-#include "Dataset.h"
-#include <algorithm>
-
-using namespace at;
-
-ShuffleDataset::ShuffleDataset(Dataset& dataset) : ResampleDataset(dataset) {
-   resample();
-}
-
-void ShuffleDataset::resample() {
-   perm_.reserve(size_);
-   for(int n = 0; n < size_; ++n)
-      perm_[n] = n;
-   std::random_shuffle(perm_.begin(), perm_.end());
-}

aten/contrib/data/ShuffleDataset.h

@@ -1,14 +0,0 @@
-#ifndef AT_SHUFFLE_DATASET_H
-#define AT_SHUFFLE_DATASET_H
-
-#include "Dataset.h"
-#include "ResampleDataset.h"
-
-class ShuffleDataset : public ResampleDataset
-{
-public:
-   ShuffleDataset(Dataset& dataset);
-   virtual void resample();
-};
-
-#endif

aten/contrib/data/TensorDataset.cc

@@ -1,27 +0,0 @@
-#include "TensorDataset.h"
-#include "ATen/ATen.h"
-#include <cassert>
-
-using namespace at;
-
-TensorDataset::TensorDataset(Tensor& t, std::string& fieldkey) {
-   t_ = t;
-   fieldkey_ = fieldkey;
-   addFieldKey(fieldkey);
-}
-
-void TensorDataset::getField(uint64_t idx, std::string& fieldkey, Tensor& field) {
-
-   // assertions:
-   assert(idx < size());
-   assert(fieldkey_.compare(fieldkey) == 0);
-
-   // get sample:
-   Tensor buffer = select(t_, 0, idx);
-   field.copy_(buffer);
-
-}
-
-uint64_t TensorDataset::size() {
-   return t_.size(0);
-}

aten/contrib/data/TensorDataset.h

@@ -1,19 +0,0 @@
-#ifndef AT_TENSOR_DATASET_H
-#define AT_TENSOR_DATASET_H
-
-#include "Dataset.h"
-#include "ATen/ATen.h"
-#include <string>
-
-class TensorDataset : public Dataset
-{
-public:
-   TensorDataset(at::Tensor& t, std::string& fieldkey);
-   virtual void getField(uint64_t idx, std::string& fieldkey, at::Tensor& field);
-   virtual uint64_t size();
-private:
-   at::Tensor t_;
-   std::string fieldkey_;
-};
-
-#endif

aten/contrib/data/TransformDataset.cc

@@ -1,25 +0,0 @@
-#include "TransformDataset.h"
-#include "ATen/ATen.h"
-#include "ATen/ATen.h"
-#include <cassert>
-
-using namespace at;
-
-TransformDataset::TransformDataset(Dataset& dataset, std::string& fieldkey, std::function<Tensor(Tensor)>& transform) {
-   assert(hasField(fieldkey));
-   dataset_ = &dataset;
-   fieldkey_ = fieldkey;
-   transform_ = transform;
-}
-
-void TransformDataset::getField(uint64_t idx, std::string& fieldkey, Tensor& field) {
-   dataset_->getField(idx, fieldkey, field);
-   if(fieldkey.compare(fieldkey_) == 0) {
-      Tensor transformed = transform_(field);
-      field.copy_(transformed);
-   }
-}
-
-uint64_t TransformDataset::size() {
-   return dataset_->size();
-}

aten/contrib/data/TransformDataset.h

@@ -1,23 +0,0 @@
-#ifndef AT_TRANSFORM_DATASET_H
-#define AT_TRANSFORM_DATASET_H
-
-#include "Dataset.h"
-#include "ATen/ATen.h"
-#include <functional>
-#include <string>
-
-using namespace at;
-
-class TransformDataset : public Dataset
-{
-public:
-   TransformDataset(Dataset& dataset, std::string& fieldkey, std::function<Tensor(Tensor)>& transform);
-   virtual void getField(uint64_t idx, std::string& fieldkey, Tensor& field);
-   virtual uint64_t size();
-private:
-   Dataset* dataset_;
-   std::string fieldkey_;
-   std::function<Tensor(Tensor)> transform_;
-};
-
-#endif

aten/contrib/data/test/basic.cc

@@ -1,22 +0,0 @@
-#include "Dataset.h"
-#include "DatasetIterator.h"
-#include "TensorDataset.h"
-#include <iostream>
-
-using namespace at;
-
-int main()
-{
-   std::cout << "hello\n";
-
-   Tensor tensor = rand(CPU(kDouble), {256,32});
-
-   TensorDataset dataset(tensor);
-   DatasetIterator datasetiterator(dataset);
-   uint64_t cnt = 0;
-   for(auto& sample : datasetiterator) {
-      std::cout << "got sample " << cnt << std:endl;
-      cnt++;
-   }
-   return 0;
-}

aten/contrib/data/threadpool/ThreadPool.cc

@@ -1,77 +0,0 @@
-// dependencies:
-#include "ThreadPool.h"
-#include <vector>
-#include <queue>
-#include <memory>
-#include <thread>
-#include <mutex>
-#include <condition_variable>
-#include <future>
-#include <functional>
-#include <stdexcept>
-#include <stdlib.h>
-#include <stdarg.h>
-
-// constructor launches the specified number of workers:
-ThreadPool::ThreadPool(uint64_t threads) : stop(false) {
-
-   // loop over all threads:
-   for(uint64_t i = 0; i < threads; ++i) {
-      workers.emplace_back(
-         [this] {
-            for(;;) {
-               std::function<void()> task;
-               {
-                  std::unique_lock<std::mutex> lock(this->queue_mutex);
-                  this->condition.wait(lock,
-                     [this]{ return this->stop || !this->tasks.empty(); });
-                  if(this->stop && this->tasks.empty())
-                     return;
-                  task = std::move(this->tasks.front());
-                  this->tasks.pop();
-               }
-               task();
-            }
-         }
-      );
-   }
-}
-
-// synchronize all the threads:
-void ThreadPool::synchronize() {
-   for(uint64_t i = 0; i < futures.size(); i++)
-      futures[i].first.wait();
-}
-
-// poll the threads for results:
-unsigned int ThreadPool::waitFor() {
-
-   // wait until a task is finished:
-   uint64_t i;
-   std::future_status status;
-   do {
-      for(i = 0; i < futures.size(); i++) {
-         status = futures[i].first.wait_for(std::chrono::microseconds(0));
-         if(status == std::future_status::ready) break;
-      }
-      std::this_thread::sleep_for(std::chrono::microseconds(1));
-   } while (status != std::future_status::ready);
-
-   // get the result and remove the future:
-   futures[i].first.get();
-   unsigned int handle = futures[i].second;
-   iter_swap(futures.begin() + i, futures.end() - 1);
-   futures.pop_back();
-   return handle;
-}
-
-// the destructor joins all threads:
-ThreadPool::~ThreadPool() {
-   {
-      std::unique_lock<std::mutex> lock(queue_mutex);
-      stop = true;
-   }
-   condition.notify_all();
-   for(std::thread &worker: workers)
-      worker.join();
-}

aten/contrib/data/threadpool/ThreadPool.h

@@ -1,65 +0,0 @@
-#ifndef AT_THREADPOOL_H
-#define AT_THREADPOOL_H
-
-// dependencies:
-#include <vector>
-#include <queue>
-#include <thread>
-#include <mutex>
-#include <condition_variable>
-#include <future>
-
-// definition of the ThreadPool class:
-class ThreadPool {
-   public:
-      explicit ThreadPool(uint64_t);
-      void synchronize();
-      unsigned int waitFor();
-      template<class F, class... Args>
-      unsigned int enqueue(F&& f, Args&&... args);
-      ~ThreadPool();
-
-   private:
-
-      // all the threads that can perform work:
-      std::vector< std::thread > workers;
-
-      // the list of futures:
-      unsigned int handle = 0;
-      std::vector< std::pair< std::future<void>, unsigned int > > futures;
-
-      // the task queue:
-      std::queue< std::function<void()> > tasks;
-
-      // synchronization:
-      std::mutex queue_mutex;
-      std::condition_variable condition;
-      bool stop;
-};
-
-// enqueue new work item into the pool:
-template<class F, class... Args>
-unsigned int ThreadPool::enqueue(F&& f, Args&&... args)
-{
-
-   // create the task:
-   auto task = std::make_shared< std::packaged_task<void()> >(
-      std::bind(std::forward<F>(f), std::forward<Args>(args)...)
-   );
-
-   // get future and enqueue the task:
-   std::future<void> future = task->get_future();
-   {
-      std::unique_lock<std::mutex> lock(queue_mutex);
-      if(stop) throw std::runtime_error("enqueue on stopped ThreadPool");
-      tasks.emplace([task](){ (*task)(); });
-   }
-   condition.notify_one();
-
-   // generate handle and store future:
-   handle++;
-   futures.push_back(std::make_pair(std::move(future), handle));
-   return handle;
-}
-
-#endif

aten/contrib/meter/APMeter.cc

@@ -1,89 +0,0 @@
-#include "APMeter.h"
-#include <math.h>
-#include <cassert>
-
-using namespace at;
-
-APMeter::APMeter() {
-   reset();
-}
-
-void APMeter::reset() {
-   outputs_ = CPU(kFloat).tensor();
-   targets_ = CPU(kFloat).tensor();
-   n_ = 0;
-}
-
-void APMeter::add(Tensor& output, Tensor& target) {
-
-   // assertions and allocations:
-   assert(output.dim() == 2 && target.dim() == 2);
-   //assert(isSameSizeAs(output, target));
-   assert(output.size(1) == outputs_.size(1));
-
-   // get current outputs and targets:
-   Tensor curoutputs = getOutputs();
-   Tensor curtargets = getTargets();
-
-   // make sure underlying storages are sufficiently large:
-   if(numel(outputs_) < numel(curoutputs) + numel(output)) {
-      long long newsize = ceil(numel(outputs_) * 1.5);
-      outputs_.resize_({newsize + numel(output)});
-      targets_.resize_({newsize + numel(output)});
-   }
-   n_ += output.size(0);
-
-   // store scores and targets:
-   uint64_t offset = (numel(curoutputs) > 0) ? curoutputs.size(0) : 0;
-   Tensor outputbuffer = outputs_.narrow( 0, offset, output.size(0));
-   Tensor targetbuffer = targets_.narrow( 0, offset, target.size(0));
-
-   outputbuffer.copy_(output);
-   targetbuffer.copy_(target);
-}
-
-Tensor APMeter::getOutputs() {
-   return outputs_.narrow(0, 0, n_);
-}
-Tensor APMeter::getTargets() {
-   return targets_.narrow(0, 0, n_);
-}
-
-void APMeter::value(Tensor& val) {
-
-   // get current outputs and targets:
-   Tensor curoutputs = getOutputs();
-   Tensor curtargets = getTargets();
-
-   // allocate some memory:
-   val.resize_({curoutputs.size(1)});
-   double * val_d = val.data<double>();
-   Tensor outputbuffer, targetbuffer, sortval, sortidx, sorttgt;
-   Tensor truepos, precision;
-   Tensor range = val.type().range(0,curoutputs.size(0));
-
-   // loop over all classes:
-   for(uint64_t k = 0; k < curoutputs.size(1); ++k) {
-
-      // sort scores:
-      outputbuffer = curoutputs.narrow( 1, k, 1);
-      targetbuffer = curtargets.narrow(1, k, 1).contiguous().toType(CPU(kDouble));
-      double * targetbuffer_d = targetbuffer.data<double>();
-      std::tie(sortval, sortidx) = sort(curoutputs, 0);
-      sorttgt = index_select(targetbuffer, 0, sortidx);
-
-      // compue true positive sums, and precision:
-      truepos = cumsum(targetbuffer,0);  // NOTE: Cast to double first?
-      precision = div(truepos, range);
-      double * precision_d = precision.data<double>();
-      // compute average precision:
-      val_d[k] = .0;
-      for(uint64_t n = 0; n < precision.size(0); ++n) {
-         if(targetbuffer_d[n] != 0.)
-            val_d[k] += precision_d[n];
-      }
-      auto norm = sum(targetbuffer).toCDouble();
-      if(norm > 0)
-        val_d[k] /= norm;
-   }
-}

aten/contrib/meter/APMeter.h

@@ -1,22 +0,0 @@
-#ifndef AT_AP_METER_H
-#define AT_AP_METER_H
-
-#include "Meter.h"
-#include "ATen/ATen.h"
-
-class APMeter : public Meter
-{
-public:
-   APMeter();
-   virtual void add(Tensor& output, Tensor& target);
-   virtual void value(Tensor& val);
-   virtual void reset();
-   virtual Tensor getOutputs();
-   virtual Tensor getTargets();
-private:
-   Tensor outputs_;
-   Tensor targets_;
-   uint64_t n_;
-};
-
-#endif

aten/contrib/meter/AUCMeter.cc

@@ -1,60 +0,0 @@
-#include "AUCMeter.h"
-#include "APMeter.h"
-#include <cassert>
-
-using namespace at;
-
-AUCMeter::AUCMeter() {
-   reset();
-}
-
-void AUCMeter::reset() {
-   meter_ = APMeter();
-}
-
-void AUCMeter::add(Tensor& output, Tensor& target) {
-   meter_.add(output, target);
-}
-
-void AUCMeter::value(Tensor& val) {
-
-   // get data from APMeter:
-   Tensor outputs = meter_.getOutputs();
-   Tensor targets = meter_.getTargets();
-
-   // sort scores:
-   Tensor sortval, sortidx, sorttgt;
-   std::tie(sortval, sortidx) = sort(outputs, 0, true);
-   sorttgt = index_select(targets, 0, sortidx);
-   int64_t * sortidx_d = sortidx.data<int64_t>();
-   int16_t * targets_d = sortidx.data<int16_t>();
-   // construct the ROC curve:
-   Tensor tpr = zeros(CPU(kDouble), {numel(outputs)});
-   Tensor fpr = zeros(CPU(kDouble), {numel(outputs)});
-
-   double * tpr_d = tpr.data<double>();
-   double * fpr_d = fpr.data<double>();
-   for(uint64_t n = 1; n <= numel(outputs); ++n) {
-      if(targets_d[sortidx_d[n - 1]] == 1) {
-         tpr_d[n] = tpr_d[n - 1] + 1.;
-         fpr_d[n] = fpr_d[n - 1];
-      } else {
-         tpr_d[n] = tpr_d[n - 1];
-         fpr_d[n] = fpr_d[n - 1] + 1.;
-      }
-   }
-   tpr.div_(sum(targets));
-   fpr.div_(sum(at::add(mul(targets, -1.), 1.)));
-
-   /**
-   local auc = torch.cmul(
-      tpr:narrow(1, 1, tpr:nElement() - 1),
-      fpr:narrow(1, 2, fpr:nElement() - 1) -
-      fpr:narrow(1, 1, fpr:nElement() - 1)):sum()
-   */
-
-   val.resize_({1}).fill_(
-     sum(mul(tpr.narrow(0, 0, numel(tpr) - 1),
-                sub(fpr.narrow(0, 1, numel(tpr) - 1),
-                     fpr.narrow(0, 0, numel(tpr) - 1)))));
-}

aten/contrib/meter/AUCMeter.h

@@ -1,19 +0,0 @@
-#ifndef AT_AUC_METER_H
-#define AT_AUC_METER_H
-
-#include "Meter.h"
-#include "APMeter.h"
-#include "ATen/ATen.h"
-
-class AUCMeter : public Meter
-{
-public:
-   AUCMeter();
-   virtual void reset();
-   virtual void add(Tensor& output, Tensor& target);
-   virtual void value(Tensor& val);
-private:
-   APMeter meter_;
-};
-
-#endif

aten/contrib/meter/CMakeLists.txt

@@ -1,25 +0,0 @@
-# requires to be compiled along xttensor
-
-include_directories(${CMAKE_CURRENT_SOURCE_DIR})
-
-# C++11
-if(CMAKE_VERSION VERSION_LESS "3.1")
-  set(CMAKE_CXX_FLAGS "--std=c++11 ${CMAKE_CXX_FLAGS}")
-else()
-  set(CMAKE_CXX_STANDARD 11)
-endif()
-
-set(src
-  APMeter.cc
-  AUCMeter.cc
-  ClassErrorMeter.cc
-  MAPMeter.cc
-  MSEMeter.cc
-)
-
-add_library(xtmeter ${TH_LINK_STYLE} ${src})
-target_link_libraries(xtmeter ATen)
-
-add_executable(test-meter test/basic.cc ${BACKWARD_ENABLE})
-# add_backward(test-meter)
-target_link_libraries(test-meter xtmeter)

aten/contrib/meter/ClassErrorMeter.cc

@@ -1,59 +0,0 @@
-#include "ClassErrorMeter.h"
-#include "ATen/ATen.h"
-#include <cassert>
-
-using namespace at;
-
-ClassErrorMeter::ClassErrorMeter() {
-   ClassErrorMeter(1);
-}
-
-ClassErrorMeter::ClassErrorMeter(const int64_t topk) {
-   topkval_ = CPU(kShort).tensor();
-   sumval_ = CPU(kShort).tensor();
-   topkval_.resize_({topk});
-   sumval_.resize_({topk});
-   reset();
-}
-
-void ClassErrorMeter::reset() {
-  range_out(topkval_, 1, numel(topkval_));
-  sumval_.fill_(0.);
-  n_ = 0;
-}
-
-void ClassErrorMeter::add(Tensor& output, Tensor& target) {
-
-   // assertions and allocations:
-   assert(output.dim() == 2 && target.dim() == 1);
-   //assert(isSameSizeAs(output, target));
-   auto sumval_d = sumval_.data<int16_t>();
-   auto target_long = target.contiguous().toType(CPU(kLong));
-   auto target_d = target_long.data<int64_t>();
-   // update counts:
-   Tensor val, idx;
-   std::tie(val, idx) = topk(output, numel(topkval_), 1, true, true);
-   for(uint64_t n = 0; n < output.size(0); ++n) {
-      bool targetseen = false;
-      Tensor idx_n = idx.select(0,n);
-      auto idx_n_d = idx_n.data<int64_t>();
-      for(uint64_t k = 0; k < numel(topkval_); ++k) {
-         n_++;
-         if(targetseen) {
-            sumval_d[k]++;
-         } else if(idx_n_d[k] == target_d[n]) {
-            targetseen = true;
-            sumval_d[k]++;
-         }
-      }
-   }
-}
-
-void ClassErrorMeter::value(Tensor& val) {
-   val.resize_({numel(topkval_)});
-   auto val_d = val.data<double>();
-   auto sumval_d = sumval_.data<int16_t>();
-   for(uint64_t k = 0; k < numel(topkval_); ++k) {
-     val_d[k] = 1.0 - (double(sumval_d[k]) / double(n_));
-   }
-}

aten/contrib/meter/ClassErrorMeter.h

@@ -1,21 +0,0 @@
-#ifndef AT_CLASS_ERROR_METER_H
-#define AT_CLASS_ERROR_METER_H
-
-#include "Meter.h"
-#include "ATen/ATen.h"
-
-class ClassErrorMeter : public Meter
-{
-public:
-   ClassErrorMeter();
-   ClassErrorMeter(const int64_t topk);
-   virtual void reset();
-   virtual void add(Tensor& output, Tensor& target);
-   virtual void value(Tensor& val);
-private:
-   Tensor topkval_;
-   Tensor sumval_;
-   uint64_t n_;
-};
-
-#endif

aten/contrib/meter/MAPMeter.cc

@@ -1,23 +0,0 @@
-#include "MAPMeter.h"
-
-using namespace at;
-
-MAPMeter::MAPMeter() {
-   reset();
-}
-
-void MAPMeter::reset() {
-   meter_.reset();
-}
-
-void MAPMeter::add(Tensor& output, Tensor& target) {
-   meter_.add(output, target);
-}
-
-void MAPMeter::value(Tensor& val) {
-   //TODO: 0-dim
-   val.resize_({1});
-   Tensor allvalues = val.type().tensor();
-   meter_.value(allvalues);
-   val.fill_(mean(allvalues));
-}

aten/contrib/meter/MAPMeter.h

@@ -1,19 +0,0 @@
-#ifndef AT_MAP_METER_H
-#define AT_MAP_METER_H
-
-#include "Meter.h"
-#include "APMeter.h"
-#include "ATen/ATen.h"
-
-class MAPMeter : public Meter
-{
-public:
-   MAPMeter();
-   virtual void reset();
-   virtual void add(Tensor& output, Tensor& target);
-   virtual void value(Tensor& val);
-private:
-   APMeter meter_;
-};
-
-#endif

aten/contrib/meter/MSEMeter.cc

@@ -1,31 +0,0 @@
-#include "MSEMeter.h"
-#include <cassert>
-#include <math.h>
-
-using namespace at;
-
-MSEMeter::MSEMeter() {
-   reset();
-}
-
-void MSEMeter::reset() {
-   n_ = 0;
-   val_ = .0;
-}
-
-void MSEMeter::add(Tensor& output, Tensor& target) {
-   //assert(isSameSizeAs(output, output
-  Tensor t = output.sub(target);
-  Tensor result = t.mul(t).contiguous().toType(CPU(kDouble));
-  double * data = result.data<double>();
-  for(uint64_t n = 0; n < numel(result); ++n) {
-    n_++;
-    val_ += ( (1. / ((double)n_ - 1.) * val_) +
-              ((1. /  (double)n_) * data[n]));
-  }
-}
-
-void MSEMeter::value(Tensor& val) {
-  //TODO: 0-dim
-  val.resize_({1}).fill_(val_);
-}

aten/contrib/meter/MSEMeter.h

@@ -1,19 +0,0 @@
-#ifndef AT_MSE_METER_H
-#define AT_MSE_METER_H
-
-#include "Meter.h"
-#include "ATen/ATen.h"
-
-class MSEMeter : public Meter
-{
-public:
-   MSEMeter();
-   virtual void reset();
-   virtual void add(Tensor& output, Tensor& target);
-   virtual void value(Tensor& val);
-private:
-   double val_;
-   uint64_t n_;
-};
-
-#endif

aten/contrib/meter/Meter.h

@@ -1,17 +0,0 @@
-#ifndef AT_METER_H
-#define AT_METER_H
-
-#include "ATen/ATen.h"
-
-using namespace at;
-
-class Meter
-{
-public:
-   virtual void add(Tensor& output, Tensor& target) = 0;
-   virtual void value(Tensor& val) = 0;
-   virtual void reset() = 0;
-  virtual ~Meter() {};
-};
-
-#endif

aten/contrib/meter/test/basic.cc

@@ -1,25 +0,0 @@
-#include "APMeter.h"
-#include <iostream>
-
-using namespace at;
-
-int main()
-{
-   auto && T = CPU(kFloat);
-   std::cout << "hello\n";
-   APMeter meter;
-   Tensor output = at::randn(T, {10, 7});
-   Tensor target = at::zeros(T, {10, 7});
-   for(uint64_t n = 0; n < 10; ++n) {
-     Tensor row = target.select(0,n);
-     auto row_d = row.data<float>();
-     row_d[rand() % 7] = 1.;
-   }
-   std::cout << output;
-   std::cout << target;
-   meter.add(output, target);
-   Tensor val;
-   meter.value(val);
-   std::cout << "value: " << val << std::endl;
-   return 0;
-}

aten/src/ATen/ATen.h

@@ -1,17 +1,26 @@
 #pragma once
 
-#include "ATen/ATenGeneral.h"
-#include "ATen/CPUGeneral.h"
+#include "ATen/core/ATenGeneral.h"
 #include "ATen/Allocator.h"
-#include "ATen/Scalar.h"
-#include "ATen/Type.h"
-#include "ATen/Generator.h"
+#include "ATen/CPUGeneral.h"
+#include "ATen/CUDAGuard.h"
 #include "ATen/Context.h"
-#include "ATen/Storage.h"
+#include "ATen/Device.h"
+#include "ATen/DeviceGuard.h"
+#include "ATen/DimVector.h"
+#include "ATen/Dispatch.h"
+#include "ATen/Formatting.h"
+#include "ATen/Functions.h"
+#include "ATen/core/Generator.h"
+#include "ATen/core/Layout.h"
+#include "ATen/OptionsGuard.h"
+#include "ATen/core/Scalar.h"
+#include "ATen/ScalarOps.h"
+#include "ATen/core/Storage.h"
 #include "ATen/Tensor.h"
 #include "ATen/TensorGeometry.h"
-#include "ATen/Functions.h"
-#include "ATen/Formatting.h"
+#include "ATen/core/TensorMethods.h"
 #include "ATen/TensorOperators.h"
-#include "ATen/TensorMethods.h"
-#include "ATen/Dispatch.h"
+#include "ATen/core/TensorOptions.h"
+#include "ATen/Type.h"
+#include "ATen/core/Error.h"

aten/src/ATen/ATenGeneral.h

@@ -1,11 +0,0 @@
-#pragma once
-
-#ifdef _WIN32
-# ifdef ATen_EXPORTS
-#  define AT_API __declspec(dllexport)
-# else
-#  define AT_API __declspec(dllimport)
-# endif
-#else
-# define AT_API
-#endif

aten/src/ATen/AccumulateType.h

@@ -0,0 +1,43 @@
+#pragma once
+#include "ATen/Config.h"
+#include "ATen/core/Half.h"
+
+// Defines the accumulation type for a scalar type.
+// Example:
+//   using accscalar_t = acc_type<scalar_t, true>;
+
+#ifdef __CUDACC__
+#include <cuda.h>
+#include <cuda_fp16.h>
+#endif
+
+namespace at {
+
+template <typename T, bool is_cuda>
+struct AccumulateType { };
+
+#ifdef __CUDACC__
+template <> struct AccumulateType<half, true> { using type = float; };
+#endif
+template <> struct AccumulateType<Half, true> { using type = float; };
+template <> struct AccumulateType<float, true> { using type = float; };
+template <> struct AccumulateType<double, true> { using type = double; };
+template <> struct AccumulateType<int8_t, true> { using type = int64_t; };
+template <> struct AccumulateType<uint8_t, true> { using type = int64_t; };
+template <> struct AccumulateType<char, true> { using type = int64_t; };
+template <> struct AccumulateType<int16_t, true> { using type = int64_t; };
+template <> struct AccumulateType<int32_t, true> { using type = int64_t; };
+template <> struct AccumulateType<int64_t, true> { using type = int64_t; };
+template <> struct AccumulateType<float, false> { using type = double; };
+template <> struct AccumulateType<double, false> { using type = double; };
+template <> struct AccumulateType<int8_t, false> { using type = int64_t; };
+template <> struct AccumulateType<uint8_t, false> { using type = int64_t; };
+template <> struct AccumulateType<char, false> { using type = int64_t; };
+template <> struct AccumulateType<int16_t, false> { using type = int64_t; };
+template <> struct AccumulateType<int32_t, false> { using type = int64_t; };
+template <> struct AccumulateType<int64_t, false> { using type = int64_t; };
+
+template<typename T, bool is_cuda>
+using acc_type = typename AccumulateType<T, is_cuda>::type;
+
+}  // namespace at

aten/src/ATen/Allocator.h

@@ -1,34 +1,2 @@
 #pragma once
-
-#include <memory>
-#include <stddef.h>
-
-#include "ATen/Retainable.h"
-
-namespace at {
-
-struct Allocator {
-  virtual ~Allocator() {}
-  virtual void* allocate(std::size_t n) const = 0;
-  virtual void deallocate(void* ptr) const = 0;
-};
-
-namespace detail {
-
-struct AllocatorRetainable : public Retainable {
-  AllocatorRetainable(std::unique_ptr<Allocator> allocator)
-    : allocator(std::move(allocator)) {}
-
-  void* allocate(std::size_t n) {
-    return allocator->allocate(n);
-  }
-  void deallocate(void* ptr) {
-    return allocator->deallocate(ptr);
-  }
-private:
-  std::unique_ptr<Allocator> allocator;
-};
-
-}  // namespace at::detail
-
-}  // namespace at
+#include <ATen/core/Allocator.h>

aten/src/ATen/ArrayRef.h

@@ -1,192 +1,2 @@
-//===--- ArrayRef.h - Array Reference Wrapper -------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// ATen: modified from llvm::ArrayRef.
-// removed llvm-specific functionality
-// removed some implicit const -> non-const conversions that rely on
-// complicated std::enable_if meta-programming
-// removed a bunch of slice variants for simplicity...
-
 #pragma once
-
-#include <ATen/Error.h>
-#include <ATen/SmallVector.h>
-
-#include <array>
-#include <iterator>
-#include <vector>
-
-namespace at {
-  /// ArrayRef - Represent a constant reference to an array (0 or more elements
-  /// consecutively in memory), i.e. a start pointer and a length.  It allows
-  /// various APIs to take consecutive elements easily and conveniently.
-  ///
-  /// This class does not own the underlying data, it is expected to be used in
-  /// situations where the data resides in some other buffer, whose lifetime
-  /// extends past that of the ArrayRef. For this reason, it is not in general
-  /// safe to store an ArrayRef.
-  ///
-  /// This is intended to be trivially copyable, so it should be passed by
-  /// value.
-  template<typename T>
-  class ArrayRef {
-  public:
-    typedef const T *iterator;
-    typedef const T *const_iterator;
-    typedef size_t size_type;
-
-    typedef std::reverse_iterator<iterator> reverse_iterator;
-
-  private:
-    /// The start of the array, in an external buffer.
-    const T *Data;
-
-    /// The number of elements.
-    size_type Length;
-
-  public:
-    /// @name Constructors
-    /// @{
-
-    /// Construct an empty ArrayRef.
-    /*implicit*/ ArrayRef() : Data(nullptr), Length(0) {}
-
-    /// Construct an ArrayRef from a single element.
-    /*implicit*/ ArrayRef(const T &OneElt)
-      : Data(&OneElt), Length(1) {}
-
-    /// Construct an ArrayRef from a pointer and length.
-    /*implicit*/ ArrayRef(const T *data, size_t length)
-      : Data(data), Length(length) {}
-
-    /// Construct an ArrayRef from a range.
-    ArrayRef(const T *begin, const T *end)
-      : Data(begin), Length(end - begin) {}
-
-    /// Construct an ArrayRef from a SmallVector. This is templated in order to
-    /// avoid instantiating SmallVectorTemplateCommon<T> whenever we
-    /// copy-construct an ArrayRef.
-    template<typename U>
-    /*implicit*/ ArrayRef(const SmallVectorTemplateCommon<T, U> &Vec)
-      : Data(Vec.data()), Length(Vec.size()) {
-    }
-
-    /// Construct an ArrayRef from a std::vector.
-    template<typename A>
-    /*implicit*/ ArrayRef(const std::vector<T, A> &Vec)
-      : Data(Vec.data()), Length(Vec.size()) {}
-
-    /// Construct an ArrayRef from a std::array
-    template <size_t N>
-    /*implicit*/ constexpr ArrayRef(const std::array<T, N> &Arr)
-        : Data(Arr.data()), Length(N) {}
-
-    /// Construct an ArrayRef from a C array.
-    template <size_t N>
-    /*implicit*/ constexpr ArrayRef(const T (&Arr)[N]) : Data(Arr), Length(N) {}
-
-    /// Construct an ArrayRef from a std::initializer_list.
-    /*implicit*/ ArrayRef(const std::initializer_list<T> &Vec)
-    : Data(Vec.begin() == Vec.end() ? (T*)nullptr : Vec.begin()),
-      Length(Vec.size()) {}
-
-    /// @}
-    /// @name Simple Operations
-    /// @{
-
-    iterator begin() const { return Data; }
-    iterator end() const { return Data + Length; }
-
-    reverse_iterator rbegin() const { return reverse_iterator(end()); }
-    reverse_iterator rend() const { return reverse_iterator(begin()); }
-
-    /// empty - Check if the array is empty.
-    bool empty() const { return Length == 0; }
-
-    const T *data() const { return Data; }
-
-    /// size - Get the array size.
-    size_t size() const { return Length; }
-
-    /// front - Get the first element.
-    const T &front() const {
-      AT_ASSERT(!empty(), "Empty list!");
-      return Data[0];
-    }
-
-    /// back - Get the last element.
-    const T &back() const {
-      AT_ASSERT(!empty(), "Empty list!");
-      return Data[Length-1];
-    }
-
-    /// equals - Check for element-wise equality.
-    bool equals(ArrayRef RHS) const {
-      if (Length != RHS.Length)
-        return false;
-      return std::equal(begin(), end(), RHS.begin());
-    }
-
-    /// slice(n, m) - Chop off the first N elements of the array, and keep M
-    /// elements in the array.
-    ArrayRef<T> slice(size_t N, size_t M) const {
-      AT_ASSERT(N+M <= size(), "Invalid specifier");
-      return ArrayRef<T>(data()+N, M);
-    }
-
-    /// slice(n) - Chop off the first N elements of the array.
-    ArrayRef<T> slice(size_t N) const { return slice(N, size() - N); }
-
-    /// @}
-    /// @name Operator Overloads
-    /// @{
-    const T &operator[](size_t Index) const {
-      return Data[Index];
-    }
-
-    /// Vector compatibility
-    const T &at(size_t Index) const {
-      AT_ASSERT(Index < Length, "Invalid index!");
-      return Data[Index];
-    }
-
-    /// Disallow accidental assignment from a temporary.
-    ///
-    /// The declaration here is extra complicated so that "arrayRef = {}"
-    /// continues to select the move assignment operator.
-    template <typename U>
-    typename std::enable_if<std::is_same<U, T>::value, ArrayRef<T>>::type &
-    operator=(U &&Temporary) = delete;
-
-    /// Disallow accidental assignment from a temporary.
-    ///
-    /// The declaration here is extra complicated so that "arrayRef = {}"
-    /// continues to select the move assignment operator.
-    template <typename U>
-    typename std::enable_if<std::is_same<U, T>::value, ArrayRef<T>>::type &
-    operator=(std::initializer_list<U>) = delete;
-
-    /// @}
-    /// @name Expensive Operations
-    /// @{
-    std::vector<T> vec() const {
-      return std::vector<T>(Data, Data+Length);
-    }
-
-    /// @}
-    /// @name Conversion operators
-    /// @{
-    operator std::vector<T>() const {
-      return std::vector<T>(Data, Data+Length);
-    }
-
-    /// @}
-  };
-
-} // end namespace at
+#include <ATen/core/ArrayRef.h>

aten/src/ATen/Backend.h

@@ -0,0 +1,2 @@
+#pragma once
+#include <ATen/core/Backend.h>

aten/src/ATen/Backtrace.h

@@ -0,0 +1,2 @@
+#pragma once
+#include <ATen/core/Backtrace.h>

aten/src/ATen/CMakeLists.txt

@@ -1,132 +1,12 @@
-
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
 SET(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
 
-# avoid some cmake warnings
-IF(POLICY CMP0026)
- CMAKE_POLICY(SET CMP0026 OLD)
-ENDIF()
-
-IF(MSVC AND NOT "${CMAKE_BUILD_TYPE}" MATCHES "Debug")
-  SET(MSVC_OPT_FLAG "/Ox /fp:strict ")
-ELSE()
-  SET(MSVC_OPT_FLAG "")
-ENDIF()
-
-########################
-# SET_SOURCE_FILES_PROPERTIES must be in the same CMakeLists.txt file as the target that includes the file
-# so we need to set these commands here rather than in src/TH
-IF(C_SSE4_1_FOUND AND C_SSE4_2_FOUND)
-  IF(MSVC)
-    SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/generic/simd/convolve5x5_sse.c PROPERTIES COMPILE_FLAGS "${MSVC_OPT_FLAG}/fp:fast")
-  ELSE(MSVC)
-    SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/generic/simd/convolve5x5_sse.c PROPERTIES COMPILE_FLAGS "-O3 -ffast-math")
-  ENDIF(MSVC)
-ENDIF(C_SSE4_1_FOUND AND C_SSE4_2_FOUND)
-IF(C_AVX_FOUND)
-  IF(MSVC)
-    SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/generic/simd/convolve5x5_avx.c PROPERTIES COMPILE_FLAGS "${MSVC_OPT_FLAG}/fp:fast ${C_AVX_FLAGS}")
-    SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/vector/AVX.c PROPERTIES COMPILE_FLAGS "${MSVC_OPT_FLAG}/arch:AVX ${C_AVX_FLAGS}")
-  ELSE(MSVC)
-    SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/generic/simd/convolve5x5_avx.c PROPERTIES COMPILE_FLAGS "-O3 -ffast-math ${C_AVX_FLAGS}")
-    SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/vector/AVX.c PROPERTIES COMPILE_FLAGS "-O3 ${C_AVX_FLAGS}")
-  ENDIF(MSVC)
-ENDIF(C_AVX_FOUND)
-
-IF(C_AVX2_FOUND)
-  IF(MSVC)
-    SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/vector/AVX2.cpp PROPERTIES COMPILE_FLAGS "${MSVC_OPT_FLAG}/arch:AVX2 ${C_AVX2_FLAGS}")
-  ELSE(MSVC)
-    SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/vector/AVX2.cpp PROPERTIES COMPILE_FLAGS "-O3 ${C_AVX2_FLAGS}")
-  ENDIF(MSVC)
-ENDIF(C_AVX2_FOUND)
-
-IF(NOT MSVC AND NOT "${CMAKE_C_COMPILER_ID}" MATCHES "Clang")
-  SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/THAtomic.c PROPERTIES COMPILE_FLAGS "-fno-openmp")
-  SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/THAllocator.c PROPERTIES COMPILE_FLAGS "-fno-openmp")
-ENDIF()
-
-FILE(GLOB cpu_kernel_cpp_in RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "native/cpu/*.cpp")
-
-LIST(APPEND CPU_CAPABILITY_NAMES "DEFAULT")
-IF(MSVC)
-  LIST(APPEND CPU_CAPABILITY_FLAGS "${MSVC_OPT_FLAG}")
-ELSE(MSVC)
-  LIST(APPEND CPU_CAPABILITY_FLAGS "-O3")
-ENDIF(MSVC)
-
-IF(CXX_AVX_FOUND)
-  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_AVX_CPU_DEFINITION")
-  LIST(APPEND CPU_CAPABILITY_NAMES "AVX")
-  IF(MSVC)
-    LIST(APPEND CPU_CAPABILITY_FLAGS "${MSVC_OPT_FLAG}/arch:AVX")
-  ELSE(MSVC)
-    LIST(APPEND CPU_CAPABILITY_FLAGS "-O3 -mavx")
-  ENDIF(MSVC)
-ENDIF(CXX_AVX_FOUND)
-
-IF(CXX_AVX2_FOUND)
-  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_AVX2_CPU_DEFINITION")
-  LIST(APPEND CPU_CAPABILITY_NAMES "AVX2")
-  IF(MSVC)
-    LIST(APPEND CPU_CAPABILITY_FLAGS "${MSVC_OPT_FLAG}/arch:AVX2")
-  ELSE(MSVC)
-    LIST(APPEND CPU_CAPABILITY_FLAGS "-O3 -mavx2")
-  ENDIF(MSVC)
-ENDIF(CXX_AVX2_FOUND)
-
-list(LENGTH CPU_CAPABILITY_NAMES NUM_CPU_CAPABILITY_NAMES)
-math(EXPR NUM_CPU_CAPABILITY_NAMES "${NUM_CPU_CAPABILITY_NAMES}-1")
-
-FOREACH(i RANGE ${NUM_CPU_CAPABILITY_NAMES})
-  FOREACH(IMPL ${cpu_kernel_cpp_in})
-    LIST(GET CPU_CAPABILITY_NAMES ${i} CPU_CAPABILITY)
-    SET(NEW_IMPL ${CMAKE_CURRENT_BINARY_DIR}/${IMPL}.${CPU_CAPABILITY}.cpp)
-    CONFIGURE_FILE(${IMPL} ${NEW_IMPL} COPYONLY)
-    SET(cpu_kernel_cpp ${NEW_IMPL} ${cpu_kernel_cpp}) # Create list of copies
-    LIST(GET CPU_CAPABILITY_FLAGS ${i} FLAGS)
-    IF(MSVC)
-      SET(MACRO_FLAG "/DCPU_CAPABILITY=${CPU_CAPABILITY} /DCPU_CAPABILITY_${CPU_CAPABILITY}")
-    ELSE(MSVC)
-      SET(MACRO_FLAG "-DCPU_CAPABILITY=${CPU_CAPABILITY} -DCPU_CAPABILITY_${CPU_CAPABILITY}")
-    ENDIF(MSVC)
-    SET_SOURCE_FILES_PROPERTIES(${NEW_IMPL} PROPERTIES COMPILE_FLAGS "${FLAGS} ${MACRO_FLAG}")
-  ENDFOREACH()
-ENDFOREACH()
-
-################################################################################
-# Helper functions
-################################################################################
-
-FUNCTION(EXCLUDE_DIR list_name dir_name)
-  # A helper that excludes all files that contain dir_name in their file path
-  SET(local_list ${${list_name}})
-  FOREACH(source ${local_list})
-    IF(${source} MATCHES ${dir_name})
-      MESSAGE(STATUS "Excluding " ${source} " from the build")
-      LIST(REMOVE_ITEM local_list ${source})
-    ENDIF()
-  ENDFOREACH()
-  SET(${list_name} ${local_list} PARENT_SCOPE)
-ENDFUNCTION()
-
-function(filter_list output input)
-    unset(result)
-    foreach(filename ${${input}})
-        foreach(pattern ${ARGN})
-            if("${filename}" MATCHES "${pattern}")
-                list(APPEND result "${filename}")
-            endif()
-        endforeach()
-    endforeach()
-    set(${output} ${result} PARENT_SCOPE)
-endfunction()
-
-
-IF ($ENV{TH_BINARY_BUILD})
-  MESSAGE(STATUS "TH_BINARY_BUILD detected. Statically linking libstdc++")
-  SET(CMAKE_CXX_FLAGS "-static-libstdc++ ${CMAKE_CXX_FLAGS}")
-ENDIF()
+IF(NOT MSVC)
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-ignored-qualifiers")
+  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-ignored-qualifiers")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-absolute-value")
+  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-absolute-value")
+ENDIF(NOT MSVC)
 
 # Can be compiled standalone
 IF(NOT AT_INSTALL_BIN_DIR OR NOT AT_INSTALL_LIB_DIR OR NOT AT_INSTALL_INCLUDE_DIR OR NOT AT_INSTALL_SHARE_DIR)
@@ -136,131 +16,94 @@ IF(NOT AT_INSTALL_BIN_DIR OR NOT AT_INSTALL_LIB_DIR OR NOT AT_INSTALL_INCLUDE_DI
   SET(AT_INSTALL_SHARE_DIR "share" CACHE PATH "AT install include subdirectory")
 ENDIF()
 
-# TODO: Maybe put this in the generated files directory
 CONFIGURE_FILE(Config.h.in "${CMAKE_CURRENT_SOURCE_DIR}/Config.h")
+CONFIGURE_FILE(cuda/CUDAConfig.h.in "${CMAKE_CURRENT_SOURCE_DIR}/cuda/CUDAConfig.h")
 
-FILE(GLOB base_h RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.h")
-FILE(GLOB base_cpp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cpp")
-FILE(GLOB native_cpp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "native/*.cpp")
-FILE(GLOB native_cudnn_cpp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "native/cudnn/*.cpp")
-FILE(GLOB native_cuda_cu RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "native/cuda/*.cu")
-FILE(GLOB native_mkl_cpp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "native/mkl/*.cpp")
-FILE(GLOB native_mkldnn_cpp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "native/mkldnn/*.cpp")
+# NB: If you edit these globs, you'll have to update setup.py package_data as well
+FILE(GLOB base_h "*.h" "detail/*.h")
+FILE(GLOB base_cpp "*.cpp" "detail/*.cpp")
+add_subdirectory(core)
+FILE(GLOB cuda_h "cuda/*.h" "cuda/detail/*.h" "cuda/*.cuh" "cuda/detail/*.cuh")
+FILE(GLOB cuda_cpp "cuda/*.cpp" "cuda/detail/*.cpp")
+FILE(GLOB cuda_cu "cuda/*.cu" "cuda/detail/*.cu")
+FILE(GLOB cudnn_h "cudnn/*.h" "cudnn/*.cuh")
+FILE(GLOB cudnn_cpp "cudnn/*.cpp")
+FILE(GLOB miopen_h "miopen/*.h")
+FILE(GLOB miopen_cpp "miopen/*.cpp")
+FILE(GLOB mkl_cpp "mkl/*.cpp")
+FILE(GLOB mkldnn_cpp "mkldnn/*.cpp")
 
-FILE(GLOB_RECURSE cuda_h
-     RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
-     "cuda/*.cuh" "cuda/*.h" "cudnn/*.cuh" "cudnn/*.h")
+FILE(GLOB native_cpp "native/*.cpp")
+FILE(GLOB native_sparse_cpp "native/sparse/*.cpp")
+FILE(GLOB native_sparse_cuda_cu "native/sparse/cuda/*.cu")
+FILE(GLOB native_sparse_cuda_cpp "native/sparse/cuda/*.cpp")
+FILE(GLOB native_cudnn_cpp "native/cudnn/*.cpp")
+FILE(GLOB native_miopen_cpp "native/miopen/*.cpp")
+FILE(GLOB native_cuda_cu "native/cuda/*.cu")
+FILE(GLOB native_cuda_cpp "native/cuda/*.cpp")
+FILE(GLOB native_mkl_cpp "native/mkl/*.cpp")
+FILE(GLOB native_mkldnn_cpp "native/mkldnn/*.cpp")
 
-FILE(GLOB cudnn_cpp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "cudnn/*.cpp")
-FILE(GLOB mkl_cpp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "mkl/*.cpp")
-FILE(GLOB mkldnn_cpp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "mkldnn/*.cpp")
-
-FILE(GLOB all_python RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.py")
-
-FILE(GLOB_RECURSE aten_cuda_cu RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "cuda/*.cu")
-
-IF(DEFINED ENV{PYTORCH_PYTHON})
-  message(STATUS "Using python found in $ENV{PYTORCH_PYTHON}")
-  SET(PYCMD "$ENV{PYTORCH_PYTHON}")
-ELSE()
-  SET(PYCMD "python")
-ENDIF()
-
-SET(GEN_COMMAND
-    ${PYCMD} ${CMAKE_CURRENT_SOURCE_DIR}/gen.py ${CUDA_FLAG}
-    -s ${CMAKE_CURRENT_SOURCE_DIR}
-    ${cwrap_files}
-)
-
-EXECUTE_PROCESS(
-    COMMAND ${GEN_COMMAND}
-      --output-dependencies ${CMAKE_CURRENT_BINARY_DIR}/generated_cpp.txt
-    RESULT_VARIABLE RETURN_VALUE
-)
-if (NOT RETURN_VALUE EQUAL 0)
-    message(STATUS ${generated_cpp})
-    message(FATAL_ERROR "Failed to get generated_cpp list")
+set(all_cpu_cpp ${base_cpp} ${ATen_CORE_SRCS} ${native_cpp} ${native_sparse_cpp} ${native_mkl_cpp} ${native_mkldnn_cpp} ${generated_cpp} ${ATen_CPU_SRCS} ${cpu_kernel_cpp})
+if(AT_MKL_ENABLED)
+  set(all_cpu_cpp ${all_cpu_cpp} ${mkl_cpp})
 endif()
-file(READ ${CMAKE_CURRENT_BINARY_DIR}/generated_cpp.txt generated_cpp)
-
-FILE(GLOB_RECURSE all_templates "templates/*")
-
-FILE(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ATen)
-
-ADD_CUSTOM_COMMAND(OUTPUT ${generated_cpp}
-COMMAND ${GEN_COMMAND}
-DEPENDS ${all_python} ${all_templates} ${cwrap_files})
-
-# Generated headers used from a cuda (.cu) file are
-# not tracked correctly in cmake . We make the libATen.so depend explicitly
-# on building the generated aten files to workaround.
-ADD_CUSTOM_TARGET(aten_files_are_generated
-  DEPENDS ${generated_cpp}
-)
-
-
-SET(all_cpp ${base_cpp} ${native_cpp} ${native_cudnn_cpp} ${native_mkl_cpp} ${native_mkldnn_cpp} ${generated_cpp} ${ATen_CPU_SRCS} ${cpu_kernel_cpp})
-
-INCLUDE_DIRECTORIES(${ATen_CPU_INCLUDE})
-IF(NOT NO_CUDA)
-  INCLUDE_DIRECTORIES(${ATen_CUDA_INCLUDE})
-  INCLUDE_DIRECTORIES("${CUDA_SDK_ROOT_DIR}/common/inc")
-  INCLUDE_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/cuda")
-  SET(ATen_CUDA_SRCS ${ATen_CUDA_SRCS} ${aten_cuda_cu} ${native_cuda_cu})
-  SET(all_cpp ${all_cpp} ${ATen_CUDA_SRCS})
-  IF(CUDNN_FOUND)
-    SET(all_cpp ${all_cpp} ${cudnn_cpp})
-  ENDIF()
-  IF(AT_MKL_ENABLED)
-    SET(all_cpp ${all_cpp} ${mkl_cpp})
-  ENDIF()
+if(AT_MKLDNN_ENABLED)
+  set(all_cpu_cpp ${all_cpu_cpp} ${mkldnn_cpp})
 endif()
 
-IF(AT_MKLDNN_ENABLED)
-  SET(all_cpp ${all_cpp} ${mkldnn_cpp})
-ENDIF()
+IF(USE_CUDA OR USE_ROCM)
+  list(APPEND ATen_CUDA_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/cuda)
+  set(ATen_CUDA_SRCS ${ATen_CUDA_SRCS} ${cuda_cu} ${native_cuda_cu} ${native_sparse_cuda_cu})
+  set(all_cuda_cpp ${native_sparse_cuda_cpp} ${cuda_cpp} ${native_cuda_cpp} ${cuda_generated_cpp} ${ATen_CUDA_SRCS})
+  IF(USE_CUDA)
+    SET(all_cuda_cpp ${native_cudnn_cpp} ${native_miopen_cpp} ${all_cuda_cpp})
+    IF(CUDNN_FOUND)
+      SET(all_cuda_cpp ${all_cuda_cpp} ${cudnn_cpp})
+    ENDIF()
+  ELSEIF(USE_ROCM)
+    SET(all_cuda_cpp ${native_cudnn_cpp} ${native_miopen_cpp} ${miopen_cpp} ${all_cuda_cpp})
+  ENDIF()
+endif()
 
 filter_list(generated_h generated_cpp "\\.h$")
+filter_list(cuda_generated_h cuda_generated_cpp "\\.h$")
 
-INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/..)
+list(APPEND ATen_CPU_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/..)
 # so the build can find the generated header files
-INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
+list(APPEND ATen_CPU_INCLUDE ${CMAKE_CURRENT_BINARY_DIR})
 IF(NOT AT_LINK_STYLE)
   SET(AT_LINK_STYLE SHARED)
 ENDIF()
-IF(CUDA_FOUND)
-  CUDA_ADD_LIBRARY(ATen ${AT_LINK_STYLE} ${all_cpp})
-ELSE()
-  ADD_LIBRARY(ATen ${AT_LINK_STYLE} ${all_cpp})
-ENDIF()
-ADD_DEPENDENCIES(ATen aten_files_are_generated)
-
-set(TBB_ROOT_DIR "${PROJECT_SOURCE_DIR}/src/ATen/cpu/tbb/tbb_remote")
-set(TBB_BUILD_STATIC ON CACHE BOOL " " FORCE)
-set(TBB_BUILD_SHARED OFF CACHE BOOL " " FORCE)
-set(TBB_BUILD_TBBMALLOC OFF CACHE BOOL " " FORCE)
-set(TBB_BUILD_TBBMALLOC_PROXY OFF CACHE BOOL " " FORCE)
-set(TBB_BUILD_TESTS OFF CACHE BOOL " " FORCE)
-add_subdirectory(${PROJECT_SOURCE_DIR}/src/ATen/cpu/tbb)
-set_property(TARGET tbb_static tbb_def_files PROPERTY FOLDER "dependencies")
-target_include_directories(tbb_static PUBLIC ${TBB_ROOT_DIR}/include)
-target_link_libraries(ATen tbb_static)
-
-if(NOT ${CMAKE_VERSION} VERSION_LESS "3.1")
-    SET_PROPERTY(TARGET ATen PROPERTY CXX_STANDARD 11)
-endif(NOT ${CMAKE_VERSION} VERSION_LESS "3.1")
 
 IF(BLAS_FOUND)
   IF ($ENV{TH_BINARY_BUILD})
     MESSAGE(STATUS "TH_BINARY_BUILD detected. Enabling special linkage.")
-    TARGET_LINK_LIBRARIES(ATen "${BLAS_LIBRARIES};${BLAS_LIBRARIES};${BLAS_LIBRARIES}")
+    list(APPEND ATen_CPU_DEPENDENCY_LIBS
+      "${BLAS_LIBRARIES};${BLAS_LIBRARIES};${BLAS_LIBRARIES}")
+    if(USE_CUDA OR USE_ROCM)
+      list(APPEND ATen_CUDA_DEPENDENCY_LIBS
+        "${BLAS_LIBRARIES};${BLAS_LIBRARIES};${BLAS_LIBRARIES}")
+    endif()
   ELSE ($ENV{TH_BINARY_BUILD})
-    TARGET_LINK_LIBRARIES(ATen ${BLAS_LIBRARIES})
+    list(APPEND ATen_CPU_DEPENDENCY_LIBS ${BLAS_LIBRARIES})
+    if(USE_CUDA OR USE_ROCM)
+      list(APPEND ATen_CUDA_DEPENDENCY_LIBS "${BLAS_LIBRARIES}")
+    endif()
   ENDIF ($ENV{TH_BINARY_BUILD})
 ENDIF(BLAS_FOUND)
 
 IF(LAPACK_FOUND)
-  TARGET_LINK_LIBRARIES(ATen ${LAPACK_LIBRARIES})
+  list(APPEND ATen_CPU_DEPENDENCY_LIBS ${LAPACK_LIBRARIES})
+  if(USE_CUDA OR USE_ROCM)
+    # Although Lapack provides CPU (and thus, one might expect that ATen_cuda
+    # would not need this at all), some of our libraries (magma in particular)
+    # backend to CPU BLAS/LAPACK implementations, and so it is very important
+    # we get the *right* implementation, because even if the symbols are the
+    # same, LAPACK implementions may have different calling conventions.
+    # This caused https://github.com/pytorch/pytorch/issues/7353
+    list(APPEND ATen_CUDA_DEPENDENCY_LIBS ${LAPACK_LIBRARIES})
+  endif()
 ENDIF(LAPACK_FOUND)
 
 IF (UNIX AND NOT APPLE)
@@ -268,7 +111,7 @@ IF (UNIX AND NOT APPLE)
    # https://github.com/libgit2/libgit2/issues/2128#issuecomment-35649830
    CHECK_LIBRARY_EXISTS(rt clock_gettime "time.h" NEED_LIBRT)
    IF(NEED_LIBRT)
-     TARGET_LINK_LIBRARIES(ATen rt)
+     list(APPEND ATen_CPU_DEPENDENCY_LIBS rt)
      SET(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} rt)
    ENDIF(NEED_LIBRT)
 ENDIF(UNIX AND NOT APPLE)
@@ -295,89 +138,258 @@ IF(UNIX)
   ENDIF(HAVE_MALLOC_USABLE_SIZE)
 ENDIF(UNIX)
 
-IF(NOT MSVC)
-  TARGET_LINK_LIBRARIES(ATen m)
-ELSE(NOT MSVC)
-  IF(AT_MKL_MT)
-    set_target_properties(ATen PROPERTIES LINK_FLAGS_RELEASE "/NODEFAULTLIB:vcomp")
-    set_target_properties(ATen PROPERTIES LINK_FLAGS_DEBUG "/NODEFAULTLIB:vcomp")
-    set_target_properties(ATen PROPERTIES STATIC_LIBRARY_FLAGS "/NODEFAULTLIB:vcomp")
-  ENDIF(AT_MKL_MT)
-ENDIF(NOT MSVC)
-
-# Is __thread supported?
-IF(NOT MSVC)
-  CHECK_C_SOURCE_COMPILES("static __thread int x = 1; int main() { return x; }" C_HAS_THREAD)
-ELSE(NOT MSVC)
-  CHECK_C_SOURCE_COMPILES("static __declspec( thread ) int x = 1; int main() { return x; }" C_HAS_THREAD)
-ENDIF(NOT MSVC)
-IF(NOT C_HAS_THREAD)
-  MESSAGE(STATUS "Warning: __thread is not supported, generating thread-unsafe code")
-ELSE(NOT C_HAS_THREAD)
-  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DTH_HAVE_THREAD")
-ENDIF(NOT C_HAS_THREAD)
+if(NOT MSVC)
+  list(APPEND ATen_CPU_DEPENDENCY_LIBS m)
+endif()
 
 if(MKLDNN_FOUND)
-  target_link_libraries(ATen ${MKLDNN_LIBRARIES})
+  list(APPEND ATen_CPU_DEPENDENCY_LIBS ${MKLDNN_LIBRARIES})
 endif(MKLDNN_FOUND)
 
-# ---[ Configure cpuinfo
-IF(NOT TARGET cpuinfo)
-  SET(CPUINFO_BUILD_TOOLS OFF CACHE BOOL "")
-  SET(CPUINFO_BUILD_UNIT_TESTS OFF CACHE BOOL "")
-  SET(CPUINFO_BUILD_MOCK_TESTS OFF CACHE BOOL "")
-  SET(CPUINFO_BUILD_BENCHMARKS OFF CACHE BOOL "")
-  ADD_SUBDIRECTORY("cpu/cpuinfo")
-ENDIF()
-TARGET_LINK_LIBRARIES(ATen cpuinfo)
+list(APPEND ATen_CPU_DEPENDENCY_LIBS cpuinfo)
 
-IF(CUDA_FOUND)
-  TARGET_LINK_LIBRARIES(ATen
-    ${CUDA_LIBRARIES}
-    ${CUDA_cusparse_LIBRARY}
-    ${CUDA_curand_LIBRARY})
-  CUDA_ADD_CUBLAS_TO_TARGET(ATen)
-  CUDA_ADD_CUFFT_TO_TARGET(ATen)
+if(NOT MSVC AND NOT EMSCRIPTEN)
+  # Preserve values for the main build
+  set(__aten_sleef_build_shared_libs ${BUILD_SHARED_LIBS})
+  set(__aten_sleef_build_tests ${BUILD_TESTS})
+
+  # Unset our restrictive C++ flags here and reset them later.
+  # Remove this once we use proper target_compile_options.
+  set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+  set(CMAKE_CXX_FLAGS)
+
+  # Bump up optimization level for sleef to -O1, since at -O0 the compiler
+  # excessively spills intermediate vector registers to the stack
+  # and makes things run impossibly slowly
+  set(OLD_CMAKE_C_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG})
+  IF(${CMAKE_C_FLAGS_DEBUG} MATCHES "-O0")
+    string(REGEX REPLACE "-O0" "-O1" CMAKE_C_FLAGS_DEBUG ${OLD_CMAKE_C_FLAGS_DEBUG})
+  ELSE()
+    set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O1")
+  ENDIF()
+
+  set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build sleef static" FORCE)
+  set(BUILD_DFT OFF CACHE BOOL "Don't build sleef DFT lib" FORCE)
+  set(BUILD_GNUABI_LIBS OFF CACHE BOOL "Don't build sleef gnuabi libs" FORCE)
+  set(BUILD_TESTS OFF CACHE BOOL "Don't build sleef tests" FORCE)
+  add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../../../third_party/sleef" ${CMAKE_BINARY_DIR}/sleef)
+  set_property(TARGET sleef PROPERTY FOLDER "dependencies")
+  list(APPEND ATen_THIRD_PARTY_INCLUDE ${CMAKE_BINARY_DIR}/include)
+  link_directories(${CMAKE_BINARY_DIR}/sleef/lib)
+  list(APPEND ATen_CPU_DEPENDENCY_LIBS sleef)
+
+  set(CMAKE_C_FLAGS_DEBUG ${OLD_CMAKE_C_FLAGS_DEBUG})
+  set(CMAKE_CXX_FLAGS ${OLD_CMAKE_CXX_FLAGS})
+
+  # Set these back. TODO: Use SLEEF_ to pass these instead
+  set(BUILD_SHARED_LIBS ${__aten_sleef_build_shared_libs} CACHE BOOL "Build shared libs" FORCE)
+  set(BUILD_TESTS ${__aten_sleef_build_tests} CACHE BOOL "Build tests" FORCE)
+endif()
+
+IF(USE_CUDA AND NOT USE_ROCM)
+  IF ($ENV{ATEN_STATIC_CUDA})
+    # CuFFT has a complicated static story (especially around CUDA < 9) because it has device callback support
+    # we first have to build a fake lib that links with no device callbacks,
+    # and then we link against this object file.
+    # This was recommended by the CuFFT team at NVIDIA
+
+    # build fake CuFFT lib in build dir
+    EXECUTE_PROCESS(COMMAND touch ${CMAKE_CURRENT_BINARY_DIR}/empty_file.cc)
+    if(${CUDA_VERSION_MAJOR} EQUAL "8")
+      SET(CUFFT_FAKELINK_OPTIONS
+	--generate-code arch=compute_35,code=sm_35
+	--generate-code arch=compute_50,code=sm_50
+	--generate-code arch=compute_60,code=sm_60)
+    elseif(${CUDA_VERSION_MAJOR} EQUAL "9")
+      SET(CUFFT_FAKELINK_OPTIONS
+	--generate-code arch=compute_35,code=sm_35
+	--generate-code arch=compute_50,code=sm_50
+	--generate-code arch=compute_60,code=sm_60
+	--generate-code arch=compute_70,code=sm_70)
+    else()
+      MESSAGE(FATAL_ERROR "Unhandled major cuda version ${CUDA_VERSION_MAJOR}")
+    endif()
+    ADD_CUSTOM_COMMAND(
+      OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cufft_static_library.a
+      COMMAND "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" -o ${CMAKE_CURRENT_BINARY_DIR}/cufft_static_library.a -Xcompiler -fPIC
+      ${CUFFT_FAKELINK_OPTIONS}
+      --device-link ${CMAKE_CURRENT_BINARY_DIR}/empty_file.cc -lcufft_static -lculibos
+      )
+    ADD_CUSTOM_TARGET(FAKELINKED_CUFFT_TARGET DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/cufft_static_library.a)
+    add_library(FAKELINKED_CUFFT STATIC IMPORTED GLOBAL)
+    add_dependencies(FAKELINKED_CUFFT FAKELINKED_CUFFT_TARGET)
+    set_target_properties(FAKELINKED_CUFFT PROPERTIES IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/cufft_static_library.a)
+
+    list(APPEND ATen_CUDA_DEPENDENCY_LIBS
+      ${CUDA_LIBRARIES}
+      ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcusparse_static.a
+      ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcurand_static.a
+      ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcublas_static.a
+      FAKELINKED_CUFFT
+      ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcufft_static.a
+      )
+  ELSE()
+    list(APPEND ATen_CUDA_DEPENDENCY_LIBS
+      ${CUDA_LIBRARIES}
+      ${CUDA_cusparse_LIBRARY}
+      ${CUDA_curand_LIBRARY})
+  ENDIF()
 
   if(CUDNN_FOUND)
-    target_link_libraries(ATen ${CUDNN_LIBRARIES})
+    list(APPEND ATen_CUDA_DEPENDENCY_LIBS ${CUDNN_LIBRARIES})
   endif(CUDNN_FOUND)
 
   IF(USE_MAGMA)
-    TARGET_LINK_LIBRARIES(ATen ${MAGMA_LIBRARIES})
+    list(APPEND ATen_CUDA_DEPENDENCY_LIBS ${MAGMA_LIBRARIES})
     IF ($ENV{TH_BINARY_BUILD})
-      TARGET_LINK_LIBRARIES(ATen "${BLAS_LIBRARIES};${BLAS_LIBRARIES};${BLAS_LIBRARIES}")
+      list(APPEND ATen_CUDA_DEPENDENCY_LIBS
+        "${BLAS_LIBRARIES};${BLAS_LIBRARIES};${BLAS_LIBRARIES}")
     ENDIF($ENV{TH_BINARY_BUILD})
   ENDIF(USE_MAGMA)
   IF ($ENV{ATEN_STATIC_CUDA})
-    TARGET_LINK_LIBRARIES(ATen "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libculibos.a")
+    list(APPEND ATen_CUDA_DEPENDENCY_LIBS "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libculibos.a")
+    list(APPEND ATen_CUDA_DEPENDENCY_LIBS "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcudart_static.a")
   ENDIF($ENV{ATEN_STATIC_CUDA})
 ENDIF()
 
-INSTALL(TARGETS ATen
-  RUNTIME DESTINATION "${AT_INSTALL_BIN_DIR}"
-  LIBRARY DESTINATION "${AT_INSTALL_LIB_DIR}"
-  ARCHIVE DESTINATION "${AT_INSTALL_LIB_DIR}")
+IF(USE_ROCM)
+ ### Link in the ROCm libraries BLAS / RNG.
+ FIND_LIBRARY(ROCBLAS_LIBRARY rocblas HINTS ${ROCBLAS_PATH}/lib)
+ FIND_LIBRARY(HIPRAND_LIBRARY hiprand HINTS ${HIPRAND_PATH}/lib)
+
+ list(APPEND ATen_CUDA_DEPENDENCY_LIBS ${ROCBLAS_LIBRARY} ${HIPRAND_LIBRARY})
+ENDIF()
+
+# Include CPU paths for CUDA as well
+list(APPEND ATen_CUDA_INCLUDE ${ATen_CPU_INCLUDE})
+
+# We have two libraries: libATen_cpu.so and libATen_cuda.so,
+# with libATen_cuda.so depending on libATen_cpu.so.  The CPU library
+# contains CPU code only.  libATen_cpu.so is invariant to the setting
+# of USE_CUDA (it always builds the same way); libATen_cuda.so is only
+# built when USE_CUDA=1 and CUDA is available.
+set(ATen_CPU_SRCS ${all_cpu_cpp})
+if(AT_LINK_STYLE STREQUAL "INTERFACE")
+  # Source code can't be added to an interface library, so it is
+  # passed back to be compiled into the containing library
+  add_library(ATen_cpu INTERFACE)
+  list(APPEND ATen_CPU_DEPENDENCY_LIBS ATEN_CPU_FILES_GEN_LIB)
+else()
+  add_library(ATen_cpu ${AT_LINK_STYLE} ${ATen_CPU_SRCS})
+  if (ATen_THIRD_PARTY_INCLUDE)
+    target_include_directories(ATen_cpu SYSTEM PRIVATE ${ATen_THIRD_PARTY_INCLUDE})
+  endif()
+  target_include_directories(ATen_cpu INTERFACE $<INSTALL_INTERFACE:include>)
+  target_include_directories(ATen_cpu PRIVATE ${ATen_CPU_INCLUDE})
+  target_link_libraries(ATen_cpu PUBLIC ${ATen_CPU_DEPENDENCY_LIBS})
+  target_link_libraries(ATen_cpu PRIVATE ATEN_CPU_FILES_GEN_LIB)
+  caffe2_interface_library(ATen_cpu ATen_cpu_library)
+  # Set standard properties on the target
+  aten_set_target_props(ATen_cpu)
+
+  # Make sure these don't get built by parent
+  set(ATen_CPU_SRCS)
+endif()
+
+if(USE_CUDA OR USE_ROCM)
+  set(ATen_CUDA_SRCS ${all_cuda_cpp})
+  if(AT_LINK_STYLE STREQUAL "INTERFACE")
+    # Source code can't be added to an interface library, so it is
+    # passed back to be compiled into the containing library
+    add_library(ATen_cuda INTERFACE)
+    list(APPEND ATen_CUDA_DEPENDENCY_LIBS ATEN_CUDA_FILES_GEN_LIB)
+  else()
+    # A hack to deal with cuda library dependencies and modern CMake: the
+    # CUDA_ADD_LIBRARY includes a target_link_libraries, and as a result,
+    # one cannot use PUBLIC/PRIVATE/INTERFACE for the target anymore. This
+    # hack adds the PRIVATE keywords to CUDA_LIBRARIES so we can deal with
+    # it. We will then manually add the cudart library as interface libs.
+    set(__tmp ${CUDA_LIBRARIES})
+    set(CUDA_LIBRARIES PRIVATE ${CUDA_LIBRARIES})
+    torch_cuda_based_add_library(ATen_cuda ${AT_LINK_STYLE} ${ATen_CUDA_SRCS})
+    set(CUDA_LIBRARIES ${__tmp})
+    target_link_libraries(ATen_cuda INTERFACE caffe2::cudart)
+
+    target_include_directories(
+        ATen_cuda INTERFACE $<INSTALL_INTERFACE:include>)
+    target_include_directories(
+        ATen_cuda PRIVATE ${ATen_THIRD_PARTY_INCLUDE})
+    target_include_directories(
+        ATen_cuda PRIVATE ${ATen_CUDA_INCLUDE})
+    target_link_libraries(
+        ATen_cuda PRIVATE ${ATen_CUDA_DEPENDENCY_LIBS} ATEN_CUDA_FILES_GEN_LIB)
+
+    # These public dependencies must go after the previous dependencies, as the
+    # order of the libraries in the linker call matters here when statically
+    # linking; libculibos and cublas must be last.
+    target_link_libraries(
+        ATen_cuda PUBLIC ATen_cpu ${ATen_PUBLIC_CUDA_DEPENDENCY_LIBS})
+
+    # Set standard properties on the target
+    aten_set_target_props(ATen_cuda)
+
+    caffe2_interface_library(ATen_cuda ATen_cuda_library)
+
+    # Make sure these don't get built by parent
+    set(ATen_CUDA_SRCS)
+  endif()
+endif()
+
+if(NOT AT_LINK_STYLE STREQUAL "INTERFACE")
+  if(USE_CUDA)
+    if (NOT $ENV{ATEN_STATIC_CUDA})
+      cuda_add_cublas_to_target(ATen_cuda)
+      cuda_add_cufft_to_target(ATen_cuda)
+    endif()
+  endif()
+
+  if(NOT MSVC)
+    aten_compile_options(ATen_cpu)
+    if(USE_CUDA OR USE_ROCM)
+      aten_compile_options(ATen_cuda)
+    endif()
+  endif()
+
+  if(NOT ${CMAKE_VERSION} VERSION_LESS "3.1")
+    set_property(TARGET ATen_cpu PROPERTY CXX_STANDARD 11)
+    if(USE_CUDA OR USE_ROCM)
+      set_property(TARGET ATen_cuda PROPERTY CXX_STANDARD 11)
+    endif()
+  endif()
+endif()
 
-GET_TARGET_PROPERTY(ATEN_OUTPUT_NAME ATen LOCATION)
-GET_FILENAME_COMPONENT(ATEN_OUTPUT_NAME ${ATEN_OUTPUT_NAME} NAME)
-SET(ATEN_LIBRARIES "${CMAKE_INSTALL_PREFIX}/${AT_INSTALL_LIB_DIR}/${ATEN_OUTPUT_NAME}")
 SET(ATEN_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/${AT_INSTALL_INCLUDE_DIR}")
 CONFIGURE_FILE(ATenConfig.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/cmake-exports/ATenConfig.cmake")
 INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/cmake-exports/ATenConfig.cmake"
   DESTINATION "${AT_INSTALL_SHARE_DIR}/cmake/ATen")
 
-FOREACH(HEADER ${base_h})
-  INSTALL(FILES ${HEADER} DESTINATION ${AT_INSTALL_INCLUDE_DIR}/ATen)
-ENDFOREACH()
-FOREACH(HEADER ${generated_h})
-  INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/${HEADER}
-  DESTINATION ${AT_INSTALL_INCLUDE_DIR}/ATen)
-ENDFOREACH()
-FOREACH(HEADER ${cuda_h})
-  # https://stackoverflow.com/questions/11096471/how-can-i-install-a-hierarchy-of-files-using-cmake
-  GET_FILENAME_COMPONENT(DIR ${HEADER} DIRECTORY)
+# https://stackoverflow.com/questions/11096471/how-can-i-install-a-hierarchy-of-files-using-cmake
+FOREACH(HEADER ${base_h} ${ATen_CORE_HEADERS} ${cuda_h} ${cudnn_h})
+  string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/" "" HEADER_SUB ${HEADER})
+  GET_FILENAME_COMPONENT(DIR ${HEADER_SUB} DIRECTORY)
   INSTALL(FILES ${HEADER} DESTINATION ${AT_INSTALL_INCLUDE_DIR}/ATen/${DIR})
 ENDFOREACH()
-INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/ATen/Declarations.yaml
+FOREACH(HEADER ${generated_h} ${cuda_generated_h})
+  # NB: Assumed to be flat
+  INSTALL(FILES ${HEADER} DESTINATION ${AT_INSTALL_INCLUDE_DIR}/ATen)
+ENDFOREACH()
+INSTALL(FILES ${CMAKE_BINARY_DIR}/aten/src/ATen/Declarations.yaml
   DESTINATION ${AT_INSTALL_SHARE_DIR}/ATen)
+
+if(ATEN_NO_TEST)
+  message("disable test because ATEN_NO_TEST is set")
+else()
+  add_subdirectory(test)
+endif()
+
+# Pass source, includes, and libs to parent
+set(ATen_CORE_SRCS ${ATen_CORE_SRCS} PARENT_SCOPE)
+set(ATen_CPU_SRCS ${ATen_CPU_SRCS} PARENT_SCOPE)
+set(ATen_CUDA_SRCS ${ATen_CUDA_SRCS} PARENT_SCOPE)
+set(ATen_CPU_TEST_SRCS ${ATen_CPU_TEST_SRCS} PARENT_SCOPE)
+set(ATen_CUDA_TEST_SRCS ${ATen_CUDA_TEST_SRCS} PARENT_SCOPE)
+set(ATen_CPU_INCLUDE ${ATen_CPU_INCLUDE} PARENT_SCOPE)
+set(ATen_THIRD_PARTY_INCLUDE ${ATen_THIRD_PARTY_INCLUDE} PARENT_SCOPE)
+set(ATen_CUDA_INCLUDE ${ATen_CUDA_INCLUDE} PARENT_SCOPE)
+set(ATen_CPU_DEPENDENCY_LIBS ${ATen_CPU_DEPENDENCY_LIBS} PARENT_SCOPE)
+set(ATen_CUDA_DEPENDENCY_LIBS ${ATen_CUDA_DEPENDENCY_LIBS} PARENT_SCOPE)

aten/src/ATen/CPUApplyUtils.h

@@ -1,18 +1,109 @@
 #pragma once
 
-#include <sstream>
+#include "ATen/Parallel.h"
 #include "ATen/TensorUtils.h"
+#include <limits>
+#include <utility>
+#include <cstring>
 
 namespace at {
 
+/*
+[collapse dims] Updates sizes, and strides to reflect a "collapse" of
+the info, possibly excluding the optional excludeDim. A "collapsed" version
+of the info is the fewest dims that order the tensor's elements in the same
+way as the original info. If excludeDim is specified, the collapse is the
+fewest dims that order the tensor's elements as the original and preserve the
+excluded dimension, unless the tensor collapses to a point.
+
+This function returns a pair of values.
+
+1) The (new) index of the preserved dimension if excludeDim is
+specified. 0 if the tensor is collapsed to a point. -1
+otherwise.
+
+2) The new number of dimensions.
+*/
+template <typename T>
+inline std::pair<int64_t, int64_t> collapse_dims(
+    T* sizes,
+    T* strides,
+    int64_t dims,
+    const int excludeDim = -1) {
+  AT_CHECK(
+      excludeDim >= -1 && excludeDim < dims,
+      "expected excluded dim between -1 and dims - 1");
+
+  int64_t stopDim = (excludeDim == -1) ? dims : excludeDim;
+  int64_t newIndex = -1;
+  int64_t oldIndex = 0;
+  int64_t remappedExcludedDim = -1;
+
+  while (oldIndex < dims) {
+    // Finds a dimension to collapse into
+    for (; oldIndex < stopDim; ++oldIndex) {
+      if (sizes[oldIndex] == 1) {
+        continue;
+      }
+
+      ++newIndex;
+      sizes[newIndex] = sizes[oldIndex];
+      strides[newIndex] = strides[oldIndex];
+      ++oldIndex;
+      break;
+    }
+
+    // Collapses dims
+    for (; oldIndex < stopDim; ++oldIndex) {
+      if (sizes[oldIndex] == 1) {
+        continue;
+      }
+
+      if (strides[newIndex] == sizes[oldIndex] * strides[oldIndex]) {
+        sizes[newIndex] *= sizes[oldIndex];
+        strides[newIndex] = strides[oldIndex];
+      } else {
+        ++newIndex;
+        sizes[newIndex] = sizes[oldIndex];
+        strides[newIndex] = strides[oldIndex];
+      }
+    }
+
+    // Handles excludeDim being set (oldIndex == excludeDim)
+    if (oldIndex != dims) {
+      // Preserves excluded dimension
+      ++newIndex;
+      sizes[newIndex] = sizes[oldIndex];
+      strides[newIndex] = strides[oldIndex];
+      remappedExcludedDim = newIndex;
+
+      // Restarts iteration after excludeDim
+      ++oldIndex;
+      stopDim = dims;
+    }
+  }
+
+  // Handles special case of all dims size 1
+  if (newIndex == -1 || (newIndex == 0 && sizes[0] == 1)) {
+    dims = 1;
+    sizes[0] = 1;
+    strides[0] = 1;
+
+    return std::pair<int64_t, int64_t>(0, 1);
+  }
+
+  dims = newIndex + 1;
+  return std::pair<int64_t, int64_t>(remappedExcludedDim, dims);
+}
+
 /*
  * The basic strategy for apply is as follows:
  *
- * 1. Starting with the outermost index, loop until we reach a dimension where the
- * data is no longer contiguous, i.e. the stride at that dimension is not equal to
- * the size of the tensor defined by the outer dimensions. Let's call this outer
- * (contiguous) tensor A. Note that if the Tensor is contiguous, then A is equal
- * to the entire Tensor. Let's call the inner tensor B.
+ * 1. Starting with the outermost index, loop until we reach a dimension where
+ * the data is no longer contiguous, i.e. the stride at that dimension is not
+ * equal to the size of the tensor defined by the outer dimensions. Let's call
+ * this outer (contiguous) tensor A. Note that if the Tensor is contiguous, then
+ * A is equal to the entire Tensor. Let's call the inner tensor B.
  *
  * 2. We loop through the indices in B, starting at its outermost dimension. For
  * example, if B is a 2x2 matrix, then we do:
@@ -22,289 +113,455 @@ namespace at {
  * B[1][0]
  * B[1][1]
  *
- * We set the offset into the underlying storage as (storageOffset + stride_B * index_B),
- * i.e. basically we compute the offset into the storage as we would normally for a
- * Tensor. But because we are guaranteed the subsequent data is contiguous in memory, we
- * can simply loop for sizeof(A) iterations and perform the operation, without having to
- * follow the order described by the strides of A.
+ * We set the offset into the underlying storage as (storageOffset + stride_B *
+ * index_B), i.e. basically we compute the offset into the storage as we would
+ * normally for a Tensor. But because we are guaranteed the subsequent data is
+ * contiguous in memory, we can simply loop for sizeof(A) iterations and perform
+ * the operation, without having to follow the order described by the strides of
+ * A.
  *
- * 3. As an optimization, we merge dimensions of A that are contiguous in memory. For
- * example, if A is a 3x3x3x3 tensor narrowed from a 3x3x4x3 tensor, then the first two
- * dimensions can be merged for the purposes of APPLY, reducing the number of nested
- * loops.
+ * 3. As an optimization, we merge dimensions of A that are contiguous in
+ * memory. For example, if A is a 3x3x3x3 tensor narrowed from a 3x3x4x3 tensor,
+ * then the first two dimensions can be merged for the purposes of APPLY,
+ * reducing the number of nested loops.
  */
 
-// TODO: turn this macro into a proper template
-#define __ATH_TENSOR_APPLYX_PREAMBLE(TYPE, ATENSOR, DIM, ALLOW_CONTIGUOUS) \
-  TYPE *ATENSOR##_data = NULL; \
-  int64_t *ATENSOR##_counter = NULL, *ATENSOR##_sizes = NULL, *ATENSOR##_strides = NULL, *ATENSOR##_dimOffset = NULL; \
-  int64_t ATENSOR##_stride = 0, ATENSOR##_size = 0, ATENSOR##_dim = 0, ATENSOR##_i; \
-  int ATENSOR##_contiguous = ALLOW_CONTIGUOUS && DIM < 0; \
-\
-  if(ATENSOR.sizes().equals({0})) \
-    TH_TENSOR_APPLY_hasFinished = true; \
-  else \
-  { \
-    ATENSOR##_data = ATENSOR.data<TYPE>(); \
-    ATENSOR##_size = 1; \
-    ATENSOR##_stride = 1; \
-    for(ATENSOR##_i = ATENSOR.dim() - 1; ATENSOR##_i >= 0; ATENSOR##_i--) { \
-      if(ATENSOR.sizes()[ATENSOR##_i] != 1) { \
-        if(ATENSOR.strides()[ATENSOR##_i] == ATENSOR##_size && ATENSOR##_i != DIM) \
-          ATENSOR##_size *= ATENSOR.sizes()[ATENSOR##_i]; \
-        else{ \
-          ATENSOR##_contiguous = 0; \
-          break; \
-        } \
-      } \
-    } \
-    if (!ATENSOR##_contiguous) { \
-      /* Find the dimension of contiguous sections */ \
-      ATENSOR##_dim = 1; \
-      for(ATENSOR##_i = ATENSOR.dim() - 2; ATENSOR##_i >= 0; ATENSOR##_i--) \
-      { \
-        if(ATENSOR.strides()[ATENSOR##_i] != ATENSOR.strides()[ATENSOR##_i+1] * ATENSOR.sizes()[ATENSOR##_i+1] || ATENSOR##_i == DIM || ATENSOR##_i+1 == DIM) \
-          ATENSOR##_dim++; \
-      } \
-      /* Allocate an array of 3*dim elements, where dim is the number of contiguous sections */ \
-      ATENSOR##_counter = new int64_t[3*ATENSOR##_dim]; \
-      ATENSOR##_sizes = ATENSOR##_counter + ATENSOR##_dim; \
-      ATENSOR##_strides = ATENSOR##_counter + 2*ATENSOR##_dim; \
-      TH_TENSOR_dim_index = ATENSOR##_dim-1; \
-      ATENSOR##_dimOffset = (DIM == ATENSOR.dim()-1) ? &ATENSOR##_i : &ATENSOR##_counter[DIM]; \
-      ATENSOR##_sizes[TH_TENSOR_dim_index] = ATENSOR.sizes()[ATENSOR.dim()-1]; \
-      ATENSOR##_strides[TH_TENSOR_dim_index] = ATENSOR.strides()[ATENSOR.dim()-1]; \
-      /* ATENSOR##_counter tracks where we are in the storage. The offset into the */ \
-      /* storage is given by storage_offset + (i * j), where i is the stride */ \
-      /* vector and j is tensor_counter vector. This sets the starting position for the loop. */ \
-      for(ATENSOR##_i = ATENSOR##_dim-1; ATENSOR##_i >= 0; --ATENSOR##_i) { \
-        ATENSOR##_counter[ATENSOR##_i] = 0; \
-      } \
-      for(ATENSOR##_i = ATENSOR.dim()-2; ATENSOR##_i >= 0; --ATENSOR##_i) { \
-        if (ATENSOR.strides()[ATENSOR##_i] == ATENSOR.strides()[ATENSOR##_i+1] * ATENSOR.sizes()[ATENSOR##_i+1] && ATENSOR##_i != DIM && ATENSOR##_i+1 != DIM) { \
-          ATENSOR##_sizes[TH_TENSOR_dim_index] = ATENSOR.sizes()[ATENSOR##_i] * ATENSOR##_sizes[TH_TENSOR_dim_index]; \
-          if (DIM != ATENSOR.dim()-1 && ATENSOR##_i < DIM) \
-            ATENSOR##_dimOffset--; \
-        } else { \
-          --TH_TENSOR_dim_index; \
-          ATENSOR##_sizes[TH_TENSOR_dim_index] = ATENSOR.sizes()[ATENSOR##_i]; \
-          ATENSOR##_strides[TH_TENSOR_dim_index] = ATENSOR.strides()[ATENSOR##_i]; \
-        } \
-      } \
-      /* Size of the inner most section */ \
-      ATENSOR##_size = ATENSOR##_sizes[ATENSOR##_dim-1]; \
-      /* Stride of the inner most section */ \
-      ATENSOR##_stride = ATENSOR##_strides[ATENSOR##_dim-1]; \
-    } \
-  } \
-  ATENSOR##_i = 0;
-
-// TODO: turn this macro into a proper template
-#define  __ATH_TENSOR_APPLYX_UPDATE_COUNTERS(ATENSOR, ALWAYS_UPDATE) \
-  if(ATENSOR##_i == ATENSOR##_size || ALWAYS_UPDATE) \
-  { \
-    if(ATENSOR##_contiguous) \
-      break; \
-\
-    if(ATENSOR##_dim == 1) \
-       break; \
-\
-    /* Reset pointer to beginning of loop */ \
-    ATENSOR##_data -= ATENSOR##_size*ATENSOR##_stride; \
-    for(ATENSOR##_i = ATENSOR##_dim-2; ATENSOR##_i >= 0; ATENSOR##_i--) \
-    { \
-      ATENSOR##_counter[ATENSOR##_i]++; \
-      /* Jump ahread by the stride of this dimension */ \
-      ATENSOR##_data += ATENSOR##_strides[ATENSOR##_i]; \
-\
-      if(ATENSOR##_counter[ATENSOR##_i]  == ATENSOR##_sizes[ATENSOR##_i]) \
-      { \
-        if(ATENSOR##_i == 0) \
-        { \
-          TH_TENSOR_APPLY_hasFinished = true; \
-          break; \
-        } \
-          else \
-        { \
-          /* Reset the pointer to the beginning of the chunk defined by this dimension */ \
-          ATENSOR##_data -= ATENSOR##_counter[ATENSOR##_i]*ATENSOR##_strides[ATENSOR##_i]; \
-          ATENSOR##_counter[ATENSOR##_i] = 0; \
-        } \
-      } \
-      else \
-        break; \
-    } \
-    ATENSOR##_i = 0; \
+inline Tensor sort_strides(Tensor& tensor_) {
+  IntList strides = tensor_.strides();
+  std::vector<int64_t> indices;
+  indices.reserve(tensor_.ndimension());
+  for (int64_t i = 0; i < tensor_.ndimension(); i++) {
+    indices.push_back(i);
   }
+  std::sort(indices.begin(), indices.end(), [&strides](int64_t i1, int64_t i2) {
+    return strides[i1] > strides[i2];
+  });
+  Tensor tensor = tensor_.permute(indices);
+  return tensor;
+}
+
+template <typename T, int N>
+struct strided_tensor_iter_fixed {
+ public:
+  T* data_ = NULL;
+  int64_t dim_ = 0;
+
+  int64_t counter_[N] = {0};
+  int64_t sizes_[N] = {0};
+  int64_t strides_[N] = {0};
+
+  strided_tensor_iter_fixed(strided_tensor_iter_fixed const&) = delete;
+  void operator=(strided_tensor_iter_fixed const& x) = delete;
+  strided_tensor_iter_fixed(strided_tensor_iter_fixed&&) = default;
+  strided_tensor_iter_fixed(Tensor& tensor, bool sort_strides = false)
+      : data_(tensor.data<T>()) {
+    std::memset(counter_, 0, sizeof(int64_t) * N);
+    std::memcpy(
+        sizes_, tensor.sizes().data(), tensor.ndimension() * sizeof(int64_t));
+    std::memcpy(
+        strides_,
+        tensor.strides().data(),
+        tensor.ndimension() * sizeof(int64_t));
+    dim_ = std::get<1>(collapse_dims(sizes_, strides_, tensor.ndimension()));
+  }
+};
+
+template <typename T>
+struct strided_tensor_iter {
+ private:
+ public:
+  T* data_ = NULL;
+  int64_t dim_;
+
+  std::vector<int64_t> counter_;
+  std::vector<int64_t> sizes_;
+  std::vector<int64_t> strides_;
+
+  strided_tensor_iter(strided_tensor_iter const&) = delete;
+  void operator=(strided_tensor_iter const& x) = delete;
+  strided_tensor_iter(strided_tensor_iter&&) = default;
+  strided_tensor_iter(Tensor& tensor)
+      : data_(tensor.data<T>()),
+        dim_(tensor.ndimension()),
+        counter_(dim_, 0),
+        sizes_(tensor.sizes().vec()),
+        strides_(tensor.strides().vec()) {
+    dim_ = std::get<1>(collapse_dims(sizes_.data(), strides_.data(), dim_));
+  }
+};
+
+inline bool _all_equal_numel(at::ArrayRef<Tensor> tensors) {
+  if (tensors.size() == 0)
+    return true;
+  int64_t all_numel = tensors[0].numel();
+  for (size_t i = 1; i < tensors.size(); i++) {
+    if (tensors[i].numel() != all_numel)
+      return false;
+  }
+  return true;
+}
+
+inline std::string _all_equal_numel_error(at::ArrayRef<Tensor> tensors) {
+  std::ostringstream oss;
+  oss << "inconsistent tensor size, expected ";
+  for (size_t i = 0; i < tensors.size() - 1; i++) {
+    oss << tensors[i].sizes() << ", ";
+  }
+  oss << "and " << tensors[tensors.size() - 1].sizes()
+      << " to have the same number of elements, but got ";
+  for (size_t i = 0; i < tensors.size() - 1; i++) {
+    oss << tensors[i].numel() << ", ";
+  }
+  oss << "and " << tensors[tensors.size() - 1].numel()
+      << " elements respectively";
+  return oss.str();
+}
+
+inline bool _apply_preamble(ArrayRef<Tensor> tensors) {
+  checkBackend("CPU_tensor_apply", tensors, Backend::CPU);
+  if (!_all_equal_numel(tensors))
+    AT_ERROR(_all_equal_numel_error(tensors));
+  // An empty tensor has no elements
+  for (auto& t : tensors)
+    if (t.numel() == 0)
+      return false;
+  return true;
+}
+
+inline int64_t _max_dim_tensors(ArrayRef<Tensor> tensors) {
+  int64_t dim = 0;
+  for (auto& t : tensors)
+    dim = std::max(dim, t.ndimension());
+  return dim;
+}
+
+inline void iterate(int64_t size){};
+
+template <typename Arg, typename... Args>
+inline void iterate(int64_t size, Arg& iter, Args&... iter_tail) {
+  iter.counter_[iter.dim_ - 1] += size;
+  iter.data_ = iter.data_ + size * iter.strides_[iter.dim_ - 1];
+  iterate(size, iter_tail...);
+}
+
+inline bool iterate_continue() {
+  return true;
+};
+
+template <typename Arg, typename... Args>
+inline bool iterate_continue(Arg& iter, Args&... iter_tail) {
+  return iter.counter_[iter.dim_ - 1] < iter.sizes_[iter.dim_ - 1] &&
+      iterate_continue(iter_tail...);
+}
+
+inline int64_t max_iterate_size() {
+  return std::numeric_limits<int64_t>::max();
+};
+
+template <typename Arg, typename... Args>
+inline int64_t max_iterate_size(Arg& iter, Args&... iter_tail) {
+  return std::min(
+      (iter.sizes_[iter.dim_ - 1] - iter.counter_[iter.dim_ - 1]),
+      max_iterate_size(iter_tail...));
+}
+
+inline void iterate_overflow(){};
+
+template <typename Arg, typename... Args>
+inline void iterate_overflow(Arg& iter, Args&... iter_tail) {
+  if (iter.counter_[iter.dim_ - 1] == iter.sizes_[iter.dim_ - 1]) {
+    for (int64_t i = iter.dim_ - 1; i > 0; i--) {
+      if (iter.counter_[i] == iter.sizes_[i]) {
+        iter.counter_[i] = 0;
+        iter.counter_[i - 1]++;
+        iter.data_ = iter.data_ - (iter.sizes_[i] * iter.strides_[i]) +
+            iter.strides_[i - 1];
+      }
+    }
+  }
+  iterate_overflow(iter_tail...);
+}
+
+inline void forward(int64_t offset){};
+
+template <typename Arg, typename... Args>
+inline void forward(int64_t offset, Arg& iter, Args&... iter_tail) {
+  int64_t multi = offset;
+  for (int64_t i = iter.dim_ - 1; i >= 0; i--) {
+    int64_t inc = multi % iter.sizes_[i];
+    multi = multi / iter.sizes_[i];
+    iter.data_ = iter.data_ + inc * iter.strides_[i];
+    iter.counter_[i] += inc;
+  }
+  forward(offset, iter_tail...);
+}
+
+inline int64_t max_dim() {
+  return 0;
+}
+
+template <typename Arg, typename... Args>
+inline int64_t max_dim(Arg& iter, Args&... iter_tail) {
+  return std::max(iter.dim_, max_dim(iter_tail...));
+}
+
+inline void apply_op(){};
+
+template <typename Op, typename... Args>
+inline void
+apply_op(int64_t numel, int64_t offset, const Op& op, Args... iters) {
+  // For 0-dim tensors
+  if (numel == 1 && max_dim(iters...) == 0) {
+    op(*iters.data_...);
+    return;
+  }
+  if (offset > 0)
+    forward(offset, iters...);
+  // Splitting this into chunks helps the compiler create faster assembly
+  for (int64_t i = 0; i < numel;) {
+    for (; iterate_continue(iters...) && i < numel;) {
+      op(*iters.data_...);
+      iterate(1, iters...);
+      i++;
+    }
+    iterate_overflow(iters...);
+  }
+}
+
+
+inline void apply_kernel(){};
+
+// TODO: Deal elegantly with 0-dim tensors. iters.strides_ of 0-dim
+// strided_tensor_iter will be of size 0 for dim 0 and iters.strides_[iters.dim_
+// - 1] will index at -1. C++14 integer_sequence could be of use here.
+template <typename Op, typename... Args>
+inline void
+apply_kernel(int64_t numel, int64_t offset, const Op& op, Args... iters) {
+  if (offset > 0)
+    forward(offset, iters...);
+  int64_t size = std::min(numel, max_iterate_size(iters...));
+  op(size, iters.data_..., iters.strides_[iters.dim_ - 1]...);
+  iterate(size, iters...);
+  iterate_overflow(iters...);
+  int64_t i = size;
+  size = std::min(numel, max_iterate_size(iters...));
+  for (; i < numel;) {
+    op(size, iters.data_..., iters.strides_[iters.dim_ - 1]...);
+    iterate(size, iters...);
+    i += size;
+    iterate_overflow(iters...);
+  }
+}
 
 template <typename scalar1, typename scalar2, typename Op>
-void CPU_tensor_apply2_dim(Tensor& tensor1, Tensor& tensor2, int64_t dim, Op op) {
-  checkBackend("CPU_tensor_apply2", {tensor1, tensor2}, Backend::CPU);
-  bool TH_TENSOR_APPLY_hasFinished = false;
-  int64_t TH_TENSOR_dim_index = 0;
-  __ATH_TENSOR_APPLYX_PREAMBLE(scalar1, tensor1, dim, 1)
-  __ATH_TENSOR_APPLYX_PREAMBLE(scalar2, tensor2, dim, 1)
-  auto t1_numel = tensor1.numel();
-  auto t2_numel = tensor2.numel();
-  if(t1_numel != t2_numel) {
-    std::ostringstream oss;
-    oss << "inconsistent tensor size, expected " << tensor1.sizes() << " and " << tensor2.sizes()
-        << " to have the same number of elements, but got " << t1_numel << " and " << t2_numel << " elements respectively";
-    throw std::runtime_error(oss.str());
+inline void
+CPU_tensor_parallel_kernel_apply2(Tensor tensor1, Tensor tensor2, const Op op) {
+  if (!_apply_preamble({tensor1, tensor2}))
+    return;
+  if (tensor1.numel() == 1) {
+    op(1, tensor1.data<scalar1>(), tensor2.data<scalar2>(), 0, 0);
+    return;
   }
-  while(!TH_TENSOR_APPLY_hasFinished)
-  {
-    /* Loop through the inner most region of the Tensor */
-    for(; tensor1_i < tensor1_size && tensor2_i < tensor2_size; tensor1_i++, tensor2_i++, tensor1_data += tensor1_stride, tensor2_data += tensor2_stride)
-    {
-      op(*tensor1_data, *tensor2_data);
-    }
-    __ATH_TENSOR_APPLYX_UPDATE_COUNTERS(tensor1, 0)
-    __ATH_TENSOR_APPLYX_UPDATE_COUNTERS(tensor2, 0)
+  if (tensor1.ndimension() < 8 && tensor2.ndimension() < 8) {
+    parallel_for(
+        0,
+        tensor1.numel(),
+        1,
+        [&tensor1, &tensor2, &op](int64_t begin, int64_t end) {
+          apply_kernel(
+              end - begin,
+              begin,
+              op,
+              strided_tensor_iter_fixed<scalar1, 8>(tensor1),
+              strided_tensor_iter_fixed<scalar2, 8>(tensor2));
+        });
+  } else {
+    parallel_for(
+        0,
+        tensor1.numel(),
+        1,
+        [&tensor1, &tensor2, &op](int64_t begin, int64_t end) {
+          apply_kernel(
+              end - begin,
+              begin,
+              op,
+              strided_tensor_iter<scalar1>(tensor1),
+              strided_tensor_iter<scalar2>(tensor2));
+        });
   }
-  if(tensor1_counter != NULL)
-    delete [] tensor1_counter;
-  if(tensor2_counter != NULL)
-    delete [] tensor2_counter;
 }
 
 /*
-  Apply a pointwise operator to two tensors.
+  Apply a pointwise operator to sequence of tensors
 
-  The calling convention for op is a function/functor that takes takes two references to
-  type scalar; at least one of these references should be non-const in order to write the output.
-  For example, to compute a = b^2, op would be of the form:
-  [](scalar &a_val, const scalar &b_val) { a_val = b_val * b_val; };
+  The calling convention for op is a function/functor that takes takes the same
+  number of pointers of type scalar as the number of given tensors. For example,
+  to compute a = b * c, op would be of the form:
+  [](scalar* a_val, const scalar* b_val, const scalar* c_val) { a_val[0] =
+  b_val[0] * c_val[0]; };
 */
-template<typename scalar1, typename scalar2, typename Op>
-void CPU_tensor_apply2(Tensor tensor1, Tensor tensor2, Op op) {
-  CPU_tensor_apply2_dim<scalar1, scalar2, Op>(tensor1, tensor2, -1, op);
+
+template <typename scalar1, typename Op>
+inline void CPU_tensor_apply1(Tensor tensor1, const Op op) {
+  if (!_apply_preamble({tensor1}))
+    return;
+  if (tensor1.ndimension() < 8) {
+    apply_op(
+        tensor1.numel(),
+        0,
+        op,
+        strided_tensor_iter_fixed<scalar1, 8>(tensor1, true));
+  } else {
+    apply_op(tensor1.numel(), 0, op, strided_tensor_iter<scalar1>(tensor1));
+  }
 }
 
-template<typename scalar1, typename scalar2, typename scalar3, typename Op>
-void CPU_tensor_apply3_dim(Tensor &tensor1, Tensor& tensor2, Tensor& tensor3, int64_t dim, Op op) {
-  checkBackend("CPU_tensor_apply3", {tensor1, tensor2, tensor3}, Backend::CPU);
-  bool TH_TENSOR_APPLY_hasFinished = false;
-  int64_t TH_TENSOR_dim_index = 0;
-  __ATH_TENSOR_APPLYX_PREAMBLE(scalar1, tensor1, dim, 1)
-  __ATH_TENSOR_APPLYX_PREAMBLE(scalar2, tensor2, dim, 1)
-  __ATH_TENSOR_APPLYX_PREAMBLE(scalar3, tensor3, dim, 1)
-
-  int elements_equal = 1;
-  auto t1_numel = tensor1.numel();
-  auto t2_numel = tensor2.numel();
-  auto t3_numel = tensor3.numel();
-  if(t1_numel!= t2_numel) {
-    elements_equal = 0;
-  } else if(t1_numel != t3_numel) {
-    elements_equal = 0;
+template <typename scalar1, typename scalar2, typename Op>
+inline void CPU_tensor_apply2(Tensor tensor1, Tensor tensor2, const Op op) {
+  if (!_apply_preamble({tensor1, tensor2}))
+    return;
+  if (_max_dim_tensors({tensor1, tensor2}) <= 8) {
+    apply_op(
+        tensor1.numel(),
+        0,
+        op,
+        strided_tensor_iter_fixed<scalar1, 8>(tensor1),
+        strided_tensor_iter_fixed<scalar2, 8>(tensor2));
+  } else {
+    apply_op(
+        tensor1.numel(),
+        0,
+        op,
+        strided_tensor_iter<scalar1>(tensor1),
+        strided_tensor_iter<scalar2>(tensor2));
   }
-  if (elements_equal == 0) {
-    std::ostringstream oss;
-    oss << "inconsistent tensor size, expected " << tensor1.sizes() << ", " << tensor2.sizes() << ", and " << tensor3.sizes()
-        << " to have the same number of elements, but got " << t1_numel << ", " << t2_numel << ", and " << t3_numel << " elements respectively";
-    throw std::runtime_error(oss.str());
-  }
-
-  while(!TH_TENSOR_APPLY_hasFinished)
-  {
-    /* Loop through the inner most region of the Tensor */
-    for(; tensor1_i <  tensor1_size && tensor2_i < tensor2_size && tensor3_i < tensor3_size; tensor1_i++, tensor2_i++, tensor3_i++, tensor1_data += tensor1_stride, tensor2_data += tensor2_stride, tensor3_data += tensor3_stride)
-    {
-      op(*tensor1_data, *tensor2_data, *tensor3_data);
-    }
-    __ATH_TENSOR_APPLYX_UPDATE_COUNTERS(tensor1, 0)
-    __ATH_TENSOR_APPLYX_UPDATE_COUNTERS(tensor2, 0)
-    __ATH_TENSOR_APPLYX_UPDATE_COUNTERS(tensor3, 0)
-  }
-  if(tensor1_counter != NULL)
-    delete [] tensor1_counter;
-  if(tensor2_counter != NULL)
-    delete [] tensor2_counter;
-  if(tensor3_counter != NULL)
-    delete [] tensor3_counter;
 }
 
-/*
-  Apply a pointwise operator to three tensors.
-
-  The calling convention for op is a function/functor that takes takes three references to
-  type scalar; at least one of these references should be non-const in order to write the output.
-  For example, to compute a = b + c, op would be of the form:
-  [](scalar &a_val, const scalar &b_val, const scalar &c_val) { a_val = b_val + c_val; };
-*/
-template<typename scalar1, typename scalar2, typename scalar3, typename Op>
-void CPU_tensor_apply3(Tensor tensor1, Tensor tensor2, Tensor tensor3, Op op) {
-  CPU_tensor_apply3_dim<scalar1, scalar2, scalar3, Op>(tensor1, tensor2, tensor3, -1, op);
-}
-
-template <typename scalar1, typename scalar2, typename scalar3, typename scalar4, typename Op>
-void CPU_tensor_apply4_dim(Tensor &tensor1, Tensor& tensor2, Tensor& tensor3, Tensor& tensor4, int64_t dim, Op op) {
-  checkBackend("CPU_tensor_apply4", {tensor1, tensor2, tensor3, tensor4}, Backend::CPU);
-  bool TH_TENSOR_APPLY_hasFinished = false;
-  int64_t TH_TENSOR_dim_index = 0;
-  __ATH_TENSOR_APPLYX_PREAMBLE(scalar1, tensor1, dim, 1)
-  __ATH_TENSOR_APPLYX_PREAMBLE(scalar2, tensor2, dim, 1)
-  __ATH_TENSOR_APPLYX_PREAMBLE(scalar3, tensor3, dim, 1)
-  __ATH_TENSOR_APPLYX_PREAMBLE(scalar4, tensor4, dim, 1)
-
-  int elements_equal = 1;
-  auto t1_numel = tensor1.numel();
-  auto t2_numel = tensor2.numel();
-  auto t3_numel = tensor3.numel();
-  auto t4_numel = tensor4.numel();
-  if(t1_numel!= t2_numel) {
-    elements_equal = 0;
-  } else if(t1_numel != t3_numel) {
-    elements_equal = 0;
-  } else if(t1_numel != t4_numel) {
-      elements_equal = 0;
+template <typename scalar1, typename scalar2, typename scalar3, typename Op>
+inline void
+CPU_tensor_apply3(Tensor tensor1, Tensor tensor2, Tensor tensor3, const Op op) {
+  if (!_apply_preamble({tensor1, tensor2, tensor3}))
+    return;
+  if (_max_dim_tensors({tensor1, tensor2, tensor3}) <= 8) {
+    apply_op(
+        tensor1.numel(),
+        0,
+        op,
+        strided_tensor_iter_fixed<scalar1, 8>(tensor1),
+        strided_tensor_iter_fixed<scalar2, 8>(tensor2),
+        strided_tensor_iter_fixed<scalar3, 8>(tensor3));
+  } else {
+    apply_op(
+        tensor1.numel(),
+        0,
+        op,
+        strided_tensor_iter<scalar1>(tensor1),
+        strided_tensor_iter<scalar2>(tensor2),
+        strided_tensor_iter<scalar3>(tensor3));
   }
-  if (elements_equal == 0) {
-    std::ostringstream oss;
-    oss << "inconsistent tensor size, expected " << tensor1.sizes() << ", " << tensor2.sizes() << ", "
-        << tensor3.sizes() << ", and " << tensor4.sizes() << " to have the same number of elements, but got "
-        << t1_numel << ", " << t2_numel << ", " << t3_numel << ", and " << t4_numel << " elements respectively";
-    throw std::runtime_error(oss.str());
+}
+
+template <
+    typename scalar1,
+    typename scalar2,
+    typename scalar3,
+    typename scalar4,
+    typename Op>
+inline void CPU_tensor_apply4(
+    Tensor tensor1,
+    Tensor tensor2,
+    Tensor tensor3,
+    Tensor tensor4,
+    const Op op) {
+  if (!_apply_preamble({tensor1, tensor2, tensor3, tensor4}))
+    return;
+  if (_max_dim_tensors({tensor1, tensor2, tensor3, tensor4}) <= 8) {
+    apply_op(
+        tensor1.numel(),
+        0,
+        op,
+        strided_tensor_iter_fixed<scalar1, 8>(tensor1),
+        strided_tensor_iter_fixed<scalar2, 8>(tensor2),
+        strided_tensor_iter_fixed<scalar3, 8>(tensor3),
+        strided_tensor_iter_fixed<scalar4, 8>(tensor4));
+  } else {
+    apply_op(
+        tensor1.numel(),
+        0,
+        op,
+        strided_tensor_iter<scalar1>(tensor1),
+        strided_tensor_iter<scalar2>(tensor2),
+        strided_tensor_iter<scalar3>(tensor3),
+        strided_tensor_iter<scalar4>(tensor4));
   }
+}
 
-  while(!TH_TENSOR_APPLY_hasFinished)
-  {
-    /* Loop through the inner most region of the Tensor */
-    for(; tensor1_i <  tensor1_size && tensor2_i < tensor2_size && tensor3_i < tensor3_size && tensor4_i < tensor4_size
-        ; tensor1_i++, tensor2_i++, tensor3_i++, tensor4_i++,
-          tensor1_data += tensor1_stride, tensor2_data += tensor2_stride, tensor3_data += tensor3_stride, tensor4_data += tensor4_stride)
-    {
-      op(*tensor1_data, *tensor2_data, *tensor3_data, *tensor4_data);
-    }
-    __ATH_TENSOR_APPLYX_UPDATE_COUNTERS(tensor1, 0)
-    __ATH_TENSOR_APPLYX_UPDATE_COUNTERS(tensor2, 0)
-    __ATH_TENSOR_APPLYX_UPDATE_COUNTERS(tensor3, 0)
-    __ATH_TENSOR_APPLYX_UPDATE_COUNTERS(tensor4, 0)
+template <typename scalar1, typename Op>
+inline void CPU_tensor_parallel_apply1(
+    Tensor tensor1,
+    const Op op,
+    int64_t grain_size = internal::GRAIN_SIZE) {
+  if (!_apply_preamble({tensor1}))
+    return;
+  if (tensor1.ndimension() < 8) {
+    parallel_for(
+        0,
+        tensor1.numel(),
+        grain_size,
+        [&tensor1, &op](int64_t begin, int64_t end) {
+          apply_op(
+              end - begin,
+              begin,
+              op,
+              strided_tensor_iter_fixed<scalar1, 8>(tensor1, true));
+        });
+  } else {
+    parallel_for(
+        0,
+        tensor1.numel(),
+        grain_size,
+        [&tensor1, &op](int64_t begin, int64_t end) {
+          apply_op(
+              end - begin, begin, op, strided_tensor_iter<scalar1>(tensor1));
+        });
   }
-  if(tensor1_counter != NULL)
-    delete [] tensor1_counter;
-  if(tensor2_counter != NULL)
-    delete [] tensor2_counter;
-  if(tensor3_counter != NULL)
-    delete [] tensor3_counter;
-  if(tensor4_counter != NULL)
-    delete [] tensor4_counter;
 }
 
-/*
-  Apply a pointwise operator to four tensors.
-
-  The calling convention for op is a function/functor that takes takes four references to
-  type scalar; at least one of these references should be non-const in order to write the output.
-  For example, to compute a = b + c * d, op would be of the form:
-  [](scalar &a_val, const scalar &b_val, const scalar &c_val, const scalar &d_val) {
-    a_val = b_val + c_val * d_val;
-  };
-*/
-template<typename scalar1, typename scalar2, typename scalar3, typename scalar4, typename Op>
-void CPU_tensor_apply4(Tensor tensor1, Tensor tensor2, Tensor tensor3, Tensor tensor4, Op op) {
-  CPU_tensor_apply4_dim<scalar1, scalar2, scalar3, scalar4, Op>(tensor1, tensor2, tensor3, tensor4, -1, op);
+template <typename scalar1, typename scalar2, typename Op>
+inline void CPU_tensor_parallel_apply2(
+    Tensor tensor1,
+    Tensor tensor2,
+    const Op op,
+    int64_t grain_size = internal::GRAIN_SIZE) {
+  if (!_apply_preamble({tensor1, tensor2}))
+    return;
+  if (tensor1.ndimension() < 8 && tensor2.ndimension() < 8) {
+    parallel_for(
+        0,
+        tensor1.numel(),
+        grain_size,
+        [&tensor1, &tensor2, &op](int64_t begin, int64_t end) {
+          apply_op(
+              end - begin,
+              begin,
+              op,
+              strided_tensor_iter_fixed<scalar1, 8>(tensor1),
+              strided_tensor_iter_fixed<scalar2, 8>(tensor2));
+        });
+  } else {
+    parallel_for(
+        0,
+        tensor1.numel(),
+        grain_size,
+        [&tensor1, &tensor2, &op](int64_t begin, int64_t end) {
+          apply_op(
+              end - begin,
+              begin,
+              op,
+              strided_tensor_iter<scalar1>(tensor1),
+              strided_tensor_iter<scalar2>(tensor2));
+        });
+  }
 }
 
-}
+} // namespace at

aten/src/ATen/CPUFixedAllocator.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include "ATen/core/Error.h"
 #include "TH/TH.h"
-#include "ATen/Error.h"
 
 // This file creates a fake allocator that just throws exceptions if
 // it is actually used.

aten/src/ATen/CPUGeneral.h

@@ -1,12 +1,12 @@
 #pragma once
 
-// Using AT_API is crucial as otherwise you'll see
+// Using CAFFE2_API is crucial as otherwise you'll see
 // linking errors using MSVC
 // See https://msdn.microsoft.com/en-us/library/a90k134d.aspx
-// This header adds this if using AT_API
-#include "ATen/ATenGeneral.h"
+// This header adds this if using CAFFE2_API
+#include "ATen/core/ATenGeneral.h"
 
 namespace at {
-AT_API void set_num_threads(int);
-AT_API int get_num_threads();
+CAFFE2_API void set_num_threads(int);
+CAFFE2_API int get_num_threads();
 }

aten/src/ATen/CPUGenerator.cpp

@@ -37,6 +37,11 @@ CPUGenerator& CPUGenerator::manualSeed(uint64_t seed) {
   return *this;
 }
 
+CPUGenerator& CPUGenerator::manualSeedAll(uint64_t seed) {
+  // There's only one CPU generator
+  return manualSeed(seed);
+}
+
 void * CPUGenerator::unsafeGetTH() {
   return generator;
 }

aten/src/ATen/CPUTypeDefault.cpp

@@ -0,0 +1,20 @@
+#include <ATen/CPUTypeDefault.h>
+
+#include <ATen/Context.h>
+#include <ATen/CPUGenerator.h>
+
+namespace at {
+
+Allocator* CPUTypeDefault::allocator() const {
+  return getCPUAllocator();
+}
+
+Device CPUTypeDefault::getDeviceFromPtr(void * data) const {
+  return DeviceType::CPU;
+}
+
+std::unique_ptr<Generator> CPUTypeDefault::generator() const {
+  return std::unique_ptr<Generator>(new CPUGenerator(&at::globalContext()));
+}
+
+} // namespace at

aten/src/ATen/CPUTypeDefault.h

@@ -0,0 +1,14 @@
+#pragma once
+#include <ATen/TypeDefault.h>
+
+namespace at {
+
+struct CAFFE2_API CPUTypeDefault : public TypeDefault {
+  CPUTypeDefault(TensorTypeId type_id, bool is_variable, bool is_undefined)
+      : TypeDefault(type_id, is_variable, is_undefined) {}
+  Allocator* allocator() const override;
+  Device getDeviceFromPtr(void * data) const override;
+  std::unique_ptr<Generator> generator() const override;
+};
+
+} // namespace at

aten/src/ATen/CUDAFixedAllocator.h

@@ -1,43 +0,0 @@
-#pragma once
-
-#include "THC/THC.h"
-#include "ATen/Error.h"
-
-// This file creates a fake allocator that just throws exceptions if
-// it is actually used.
-
-// state passed to the allocator is the std::function<void(void*)> called
-// when the blob is release by ATen
-
-namespace at {
-
-static cuda_fixed_malloc(void *, void**, size_t, cudaStream_t) {
-  AT_ERROR("attempting to resize a tensor view of an external blob");
-}
-
-static cpu_fixed_realloc(void*, void**, size_t, size_t, cudaStream_t) {
-  AT_ERROR("attempting to resize a tensor view of an external blob");
-}
-
-static cuda_fixed_free(void * state, void * allocation) {
-    auto on_release = static_cast<std::function<void(void*)>*>(state);
-    (*on_release)(allocation);
-    delete on_release;
-}
-
-static cuda_fixed_emptyCache(void*) {
-  AT_ERROR("?? attempting to resize a tensor view of an external blob");
-}
-
-static cuda_fixed_cacheInfo(void*, int, size_t*, size_t*) {
-  AT_ERROR("?? attempting to resize a tensor view of an external blob");
-}
-
-
-static THCDeviceAllocator CUDA_fixed_allocator = {
-  cuda_fixed_malloc, cuda_fixed_realloc, cuda_fixed_free, cuda_fixed_emptyCache,
-  cuda_fixed_cacheInfo
-};
-
-
-}