.circleci: Specify setup job to run on everything (#35013)

Summary:
CircleCI by default, chooses to run 0 jobs on tags meaning that when we
tag a build that no job is run if a dependent job does not contain the
correct filters.

This adds an explicit configuration to run the setup job on every branch
and every tag that CircleCI can run on.

For more information on CircleCI filters and what they do (and more
importantly what they do not do) visit:

https://circleci.com/docs/2.0/configuration-reference/#filters-1

Signed-off-by: Eli Uriegas <eliuriegas@fb.com>
Pull Request resolved: https://github.com/pytorch/pytorch/pull/35013

Differential Revision: D20535560

Pulled By: seemethere

fbshipit-source-id: 7ee5dddbc0a9416fd76ed198e5447318c53e1873

.circleci/cimodel/data/binary_build_data.py

@@ -39,6 +39,7 @@ LINUX_PACKAGE_VARIANTS = OrderedDict(
         "3.5m",
         "3.6m",
         "3.7m",
+        "3.8m",
     ],
     conda=dimensions.STANDARD_PYTHON_VERSIONS,
     libtorch=[

.circleci/cimodel/data/binary_build_definitions.py

@@ -24,7 +24,7 @@ class Conf(object):
 
     def gen_docker_image(self):
         if self.gcc_config_variant == 'gcc5.4_cxx11-abi':
-            return miniutils.quote("pytorch/conda-cuda-cxx11-ubuntu1604:latest")
+            return miniutils.quote("pytorch/pytorch-binary-docker-image-ubuntu16.04:latest")
 
         docker_word_substitution = {
             "manywheel": "manylinux",
@@ -33,11 +33,9 @@ class Conf(object):
 
         docker_distro_prefix = miniutils.override(self.pydistro, docker_word_substitution)
 
-        # The cpu nightlies are built on the pytorch/manylinux-cuda100 docker image
-        alt_docker_suffix = self.cuda_version or "100"
+        # The cpu nightlies are built on the pytorch/manylinux-cuda102 docker image
+        alt_docker_suffix = self.cuda_version or "102"
         docker_distro_suffix = "" if self.pydistro == "conda" else alt_docker_suffix
-        if self.cuda_version == "101":
-            return "soumith/manylinux-cuda101@sha256:5d62be90d5b7777121180e6137c7eed73d37aaf9f669c51b783611e37e0b4916"
         return miniutils.quote("pytorch/" + docker_distro_prefix + "-cuda" + docker_distro_suffix)
 
     def get_name_prefix(self):
@@ -69,7 +67,17 @@ class Conf(object):
             job_def["requires"].append("update_s3_htmls_for_nightlies_devtoolset7")
             job_def["filters"] = {"branches": {"only": "postnightly"}}
         else:
-            job_def["filters"] = {"branches": {"only": "nightly"}}
+            job_def["filters"] = {
+                "branches": {
+                    "only": "nightly"
+                },
+                # Will run on tags like v1.5.0-rc1, etc.
+                "tags": {
+                    # Using a raw string here to avoid having to escape
+                    # anything
+                    "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
+                }
+            }
         if self.libtorch_variant:
             job_def["libtorch_variant"] = miniutils.quote(self.libtorch_variant)
         if phase == "test":

.circleci/cimodel/data/caffe2_build_data.py

@@ -5,7 +5,9 @@ from cimodel.lib.conf_tree import Ver
 CONFIG_TREE_DATA = [
     (Ver("ubuntu", "16.04"), [
         ([Ver("gcc", "5")], [XImportant("onnx_py2")]),
-        ([Ver("clang", "7")], [XImportant("onnx_py3.6")]),
+        ([Ver("clang", "7")], [XImportant("onnx_main_py3.6"),
+                               XImportant("onnx_ort1_py3.6"),
+                               XImportant("onnx_ort2_py3.6")]),
     ]),
 ]
 
@@ -27,7 +29,9 @@ class TreeConfigNode(ConfigNode):
         return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
 
     def is_build_only(self):
-        if str(self.find_prop("language_version")) == "onnx_py3.6":
+        if str(self.find_prop("language_version")) == "onnx_main_py3.6" or \
+                str(self.find_prop("language_version")) == "onnx_ort1_py3.6" or \
+                str(self.find_prop("language_version")) == "onnx_ort2_py3.6":
             return False
         return set(str(c) for c in self.find_prop("compiler_version")).intersection({
             "clang3.8",
@@ -36,6 +40,12 @@ class TreeConfigNode(ConfigNode):
             "android",
         }) or self.find_prop("distro_version").name == "macos"
 
+    def is_test_only(self):
+        if str(self.find_prop("language_version")) == "onnx_ort1_py3.6" or \
+                str(self.find_prop("language_version")) == "onnx_ort2_py3.6":
+            return True
+        return False
+
 
 class TopLevelNode(TreeConfigNode):
     def __init__(self, node_name, subtree):
@@ -68,6 +78,7 @@ class LanguageConfigNode(TreeConfigNode):
     def init2(self, node_name):
         self.props["language_version"] = node_name
         self.props["build_only"] = self.is_build_only()
+        self.props["test_only"] = self.is_test_only()
 
     def child_constructor(self):
         return ImportantConfigNode

.circleci/cimodel/data/caffe2_build_definitions.py

@@ -12,7 +12,7 @@ from dataclasses import dataclass
 
 DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
 
-DOCKER_IMAGE_VERSION = 345
+DOCKER_IMAGE_VERSION = "345"
 
 
 @dataclass
@@ -23,6 +23,7 @@ class Conf:
     # for gpu files and host compiler (gcc/clang) for cpu files)
     compilers: [Ver]
     build_only: bool
+    test_only: bool
     is_important: bool
 
     @property
@@ -33,7 +34,9 @@ class Conf:
     def get_cudnn_insertion(self):
 
         omit = self.language == "onnx_py2" \
-            or self.language == "onnx_py3.6" \
+            or self.language == "onnx_main_py3.6" \
+            or self.language == "onnx_ort1_py3.6" \
+            or self.language == "onnx_ort2_py3.6" \
             or set(self.compiler_names).intersection({"android", "mkl", "clang"}) \
             or str(self.distro) in ["ubuntu14.04", "macos10.13"]
 
@@ -50,6 +53,13 @@ class Conf:
 
     def construct_phase_name(self, phase):
         root_parts = self.get_build_name_root_parts()
+
+        build_name_substitutions = {
+            "onnx_ort1_py3.6": "onnx_main_py3.6",
+            "onnx_ort2_py3.6": "onnx_main_py3.6",
+        }
+        if phase == "build":
+            root_parts = [miniutils.override(r, build_name_substitutions) for r in root_parts]
         return "_".join(root_parts + [phase]).replace(".", "_")
 
     def get_platform(self):
@@ -62,7 +72,9 @@ class Conf:
 
         lang_substitutions = {
             "onnx_py2": "py2",
-            "onnx_py3.6": "py3.6",
+            "onnx_main_py3.6": "py3.6",
+            "onnx_ort1_py3.6": "py3.6",
+            "onnx_ort2_py3.6": "py3.6",
             "cmake": "py2",
         }
 
@@ -74,7 +86,9 @@ class Conf:
         parameters = OrderedDict()
         lang_substitutions = {
             "onnx_py2": "onnx-py2",
-            "onnx_py3.6": "onnx-py3.6",
+            "onnx_main_py3.6": "onnx-main-py3.6",
+            "onnx_ort1_py3.6": "onnx-ort1-py3.6",
+            "onnx_ort2_py3.6": "onnx-ort2-py3.6",
         }
 
         lang = miniutils.override(self.language, lang_substitutions)
@@ -136,6 +150,7 @@ def instantiate_configs():
             distro=fc.find_prop("distro_version"),
             compilers=fc.find_prop("compiler_version"),
             build_only=fc.find_prop("build_only"),
+            test_only=fc.find_prop("test_only"),
             is_important=fc.find_prop("important"),
         )
 
@@ -150,10 +165,11 @@ def get_workflow_jobs():
 
     x = []
     for conf_options in configs:
-
         phases = ["build"]
         if not conf_options.build_only:
             phases = dimensions.PHASES
+        if conf_options.test_only:
+            phases = ["test"]
 
         for phase in phases:
             x.append(conf_options.gen_workflow_job(phase))

.circleci/cimodel/data/dimensions.py

@@ -3,8 +3,8 @@ PHASES = ["build", "test"]
 CUDA_VERSIONS = [
     None,  # cpu build
     "92",
-    "100",
     "101",
+    "102",
 ]
 
 STANDARD_PYTHON_VERSIONS = [
@@ -12,4 +12,5 @@ STANDARD_PYTHON_VERSIONS = [
     "3.5",
     "3.6",
     "3.7",
+    "3.8"
 ]

.circleci/cimodel/data/pytorch_build_data.py

@@ -4,17 +4,15 @@ from cimodel.lib.conf_tree import ConfigNode, X, XImportant
 CONFIG_TREE_DATA = [
     ("xenial", [
         (None, [
-            XImportant("2.7.9"),
-            X("2.7"),
-            XImportant("3.5"),  # Not run on all PRs, but should be included on [test all]
+            X("3.5"),
             X("nightly"),
         ]),
         ("gcc", [
             ("5.4", [  # All this subtree rebases to master and then build
                 XImportant("3.6"),
                 ("3.6", [
-                    ("parallel_tbb", [XImportant(True)]),
-                    ("parallel_native", [XImportant(True)]),
+                    ("parallel_tbb", [X(True)]),
+                    ("parallel_native", [X(True)]),
                 ]),
             ]),
             # TODO: bring back libtorch test
@@ -24,11 +22,11 @@ CONFIG_TREE_DATA = [
             ("5", [
                 XImportant("3.6"),  # This is actually the ASAN build
             ]),
-            # ("7", [
-            #     ("3.6", [
-            #         ("xla", [XImportant(True)]),
-            #     ]),
-            # ]),
+            ("7", [
+                ("3.6", [
+                    ("xla", [XImportant(True)]),
+                ]),
+            ]),
         ]),
         ("cuda", [
             ("9", [
@@ -39,14 +37,16 @@ CONFIG_TREE_DATA = [
                 # and
                 # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L153
                 # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453144)
+                X("3.6"),
+            ]),
+            ("9.2", [X("3.6")]),
+            ("10.1", [X("3.6")]),
+            ("10.2", [
                 XImportant("3.6"),
                 ("3.6", [
                     ("libtorch", [XImportant(True)])
                 ]),
             ]),
-            ("9.2", [X("3.6")]),
-            ("10", [X("3.6")]),
-            ("10.1", [X("3.6")]),
         ]),
         ("android", [
             ("r19c", [

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -13,7 +13,7 @@ DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/"
 
 # ARE YOU EDITING THIS NUMBER?  MAKE SURE YOU READ THE GUIDANCE AT THE
 # TOP OF .circleci/config.yml
-DOCKER_IMAGE_VERSION = 405
+DOCKER_IMAGE_VERSION = "f990c76a-a798-42bb-852f-5be5006f8026"
 
 
 @dataclass
@@ -160,6 +160,11 @@ def gen_dependent_configs(xenial_parent_config):
 
         configs.append(c)
 
+    return configs
+
+def gen_docs_configs(xenial_parent_config):
+    configs = []
+
     for x in ["pytorch_python_doc_push", "pytorch_cpp_doc_push"]:
         configs.append(HiddenConf(x, parent_build=xenial_parent_config))
 
@@ -210,7 +215,6 @@ def instantiate_configs():
             android_abi = fc.find_prop("android_abi")
             parms_list_ignored_for_docker_image.append(android_abi)
             restrict_phases = ["build"]
-            fc.props["is_important"] = True
 
         elif compiler_name:
             gcc_version = compiler_name + (fc.find_prop("compiler_version") or "")
@@ -222,7 +226,7 @@ def instantiate_configs():
                 python_version = fc.find_prop("pyver")
                 parms_list[0] = fc.find_prop("abbreviated_pyver")
 
-        if cuda_version in ["9.2", "10", "10.1"]:
+        if cuda_version in ["9.2", "10", "10.1", "10.2"]:
             # TODO The gcc version is orthogonal to CUDA version?
             parms_list.append("gcc7")
 
@@ -248,7 +252,16 @@ def instantiate_configs():
             parallel_backend=parallel_backend,
         )
 
-        if cuda_version == "9" and python_version == "3.6" and not is_libtorch:
+        # run docs builds on "pytorch-linux-xenial-py3.6-gcc5.4". Docs builds
+        # should run on a CPU-only build that runs on all PRs.
+        if distro_name == 'xenial' and fc.find_prop("pyver") == '3.6' \
+                and cuda_version is None \
+                and parallel_backend is None \
+                and compiler_name == 'gcc' \
+                and fc.find_prop('compiler_version') == '5.4':
+            c.dependent_tests = gen_docs_configs(c)
+
+        if cuda_version == "10.1" and python_version == "3.6" and not is_libtorch:
             c.dependent_tests = gen_dependent_configs(c)
 
         if (compiler_name == "gcc"
@@ -275,7 +288,7 @@ def get_workflow_jobs():
 
     config_list = instantiate_configs()
 
-    x = ["setup"]
+    x = []
     for conf_options in config_list:
 
         phases = conf_options.restrict_phases or dimensions.PHASES

.circleci/docker/build.sh

@@ -27,6 +27,8 @@ elif [[ "$image" == *-bionic* ]]; then
   UBUNTU_VERSION=18.04
 fi
 
+TRAVIS_DL_URL_PREFIX="https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/14.04/x86_64"
+
 # It's annoying to rename jobs every time you want to rewrite a
 # configuration, so we hardcode everything here rather than do it
 # from scratch
@@ -54,6 +56,13 @@ case "$image" in
     GCC_VERSION=7
     # Do not install PROTOBUF, DB, and VISION as a test
     ;;
+  pytorch-linux-xenial-py3.8)
+    # TODO: This is a hack, get rid of this as soon as you get rid of the travis downloads
+    TRAVIS_DL_URL_PREFIX="https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/16.04/x86_64"
+    TRAVIS_PYTHON_VERSION=3.8
+    GCC_VERSION=7
+    # Do not install PROTOBUF, DB, and VISION as a test
+    ;;
   pytorch-linux-xenial-py3.6-gcc4.8)
     ANACONDA_PYTHON_VERSION=3.6
     GCC_VERSION=4.8
@@ -67,6 +76,7 @@ case "$image" in
     PROTOBUF=yes
     DB=yes
     VISION=yes
+    KATEX=yes
     ;;
   pytorch-linux-xenial-py3.6-gcc7.2)
     ANACONDA_PYTHON_VERSION=3.6
@@ -87,22 +97,6 @@ case "$image" in
     DB=yes
     VISION=yes
     ;;
-  pytorch-linux-xenial-cuda8-cudnn7-py2)
-    CUDA_VERSION=8.0
-    CUDNN_VERSION=7
-    ANACONDA_PYTHON_VERSION=2.7
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ;;
-  pytorch-linux-xenial-cuda8-cudnn7-py3)
-    CUDA_VERSION=8.0
-    CUDNN_VERSION=7
-    ANACONDA_PYTHON_VERSION=3.6
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ;;
   pytorch-linux-xenial-cuda9-cudnn7-py2)
     CUDA_VERSION=9.0
     CUDNN_VERSION=7
@@ -118,7 +112,6 @@ case "$image" in
     PROTOBUF=yes
     DB=yes
     VISION=yes
-    KATEX=yes
     ;;
   pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7)
     CUDA_VERSION=9.2
@@ -146,6 +139,17 @@ case "$image" in
     PROTOBUF=yes
     DB=yes
     VISION=yes
+    KATEX=yes
+    ;;
+  pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7)
+    CUDA_VERSION=10.2
+    CUDNN_VERSION=7
+    ANACONDA_PYTHON_VERSION=3.6
+    GCC_VERSION=7
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    KATEX=yes
     ;;
   pytorch-linux-xenial-py3-clang5-asan)
     ANACONDA_PYTHON_VERSION=3.6
@@ -157,6 +161,7 @@ case "$image" in
   pytorch-linux-xenial-py3-clang5-android-ndk-r19c)
     ANACONDA_PYTHON_VERSION=3.6
     CLANG_VERSION=5.0
+    LLVMDEV=yes
     PROTOBUF=yes
     ANDROID=yes
     ANDROID_NDK_VERSION=r19c
@@ -184,6 +189,7 @@ tmp_tag="tmp-$(cat /dev/urandom | tr -dc 'a-z' | fold -w 32 | head -n 1)"
 # Build image
 docker build \
        --no-cache \
+       --build-arg "TRAVIS_DL_URL_PREFIX=${TRAVIS_DL_URL_PREFIX}" \
        --build-arg "BUILD_ENVIRONMENT=${image}" \
        --build-arg "PROTOBUF=${PROTOBUF:-}" \
        --build-arg "THRIFT=${THRIFT:-}" \

.circleci/docker/common/install_android.sh

@@ -10,7 +10,7 @@ apt-get autoclean && apt-get clean
 rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 pushd /tmp
-curl -Os https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK}-linux-x86_64.zip
+curl -Os --retry 3 https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK}-linux-x86_64.zip
 popd
 _ndk_dir=/opt/ndk
 mkdir -p "$_ndk_dir"

.circleci/docker/common/install_cache.sh

@@ -8,7 +8,7 @@ sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment
 export PATH="/opt/cache/bin:$PATH"
 
 # Setup compiler cache
-curl https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
+curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
 chmod a+x /opt/cache/bin/sccache
 
 function write_sccache_stub() {

.circleci/docker/common/install_cmake.sh

@@ -10,7 +10,7 @@ file="cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz"
 
 # Download and install specific CMake version in /usr/local
 pushd /tmp
-curl -Os "https://cmake.org/files/${path}/${file}"
+curl -Os --retry 3 "https://cmake.org/files/${path}/${file}"
 tar -C /usr/local --strip-components 1 --no-same-owner -zxf cmake-*.tar.gz
 rm -f cmake-*.tar.gz
 popd

.circleci/docker/common/install_conda.sh

@@ -65,9 +65,7 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   # DO NOT install cmake here as it would install a version newer than 3.5, but
   # we want to pin to version 3.5.
   conda_install numpy pyyaml mkl mkl-include setuptools cffi typing future six
-  if [[ "$CUDA_VERSION" == 8.0* ]]; then
-    conda_install magma-cuda80 -c pytorch
-  elif [[ "$CUDA_VERSION" == 9.0* ]]; then
+  if [[ "$CUDA_VERSION" == 9.0* ]]; then
     conda_install magma-cuda90 -c pytorch
   elif [[ "$CUDA_VERSION" == 9.1* ]]; then
     conda_install magma-cuda91 -c pytorch
@@ -88,7 +86,7 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   # scikit-learn is pinned because of
   # https://github.com/scikit-learn/scikit-learn/issues/14485 (affects gcc 5.5
   # only)
-  as_jenkins pip install --progress-bar off pytest scipy==1.1.0 scikit-learn==0.20.3 scikit-image librosa>=0.6.2 psutil numba==0.43.1 llvmlite==0.28.0
+  as_jenkins pip install --progress-bar off pytest scipy==1.1.0 scikit-learn==0.20.3 scikit-image librosa>=0.6.2 psutil numba==0.46.0 llvmlite==0.30.0
 
   popd
 fi

.circleci/docker/common/install_travis_python.sh

@@ -14,7 +14,7 @@ if [ -n "$TRAVIS_PYTHON_VERSION" ]; then
 
   # Download Python binary from Travis
   pushd tmp
-  as_jenkins wget --quiet https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/14.04/x86_64/python-$TRAVIS_PYTHON_VERSION.tar.bz2
+  as_jenkins wget --quiet ${TRAVIS_DL_URL_PREFIX}/python-$TRAVIS_PYTHON_VERSION.tar.bz2
   # NB: The tarball also comes with /home/travis virtualenv that we
   # don't care about.  (Maybe we should, but we've worked around the
   # "how do I install to python" issue by making this entire directory
@@ -88,6 +88,9 @@ if [ -n "$TRAVIS_PYTHON_VERSION" ]; then
   # Install psutil for dataloader tests
   as_jenkins pip install psutil
 
+  # Install dill for serialization tests
+  as_jenkins pip install "dill>=0.3.1"
+
   # Cleanup package manager
   apt-get autoclean && apt-get clean
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

.circleci/docker/ubuntu/Dockerfile

@@ -46,6 +46,7 @@ RUN bash ./install_gcc.sh && rm install_gcc.sh
 
 # Install non-standard Python versions (via Travis binaries)
 ARG TRAVIS_PYTHON_VERSION
+ARG TRAVIS_DL_URL_PREFIX
 ENV PATH /opt/python/$TRAVIS_PYTHON_VERSION/bin:$PATH
 ADD ./common/install_travis_python.sh install_travis_python.sh
 RUN bash ./install_travis_python.sh && rm install_travis_python.sh

.circleci/ecr_gc_docker/Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
+
+ADD requirements.txt /requirements.txt
+
+RUN pip install -r /requirements.txt
+
+ADD gc.py /usr/bin/gc.py
+
+ADD docker_hub.py /usr/bin/docker_hub.py
+
+ENTRYPOINT ["/usr/bin/gc.py"]

.circleci/ecr_gc_docker/docker_hub.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+
+from collections import namedtuple
+
+import boto3
+import requests
+import os
+
+
+IMAGE_INFO = namedtuple(
+    "IMAGE_INFO", ("repo", "tag", "size", "last_updated_at", "last_updated_by")
+)
+
+
+def build_access_token(username, passwordtr):
+    r = requests.post(
+        "https://hub.docker.com/v2/users/login/",
+        data={"username": username, "password": password},
+    )
+    r.raise_for_status()
+    token = r.json().get("token")
+    return {"Authorization": "JWT " + token}
+
+
+def list_repos(user, token):
+    r = requests.get("https://hub.docker.com/v2/repositories/" + user, headers=token)
+    r.raise_for_status()
+    ret = sorted(
+        repo["user"] + "/" + repo["name"] for repo in r.json().get("results", [])
+    )
+    if ret:
+        print("repos found:")
+        print("".join("\n\t" + r for r in ret))
+    return ret
+
+
+def list_tags(repo, token):
+    r = requests.get(
+        "https://hub.docker.com/v2/repositories/" + repo + "/tags", headers=token
+    )
+    r.raise_for_status()
+    return [
+        IMAGE_INFO(
+            repo=repo,
+            tag=t["name"],
+            size=t["full_size"],
+            last_updated_at=t["last_updated"],
+            last_updated_by=t["last_updater_username"],
+        )
+        for t in r.json().get("results", [])
+    ]
+
+
+def save_to_s3(tags):
+    table_content = ""
+    client = boto3.client("s3")
+    for t in tags:
+        table_content += (
+            "<tr><td>{repo}</td><td>{tag}</td><td>{size}</td>"
+            "<td>{last_updated_at}</td><td>{last_updated_by}</td></tr>"
+        ).format(
+            repo=t.repo,
+            tag=t.tag,
+            size=t.size,
+            last_updated_at=t.last_updated_at,
+            last_updated_by=t.last_updated_by,
+        )
+    html_body = """
+    <html>
+        <head>
+            <link rel="stylesheet"
+                href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css"
+                integrity="sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh"
+                crossorigin="anonymous">
+            <link rel="stylesheet" type="text/css"
+                href="https://cdn.datatables.net/1.10.20/css/jquery.dataTables.css">
+            <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js">
+            </script>
+            <script type="text/javascript" charset="utf8"
+                src="https://cdn.datatables.net/1.10.20/js/jquery.dataTables.js"></script>
+            <title> docker image info</title>
+        </head>
+        <body>
+            <table class="table table-striped table-hover" id="docker">
+            <caption>Docker images on docker hub</caption>
+            <thead class="thead-dark">
+                <tr>
+                <th scope="col">repo</th>
+                <th scope="col">tag</th>
+                <th scope="col">size</th>
+                <th scope="col">last_updated_at</th>
+                <th scope="col">last_updated_by</th>
+                </tr>
+            </thead>
+            <tbody>
+                {table_content}
+            </tbody>
+            </table>
+        </body>
+        <script>
+            $(document).ready( function () {{
+                $('#docker').DataTable({{paging: false}});
+            }} );py
+        </script>
+    </html>
+    """.format(
+        table_content=table_content
+    )
+    client.put_object(
+        Bucket="docker.pytorch.org",
+        ACL="public-read",
+        Key="docker_hub.html",
+        Body=html_body,
+        ContentType="text/html",
+    )
+
+
+if __name__ == "__main__":
+    username = os.environ.get("DOCKER_HUB_USERNAME")
+    password = os.environ.get("DOCKER_HUB_PASSWORD")
+    token = build_access_token(username, password)
+    tags = []
+    for repo in list_repos("pytorch", token):
+        tags.extend(list_tags(repo, token))
+    save_to_s3(tags)

.circleci/ecr_gc_docker/gc.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python
+
+import argparse
+import datetime
+import boto3
+import pytz
+import sys
+
+
+def save_to_s3(project, data):
+    table_content = ""
+    client = boto3.client("s3")
+    for repo, tag, window, age, pushed in data:
+        table_content += "<tr><td>{repo}</td><td>{tag}</td><td>{window}</td><td>{age}</td><td>{pushed}</td></tr>".format(
+            repo=repo, tag=tag, window=window, age=age, pushed=pushed
+        )
+    html_body = """
+    <html>
+        <head>
+            <link rel="stylesheet"
+                href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css"
+                integrity="sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh"
+                crossorigin="anonymous">
+            <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.20/css/jquery.dataTables.css">
+            <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>
+            <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.20/js/jquery.dataTables.js"></script>
+            <title>{project} nightly and permanent docker image info</title>
+        </head>
+        <body>
+            <table class="table table-striped table-hover" id="docker">
+            <thead class="thead-dark">
+                <tr>
+                <th scope="col">repo</th>
+                <th scope="col">tag</th>
+                <th scope="col">keep window</th>
+                <th scope="col">age</th>
+                <th scope="col">pushed at</th>
+                </tr>
+            </thead>
+            <tbody>
+                {table_content}
+            </tbody>
+            </table>
+        </body>
+        <script>
+            $(document).ready( function () {{
+                $('#docker').DataTable({{paging: false}});
+            }} );
+        </script>
+    </html>
+    """.format(
+        project=project, table_content=table_content
+    )
+
+    # for pytorch, file can be found at
+    # http://ossci-docker.s3-website.us-east-1.amazonaws.com/pytorch.html
+    # and later one we can config docker.pytorch.org to point to the location
+
+    client.put_object(
+        Bucket="docker.pytorch.org",
+        ACL="public-read",
+        Key="{project}.html".format(project=project),
+        Body=html_body,
+        ContentType="text/html",
+    )
+
+
+def repos(client):
+    paginator = client.get_paginator("describe_repositories")
+    pages = paginator.paginate(registryId="308535385114")
+    for page in pages:
+        for repo in page["repositories"]:
+            yield repo
+
+
+def images(client, repository):
+    paginator = client.get_paginator("describe_images")
+    pages = paginator.paginate(
+        registryId="308535385114", repositoryName=repository["repositoryName"]
+    )
+    for page in pages:
+        for image in page["imageDetails"]:
+            yield image
+
+
+parser = argparse.ArgumentParser(description="Delete old Docker tags from registry")
+parser.add_argument(
+    "--dry-run", action="store_true", help="Dry run; print tags that would be deleted"
+)
+parser.add_argument(
+    "--keep-stable-days",
+    type=int,
+    default=14,
+    help="Days of stable Docker tags to keep (non per-build images)",
+)
+parser.add_argument(
+    "--keep-unstable-days",
+    type=int,
+    default=1,
+    help="Days of unstable Docker tags to keep (per-build images)",
+)
+parser.add_argument(
+    "--filter-prefix",
+    type=str,
+    default="",
+    help="Only run cleanup for repositories with this prefix",
+)
+parser.add_argument(
+    "--ignore-tags",
+    type=str,
+    default="",
+    help="Never cleanup these tags (comma separated)",
+)
+args = parser.parse_args()
+
+if not args.ignore_tags or not args.filter_prefix:
+    print(
+        """
+Missing required arguments --ignore-tags and --filter-prefix
+
+You must specify --ignore-tags and --filter-prefix to avoid accidentally
+pruning a stable Docker tag which is being actively used.  This will
+make you VERY SAD.  So pay attention.
+
+First, which filter-prefix do you want?  The list of valid prefixes
+is in jobs/private.groovy under the 'docker-registry-cleanup' job.
+You probably want either pytorch or caffe2.
+
+Second, which ignore-tags do you want?  It should be whatever the most
+up-to-date DockerVersion for the repository in question is.  Follow
+the imports of jobs/pytorch.groovy to find them.
+"""
+    )
+    sys.exit(1)
+
+client = boto3.client("ecr", region_name="us-east-1")
+stable_window = datetime.timedelta(days=args.keep_stable_days)
+unstable_window = datetime.timedelta(days=args.keep_unstable_days)
+now = datetime.datetime.now(pytz.UTC)
+ignore_tags = args.ignore_tags.split(",")
+
+
+def chunks(chunkable, n):
+    """ Yield successive n-sized chunks from l.
+    """
+    for i in range(0, len(chunkable), n):
+        yield chunkable[i : i + n]
+
+
+stable_window_tags = []
+for repo in repos(client):
+    repositoryName = repo["repositoryName"]
+    if not repositoryName.startswith(args.filter_prefix):
+        continue
+
+    # Keep list of image digests to delete for this repository
+    digest_to_delete = []
+    print(repositoryName)
+
+    for image in images(client, repo):
+        tags = image.get("imageTags")
+        if not isinstance(tags, (list,)) or len(tags) == 0:
+            continue
+
+        tag = tags[0]
+        created = image["imagePushedAt"]
+        age = now - created
+        # new images build on circle ci use workflow ID as tag, which has 4 "-"
+        if tag.isdigit() or tag.count("-") == 4 or tag in ignore_tags:
+            window = stable_window
+            if tag in ignore_tags:
+                stable_window_tags.append((repositoryName, tag, "", age, created))
+            elif age < window:
+                stable_window_tags.append((repositoryName, tag, window, age, created))
+        else:
+            window = unstable_window
+
+        if tag in ignore_tags:
+            print("Ignoring tag {} (age: {})".format(tag, age))
+            continue
+        if age < window:
+            print("Not deleting manifest for tag {} (age: {})".format(tag, age))
+            continue
+
+        if args.dry_run:
+            print("(dry run) Deleting manifest for tag {} (age: {})".format(tag, age))
+        else:
+            print("Deleting manifest for tag {} (age: {})".format(tag, age))
+            digest_to_delete.append(image["imageDigest"])
+
+    # Issue batch delete for all images to delete for this repository
+    # Note that as of 2018-07-25, the maximum number of images you can
+    # delete in a single batch is 100, so chunk our list into batches of
+    # 100
+    for c in chunks(digest_to_delete, 100):
+        client.batch_delete_image(
+            registryId="308535385114",
+            repositoryName=repositoryName,
+            imageIds=[{"imageDigest": digest} for digest in c],
+        )
+
+    save_to_s3(args.filter_prefix, stable_window_tags)

.circleci/ecr_gc_docker/requirements.txt

@@ -0,0 +1,3 @@
+boto3
+pytz
+requests

.circleci/generate_config_yml.py

@@ -88,8 +88,11 @@ YAML_SOURCES = [
     File("job-specs-custom.yml"),
     File("binary_update_htmls.yml"),
     File("binary-build-tests.yml"),
-    File("docker_build_job.yml"),
+    File("docker_jobs.yml"),
     File("workflows.yml"),
+
+    File("workflows-setup-job.yml"),
+    File("windows-build-test.yml"),
     Listgen(pytorch_build_definitions.get_workflow_jobs, 3),
     File("workflows-pytorch-macos-builds.yml"),
     File("workflows-pytorch-android-gradle-build.yml"),
@@ -102,12 +105,14 @@ YAML_SOURCES = [
     Listgen(binary_build_definitions.get_binary_build_jobs, 3),
     File("workflows-nightly-ios-binary-builds.yml"),
     File("workflows-nightly-android-binary-builds.yml"),
+
     Header("Nightly tests"),
     Listgen(binary_build_definitions.get_nightly_tests, 3),
     File("workflows-nightly-uploads-header.yml"),
     Listgen(binary_build_definitions.get_nightly_uploads, 3),
     File("workflows-s3-html.yml"),
-    File("workflows-docker-builder.yml")
+    File("workflows-docker-builder.yml"),
+    File("workflows-ecr-gc.yml"),
 ]
 
 

.circleci/scripts/binary_checkout.sh

@@ -1,5 +1,11 @@
 #!/bin/bash
 set -eux -o pipefail
+
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+
 # This step runs on multiple executors with different envfile locations
 if [[ "$(uname)" == Darwin ]]; then
   # macos executor (builds and tests)
@@ -17,7 +23,7 @@ export PYTORCH_ROOT="$workdir/pytorch"
 export BUILDER_ROOT="$workdir/builder"
 
 # Clone the Pytorch branch
-git clone https://github.com/pytorch/pytorch.git "$PYTORCH_ROOT"
+retry git clone https://github.com/pytorch/pytorch.git "$PYTORCH_ROOT"
 pushd "$PYTORCH_ROOT"
 if [[ -n "${CIRCLE_PR_NUMBER:-}" ]]; then
   # "smoke" binary build on PRs
@@ -33,13 +39,13 @@ else
   echo "Can't tell what to checkout"
   exit 1
 fi
-git submodule update --init --recursive --quiet
+retry git submodule update --init --recursive
 echo "Using Pytorch from "
 git --no-pager log --max-count 1
 popd
 
 # Clone the Builder master repo
-git clone -q https://github.com/pytorch/builder.git "$BUILDER_ROOT"
+retry git clone -q https://github.com/pytorch/builder.git "$BUILDER_ROOT"
 pushd "$BUILDER_ROOT"
 echo "Using builder from "
 git --no-pager log --max-count 1

.circleci/scripts/binary_install_miniconda.sh

@@ -31,9 +31,9 @@ fi
 
 conda_sh="$workdir/install_miniconda.sh"
 if [[ "$(uname)" == Darwin ]]; then
-  retry curl -o "$conda_sh" https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+  curl --retry 3 -o "$conda_sh" https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
 else
-  retry curl -o "$conda_sh" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+  curl --retry 3 -o "$conda_sh" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
 fi
 chmod +x "$conda_sh"
 "$conda_sh" -b -p "$MINICONDA_ROOT"

.circleci/scripts/binary_ios_build.sh

@@ -5,20 +5,24 @@ echo ""
 echo "DIR: $(pwd)"
 WORKSPACE=/Users/distiller/workspace
 PROJ_ROOT=/Users/distiller/project
-export TCLLIBPATH="/usr/local/lib" 
+export TCLLIBPATH="/usr/local/lib"
+
 # Install conda
-curl -o ~/Downloads/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
-chmod +x ~/Downloads/conda.sh
-/bin/bash ~/Downloads/conda.sh -b -p ~/anaconda
+curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+chmod +x ~/conda.sh
+/bin/bash ~/conda.sh -b -p ~/anaconda
 export PATH="~/anaconda/bin:${PATH}"
 source ~/anaconda/bin/activate
+
 # Install dependencies
 conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing requests --yes
 export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
+
 # sync submodules
 cd ${PROJ_ROOT}
 git submodule sync
 git submodule update --init --recursive
+
 # run build script
 chmod a+x ${PROJ_ROOT}/scripts/build_ios.sh
 echo "########################################################"
@@ -26,13 +30,13 @@ cat ${PROJ_ROOT}/scripts/build_ios.sh
 echo "########################################################"
 echo "IOS_ARCH: ${IOS_ARCH}"
 echo "IOS_PLATFORM: ${IOS_PLATFORM}"
-export BUILD_PYTORCH_MOBILE=1
 export IOS_ARCH=${IOS_ARCH}
 export IOS_PLATFORM=${IOS_PLATFORM}
 unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts
+
 #store the binary
 cd ${WORKSPACE}
 DEST_DIR=${WORKSPACE}/ios
 mkdir -p ${DEST_DIR}
 cp -R ${PROJ_ROOT}/build_ios/install ${DEST_DIR}
-mv ${DEST_DIR}/install ${DEST_DIR}/${IOS_ARCH}
+mv ${DEST_DIR}/install ${DEST_DIR}/${IOS_ARCH}

.circleci/scripts/binary_ios_upload.sh

@@ -14,11 +14,13 @@ mkdir -p ${ZIP_DIR}/src
 cp -R ${ARTIFACTS_DIR}/arm64/include ${ZIP_DIR}/install/
 # build a FAT bianry
 cd ${ZIP_DIR}/install/lib
-target_libs=(libc10.a libclog.a libcpuinfo.a libeigen_blas.a libpytorch_qnnpack.a libtorch.a)
+target_libs=(libc10.a libclog.a libcpuinfo.a libeigen_blas.a libpytorch_qnnpack.a libtorch_cpu.a libtorch.a libXNNPACK.a)
 for lib in ${target_libs[*]}
 do
-    libs=(${ARTIFACTS_DIR}/x86_64/lib/${lib} ${ARTIFACTS_DIR}/arm64/lib/${lib})
-    lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/${lib}
+    if [ -f "${ARTIFACTS_DIR}/x86_64/lib/${lib}" ] && [ -f "${ARTIFACTS_DIR}/arm64/lib/${lib}" ]; then
+        libs=("${ARTIFACTS_DIR}/x86_64/lib/${lib}" "${ARTIFACTS_DIR}/arm64/lib/${lib}")
+        lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/${lib}
+    fi
 done
 # for nnpack, we only support arm64 build
 cp ${ARTIFACTS_DIR}/arm64/lib/libnnpack.a ./

.circleci/scripts/binary_linux_test.sh

@@ -11,11 +11,15 @@ if [[ "$PACKAGE_TYPE" == conda ]]; then
   source activate testenv >/dev/null
 elif [[ "$DESIRED_PYTHON" == 2.7mu ]]; then
   export PATH="/opt/python/cp27-cp27mu/bin:\$PATH"
-elif [[ "$DESIRED_PYTHON" == 3.8m ]]; then
-  export PATH="/opt/python/cp38-cp38/bin:\$PATH"
 elif [[ "$PACKAGE_TYPE" != libtorch ]]; then
   python_nodot="\$(echo $DESIRED_PYTHON | tr -d m.u)"
-  export PATH="/opt/python/cp\$python_nodot-cp\${python_nodot}m/bin:\$PATH"
+  python_path="/opt/python/cp\$python_nodot-cp\${python_nodot}"
+  # Prior to Python 3.8 paths were suffixed with an 'm'
+  if [[ -d  "\${python_path}/bin" ]]; then
+    export PATH="\${python_path}/bin:\$PATH"
+  elif [[ -d "\${python_path}m/bin" ]]; then
+    export PATH="\${python_path}m/bin:\$PATH"
+  fi
 fi
 
 # Install the package
@@ -28,11 +32,11 @@ pkg="/final_pkgs/\$(ls /final_pkgs)"
 if [[ "$PACKAGE_TYPE" == conda ]]; then
   conda install -y "\$pkg" --offline
   if [[ "$DESIRED_CUDA" == 'cpu' ]]; then
-    conda install -y cpuonly -c pytorch
+    retry conda install -y cpuonly -c pytorch
   fi
   retry conda install -yq future numpy protobuf six
   if [[ "$DESIRED_CUDA" != 'cpu' ]]; then
-    # DESIRED_CUDA is in format cu90 or cu100
+    # DESIRED_CUDA is in format cu90 or cu102
     if [[ "${#DESIRED_CUDA}" == 4 ]]; then
       cu_ver="${DESIRED_CUDA:2:1}.${DESIRED_CUDA:3}"
     else

.circleci/scripts/binary_linux_upload.sh

@@ -5,15 +5,6 @@ set -eu -o pipefail
 set +x
 declare -x "AWS_ACCESS_KEY_ID=${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
 declare -x "AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
-cat >/home/circleci/project/login_to_anaconda.sh <<EOL
-set +x
-echo "Trying to login to Anaconda"
-yes | anaconda login \
-    --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
-    --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
-set -x
-EOL
-chmod +x /home/circleci/project/login_to_anaconda.sh
 
 #!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
 # DO NOT TURN -x ON BEFORE THIS LINE
@@ -21,12 +12,18 @@ chmod +x /home/circleci/project/login_to_anaconda.sh
 set -eux -o pipefail
 export PATH="$MINICONDA_ROOT/bin:$PATH"
 
+# This gets set in binary_populate_env.sh, but lets have a sane default just in case
+PIP_UPLOAD_FOLDER=${PIP_UPLOAD_FOLDER:-nightly}
+# TODO: Combine CONDA_UPLOAD_CHANNEL and PIP_UPLOAD_FOLDER into one variable
+#       The only difference is the trailing slash
+# Strip trailing slashes if there
+CONDA_UPLOAD_CHANNEL=$(echo "${PIP_UPLOAD_FOLDER}" | sed 's:/*$::')
+
 # Upload the package to the final location
 pushd /home/circleci/project/final_pkgs
 if [[ "$PACKAGE_TYPE" == conda ]]; then
   retry conda install -yq anaconda-client
-  retry timeout 30 /home/circleci/project/login_to_anaconda.sh
-  anaconda upload "$(ls)" -u pytorch-nightly --label main --no-progress --force
+  anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload  "$(ls)" -u "pytorch-${CONDA_UPLOAD_CHANNEL}" --label main --no-progress --force
 elif [[ "$PACKAGE_TYPE" == libtorch ]]; then
   retry pip install -q awscli
   s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"

.circleci/scripts/binary_macos_upload.sh

@@ -4,15 +4,6 @@ set -eu -o pipefail
 set +x
 export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
 export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
-cat >/Users/distiller/project/login_to_anaconda.sh <<EOL
-set +x
-echo "Trying to login to Anaconda"
-yes | anaconda login \
-    --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
-    --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
-set -x
-EOL
-chmod +x /Users/distiller/project/login_to_anaconda.sh
 
 #!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
 # DO NOT TURN -x ON BEFORE THIS LINE
@@ -22,11 +13,17 @@ set -eux -o pipefail
 source "/Users/distiller/project/env"
 export "PATH=$workdir/miniconda/bin:$PATH"
 
+# This gets set in binary_populate_env.sh, but lets have a sane default just in case
+PIP_UPLOAD_FOLDER=${PIP_UPLOAD_FOLDER:-nightly}
+# TODO: Combine CONDA_UPLOAD_CHANNEL and PIP_UPLOAD_FOLDER into one variable
+#       The only difference is the trailing slash
+# Strip trailing slashes if there
+CONDA_UPLOAD_CHANNEL=$(echo "${PIP_UPLOAD_FOLDER}" | sed 's:/*$::')
+
 pushd "$workdir/final_pkgs"
 if [[ "$PACKAGE_TYPE" == conda ]]; then
   retry conda install -yq anaconda-client
-  retry /Users/distiller/project/login_to_anaconda.sh
-  retry anaconda upload "$(ls)" -u pytorch-nightly --label main --no-progress --force
+  retry anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload "$(ls)" -u "pytorch-${CONDA_UPLOAD_CHANNEL}" --label main --no-progress --force
 elif [[ "$PACKAGE_TYPE" == libtorch ]]; then
   retry pip install -q awscli
   s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"

.circleci/scripts/binary_populate_env.sh

@@ -40,25 +40,25 @@ if [[ -z "$DOCKER_IMAGE" ]]; then
   fi
 fi
 
-# Upload to parallel folder for devtoolsets
-# All nightlies used to be devtoolset3, then devtoolset7 was added as a build
-# option, so the upload was redirected to nightly/devtoolset7 to avoid
-# conflicts with other binaries (there shouldn't be any conflicts). Now we are
-# making devtoolset7 the default.
-if [[ "$DESIRED_DEVTOOLSET" == 'devtoolset7' || "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* || "$(uname)" == 'Darwin' ]]; then
-  export PIP_UPLOAD_FOLDER='nightly/'
-else
-  # On linux machines, this shouldn't actually be called anymore. This is just
-  # here for extra safety.
-  export PIP_UPLOAD_FOLDER='nightly/devtoolset3/'
-fi
-
+# Default to nightly, since that's where this normally uploads to
+PIP_UPLOAD_FOLDER='nightly/'
 # We put this here so that OVERRIDE_PACKAGE_VERSION below can read from it
 export DATE="$(date -u +%Y%m%d)"
+#TODO: We should be pulling semver version from the base version.txt
+BASE_BUILD_VERSION="1.5.0.dev$DATE"
+# Change BASE_BUILD_VERSION to git tag when on a git tag
+if git describe --tags --exact >/dev/null 2>/dev/null; then
+  # Switch upload folder to 'test/' if we are on a tag
+  PIP_UPLOAD_FOLDER='test/'
+  # Grab git tag, remove prefixed v and remove everything after -
+  # Used to clean up tags that are for release candidates like v1.5.0-rc1
+  # Turns tag v1.5.0-rc1 -> v1.5.0
+  BASE_BUILD_VERSION="$(git describe --tags | sed -e 's/^v//' -e 's/-.*$//')"
+fi
 if [[ "$(uname)" == 'Darwin' ]] || [[ "$DESIRED_CUDA" == "cu101" ]] || [[ "$PACKAGE_TYPE" == conda ]]; then
-  export PYTORCH_BUILD_VERSION="1.4.0.dev$DATE"
+  export PYTORCH_BUILD_VERSION="${BASE_BUILD_VERSION}"
 else
-  export PYTORCH_BUILD_VERSION="1.4.0.dev$DATE+$DESIRED_CUDA"
+  export PYTORCH_BUILD_VERSION="${BASE_BUILD_VERSION}+$DESIRED_CUDA"
 fi
 export PYTORCH_BUILD_NUMBER=1
 
@@ -96,7 +96,7 @@ export BUILD_PYTHONLESS="${BUILD_PYTHONLESS:-}"
 export DESIRED_DEVTOOLSET="$DESIRED_DEVTOOLSET"
 
 export DATE="$DATE"
-export NIGHTLIES_DATE_PREAMBLE=1.4.0.dev
+export NIGHTLIES_DATE_PREAMBLE=1.5.0.dev
 export PYTORCH_BUILD_VERSION="$PYTORCH_BUILD_VERSION"
 export PYTORCH_BUILD_NUMBER="$PYTORCH_BUILD_NUMBER"
 export OVERRIDE_PACKAGE_VERSION="$PYTORCH_BUILD_VERSION"

.circleci/scripts/binary_run_in_docker.sh

@@ -16,31 +16,12 @@ set -eux -o pipefail
 # Expect actual code to be written to this file
 chmod +x /home/circleci/project/ci_test_script.sh
 
+VOLUME_MOUNTS="-v /home/circleci/project/:/circleci_stuff -v /home/circleci/project/final_pkgs:/final_pkgs -v ${PYTORCH_ROOT}:/pytorch -v ${BUILDER_ROOT}:/builder"
 # Run the docker
 if [ -n "${USE_CUDA_DOCKER_RUNTIME:-}" ]; then
-  export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d "${DOCKER_IMAGE}")
+  export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia ${VOLUME_MOUNTS} -t -d "${DOCKER_IMAGE}")
 else
-  export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d "${DOCKER_IMAGE}")
-fi
-
-# Copy the envfile and script with all the code to run into the docker.
-docker cp /home/circleci/project/. "$id:/circleci_stuff"
-
-# Copy built packages into the docker to test. This should only exist on the
-# binary test jobs. The package should've been created from a binary build job,
-# whhich persisted the package to a CircleCI workspace, which this job then
-# copies into a GPU enabled docker for testing
-if [[ -d "/home/circleci/project/final_pkgs" ]]; then
-  docker cp /home/circleci/project/final_pkgs "$id:/final_pkgs"
-fi
-
-# Copy the needed repos into the docker. These do not exist in the smoke test
-# jobs, since the smoke test jobs do not need the Pytorch source code.
-if [[ -d "$PYTORCH_ROOT" ]]; then
-  docker cp "$PYTORCH_ROOT" "$id:/pytorch"
-fi
-if [[ -d "$BUILDER_ROOT" ]]; then
-  docker cp "$BUILDER_ROOT" "$id:/builder"
+  export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined ${VOLUME_MOUNTS} -t -d "${DOCKER_IMAGE}")
 fi
 
 # Execute the test script that was populated by an earlier section

.circleci/scripts/cpp_doc_push_script.sh

@@ -57,7 +57,6 @@ time python aten/src/ATen/gen.py \
   -s aten/src/ATen \
   -d build/aten/src/ATen \
   aten/src/ATen/Declarations.cwrap \
-  aten/src/THNN/generic/THNN.h \
   aten/src/THCUNN/generic/THCUNN.h \
   aten/src/ATen/nn.yaml \
   aten/src/ATen/native/native_functions.yaml
@@ -73,10 +72,10 @@ time python tools/setup_helpers/generate_code.py \
 
 # Build the docs
 pushd docs/cpp
-pip install breathe==4.11.1 bs4 lxml six
+pip install breathe>=4.13.0 bs4 lxml six
 pip install --no-cache-dir -e "git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme"
 pip install exhale>=0.2.1
-pip install sphinx==1.8.5
+pip install sphinx>=2.0
 # Uncomment once it is fixed
 # pip install -r requirements.txt
 time make VERBOSE=1 html -j

.circleci/scripts/python_doc_push_script.sh

@@ -90,6 +90,12 @@ else
   find "$install_path" -name "*.html" -print0 | xargs -0 perl -pi -w -e "s@master\s+\((\d\.\d\.[A-Fa-f0-9]+\+[A-Fa-f0-9]+)\s+\)@<a href='http://pytorch.org/docs/versions.html'>$version \&#x25BC</a>@g"
 fi
 
+# Prevent Google from indexing $install_path/_modules. This folder contains
+# generated source files.
+# NB: the following only works on gnu sed. The sed shipped with mac os is different.
+# One can `brew install gnu-sed` on a mac and then use "gsed" instead of "sed".
+find "$install_path/_modules" -name "*.html" -print0 | xargs -0 sed -i '/<head>/a \ \ <meta name="robots" content="noindex">'
+
 git add "$install_path" || true
 git status
 git config user.email "soumith+bot@pytorch.org"

.circleci/scripts/setup_ci_environment.sh

@@ -2,7 +2,7 @@
 set -ex -o pipefail
 
 # Set up NVIDIA docker repo
-curl -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+curl -s -L --retry 3 https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
 echo "deb https://nvidia.github.io/libnvidia-container/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
 echo "deb https://nvidia.github.io/nvidia-container-runtime/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
 echo "deb https://nvidia.github.io/nvidia-docker/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
@@ -45,7 +45,7 @@ retry () {
 retry sudo pip -q install awscli==1.16.35
 
 if [ -n "${USE_CUDA_DOCKER_RUNTIME:-}" ]; then
-  DRIVER_FN="NVIDIA-Linux-x86_64-430.40.run"
+  DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
   wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
   sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
   nvidia-smi

.circleci/scripts/setup_linux_system_environment.sh

@@ -2,7 +2,7 @@
 set -eux -o pipefail
 
 # Set up CircleCI GPG keys for apt, if needed
-curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
+curl --retry 3 -s -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
 
 # Stop background apt updates.  Hypothetically, the kill should not
 # be necessary, because stop is supposed to send a kill signal to

.circleci/scripts/should_run_job.py

@@ -1,140 +0,0 @@
-import argparse
-import re
-import sys
-
-# Modify this variable if you want to change the set of default jobs
-# which are run on all pull requests.
-#
-# WARNING: Actually, this is a lie; we're currently also controlling
-# the set of jobs to run via the Workflows filters in CircleCI config.
-
-default_set = set([
-    # PyTorch CPU
-    # Selected oldest Python 2 version to ensure Python 2 coverage
-    'pytorch-linux-xenial-py2.7.9',
-    # PyTorch CUDA
-    'pytorch-linux-xenial-cuda9-cudnn7-py3',
-    # PyTorch ASAN
-    'pytorch-linux-xenial-py3-clang5-asan',
-    # PyTorch DEBUG
-    'pytorch-linux-xenial-py3.6-gcc5.4',
-    # LibTorch
-    'pytorch-libtorch-linux-xenial-cuda9-cudnn7-py3',
-
-    # Caffe2 CPU
-    'caffe2-py2-mkl-ubuntu16.04',
-    # Caffe2 CUDA
-    'caffe2-py3.5-cuda10.1-cudnn7-ubuntu16.04',
-    # Caffe2 ONNX
-    'caffe2-onnx-py2-gcc5-ubuntu16.04',
-    'caffe2-onnx-py3.6-clang7-ubuntu16.04',
-    # Caffe2 Clang
-    'caffe2-py2-clang7-ubuntu16.04',
-    # Caffe2 CMake
-    'caffe2-cmake-cuda9.0-cudnn7-ubuntu16.04',
-    # Caffe2 CentOS
-    'caffe2-py3.6-devtoolset7-cuda9.0-cudnn7-centos7',
-
-    # Binaries
-    'manywheel 2.7mu cpu devtoolset7',
-    'libtorch 2.7m cpu devtoolset7',
-    'libtorch 2.7m cpu gcc5.4_cxx11-abi',
-    'libtorch 2.7 cpu',
-    'libtorch-ios-11.2.1-nightly-x86_64-build',
-    'libtorch-ios-11.2.1-nightly-arm64-build',
-    'libtorch-ios-11.2.1-nightly-binary-build-upload',
-
-    # Caffe2 Android
-    'caffe2-py2-android-ubuntu16.04',
-    # Caffe2 OSX
-    'caffe2-py2-system-macos10.13',
-    # PyTorch OSX
-    'pytorch-macos-10.13-py3',
-    'pytorch-macos-10.13-cuda9.2-cudnn7-py3',
-    # PyTorch Android
-    'pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32-build',
-    'pytorch-linux-xenial-py3-clang5-android-ndk-r19',
-    # PyTorch Android gradle
-    'pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-only-x86_32',
-
-    # Pytorch iOS builds
-    'pytorch-ios-11.2.1-x86_64_build',
-    'pytorch-ios-11.2.1-arm64_build',
-    # PyTorch Mobile builds
-    'pytorch-linux-xenial-py3-clang5-mobile-build',
-
-    # Pytorch backward compatibility check
-    'pytorch-linux-backward-compatibility-check-test',
-
-    # XLA
-    'pytorch-xla-linux-xenial-py3.6-clang7',
-
-    # GraphExecutor config jobs
-    'pytorch-linux-xenial-py3.6-gcc5.4-ge_config_simple-test',
-    'pytorch-linux-xenial-py3.6-gcc5.4-ge_config_legacy-test',
-
-    # Other checks
-    'pytorch-short-perf-test-gpu',
-    'pytorch-python-doc-push',
-    'pytorch-cpp-doc-push',
-])
-
-# Collection of jobs that are *temporarily* excluded from running on PRs.
-# Use this if there is a long-running job breakage that we can't fix with a
-# single revert.
-skip_override = {
-    # example entry:
-    # 'pytorch-cpp-doc-push': "https://github.com/pytorch/pytorch/issues/<related issue>"
-}
-
-# Takes in commit message to analyze via stdin
-#
-# This script will query Git and attempt to determine if we should
-# run the current CI job under question
-#
-# NB: Try to avoid hard-coding names here, so there's less place to update when jobs
-# are updated/renamed
-#
-# Semantics in the presence of multiple tags:
-#   - Let D be the set of default builds
-#   - Let S be the set of explicitly specified builds
-#   - Let O be the set of temporarily skipped builds
-#   - Run S \/ (D - O)
-
-parser = argparse.ArgumentParser()
-parser.add_argument('build_environment')
-args = parser.parse_args()
-
-commit_msg = sys.stdin.read()
-
-# Matches anything that looks like [foo ci] or [ci foo] or [foo test]
-# or [test foo]
-RE_MARKER = re.compile(r'\[(?:([^ \[\]]+) )?(?:ci|test)(?: ([^ \[\]]+))?\]')
-
-markers = RE_MARKER.finditer(commit_msg)
-
-for m in markers:
-    if m.group(1) and m.group(2):
-        print("Unrecognized marker: {}".format(m.group(0)))
-        continue
-    spec = m.group(1) or m.group(2)
-    if spec is None:
-        print("Unrecognized marker: {}".format(m.group(0)))
-        continue
-    if spec in args.build_environment or spec == 'all':
-        print("Accepting {} due to commit marker {}".format(args.build_environment, m.group(0)))
-        sys.exit(0)
-
-skip_override_set = set(skip_override.keys())
-should_run_set = default_set - skip_override_set
-for spec in should_run_set:
-    if spec in args.build_environment:
-        print("Accepting {} as part of default set".format(args.build_environment))
-        sys.exit(0)
-
-print("Rejecting {}".format(args.build_environment))
-for spec, issue in skip_override.items():
-    if spec in args.build_environment:
-        print("This job is temporarily excluded from running on PRs. Reason: {}".format(issue))
-        break
-sys.exit(1)

.circleci/scripts/should_run_job.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-set -exu -o pipefail
-
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-# Check if we should actually run
-echo "BUILD_ENVIRONMENT: ${BUILD_ENVIRONMENT:-}"
-echo "CIRCLE_PULL_REQUEST: ${CIRCLE_PULL_REQUEST:-}"
-if [ -z "${BUILD_ENVIRONMENT:-}" ]; then
-  echo "Cannot run should_run_job.sh if BUILD_ENVIRONMENT is not defined!"
-  echo "CircleCI scripts are probably misconfigured."
-  exit 1
-fi
-if ! [ -e "$SCRIPT_DIR/COMMIT_MSG" ]; then
-  echo "Cannot run should_run_job.sh if you don't have COMMIT_MSG"
-  echo "written out.  Are you perhaps running the wrong copy of this script?"
-  echo "You should be running the copy in ~/workspace; SCRIPT_DIR=$SCRIPT_DIR"
-  exit 1
-fi
-if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then
-  if [[ $CIRCLE_BRANCH != "ci-all/"* ]] && [[ $CIRCLE_BRANCH != "nightly" ]] &&  [[ $CIRCLE_BRANCH != "postnightly" ]] ; then
-    # Don't swallow "script doesn't exist
-    [ -e "$SCRIPT_DIR/should_run_job.py"  ]
-    if ! python "$SCRIPT_DIR/should_run_job.py" "${BUILD_ENVIRONMENT:-}" < "$SCRIPT_DIR/COMMIT_MSG" ; then
-      circleci step halt
-      exit
-    fi
-  fi
-fi

.circleci/scripts/upload_binary_size_to_scuba.py

@@ -0,0 +1,87 @@
+import glob
+import json
+import logging
+import os
+import os.path
+import re
+import sys
+import time
+
+import requests
+
+
+def get_size(file_dir):
+    try:
+        # we should only expect one file, if no, something is wrong
+        file_name = glob.glob(os.path.join(file_dir, "*"))[0]
+        return os.stat(file_name).st_size
+    except:
+        logging.exception(f"error getting file from: {file_dir}")
+        return 0
+
+
+def build_message(size):
+    pkg_type, py_ver, cu_ver, *_ = os.environ.get("BUILD_ENVIRONMENT", "").split() + [
+        None,
+        None,
+        None,
+    ]
+    os_name = os.uname()[0].lower()
+    if os_name == "darwin":
+        os_name = "macos"
+    return {
+        "normal": {
+            "os": os_name,
+            "pkg_type": pkg_type,
+            "py_ver": py_ver,
+            "cu_ver": cu_ver,
+            "pr": os.environ.get("CIRCLE_PR_NUMBER"),
+            "build_num": os.environ.get("CIRCLE_BUILD_NUM"),
+            "sha1": os.environ.get("CIRCLE_SHA1"),
+            "branch": os.environ.get("CIRCLE_BRANCH"),
+        },
+        "int": {
+            "time": int(time.time()),
+            "size": size,
+            "commit_time": int(os.environ.get("COMMIT_TIME", "0")),
+        },
+    }
+
+
+def send_message(message):
+    access_token = os.environ.get("SCRIBE_GRAPHQL_ACCESS_TOKEN")
+    if not access_token:
+        raise ValueError("Can't find access token from environment variable")
+    url = "https://graph.facebook.com/scribe_logs"
+    r = requests.post(
+        url,
+        data={
+            "access_token": access_token,
+            "logs": json.dumps(
+                [
+                    {
+                        "category": "perfpipe_pytorch_binary_size",
+                        "message": json.dumps(message),
+                        "line_escape": False,
+                    }
+                ]
+            ),
+        },
+    )
+    print(r.text)
+    r.raise_for_status()
+
+
+if __name__ == "__main__":
+    file_dir = os.environ.get(
+        "PYTORCH_FINAL_PACKAGE_DIR", "/home/circleci/project/final_pkgs"
+    )
+    if len(sys.argv) == 2:
+        file_dir = sys.argv[1]
+    print("checking dir: " + file_dir)
+    size = get_size(file_dir)
+    if size != 0:
+        try:
+            send_message(build_message(size))
+        except:
+            logging.exception("can't send message")

.circleci/scripts/vs_install.ps1

@@ -0,0 +1,25 @@
+$VS_DOWNLOAD_LINK = "https://aka.ms/vs/15/release/vs_buildtools.exe"
+$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
+                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.14.11",
+                                                     "--add Microsoft.Component.MSBuild",
+                                                     "--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
+                                                     "--add Microsoft.VisualStudio.Component.TextTemplating",
+                                                     "--add Microsoft.VisualStudio.Component.VC.CoreIde",
+                                                     "--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
+                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
+                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
+                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81")
+
+curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
+if ($LASTEXITCODE -ne 0) {
+    echo "Download of the VS 2017 installer failed"
+    exit 1
+}
+
+$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
+Remove-Item -Path vs_installer.exe -Force
+$exitCode = $process.ExitCode
+if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
+    echo "VS 2017 installer exited with code $exitCode, which should be one of [0, 3010]."
+    exit 1
+}

.circleci/validate-docker-version.py

@@ -1,43 +1,43 @@
 #!/usr/bin/env python3
-
-import urllib.request
-import re
-
-import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
 import cimodel.data.caffe2_build_definitions as caffe2_build_definitions
+import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
+from yaml import load
 
-RE_VERSION = re.compile(r'allDeployedVersions = "([0-9,]+)"')
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
 
-URL_TEMPLATE = (
-    "https://raw.githubusercontent.com/pytorch/ossci-job-dsl/"
-    "master/src/main/groovy/ossci/{}/DockerVersion.groovy"
-)
 
-def check_version(job, expected_version):
-    url = URL_TEMPLATE.format(job)
-    with urllib.request.urlopen(url) as f:
-        contents = f.read().decode('utf-8')
-        m = RE_VERSION.search(contents)
-        if not m:
-            raise RuntimeError(
-                "Unbelievable! I could not find the variable allDeployedVersions in "
-                "{}; did the organization of ossci-job-dsl change?\n\nFull contents:\n{}"
-                .format(url, contents)
-            )
-        valid_versions = [int(v) for v in m.group(1).split(',')]
-        if expected_version not in valid_versions:
-            raise RuntimeError(
-                "We configured {} to use Docker version {}; but this "
-                "version is not deployed in {}.  Non-deployed versions will be "
-                "garbage collected two weeks after they are created.  DO NOT LAND "
-                "THIS TO MASTER without also updating ossci-job-dsl with this version."
-                "\n\nDeployed versions: {}"
-                .format(job, expected_version, url, m.group(1))
-            )
+def load_config(filename=".circleci/config.yml"):
+    with open(filename, "r") as fh:
+        return load("".join(fh.readlines()), Loader)
+
+
+def load_tags_for_projects(workflow_config):
+    return {
+        v["ecr_gc_job"]["project"]: v["ecr_gc_job"]["tags_to_keep"]
+        for v in workflow_config["workflows"]["ecr_gc"]["jobs"]
+        if isinstance(v, dict) and "ecr_gc_job" in v
+    }
+
+
+def check_version(job, tags, expected_version):
+    valid_versions = tags[job].split(",")
+    if expected_version not in valid_versions:
+        raise RuntimeError(
+            "We configured {} to use Docker version {}; but this "
+            "version is not configured in job ecr_gc_job_for_{}.  Non-deployed versions will be "
+            "garbage collected two weeks after they are created.  DO NOT LAND "
+            "THIS TO MASTER without also updating ossci-job-dsl with this version."
+            "\n\nDeployed versions: {}".format(job, expected_version, job, tags[job])
+        )
+
 
 def validate_docker_version():
-    check_version('pytorch', pytorch_build_definitions.DOCKER_IMAGE_VERSION)
-    check_version('caffe2', caffe2_build_definitions.DOCKER_IMAGE_VERSION)
+    tags = load_tags_for_projects(load_config())
+    check_version("pytorch", tags, pytorch_build_definitions.DOCKER_IMAGE_VERSION)
+    check_version("caffe2", tags, caffe2_build_definitions.DOCKER_IMAGE_VERSION)
 
 
 if __name__ == "__main__":

.circleci/verbatim-sources/binary-build-tests.yml

@@ -12,9 +12,3 @@
 #    resource_class: gpu.medium
 #    <<: *binary_linux_test
 #
-#  binary_linux_libtorch_2.7m_cu100_test:
-#    environment:
-#      BUILD_ENVIRONMENT: "libtorch 2.7m cu100"
-#    resource_class: gpu.medium
-#    <<: *binary_linux_test
-

.circleci/verbatim-sources/binary-job-specs.yml

@@ -2,7 +2,7 @@
     <<: *binary_linux_build_params
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     - run:
         <<: *binary_checkout
     - run:
@@ -19,8 +19,8 @@
             elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then
               retry apt-get update
               retry apt-get -y install expect moreutils
-              conda install -y -c eumetsat expect
-              conda install -y cmake
+              retry conda install -y -c eumetsat expect
+              retry conda install -y cmake
             fi
     - run:
         name: Update compiler to devtoolset7
@@ -41,6 +41,16 @@
         no_output_timeout: "1h"
         command: |
             source "/pytorch/.circleci/scripts/binary_linux_build.sh"
+    - run:
+        name: save binary size
+        no_output_timeout: "5m"
+        command: |
+            source /env
+            cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
+            pip3 install requests && \
+            SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
+            python3 /pytorch/.circleci/scripts/upload_binary_size_to_scuba.py || exit 0
+
     - persist_to_workspace:
         root: /
         paths: final_pkgs
@@ -56,7 +66,7 @@
         image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     # TODO: We shouldn't attach the workspace multiple times
     - attach_workspace:
         at: /home/circleci/project
@@ -79,7 +89,7 @@
         image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     - setup_linux_system_environment
     - setup_ci_environment
     - attach_workspace:
@@ -130,7 +140,7 @@
   smoke_mac_test:
     <<: *binary_linux_test_upload_params
     macos:
-      xcode: "9.0"
+      xcode: "9.4.1"
     steps:
       - attach_workspace:
           at: ~/workspace
@@ -158,10 +168,10 @@
   binary_mac_build:
     <<: *binary_mac_params
     macos:
-      xcode: "9.0"
+      xcode: "9.4.1"
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     - run:
         <<: *binary_checkout
     - run:
@@ -199,10 +209,10 @@
   binary_mac_upload: &binary_mac_upload
     <<: *binary_mac_params
     macos:
-      xcode: "9.0"
+      xcode: "9.4.1"
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     - run:
         <<: *binary_checkout
     - run:
@@ -227,7 +237,7 @@
     steps:
     - attach_workspace:
         at: ~/workspace
-    - should_run_job
+    - attach_scripts
     - checkout
     - run_brew_for_ios_build
     - run:
@@ -247,15 +257,15 @@
     - persist_to_workspace:
         root: /Users/distiller/workspace/
         paths: ios
-  
-  binary_ios_upload: 
+
+  binary_ios_upload:
     <<: *pytorch_ios_params
     macos:
       xcode: "11.2.1"
     steps:
     - attach_workspace:
         at: ~/workspace
-    - should_run_job
+    - attach_scripts
     - checkout
     - run_brew_for_ios_build
     - run:

.circleci/verbatim-sources/caffe2-job-specs.yml

@@ -4,7 +4,7 @@
       image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     - setup_linux_system_environment
     - checkout
     - setup_ci_environment
@@ -64,7 +64,7 @@
       image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -124,10 +124,10 @@
   caffe2_macos_build:
     <<: *caffe2_params
     macos:
-      xcode: "9.0"
+      xcode: "9.4.1"
     steps:
       # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-      - should_run_job
+      - attach_scripts
       - checkout
       - run_brew_for_macos_build
       - run:
@@ -151,7 +151,7 @@
             # Install Anaconda if we need to
             if [ -n "${CAFFE2_USE_ANACONDA}" ]; then
               rm -rf ${TMPDIR}/anaconda
-              curl -o ${TMPDIR}/conda.sh https://repo.continuum.io/miniconda/Miniconda${ANACONDA_VERSION}-latest-MacOSX-x86_64.sh
+              curl --retry 3 -o ${TMPDIR}/conda.sh https://repo.continuum.io/miniconda/Miniconda${ANACONDA_VERSION}-latest-MacOSX-x86_64.sh
               chmod +x ${TMPDIR}/conda.sh
               /bin/bash ${TMPDIR}/conda.sh -b -p ${TMPDIR}/anaconda
               rm -f ${TMPDIR}/conda.sh
@@ -162,7 +162,7 @@
             pip -q install numpy
 
             # Install sccache
-            sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+            sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
             sudo chmod +x /usr/local/bin/sccache
             export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
 

.circleci/verbatim-sources/commands.yml

@@ -3,16 +3,12 @@ commands:
   # attaches the workspace at ~/workspace; this workspace is generated
   # by the setup job. Note that ~/workspace is not the default working
   # directory (that's ~/project).
-  should_run_job:
-    description: "Test if the job should run or not"
+  attach_scripts:
+    description: "Attach the scripts that power everything else"
     steps:
       - attach_workspace:
           name: Attaching workspace
           at: ~/workspace
-      - run:
-          name: Should run job
-          no_output_timeout: "2m"
-          command: ~/workspace/.circleci/scripts/should_run_job.sh
 
   # This system setup script is meant to run before the CI-related scripts, e.g.,
   # installing Git client, checking out code, setting up CI env, and

.circleci/verbatim-sources/docker_build_job.yml

@@ -1,21 +0,0 @@
-  docker_build_job:
-      parameters:
-        image_name:
-          type: string
-          default: ""
-      machine:
-        image: ubuntu-1604:201903-01
-      resource_class: large
-      environment:
-        IMAGE_NAME: << parameters.image_name >>
-      steps:
-        - checkout
-        - run:
-            name: build_docker_image_<< parameters.image_name >>
-            no_output_timeout: "1h"
-            command: |
-              set +x
-              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
-              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-              set -x
-              cd .circleci/docker && ./build_docker.sh

.circleci/verbatim-sources/docker_jobs.yml

@@ -0,0 +1,84 @@
+  docker_build_job:
+      parameters:
+        image_name:
+          type: string
+          default: ""
+      machine:
+        image: ubuntu-1604:201903-01
+      resource_class: large
+      environment:
+        IMAGE_NAME: << parameters.image_name >>
+      steps:
+        - checkout
+        - run:
+            name: build_docker_image_<< parameters.image_name >>
+            no_output_timeout: "1h"
+            command: |
+              set +x
+              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
+              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
+              set -x
+              cd .circleci/docker && ./build_docker.sh
+  docker_for_ecr_gc_build_job:
+      machine:
+        image: ubuntu-1604:201903-01
+      steps:
+        - checkout
+        - run:
+            name: build_docker_image_for_ecr_gc
+            no_output_timeout: "1h"
+            command: |
+              cd .circleci/ecr_gc_docker
+              docker build . -t 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
+              set +x
+              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
+              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
+              eval $(aws ecr get-login --no-include-email --region us-east-1)
+              set -x
+              docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
+  ecr_gc_job:
+      parameters:
+        project:
+          type: string
+          default: "pytorch"
+        tags_to_keep:  # comma separate values
+          type: string
+      environment:
+        PROJECT: << parameters.project >>
+        IMAGE_TAG: << parameters.tags_to_keep >>
+      docker:
+        - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
+          aws_auth:
+            aws_access_key_id: ${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
+            aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
+
+      steps:
+        - run:
+            name: garbage collecting for ecr images
+            no_output_timeout: "1h"
+            command: |
+              set +x
+              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
+              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
+              set -x
+              /usr/bin/gc.py --filter-prefix ${PROJECT}  --ignore-tags ${IMAGE_TAG}
+
+  docker_hub_index_job:
+      docker:
+        - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
+          aws_auth:
+            aws_access_key_id: ${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
+            aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
+
+      steps:
+        - run:
+            name: garbage collecting for ecr images
+            no_output_timeout: "1h"
+            command: |
+              set +x
+              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
+              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
+              export DOCKER_HUB_USERNAME=${CIRCLECI_DOCKER_HUB_USERNAME}
+              export DOCKER_HUB_PASSWORD=${CIRCLECI_DOCKER_HUB_PASSWORD}
+              set -x
+              /usr/bin/docker_hub.py

.circleci/verbatim-sources/header-section.yml

@@ -1,15 +1,9 @@
 # WARNING: DO NOT EDIT THIS FILE DIRECTLY!!!
 # See the README.md in this directory.
 
-# IMPORTANT: To update Docker image version, please first update
-# https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/pytorch/DockerVersion.groovy and
-# https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/caffe2/DockerVersion.groovy,
-# and then update DOCKER_IMAGE_VERSION at the top of the following files:
-# * cimodel/data/pytorch_build_definitions.py
-# * cimodel/data/caffe2_build_definitions.py
-# And the inline copies of the variable in
-# * verbatim-sources/job-specs-custom.yml
-#   (grep for DOCKER_IMAGE)
+# IMPORTANT: To update Docker image version, please follow
+# the instructions at
+# https://github.com/pytorch/pytorch/wiki/Docker-image-build-on-CircleCI
 
 version: 2.1
 
@@ -19,3 +13,17 @@ docker_config_defaults: &docker_config_defaults
     # This IAM user only allows read-write access to ECR
     aws_access_key_id: ${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_WRITE_V4}
     aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_WRITE_V4}
+
+executors:
+  windows-with-nvidia-gpu:
+    machine:
+      resource_class: windows.gpu.nvidia.medium
+      image: windows-server-2019-nvidia:stable
+      shell: bash.exe
+
+  windows-cpu-with-nvidia-cuda:
+    machine:
+      # we will change to CPU host when it's ready
+      resource_class: windows.xlarge
+      image: windows-server-2019-vs2019:stable
+      shell: bash.exe

.circleci/verbatim-sources/job-specs-custom.yml

@@ -2,13 +2,13 @@
     environment:
       BUILD_ENVIRONMENT: pytorch-python-doc-push
       # TODO: stop hardcoding this
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3:405"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:f990c76a-a798-42bb-852f-5be5006f8026"
     resource_class: large
     machine:
       image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -47,13 +47,13 @@
   pytorch_cpp_doc_push:
     environment:
       BUILD_ENVIRONMENT: pytorch-cpp-doc-push
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3:405"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:f990c76a-a798-42bb-852f-5be5006f8026"
     resource_class: large
     machine:
       image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -93,10 +93,10 @@
     environment:
       BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-build
     macos:
-      xcode: "9.0"
+      xcode: "9.4.1"
     steps:
       # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-      - should_run_job
+      - attach_scripts
       - checkout
       - run_brew_for_macos_build
       - run:
@@ -107,7 +107,7 @@
             export IN_CIRCLECI=1
 
             # Install sccache
-            sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+            sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
             sudo chmod +x /usr/local/bin/sccache
             export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
 
@@ -133,11 +133,11 @@
     environment:
       BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-test
     macos:
-      xcode: "9.0"
+      xcode: "9.4.1"
     steps:
       # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
       # This workspace also carries binaries from the build job
-      - should_run_job
+      - attach_scripts
       - run_brew_for_macos_build
       - run:
           name: Test
@@ -154,64 +154,16 @@
       - store_test_results:
           path: test/test-reports
 
-  pytorch_macos_10_13_cuda9_2_cudnn7_py3_build:
-    environment:
-      BUILD_ENVIRONMENT: pytorch-macos-10.13-cuda9.2-cudnn7-py3-build
-    macos:
-      xcode: "9.0"
-    steps:
-      # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-      - should_run_job
-      - checkout
-      - run_brew_for_macos_build
-      - run:
-          name: Build
-          no_output_timeout: "1h"
-          command: |
-            set -e
-
-            export IN_CIRCLECI=1
-
-            # Install CUDA 9.2
-            sudo rm -rf ~/cuda_9.2.64_mac_installer.app || true
-            curl https://s3.amazonaws.com/ossci-macos/cuda_9.2.64_mac_installer.zip -o ~/cuda_9.2.64_mac_installer.zip
-            unzip ~/cuda_9.2.64_mac_installer.zip -d ~/
-            sudo ~/cuda_9.2.64_mac_installer.app/Contents/MacOS/CUDAMacOSXInstaller --accept-eula --no-window
-            sudo cp /usr/local/cuda/lib/libcuda.dylib /Developer/NVIDIA/CUDA-9.2/lib/libcuda.dylib
-            sudo rm -rf /usr/local/cuda || true
-
-            # Install cuDNN 7.1 for CUDA 9.2
-            curl https://s3.amazonaws.com/ossci-macos/cudnn-9.2-osx-x64-v7.1.tgz -o ~/cudnn-9.2-osx-x64-v7.1.tgz
-            rm -rf ~/cudnn-9.2-osx-x64-v7.1 && mkdir ~/cudnn-9.2-osx-x64-v7.1
-            tar -xzvf ~/cudnn-9.2-osx-x64-v7.1.tgz -C ~/cudnn-9.2-osx-x64-v7.1
-            sudo cp ~/cudnn-9.2-osx-x64-v7.1/cuda/include/cudnn.h /Developer/NVIDIA/CUDA-9.2/include/
-            sudo cp ~/cudnn-9.2-osx-x64-v7.1/cuda/lib/libcudnn* /Developer/NVIDIA/CUDA-9.2/lib/
-            sudo chmod a+r /Developer/NVIDIA/CUDA-9.2/include/cudnn.h /Developer/NVIDIA/CUDA-9.2/lib/libcudnn*
-
-            # Install sccache
-            sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
-            sudo chmod +x /usr/local/bin/sccache
-            export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
-            # This IAM user allows write access to S3 bucket for sccache
-            set +x
-            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}
-            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}
-            set -x
-
-            git submodule sync && git submodule update -q --init --recursive
-            chmod a+x .jenkins/pytorch/macos-build.sh
-            unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
-
   pytorch_android_gradle_build:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:f990c76a-a798-42bb-852f-5be5006f8026"
       PYTHON_VERSION: "3.6"
     resource_class: large
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    - should_run_job
+    - attach_scripts
     - setup_linux_system_environment
     - checkout
     - setup_ci_environment
@@ -291,13 +243,13 @@
   pytorch_android_publish_snapshot:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-publish-snapshot
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:f990c76a-a798-42bb-852f-5be5006f8026"
       PYTHON_VERSION: "3.6"
     resource_class: large
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    - should_run_job
+    - attach_scripts
     - setup_linux_system_environment
     - checkout
     - setup_ci_environment
@@ -327,13 +279,13 @@
   pytorch_android_gradle_build-x86_32:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-only-x86_32
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:f990c76a-a798-42bb-852f-5be5006f8026"
       PYTHON_VERSION: "3.6"
     resource_class: large
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    - should_run_job
+    - attach_scripts
     - run:
         name: filter out not PR runs
         no_output_timeout: "5m"
@@ -376,9 +328,9 @@
       xcode: "11.2.1"
     steps:
       # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-      - should_run_job
+      - attach_scripts
       - checkout
-      - run_brew_for_ios_build    
+      - run_brew_for_ios_build
       - run:
           name: Run Fastlane
           no_output_timeout: "1h"
@@ -410,30 +362,44 @@
             WORKSPACE=/Users/distiller/workspace
             PROJ_ROOT=/Users/distiller/project
             export TCLLIBPATH="/usr/local/lib"
+
             # Install conda
-            curl -o ~/Downloads/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
-            chmod +x ~/Downloads/conda.sh
-            /bin/bash ~/Downloads/conda.sh -b -p ~/anaconda
+            curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            chmod +x ~/conda.sh
+            /bin/bash ~/conda.sh -b -p ~/anaconda
             export PATH="~/anaconda/bin:${PATH}"
             source ~/anaconda/bin/activate
+
             # Install dependencies
-            conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing requests --yes
+            retry () {
+                $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+            }
+
+            retry conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing requests --yes
+
             # sync submodules
             cd ${PROJ_ROOT}
             git submodule sync
             git submodule update --init --recursive
+
             # export
             export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
+
             # run build script
             chmod a+x ${PROJ_ROOT}/scripts/build_ios.sh
             echo "IOS_ARCH: ${IOS_ARCH}"
             echo "IOS_PLATFORM: ${IOS_PLATFORM}"
-            export BUILD_PYTORCH_MOBILE=1
+
+            #check the custom build flag
+            echo "SELECTED_OP_LIST: ${SELECTED_OP_LIST}"
+            if [ -n "${SELECTED_OP_LIST}" ]; then
+                export SELECTED_OP_LIST="${PROJ_ROOT}/ios/TestApp/custom_build/${SELECTED_OP_LIST}"
+            fi
             export IOS_ARCH=${IOS_ARCH}
             export IOS_PLATFORM=${IOS_PLATFORM}
             unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts
       - run:
-          name: Run Build Tests
+          name: Run Build Test
           no_output_timeout: "30m"
           command: |
             set -e
@@ -445,7 +411,11 @@
               exit 1
             fi
             echo ${IOS_DEV_TEAM_ID}
-            ruby ${PROJ_ROOT}/scripts/xcode_build.rb -i ${PROJ_ROOT}/build_ios/install -x ${PROJ_ROOT}/ios/TestApp/TestApp.xcodeproj -p ${IOS_PLATFORM} -c ${PROFILE} -t ${IOS_DEV_TEAM_ID}
+            if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then
+              ruby ${PROJ_ROOT}/scripts/xcode_build.rb -i ${PROJ_ROOT}/build_ios/install -x ${PROJ_ROOT}/ios/TestApp/TestApp.xcodeproj -p ${IOS_PLATFORM} -c ${PROFILE} -t ${IOS_DEV_TEAM_ID}
+            else
+              ruby ${PROJ_ROOT}/scripts/xcode_build.rb -i ${PROJ_ROOT}/build_ios/install -x ${PROJ_ROOT}/ios/TestApp/TestApp.xcodeproj -p ${IOS_PLATFORM}
+            fi
             if ! [ "$?" -eq "0" ]; then
               echo 'xcodebuild failed!'
               exit 1
@@ -455,15 +425,14 @@
           no_output_timeout: "2h"
           command: |
             set -e
-            if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then 
+            if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then
               echo "not SIMULATOR build, skip it."
               exit 0
             fi
             WORKSPACE=/Users/distiller/workspace
             PROJ_ROOT=/Users/distiller/project
             source ~/anaconda/bin/activate
-            #install the latest version of PyTorch and TorchVision
-            pip install torch torchvision
+            pip install torch torchvision --progress-bar off
             #run unit test
             cd ${PROJ_ROOT}/ios/TestApp/benchmark
             python trace_model.py
@@ -471,4 +440,3 @@
             cd ${PROJ_ROOT}/ios/TestApp
             instruments -s -devices
             fastlane scan
-            

.circleci/verbatim-sources/pytorch-build-params.yml

@@ -12,10 +12,14 @@ pytorch_params: &pytorch_params
     use_cuda_docker_runtime:
       type: string
       default: ""
+    build_only:
+      type: string
+      default: ""
   environment:
     BUILD_ENVIRONMENT: << parameters.build_environment >>
     DOCKER_IMAGE: << parameters.docker_image >>
     USE_CUDA_DOCKER_RUNTIME: << parameters.use_cuda_docker_runtime >>
+    BUILD_ONLY: << parameters.build_only >>
   resource_class: << parameters.resource_class >>
 
 pytorch_ios_params: &pytorch_ios_params
@@ -29,11 +33,46 @@ pytorch_ios_params: &pytorch_ios_params
     ios_platform:
       type: string
       default: ""
+    op_list:
+      type: string
+      default: ""
   environment:
     BUILD_ENVIRONMENT: << parameters.build_environment >>
     IOS_ARCH: << parameters.ios_arch >>
     IOS_PLATFORM: << parameters.ios_platform >>
+    SELECTED_OP_LIST: << parameters.op_list >>
 
-
-
-
+pytorch_windows_params: &pytorch_windows_params
+  parameters:
+    test_name:
+      type: string
+      default: ""
+    cuda_version:
+      type: string
+      default: "10"
+    python_version:
+      type: string
+      default: "3.6"
+    vc_version:
+      type: string
+      default: "14.11"
+    vc_year:
+      type: string
+      default: "2017"
+    vc_product:
+      type: string
+      default: "BuildTools"
+    use_cuda:
+      type: string
+      default: ""
+  environment:
+    BUILD_ENVIRONMENT: "pytorch-win-ws2019-cuda10-cudnn7-py3"
+    SCCACHE_BUCKET: "ossci-compiler-cache"
+    CUDA_VERSION: <<parameters.cuda_version>>
+    PYTHON_VERSION: <<parameters.python_version>>
+    VC_VERSION: <<parameters.vc_version>>
+    VC_YEAR: <<parameters.vc_year>>
+    VC_PRODUCT: <<parameters.vc_product>>
+    USE_CUDA: <<parameters.use_cuda>>
+    TORCH_CUDA_ARCH_LIST: "7.5"
+    JOB_BASE_NAME: <<parameters.test_name>>

.circleci/verbatim-sources/pytorch-job-specs.yml

@@ -5,7 +5,7 @@ jobs:
       image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     - setup_linux_system_environment
     - checkout
     - setup_ci_environment
@@ -19,28 +19,27 @@ jobs:
           time docker pull ${DOCKER_IMAGE} >/dev/null
           export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
 
-          # NB: Temporarily disable the rebase logic in v1.4.0, don't merge this change into master
-          # # TODO We may want to move the rebase logic to a separate step after checkout
-          # # Rebase to master only if in xenial_py3_6_gcc5_4 case
-          # if [[ "${CIRCLE_BRANCH}" != "master" && "${BUILD_ENVIRONMENT}" == *"gcc5"* ]]; then
-          #   echo "Merge master branch into $CIRCLE_BRANCH before build in environment $BUILD_ENVIRONMENT"
-          #   set -x
-          #   git config --global user.email "circleci.ossci@gmail.com"
-          #   git config --global user.name "CircleCI"
-          #   git config remote.origin.url https://github.com/pytorch/pytorch.git
-          #   git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master
-          #   git fetch --tags --progress https://github.com/pytorch/pytorch.git +refs/heads/master:refs/remotes/origin/master --depth=100 --quiet
-          #   export GIT_MERGE_TARGET=`git log -n 1 --pretty=format:"%H" origin/master`
-          #   echo "GIT_MERGE_TARGET: " ${GIT_MERGE_TARGET}
-          #   export GIT_COMMIT=${CIRCLE_SHA1}
-          #   echo "GIT_COMMIT: " ${GIT_COMMIT}
-          #   git checkout -f ${GIT_COMMIT}
-          #   git reset --hard ${GIT_COMMIT}
-          #   git merge --allow-unrelated-histories --no-edit --no-ff ${GIT_MERGE_TARGET}
-          #   set +x
-          # else
-          #   echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
-          # fi
+          # TODO We may want to move the rebase logic to a separate step after checkout
+          # Rebase to master only if in xenial_py3_6_gcc5_4 case
+          if [[ "${CIRCLE_BRANCH}" != "master" && "${BUILD_ENVIRONMENT}" == *"gcc5"* ]]; then
+            echo "Merge master branch into $CIRCLE_BRANCH before build in environment $BUILD_ENVIRONMENT"
+            set -x
+            git config --global user.email "circleci.ossci@gmail.com"
+            git config --global user.name "CircleCI"
+            git config remote.origin.url https://github.com/pytorch/pytorch.git
+            git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master
+            git fetch --tags --progress https://github.com/pytorch/pytorch.git +refs/heads/master:refs/remotes/origin/master --depth=100 --quiet
+            export GIT_MERGE_TARGET=`git log -n 1 --pretty=format:"%H" origin/master`
+            echo "GIT_MERGE_TARGET: " ${GIT_MERGE_TARGET}
+            export GIT_COMMIT=${CIRCLE_SHA1}
+            echo "GIT_COMMIT: " ${GIT_COMMIT}
+            git checkout -f ${GIT_COMMIT}
+            git reset --hard ${GIT_COMMIT}
+            git merge --allow-unrelated-histories --no-edit --no-ff ${GIT_MERGE_TARGET}
+            set +x
+          else
+            echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
+          fi
 
           git submodule sync && git submodule update -q --init --recursive
 
@@ -89,7 +88,7 @@ jobs:
       image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - should_run_job
+    - attach_scripts
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -132,10 +131,119 @@ jobs:
           if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
             export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "${PARALLEL_FLAGS}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
           else
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "${PARALLEL_FLAGS}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}" && echo "${PARALLEL_FLAGS}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
           fi
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
           retrieve_test_reports
     - store_test_results:
         path: test-reports
+
+  pytorch_windows_build:
+    <<: *pytorch_windows_params
+    parameters:
+      test_name:
+        type: string
+        default: ""
+      cuda_version:
+        type: string
+        default: "10"
+      python_version:
+        type: string
+        default: "3.6"
+      vc_version:
+        type: string
+        default: "14.11"
+      vc_year:
+        type: string
+        default: "2017"
+      vc_product:
+        type: string
+        default: "BuildTools"
+      use_cuda:
+        type: string
+        default: ""
+    executor: windows-cpu-with-nvidia-cuda
+    steps:
+      - checkout
+      - run:
+          name: Install VS2017
+          command: |
+            if [[ "${VC_YEAR}" == "2017" ]]; then
+              powershell .circleci/scripts/vs_install.ps1
+            fi
+      - run:
+          name: Install Cuda
+          no_output_timeout: 30m
+          command: |
+            curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe
+            mkdir cuda_install_logs
+            ./cuda_10.1.243_426.00_win10.exe -s -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
+            cat cuda_install_logs/LOG.setup.exe.log
+            if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe"
+            then
+              echo "CUDA installation failed"
+              exit 1
+            fi
+            rm -rf ./cuda_install_logs
+            rm -f ./cuda_10.1.243_426.00_win10.exe
+      - run:
+          name: Install Cudnn
+          command : |
+            cd c:/
+            curl --retry 3 -O https://ossci-windows.s3.amazonaws.com/cudnn-10.1-windows10-x64-v7.6.4.38.zip
+            7z x cudnn-10.1-windows10-x64-v7.6.4.38.zip -ocudnn
+            cp -r cudnn/cuda/* "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/"
+      - run:
+          name: Build
+          no_output_timeout: "90m"
+          command: |
+            set -e
+            set +x
+            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_WIN_BUILD_V1}
+            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_WIN_BUILD_V1}
+            set -x
+            .jenkins/pytorch/win-build.sh
+  pytorch_windows_test:
+    <<: *pytorch_windows_params
+    parameters:
+      test_name:
+        type: string
+        default: ""
+      cuda_version:
+        type: string
+        default: "10"
+      python_version:
+        type: string
+        default: "3.6"
+      vc_version:
+        type: string
+        default: "14.11"
+      vc_year:
+        type: string
+        default: "2017"
+      vc_product:
+        type: string
+        default: "BuildTools"
+      use_cuda:
+        type: string
+        default: ""
+    executor: windows-with-nvidia-gpu
+    steps:
+      - checkout
+      - run:
+          name: Install VS2017
+          command: |
+            if [[ "${VC_YEAR}" == "2017" ]]; then
+              powershell .circleci/scripts/vs_install.ps1
+            fi
+      - run:
+          name: Test
+          no_output_timeout: "30m"
+          command: |
+            set -e
+            set +x
+            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_WIN_BUILD_V1}
+            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_WIN_BUILD_V1}
+            set -x
+            .jenkins/pytorch/win-test.sh

.circleci/verbatim-sources/windows-build-test.yml

@@ -0,0 +1,134 @@
+      # Warning: indentation here matters!
+
+      - pytorch_windows_build:
+          name: pytorch_windows_vs2017_14.11_py36_cuda10.1_build
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: "14.11"
+          vc_year: "2017"
+          vc_product: "BuildTools"
+          use_cuda: "1"
+          requires:
+            - setup
+          filters:
+            branches:
+              only:
+                - master
+                - /ci-all\/.*/
+      - pytorch_windows_test:
+          name: pytorch_windows_vs2017_14.11_py36_cuda10.1_test1
+          test_name: pytorch-windows-test1
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: "14.11"
+          vc_year: "2017"
+          vc_product: "BuildTools"
+          use_cuda: "1"
+          requires:
+            - setup
+            - pytorch_windows_vs2017_14.11_py36_cuda10.1_build
+          filters:
+            branches:
+              only:
+                - master
+                - /ci-all\/.*/
+      - pytorch_windows_test:
+          name: pytorch_windows_vs2017_14.11_py36_cuda10.1_test2
+          test_name: pytorch-windows-test2
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: "14.11"
+          vc_year: "2017"
+          vc_product: "BuildTools"
+          use_cuda: "1"
+          requires:
+            - setup
+            - pytorch_windows_vs2017_14.11_py36_cuda10.1_build
+          filters:
+            branches:
+              only:
+                - master
+                - /ci-all\/.*/
+      - pytorch_windows_build:
+          name: pytorch_windows_vs2017_14.16_py36_cuda10.1_build
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: "14.16"
+          vc_year: "2017"
+          vc_product: "BuildTools"
+          use_cuda: "1"
+          requires:
+            - setup
+          filters:
+            branches:
+              only:
+                - master
+                - /ci-all\/.*/
+      - pytorch_windows_test:
+          name: pytorch_windows_vs2017_14.16_py36_cuda10.1_test1
+          test_name: pytorch-windows-test1
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: "14.16"
+          vc_year: "2017"
+          vc_product: "BuildTools"
+          use_cuda: "1"
+          requires:
+            - setup
+            - pytorch_windows_vs2017_14.16_py36_cuda10.1_build
+          filters:
+            branches:
+              only:
+                - master
+                - /ci-all\/.*/
+      - pytorch_windows_test:
+          name: pytorch_windows_vs2017_14.16_py36_cuda10.1_test2
+          test_name: pytorch-windows-test2
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: "14.16"
+          vc_year: "2017"
+          vc_product: "BuildTools"
+          use_cuda: "1"
+          requires:
+            - setup
+            - pytorch_windows_vs2017_14.16_py36_cuda10.1_build
+          filters:
+            branches:
+              only:
+                - master
+                - /ci-all\/.*/
+      - pytorch_windows_build:
+          name: pytorch_windows_vs2019_py36_cuda10.1_build
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: ""
+          vc_year: "2019"
+          vc_product: "Community"
+          use_cuda: "1"
+          requires:
+            - setup
+      - pytorch_windows_test:
+          name: pytorch_windows_vs2019_py36_cuda10.1_test1
+          test_name: pytorch-windows-test1
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: ""
+          vc_year: "2019"
+          vc_product: "Community"
+          use_cuda: "1"
+          requires:
+            - setup
+            - pytorch_windows_vs2019_py36_cuda10.1_build
+      - pytorch_windows_test:
+          name: pytorch_windows_vs2019_py36_cuda10.1_test2
+          test_name: pytorch-windows-test2
+          cuda_version: "10"
+          python_version: "3.6"
+          vc_version: ""
+          vc_year: "2019"
+          vc_product: "Community"
+          use_cuda: "1"
+          requires:
+            - setup
+            - pytorch_windows_vs2019_py36_cuda10.1_build

.circleci/verbatim-sources/workflows-binary-builds-smoke-subset.yml

@@ -1,3 +1,5 @@
+      # TODO: Refactor circleci/cimodel/data/binary_build_data.py to generate this file
+      #       instead of doing one offs here
       # Binary builds (subset, to smoke test that they'll work)
       #
       # NB: If you modify this file, you need to also modify
@@ -10,13 +12,17 @@
           build_environment: "manywheel 2.7mu cpu devtoolset7"
           requires:
             - setup
-          docker_image: "pytorch/manylinux-cuda100"
+          docker_image: "pytorch/manylinux-cuda102"
       - binary_linux_build:
-          name: binary_linux_manywheel_3_7m_cu100_devtoolset7_build
-          build_environment: "manywheel 3.7m cu100 devtoolset7"
+          name: binary_linux_manywheel_3_7m_cu102_devtoolset7_build
+          build_environment: "manywheel 3.7m cu102 devtoolset7"
           requires:
             - setup
-          docker_image: "pytorch/manylinux-cuda100"
+          docker_image: "pytorch/manylinux-cuda102"
+          filters:
+            branches:
+              only:
+                - master
       - binary_linux_build:
           name: binary_linux_conda_2_7_cpu_devtoolset7_build
           build_environment: "conda 2.7 cpu devtoolset7"
@@ -31,7 +37,7 @@
           requires:
             - setup
           libtorch_variant: "shared-with-deps"
-          docker_image: "pytorch/manylinux-cuda100"
+          docker_image: "pytorch/manylinux-cuda102"
       - binary_linux_build:
           name: binary_linux_libtorch_2_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build
           build_environment: "libtorch 2.7m cpu gcc5.4_cxx11-abi"
@@ -46,33 +52,54 @@
           build_environment: "wheel 3.6 cpu"
           requires:
             - setup
+          filters:
+            branches:
+              only:
+                - master
       - binary_mac_build:
           name: binary_macos_conda_2_7_cpu_build
           build_environment: "conda 2.7 cpu"
           requires:
             - setup
+          filters:
+            branches:
+              only:
+                - master
+      # This job has an average run time of 3 hours o.O
+      # Now only running this on master to reduce overhead
       - binary_mac_build:
           name: binary_macos_libtorch_2_7_cpu_build
           build_environment: "libtorch 2.7 cpu"
           requires:
             - setup
-
+          filters:
+            branches:
+              only:
+                - master
       - binary_linux_test:
           name: binary_linux_manywheel_2_7mu_cpu_devtoolset7_test
           build_environment: "manywheel 2.7mu cpu devtoolset7"
           requires:
             - setup
             - binary_linux_manywheel_2_7mu_cpu_devtoolset7_build
-          docker_image: "pytorch/manylinux-cuda100"
+          docker_image: "pytorch/manylinux-cuda102"
+          filters:
+            branches:
+              only:
+                - master
       - binary_linux_test:
-          name: binary_linux_manywheel_3_7m_cu100_devtoolset7_test
-          build_environment: "manywheel 3.7m cu100 devtoolset7"
+          name: binary_linux_manywheel_3_7m_cu102_devtoolset7_test
+          build_environment: "manywheel 3.7m cu102 devtoolset7"
           requires:
             - setup
-            - binary_linux_manywheel_3_7m_cu100_devtoolset7_build
-          docker_image: "pytorch/manylinux-cuda100"
+            - binary_linux_manywheel_3_7m_cu102_devtoolset7_build
+          docker_image: "pytorch/manylinux-cuda102"
           use_cuda_docker_runtime: "1"
           resource_class: gpu.medium
+          filters:
+            branches:
+              only:
+                - master
       - binary_linux_test:
           name: binary_linux_conda_2_7_cpu_devtoolset7_test
           build_environment: "conda 2.7 cpu devtoolset7"
@@ -89,7 +116,7 @@
             - setup
             - binary_linux_libtorch_2_7m_cpu_devtoolset7_shared-with-deps_build
           libtorch_variant: "shared-with-deps"
-          docker_image: "pytorch/manylinux-cuda100"
+          docker_image: "pytorch/manylinux-cuda102"
       - binary_linux_test:
           name: binary_linux_libtorch_2_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_test
           build_environment: "libtorch 2.7m cpu gcc5.4_cxx11-abi"

.circleci/verbatim-sources/workflows-docker-builder.yml

@@ -7,6 +7,7 @@
               only:
                 - master
     jobs:
+      - docker_for_ecr_gc_build_job
       - docker_build_job:
           name: "pytorch-linux-bionic-clang9-thrift-llvmdev"
           image_name: "pytorch-linux-bionic-clang9-thrift-llvmdev"
@@ -17,11 +18,8 @@
           name: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
           image_name: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
       - docker_build_job:
-          name: "pytorch-linux-xenial-cuda8-cudnn7-py2"
-          image_name: "pytorch-linux-xenial-cuda8-cudnn7-py2"
-      - docker_build_job:
-          name: "pytorch-linux-xenial-cuda8-cudnn7-py3"
-          image_name: "pytorch-linux-xenial-cuda8-cudnn7-py3"
+          name: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
+          image_name: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
       - docker_build_job:
           name: "pytorch-linux-xenial-cuda9-cudnn7-py2"
           image_name: "pytorch-linux-xenial-cuda9-cudnn7-py2"
@@ -46,6 +44,9 @@
       - docker_build_job:
           name: "pytorch-linux-xenial-py3.5"
           image_name: "pytorch-linux-xenial-py3.5"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py3.8"
+          image_name: "pytorch-linux-xenial-py3.8"
       - docker_build_job:
           name: "pytorch-linux-xenial-py3.6-clang7"
           image_name: "pytorch-linux-xenial-py3.6-clang7"

.circleci/verbatim-sources/workflows-ecr-gc.yml

@@ -0,0 +1,28 @@
+  ecr_gc:
+    triggers:
+      - schedule:
+          cron: "45 * * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - ecr_gc_job:
+            name: ecr_gc_job_for_pytorch
+            project: pytorch
+            tags_to_keep: "271,262,256,278,282,291,300,323,327,347,389,401,402,403,405,a8006f9a-272d-4478-b137-d121c6f05c83,6e7b11da-a919-49e5-b2ba-da66e3d4bb0a,f990c76a-a798-42bb-852f-5be5006f8026"
+      - ecr_gc_job:
+            name: ecr_gc_job_for_caffe2
+            project: caffe2
+            tags_to_keep: "348,345,336,325,324,315,306,301,287,283,276,273,266,253,248,238,230,213"
+      - ecr_gc_job:
+            name: ecr_gc_job_for_translate
+            project: translate
+            tags_to_keep: "8"
+      - ecr_gc_job:
+            name: ecr_gc_job_for_tensorcomp
+            project: tensorcomp
+            tags_to_keep: "34"
+      - docker_hub_index_job
+
+

.circleci/verbatim-sources/workflows-nightly-android-binary-builds.yml

@@ -3,7 +3,7 @@
           build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32"
           requires:
             - setup
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:f990c76a-a798-42bb-852f-5be5006f8026"
           filters:
             branches:
               only: nightly
@@ -12,7 +12,7 @@
           build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64"
           requires:
             - setup
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:f990c76a-a798-42bb-852f-5be5006f8026"
           filters:
             branches:
               only: nightly
@@ -21,7 +21,7 @@
           build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a"
           requires:
             - setup
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:f990c76a-a798-42bb-852f-5be5006f8026"
           filters:
             branches:
               only: nightly
@@ -30,7 +30,7 @@
           build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a"
           requires:
             - setup
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:f990c76a-a798-42bb-852f-5be5006f8026"
           filters:
             branches:
               only: nightly

.circleci/verbatim-sources/workflows-nightly-uploads-header.yml

@@ -4,8 +4,5 @@
       #- binary_linux_libtorch_2.7m_cu90_test:
       #    requires:
       #      - binary_linux_libtorch_2.7m_cu90_build
-      #- binary_linux_libtorch_2.7m_cu100_test:
-      #    requires:
-      #      - binary_linux_libtorch_2.7m_cu100_build
 
       # Nightly uploads

.circleci/verbatim-sources/workflows-pytorch-android-gradle-build.yml

@@ -1,10 +1,18 @@
       - pytorch_android_gradle_build-x86_32:
           name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32
+          filters:
+            branches:
+              only:
+                - master
           requires:
             - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build
 
       - pytorch_android_gradle_build:
           name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build
+          filters:
+            branches:
+              only:
+                - master
           requires:
             - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build
             - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build

.circleci/verbatim-sources/workflows-pytorch-ge-config-tests.yml

@@ -4,7 +4,7 @@
             - setup
             - pytorch_linux_xenial_py3_6_gcc5_4_build
           build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_legacy-test"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:405"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:f990c76a-a798-42bb-852f-5be5006f8026"
           resource_class: large
       - pytorch_linux_test:
           name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_simple_test
@@ -12,5 +12,5 @@
             - setup
             - pytorch_linux_xenial_py3_6_gcc5_4_build
           build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_simple-test"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:405"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:f990c76a-a798-42bb-852f-5be5006f8026"
           resource_class: large

.circleci/verbatim-sources/workflows-pytorch-ios-builds.yml

@@ -1,7 +1,6 @@
       # Pytorch iOS PR builds
       - pytorch_ios_build:
           name: pytorch_ios_11_2_1_x86_64_build
-          context: org-member
           build_environment: "pytorch-ios-11.2.1-x86_64_build"
           ios_arch: "x86_64"
           ios_platform: "SIMULATOR"
@@ -15,3 +14,13 @@
           ios_platform: "OS"
           requires:
             - setup
+      - pytorch_ios_build:
+          name: pytorch_ios_11_2_1_arm64_custom_build
+          context: org-member
+          build_environment: "pytorch-ios-11.2.1-arm64_custom_build"
+          ios_arch: "arm64"
+          ios_platform: "OS"
+          op_list: "mobilenetv2.yaml"
+          requires:
+            - setup
+

.circleci/verbatim-sources/workflows-pytorch-macos-builds.yml

@@ -8,6 +8,3 @@
           requires:
             - setup
             - pytorch_macos_10_13_py3_build
-      - pytorch_macos_10_13_cuda9_2_cudnn7_py3_build:
-          requires:
-            - setup

.circleci/verbatim-sources/workflows-pytorch-mobile-builds.yml

@@ -4,4 +4,34 @@
           requires:
             - setup
           build_environment: "pytorch-linux-xenial-py3-clang5-mobile-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:405"
+          build_only: "1"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:f990c76a-a798-42bb-852f-5be5006f8026"
+      - pytorch_linux_build:
+          name: pytorch_linux_xenial_py3_clang5_mobile_custom_build_static
+          requires:
+            - setup
+          build_environment: "pytorch-linux-xenial-py3-clang5-mobile-custom-build-static"
+          build_only: "1"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:f990c76a-a798-42bb-852f-5be5006f8026"
+      - pytorch_linux_build:
+          name: pytorch_linux_xenial_py3_clang5_mobile_custom_build_dynamic
+          requires:
+            - setup
+          build_environment: "pytorch-linux-xenial-py3-clang5-mobile-custom-build-dynamic"
+          build_only: "1"
+          # Use LLVM-DEV toolchain in android-ndk-r19c docker image
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:f990c76a-a798-42bb-852f-5be5006f8026"
+      - pytorch_linux_build:
+          name: pytorch_linux_xenial_py3_clang5_mobile_code_analysis
+          requires:
+            - setup
+          # Most of this CI is already covered by "mobile-custom-build-dynamic" job
+          filters:
+            branches:
+              only:
+                - master
+                - /ci-all\/.*/
+          build_environment: "pytorch-linux-xenial-py3-clang5-mobile-code-analysis"
+          build_only: "1"
+          # Use LLVM-DEV toolchain in android-ndk-r19c docker image
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:f990c76a-a798-42bb-852f-5be5006f8026"

.circleci/verbatim-sources/workflows-setup-job.yml

@@ -0,0 +1,8 @@
+      - setup:
+          # Run this job on everything since it is
+          # the dependency for everything.
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: /.*/

.flake8

@@ -8,6 +8,6 @@ ignore =
     # these ignores are from flake8-bugbear; please fix!
     B007,B008,
     # these ignores are from flake8-comprehensions; please fix!
-    C400,C401,C402,C403,C404,C405,C407,C411,
+    C400,C401,C402,C403,C404,C405,C407,C411,C413,C414,C415
 per-file-ignores = __init__.py: F401
 exclude = docs/src,venv,third_party,caffe2,scripts,docs/caffe2,torch/lib/include,torch/lib/tmp_install,build,torch/include,*.pyi,.git

.github/workflows/lint.yml

@@ -22,7 +22,9 @@ jobs:
           pip install -r requirements.txt
           cd .circleci && ./ensure-consistency.py
       - name: Ensure Docker version is correctly deployed
-        run: .circleci/validate-docker-version.py
+        run: |
+          pip install pyyaml
+          .circleci/validate-docker-version.py
       - name: Shellcheck Jenkins scripts
         run: |
           sudo apt-get install -y shellcheck
@@ -65,7 +67,8 @@ jobs:
       - name: Run flake8
         run: |
           set -eux
-          pip install flake8
+          pip install flake8 flake8-mypy flake8-bugbear flake8-comprehensions flake8-executable flake8-pyi mccabe pycodestyle pyflakes
+          flake8 --version
           flake8 --exit-zero > ${GITHUB_WORKSPACE}/flake8-output.txt
           cat ${GITHUB_WORKSPACE}/flake8-output.txt
       - name: Add annotations
@@ -102,7 +105,7 @@ jobs:
         run: |
           set -eux
           pip install flake8
-          rm -rf .circleci
+          rm -rf .circleci tools/clang_format_new.py
           flake8 --exit-zero > ${GITHUB_WORKSPACE}/flake8-output.txt
           cat ${GITHUB_WORKSPACE}/flake8-output.txt
       - name: Add annotations
@@ -150,7 +153,7 @@ jobs:
           # Install dependencies
           pip install pyyaml
           wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-          sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main"
+          sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main"
           sudo apt-get update
           sudo apt-get install -y clang-tidy-8
           sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-8 1000
@@ -175,7 +178,6 @@ jobs:
               -s aten/src/ATen \
               -d build/aten/src/ATen \
               aten/src/ATen/Declarations.cwrap \
-              aten/src/THNN/generic/THNN.h \
               aten/src/THCUNN/generic/THCUNN.h \
               aten/src/ATen/nn.yaml \
               aten/src/ATen/native/native_functions.yaml

.gitignore

@@ -57,11 +57,6 @@ torch/csrc/jit/generated/*
 torch/csrc/jit/fuser/config.h
 torch/csrc/nn/THCUNN.cpp
 torch/csrc/nn/THCUNN.cwrap
-torch/csrc/nn/THNN_generic.cpp
-torch/csrc/nn/THNN_generic.cwrap
-torch/csrc/nn/THNN_generic.h
-torch/csrc/nn/THNN.cpp
-torch/csrc/nn/THNN.cwrap
 torch/bin/
 torch/cmake/
 torch/lib/*.a*
@@ -250,3 +245,13 @@ GSYMS
 GPATH
 tags
 TAGS
+
+
+# ccls file
+.ccls-cache/
+
+# clang-format storage location used by apply_clang_format.py
+.clang-format-bin
+
+# clangd background index
+.clangd/

.gitmodules

@@ -111,10 +111,14 @@
     path = third_party/foxi
     url = https://github.com/houseroad/foxi.git
 [submodule "third_party/tbb"]
-	path = third_party/tbb
-	url = https://github.com/01org/tbb
-	branch = tbb_2018
+    path = third_party/tbb
+    url = https://github.com/01org/tbb
+    branch = tbb_2018
 [submodule "android/libs/fbjni"]
     ignore = dirty
     path = android/libs/fbjni
     url = https://github.com/facebookincubator/fbjni.git
+[submodule "third_party/XNNPACK"]
+    ignore = dirty
+    path = third_party/XNNPACK
+    url = https://github.com/google/XNNPACK.git

.jenkins/caffe2/build.sh

@@ -104,7 +104,7 @@ if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
   build_args+=("BUILD_TEST=ON")
   build_args+=("USE_OBSERVERS=ON")
   build_args+=("USE_ZSTD=ON")
-  "${ROOT_DIR}/scripts/build_android.sh" $(build_to_cmake ${build_args[@]}) "$@"
+  BUILD_CAFFE2_MOBILE=1 "${ROOT_DIR}/scripts/build_android.sh" $(build_to_cmake ${build_args[@]}) "$@"
   exit 0
 fi
 
@@ -130,7 +130,7 @@ if [[ $BUILD_ENVIRONMENT == *py2-cuda9.0-cudnn7-ubuntu16.04* ]]; then
   # removing http:// duplicate in favor of nvidia-ml.list
   # which is https:// version of the same repo
   sudo rm -f /etc/apt/sources.list.d/nvidia-machine-learning.list
-  curl -o ./nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda9.0_1-1_amd64.deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda9.0_1-1_amd64.deb
+  curl --retry 3 -o ./nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda9.0_1-1_amd64.deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda9.0_1-1_amd64.deb
   sudo dpkg -i ./nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda9.0_1-1_amd64.deb
   sudo apt-key add /var/nvinfer-runtime-trt-repo-5.0.2-ga-cuda9.0/7fa2af80.pub
   sudo apt-get -qq update
@@ -175,18 +175,12 @@ if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
   ${PYTHON} "${ROOT_DIR}/tools/amd_build/build_amd.py"
 fi
 
-# building bundled nccl in this config triggers a bug in nvlink. For
-# more, see https://github.com/pytorch/pytorch/issues/14486
-if [[ "${BUILD_ENVIRONMENT}" == *-cuda8*-cudnn7* ]]; then
-    build_args+=("USE_SYSTEM_NCCL=ON")
-fi
-
 # Try to include Redis support for Linux builds
 if [ "$(uname)" == "Linux" ]; then
   build_args+=("USE_REDIS=ON")
 fi
 
-# Use a speciallized onnx namespace in CI to catch hardcoded onnx namespace
+# Use a specialized onnx namespace in CI to catch hardcoded onnx namespace
 build_args+=("ONNX_NAMESPACE=ONNX_NAMESPACE_FOR_C2_CI")
 
 ###############################################################################

.jenkins/caffe2/test.sh

@@ -40,6 +40,9 @@ for test in $(find "$cpp_test_dir" -executable -type f); do
         LD_LIBRARY_PATH="$ld_library_path" "$test"
       fi
       ;;
+    */*_benchmark)
+      LD_LIBRARY_PATH="$ld_library_path" "$test" --benchmark_color=false
+      ;;
     *)
       # Currently, we use a mixture of gtest (caffe2) and Catch2 (ATen). While
       # planning to migrate to gtest as the common PyTorch c++ test suite, we
@@ -82,7 +85,7 @@ fi
 EXTRA_TESTS=()
 
 # CUDA builds always include NCCL support
-if [[ "$BUILD_ENVIRONMENT" == *-cuda* ]]; then
+if [[ "$BUILD_ENVIRONMENT" == *-cuda* ]] || [[ "$BUILD_ENVIRONMENT" == *-rocm* ]]; then
   EXTRA_TESTS+=("$caffe2_pypath/contrib/nccl")
 fi
 

.jenkins/pytorch/build-mobile-code-analysis.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# DO NOT ADD 'set -x' not to reveal CircleCI secret context environment variables
+set -eu -o pipefail
+
+# This script builds and runs code analyzer tool to generate aten op dependency
+# graph for custom mobile build.
+
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+echo "Clang version:"
+clang --version
+
+export LLVM_DIR="$(llvm-config-5.0 --prefix)"
+echo "LLVM_DIR: ${LLVM_DIR}"
+
+# Run the following 2 steps together because they share the same (reusable) time
+# consuming process to build LibTorch into LLVM assembly.
+
+# 1. Run code analysis test first to fail fast
+time ANALYZE_TEST=1 CHECK_RESULT=1 tools/code_analyzer/build.sh
+
+# 2. Run code analysis on mobile LibTorch
+time ANALYZE_TORCH=1 tools/code_analyzer/build.sh -closure=false

.jenkins/pytorch/build-mobile.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# DO NOT ADD 'set -x' not to reveal CircleCI secret context environment variables
+set -eu -o pipefail
+
+# This script uses linux host toolchain + mobile build options in order to
+# build & test mobile libtorch without having to setup Android/iOS
+# toolchain/simulator.
+
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+# Install torch & torchvision - used to download & trace test model.
+pip install torch torchvision --progress-bar off
+
+# Run end-to-end process of building mobile library, linking into the predictor
+# binary, and running forward pass with a real model.
+if [[ "$BUILD_ENVIRONMENT" == *-mobile-custom-build-static* ]]; then
+  TEST_CUSTOM_BUILD_STATIC=1 test/mobile/custom_build/build.sh
+elif [[ "$BUILD_ENVIRONMENT" == *-mobile-custom-build-dynamic* ]]; then
+  export LLVM_DIR="$(llvm-config-5.0 --prefix)"
+  echo "LLVM_DIR: ${LLVM_DIR}"
+  TEST_CUSTOM_BUILD_DYNAMIC=1 test/mobile/custom_build/build.sh
+else
+  TEST_DEFAULT_BUILD=1 test/mobile/custom_build/build.sh
+fi

.jenkins/pytorch/build.sh

@@ -14,13 +14,13 @@ source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 # (2) build with NCCL and MPI
 # (3) build with only MPI
 # (4) build with neither
-if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]]; then
+if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda10.1-* ]]; then
   # TODO: move this to Docker
   sudo apt-get -qq update
-  sudo apt-get -qq install --allow-downgrades --allow-change-held-packages libnccl-dev=2.2.13-1+cuda9.0 libnccl2=2.2.13-1+cuda9.0
+  sudo apt-get -qq install --allow-downgrades --allow-change-held-packages libnccl-dev=2.5.6-1+cuda10.1 libnccl2=2.5.6-1+cuda10.1
 fi
 
-if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9*gcc7* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]] || [[ "$BUILD_ENVIRONMENT" == *-trusty-py2.7.9* ]]; then
+if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9*gcc7* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda10.1-* ]] || [[ "$BUILD_ENVIRONMENT" == *-trusty-py2.7.9* ]]; then
   # TODO: move this to Docker
   sudo apt-get -qq update
   if [[ "$BUILD_ENVIRONMENT" == *-trusty-py2.7.9* ]]; then
@@ -36,10 +36,12 @@ if [[ "$BUILD_ENVIRONMENT" == *-linux-xenial-py3-clang5-asan* ]]; then
   exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" "$@"
 fi
 
-if [[ "$BUILD_ENVIRONMENT" == *-linux-xenial-py3-clang5-mobile* ]]; then
-  # Use linux host toolchain + mobile build options in order to build & test
-  # mobile libtorch without having to setup Android/iOS toolchain/simulator.
-  exec ./scripts/build_mobile.sh -DBUILD_BINARY=ON "$@"
+if [[ "$BUILD_ENVIRONMENT" == *-mobile-*build* ]]; then
+  exec "$(dirname "${BASH_SOURCE[0]}")/build-mobile.sh" "$@"
+fi
+
+if [[ "$BUILD_ENVIRONMENT" == *-mobile-code-analysis* ]]; then
+  exec "$(dirname "${BASH_SOURCE[0]}")/build-mobile-code-analysis.sh" "$@"
 fi
 
 echo "Python version:"
@@ -51,6 +53,11 @@ gcc --version
 echo "CMake version:"
 cmake --version
 
+if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
+  echo "NVCC version:"
+  nvcc --version
+fi
+
 # TODO: Don't run this...
 pip_install -r requirements.txt || true
 
@@ -59,7 +66,7 @@ if ! which conda; then
   # In ROCm CIs, we are doing cross compilation on build machines with
   # intel cpu and later run tests on machines with amd cpu.
   # Also leave out two builds to make sure non-mkldnn builds still work.
-  if [[ "$BUILD_ENVIRONMENT" != *rocm* && "$BUILD_ENVIRONMENT" != *-trusty-py3.5-* && "$BUILD_ENVIRONMENT" != *-xenial-cuda9-cudnn7-py3-* ]]; then
+  if [[ "$BUILD_ENVIRONMENT" != *rocm* && "$BUILD_ENVIRONMENT" != *-trusty-py3.5-* && "$BUILD_ENVIRONMENT" != *-xenial-cuda10.1-cudnn7-py3-* ]]; then
     pip_install mkl mkl-devel
     export USE_MKLDNN=1
   else
@@ -98,7 +105,6 @@ if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
   elif [[ "${BUILD_ENVIRONMENT}" == *-x86_64* ]]; then
     build_args+=("-DANDROID_ABI=x86_64")
   fi
-  export BUILD_PYTORCH_MOBILE=1
   exec ./scripts/build_android.sh "${build_args[@]}" "$@"
 fi
 
@@ -198,16 +204,6 @@ if [[ "$BUILD_ENVIRONMENT" != *libtorch* ]]; then
 
   assert_git_not_dirty
 
-  # Test documentation build
-  if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda9-cudnn7-py3* ]]; then
-    pushd docs
-    # TODO: Don't run this here
-    pip_install -r requirements.txt || true
-    LC_ALL=C make html
-    popd
-    assert_git_not_dirty
-  fi
-
   # Build custom operator tests.
   CUSTOM_OP_BUILD="$PWD/../custom-op-build"
   CUSTOM_OP_TEST="$PWD/test/custom_operator"
@@ -221,7 +217,7 @@ if [[ "$BUILD_ENVIRONMENT" != *libtorch* ]]; then
   assert_git_not_dirty
 else
   # Test standalone c10 build
-  if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda9-cudnn7-py3* ]]; then
+  if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda10.1-cudnn7-py3* ]]; then
     mkdir -p c10/build
     pushd c10/build
     cmake ..
@@ -248,30 +244,24 @@ if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
   pip_install lark-parser
 
   # Bazel doesn't work with sccache gcc. https://github.com/bazelbuild/bazel/issues/3642
-  sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main"
+  sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main"
   wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
   sudo apt-get -qq update
 
-  # Install clang-7 clang++-7 for xla
-  sudo apt-get -qq install clang-7 clang++-7
+  # Install clang-8 clang++-8 for xla
+  sudo apt-get -qq install clang-8 clang++-8
 
-  # Bazel dependencies
-  sudo apt-get -qq install pkg-config zip zlib1g-dev unzip
-  # XLA build requires Bazel
-  wget https://github.com/bazelbuild/bazel/releases/download/0.24.1/bazel-0.24.1-installer-linux-x86_64.sh
-  chmod +x bazel-*.sh
-  sudo ./bazel-*.sh
-  BAZEL="$(which bazel)"
-  if [ -z "${BAZEL}" ]; then
-    echo "Unable to find bazel..."
-    exit 1
-  fi
-
-  # Install bazels3cache for cloud cache
   sudo apt-get -qq install npm
   npm config set strict-ssl false
-  curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -
+  curl -sL --retry 3 https://deb.nodesource.com/setup_6.x | sudo -E bash -
   sudo apt-get install -qq nodejs
+
+  # XLA build requires Bazel
+  # We use bazelisk to avoid updating Bazel version manually.
+  sudo npm install -g @bazel/bazelisk
+  sudo ln -s "$(command -v bazelisk)" /usr/bin/bazel
+
+  # Install bazels3cache for cloud cache
   sudo npm install -g bazels3cache
   BAZELS3CACHE="$(which bazels3cache)"
   if [ -z "${BAZELS3CACHE}" ]; then
@@ -281,7 +271,7 @@ if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
 
   bazels3cache --bucket=${XLA_CLANG_CACHE_S3_BUCKET_NAME} --maxEntrySizeBytes=0
   pushd xla
-  export CC=clang-7 CXX=clang++-7
+  export CC=clang-8 CXX=clang++-8
   # Use cloud cache to build when available.
   sed -i '/bazel build/ a --remote_http_cache=http://localhost:7777 \\' build_torch_xla_libs.sh
 

.jenkins/pytorch/common.sh

@@ -128,7 +128,7 @@ if [ -z "$COMPACT_JOB_NAME" ]; then
   exit 1
 fi
 
-if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-cuda9-cudnn7-py3* ]] || \
+if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-cuda10.1-cudnn7-py3* ]] || \
    [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-trusty-py3.6-gcc7* ]] || \
    [[ "$BUILD_ENVIRONMENT" == *pytorch_macos* ]]; then
   BUILD_TEST_LIBTORCH=1
@@ -140,7 +140,7 @@ fi
 # min version 3.5, so we only do it in two builds that we know should use conda.
 if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-cuda* ]]; then
   if [[ "$BUILD_ENVIRONMENT" == *cuda9-cudnn7-py2* ]] || \
-     [[ "$BUILD_ENVIRONMENT" == *cuda9-cudnn7-py3* ]]; then
+     [[ "$BUILD_ENVIRONMENT" == *cuda10.1-cudnn7-py3* ]]; then
     if ! which conda; then
       echo "Expected ${BUILD_ENVIRONMENT} to use conda, but 'which conda' returns empty"
       exit 1
@@ -179,3 +179,11 @@ function get_exit_code() {
   set -e
   return $retcode
 }
+
+function file_diff_from_base() {
+  # The fetch may fail on Docker hosts, but it's not always necessary.
+  set +e
+  git fetch origin master --quiet
+  set -e
+  git diff --name-only "$(git merge-base origin master HEAD)" > "$1"
+}

.jenkins/pytorch/macos-common.sh

@@ -13,7 +13,7 @@ mkdir -p ${WORKSPACE_DIR}
 # If a local installation of conda doesn't exist, we download and install conda
 if [ ! -d "${WORKSPACE_DIR}/miniconda3" ]; then
   mkdir -p ${WORKSPACE_DIR}
-  retry curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ${WORKSPACE_DIR}/miniconda3.sh
+  curl --retry 3 https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ${WORKSPACE_DIR}/miniconda3.sh
   retry bash ${WORKSPACE_DIR}/miniconda3.sh -b -p ${WORKSPACE_DIR}/miniconda3
 fi
 export PATH="${WORKSPACE_DIR}/miniconda3/bin:$PATH"

.jenkins/pytorch/macos-test.sh

@@ -54,7 +54,14 @@ test_python_all() {
   # using the address associated with the loopback interface.
   export GLOO_SOCKET_IFNAME=lo0
   echo "Ninja version: $(ninja --version)"
-  python test/run_test.py --verbose
+
+  if [ -n "$CIRCLE_PULL_REQUEST" ]; then
+    DETERMINE_FROM=$(mktemp)
+    file_diff_from_base "$DETERMINE_FROM"
+  fi
+
+  python test/run_test.py --verbose --determine-from="$DETERMINE_FROM"
+
   assert_git_not_dirty
 }
 
@@ -102,7 +109,6 @@ test_custom_script_ops() {
 
   # Run tests Python-side and export a script module.
   python test_custom_ops.py -v
-  python test_custom_classes.py -v
   python model.py --export-script-module=model.pt
   # Run tests C++-side and load the exported script module.
   build/test_custom_ops ./model.pt

.jenkins/pytorch/multigpu-test.sh

@@ -14,13 +14,13 @@ if [ -n "${IN_CIRCLECI}" ]; then
   # TODO move this to docker
   pip_install unittest-xml-reporting
 
-  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]]; then
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda10.1-* ]]; then
     # TODO: move this to Docker
     sudo apt-get update
-    sudo apt-get install -y --allow-downgrades --allow-change-held-packages libnccl-dev=2.2.13-1+cuda9.0 libnccl2=2.2.13-1+cuda9.0
+    sudo apt-get install -y --allow-downgrades --allow-change-held-packages libnccl-dev=2.5.6-1+cuda10.1 libnccl2=2.5.6-1+cuda10.1
   fi
 
-  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda8-* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]]; then
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]]; then
     # TODO: move this to Docker
     sudo apt-get update
     sudo apt-get install -y --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
@@ -31,7 +31,7 @@ fi
 
 python tools/download_mnist.py --quiet -d test/cpp/api/mnist
 OMP_NUM_THREADS=2 TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" build/bin/test_api
-time python test/run_test.py --verbose -i distributed
-time python test/run_test.py --verbose -i c10d
-time python test/run_test.py --verbose -i c10d_spawn
+time python test/run_test.py --verbose -i distributed/test_distributed
+time python test/run_test.py --verbose -i distributed/test_c10d
+time python test/run_test.py --verbose -i distributed/test_c10d_spawn
 assert_git_not_dirty

.jenkins/pytorch/test.sh

@@ -15,13 +15,13 @@ if [ -n "${IN_CIRCLECI}" ]; then
   # TODO move this to docker
   pip_install unittest-xml-reporting
 
-  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]]; then
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda10.1-* ]]; then
     # TODO: move this to Docker
     sudo apt-get -qq update
-    sudo apt-get -qq install --allow-downgrades --allow-change-held-packages libnccl-dev=2.2.13-1+cuda9.0 libnccl2=2.2.13-1+cuda9.0
+    sudo apt-get -qq install --allow-downgrades --allow-change-held-packages libnccl-dev=2.5.6-1+cuda10.1 libnccl2=2.5.6-1+cuda10.1
   fi
 
-  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda8-* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]]; then
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]]; then
     # TODO: move this to Docker
     sudo apt-get -qq update
     sudo apt-get -qq install --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
@@ -37,6 +37,7 @@ fi
 
 if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
   # TODO: Move this to Docker
+  sudo apt-get -qq install --no-install-recommends apt-transport-https ca-certificates
   sudo apt-get -qq update
   sudo apt-get -qq install --no-install-recommends libsndfile1
 fi
@@ -48,7 +49,8 @@ if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]]; then
   # ninja is installed in /var/lib/jenkins/.local/bin
   export PATH="/var/lib/jenkins/.local/bin:$PATH"
 
-  # TODO: move this to Docker
+  # TODO: Please move this to Docker
+  # The version is fixed to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136
   pip_install --user "hypothesis==4.53.2"
 
   # TODO: move this to Docker
@@ -74,27 +76,49 @@ fi
 # if you're not careful.  Check this if you made some changes and the
 # ASAN test is not working
 if [[ "$BUILD_ENVIRONMENT" == *asan* ]]; then
+    # Suppress vptr violations arising from multiple copies of pybind11
     export ASAN_OPTIONS=detect_leaks=0:symbolize=1:strict_init_order=true
-    # We suppress the vptr volation, since we have separate copies of
-    # libprotobuf in both libtorch.so and libcaffe2.so, and it causes
-    # the following problem:
-    #    test_cse (__main__.TestJit) ... torch/csrc/jit/export.cpp:622:38:
-    #        runtime error: member call on address ... which does not point
-    #        to an object of type 'google::protobuf::MessageLite'
-    #        ...: note: object is of type 'onnx_torch::ModelProto'
-    #
-    # This problem should be solved when libtorch.so and libcaffe2.so are
-    # merged.
     export UBSAN_OPTIONS=print_stacktrace=1:suppressions=$PWD/ubsan.supp
     export PYTORCH_TEST_WITH_ASAN=1
     export PYTORCH_TEST_WITH_UBSAN=1
     # TODO: Figure out how to avoid hard-coding these paths
     export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-5.0/bin/llvm-symbolizer
+    export TORCH_USE_RTLD_GLOBAL=1
+    # NB: We load libtorch.so with RTLD_GLOBAL for UBSAN, unlike our
+    # default behavior.
+    #
+    # The reason for this is that without RTLD_GLOBAL, if we load multiple
+    # libraries that depend on libtorch (as is the case with C++ extensions), we
+    # will get multiple copies of libtorch in our address space.  When UBSAN is
+    # turned on, it will do a bunch of virtual pointer consistency checks which
+    # won't work correctly.  When this happens, you get a violation like:
+    #
+    #    member call on address XXXXXX which does not point to an object of
+    #    type 'std::_Sp_counted_base<__gnu_cxx::_Lock_policy::_S_atomic>'
+    #    XXXXXX note: object is of type
+    #    'std::_Sp_counted_ptr<torch::nn::LinearImpl*, (__gnu_cxx::_Lock_policy)2>'
+    #
+    # (NB: the textual types of the objects here are misleading, because
+    # they actually line up; it just so happens that there's two copies
+    # of the type info floating around in the address space, so they
+    # don't pointer compare equal.  See also
+    #   https://github.com/google/sanitizers/issues/1175
+    #
+    # UBSAN is kind of right here: if we relied on RTTI across C++ extension
+    # modules they would indeed do the wrong thing;  but in our codebase, we
+    # don't use RTTI (because it doesn't work in mobile).  To appease
+    # UBSAN, however, it's better if we ensure all the copies agree!
+    #
+    # By the way, an earlier version of this code attempted to load
+    # libtorch_python.so with LD_PRELOAD, which has a similar effect of causing
+    # it to be loaded globally.  This isn't really a good idea though, because
+    # it depends on a ton of dynamic libraries that most programs aren't gonna
+    # have, and it applies to child processes.
     export LD_PRELOAD=/usr/lib/llvm-5.0/lib/clang/5.0.0/lib/linux/libclang_rt.asan-x86_64.so
     # Increase stack size, because ASAN red zones use more stack
     ulimit -s 81920
 
-    (cd test && python -c "import torch")
+    (cd test && python -c "import torch; print(torch.__version__, torch.version.git_version)")
     echo "The next three invocations are expected to crash; if they don't that means ASAN/UBSAN is misconfigured"
     (cd test && ! get_exit_code python -c "import torch; torch._C._crash_if_csrc_asan(3)")
     (cd test && ! get_exit_code python -c "import torch; torch._C._crash_if_csrc_ubsan(0)")
@@ -107,23 +131,28 @@ elif [[ "${BUILD_ENVIRONMENT}" == *-NO_AVX2-* ]]; then
   export ATEN_CPU_CAPABILITY=avx
 fi
 
+if [ -n "$CIRCLE_PULL_REQUEST" ]; then
+  DETERMINE_FROM=$(mktemp)
+  file_diff_from_base "$DETERMINE_FROM"
+fi
+
 test_python_nn() {
-  time python test/run_test.py --include nn --verbose
+  time python test/run_test.py --include test_nn --verbose --determine-from="$DETERMINE_FROM"
   assert_git_not_dirty
 }
 
 test_python_ge_config_simple() {
-  time python test/run_test.py --include jit_simple --verbose
+  time python test/run_test.py --include test_jit_simple --verbose --determine-from="$DETERMINE_FROM"
   assert_git_not_dirty
 }
 
 test_python_ge_config_legacy() {
-  time python test/run_test.py --include jit_legacy jit_fuser_legacy --verbose
+  time python test/run_test.py --include test_jit_legacy test_jit_fuser_legacy --verbose --determine-from="$DETERMINE_FROM"
   assert_git_not_dirty
 }
 
 test_python_all_except_nn() {
-  time python test/run_test.py --exclude nn jit_simple jit_legacy jit_fuser_legacy --verbose --bring-to-front quantization quantized quantized_tensor quantized_nn_mods
+  time python test/run_test.py --exclude test_nn test_jit_simple test_jit_legacy test_jit_fuser_legacy --verbose --bring-to-front test_quantization test_quantized test_quantized_tensor test_quantized_nn_mods --determine-from="$DETERMINE_FROM"
   assert_git_not_dirty
 }
 
@@ -153,7 +182,7 @@ test_aten() {
 }
 
 test_torchvision() {
-  pip_install --user git+https://github.com/pytorch/vision.git@44a5bae933655ed7ff798669a43452b833f9ce01
+  pip_install --user git+https://github.com/pytorch/vision.git@43e94b39bcdda519c093ca11d99dfa2568aa7258
 }
 
 test_libtorch() {
@@ -180,7 +209,6 @@ test_custom_script_ops() {
     cp -a "$CUSTOM_OP_BUILD" build
     # Run tests Python-side and export a script module.
     python test_custom_ops.py -v
-    python test_custom_classes.py -v
     python model.py --export-script-module=model.pt
     # Run tests C++-side and load the exported script module.
     build/test_custom_ops ./model.pt
@@ -191,7 +219,9 @@ test_custom_script_ops() {
 
 test_xla() {
   export XLA_USE_XRT=1 XRT_DEVICE_MAP="CPU:0;/job:localservice/replica:0/task:0/device:XLA_CPU:0"
-  export XRT_WORKERS="localservice:0;grpc://localhost:40934"
+  # Issue #30717: randomize the port of XLA/gRPC workers is listening on to reduce flaky tests.
+  XLA_PORT=`shuf -i 40701-40999 -n 1`
+  export XRT_WORKERS="localservice:0;grpc://localhost:$XLA_PORT"
   pushd xla
   echo "Running Python Tests"
   ./test/run_tests.sh
@@ -214,7 +244,7 @@ test_backward_compatibility() {
   pushd test/backward_compatibility
   python dump_all_function_schemas.py --filename new_schemas.txt
   pip_uninstall torch
-  pip_install torch==1.3.1+cpu -f https://download.pytorch.org/whl/torch_stable.html
+  pip_install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
   python check_backward_compatibility.py --new-schemas new_schemas.txt
   popd
   set +x
@@ -240,9 +270,9 @@ elif [[ "${BUILD_ENVIRONMENT}" == *libtorch* ]]; then
   # TODO: run some C++ tests
   echo "no-op at the moment"
 elif [[ "${BUILD_ENVIRONMENT}" == *-test1 || "${JOB_BASE_NAME}" == *-test1 ]]; then
-  test_torchvision
   test_python_nn
 elif [[ "${BUILD_ENVIRONMENT}" == *-test2 || "${JOB_BASE_NAME}" == *-test2 ]]; then
+  test_torchvision
   test_python_all_except_nn
   test_aten
   test_libtorch

.jenkins/pytorch/win-build.sh

@@ -10,7 +10,7 @@ if [ ! -f setup.py ]; then
 fi
 
 # shellcheck disable=SC2034
-COMPACT_JOB_NAME=pytorch-win-ws2016-cuda9-cudnn7-py3-build
+COMPACT_JOB_NAME=pytorch-win-ws2019-cuda10-cudnn7-py3-build
 
 SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 source "$SCRIPT_PARENT_DIR/common.sh"

.jenkins/pytorch/win-test-helpers/build_pytorch.bat

@@ -4,7 +4,7 @@ if "%DEBUG%" == "1" (
   set BUILD_TYPE=release
 )
 
-set PATH=C:\Program Files\CMake\bin;C:\Program Files\7-Zip;C:\ProgramData\chocolatey\bin;C:\Program Files\Git\cmd;C:\Program Files\Amazon\AWSCLI;%PATH%
+set PATH=C:\Program Files\CMake\bin;C:\Program Files\7-Zip;C:\ProgramData\chocolatey\bin;C:\Program Files\Git\cmd;C:\Program Files\Amazon\AWSCLI;C:\Program Files\Amazon\AWSCLI\bin;%PATH%
 
 :: This inflates our log size slightly, but it is REALLY useful to be
 :: able to see what our cl.exe commands are (since you can actually
@@ -35,17 +35,28 @@ goto cuda_build_end
 
 :: Override VS env here
 pushd .
-call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" x64
+if "%VC_VERSION%" == "" (
+    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64
+) else (
+    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64 -vcvars_ver=%VC_VERSION%
+)
 @echo on
 popd
-set DISTUTILS_USE_SDK=1
 
-set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0
-set CUDA_PATH_V9_0=%CUDA_PATH%
+set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.2
+set CUDA_PATH_V9_2=%CUDA_PATH%
 
 goto cuda_build_common
 
 :cuda_build_10
+pushd .
+if "%VC_VERSION%" == "" (
+    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64
+) else (
+    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64 -vcvars_ver=%VC_VERSION%
+)
+@echo on
+popd
 
 set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1
 set CUDA_PATH_V10_1=%CUDA_PATH%
@@ -54,6 +65,8 @@ goto cuda_build_common
 
 :cuda_build_common
 
+set DISTUTILS_USE_SDK=1
+
 set CUDNN_LIB_DIR=%CUDA_PATH%\lib\x64
 set CUDA_TOOLKIT_ROOT_DIR=%CUDA_PATH%
 set CUDNN_ROOT_DIR=%CUDA_PATH%
@@ -64,14 +77,16 @@ set PATH=%CUDA_PATH%\bin;%CUDA_PATH%\libnvvp;%PATH%
 
 set PATH=%TMP_DIR_WIN%\bin;%PATH%
 
-:: Target only our CI GPU machine's CUDA arch to speed up the build
-set TORCH_CUDA_ARCH_LIST=5.2
+:: Target only our CI GPU machine's CUDA arch to speed up the build, we can overwrite with env var
+:: default on circleci is Tesla T4 which has capability of 7.5, ref: https://developer.nvidia.com/cuda-gpus
+:: jenkins has M40, which is 5.2
+if "%TORCH_CUDA_ARCH_LIST%" == "" set TORCH_CUDA_ARCH_LIST=5.2
 
 sccache --stop-server
 sccache --start-server
 sccache --zero-stats
-set CC=sccache cl
-set CXX=sccache cl
+set CC=sccache-cl
+set CXX=sccache-cl
 
 set CMAKE_GENERATOR=Ninja
 
@@ -107,7 +122,19 @@ if not "%USE_CUDA%"=="0" (
     copy %TMP_DIR_WIN%\bin\sccache.exe %TMP_DIR_WIN%\bin\nvcc.exe
   )
 
-  set CUDA_NVCC_EXECUTABLE=%TMP_DIR_WIN%\bin\nvcc
+  :: randomtemp is used to resolve the intermittent build error related to CUDA.
+  :: code: https://github.com/peterjc123/randomtemp
+  :: issue: https://github.com/pytorch/pytorch/issues/25393
+  ::
+  :: Previously, CMake uses CUDA_NVCC_EXECUTABLE for finding nvcc and then
+  :: the calls are redirected to sccache. sccache looks for the actual nvcc
+  :: in PATH, and then pass the arguments to it.
+  :: Currently, randomtemp is placed before sccache (%TMP_DIR_WIN%\bin\nvcc)
+  :: so we are actually pretending sccache instead of nvcc itself.
+  curl -kL https://github.com/peterjc123/randomtemp/releases/download/v0.2/randomtemp.exe --output %TMP_DIR_WIN%\bin\randomtemp.exe
+  set RANDOMTEMP_EXECUTABLE=%TMP_DIR_WIN%\bin\nvcc.exe
+  set CUDA_NVCC_EXECUTABLE=%TMP_DIR_WIN%\bin\randomtemp.exe
+  set RANDOMTEMP_BASEDIR=%TMP_DIR_WIN%\bin
 
   if "%REBUILD%"=="" set USE_CUDA=1
 

.jenkins/pytorch/win-test-helpers/installation-helpers/install_magma.bat

@@ -1,4 +1,4 @@
-if "%CUDA_VERSION%" == "9" set CUDA_SUFFIX=cuda90
+if "%CUDA_VERSION%" == "9" set CUDA_SUFFIX=cuda92
 if "%CUDA_VERSION%" == "10" set CUDA_SUFFIX=cuda101
 
 if "%CUDA_SUFFIX%" == "" (
@@ -8,10 +8,10 @@ if "%CUDA_SUFFIX%" == "" (
 
 if "%REBUILD%"=="" (
   if "%BUILD_ENVIRONMENT%"=="" (
-    curl -k https://s3.amazonaws.com/ossci-windows/magma_2.5.0_%CUDA_SUFFIX%_%BUILD_TYPE%.7z --output %TMP_DIR_WIN%\magma_2.5.0_%CUDA_SUFFIX%_%BUILD_TYPE%.7z
+    curl --retry 3 -k https://s3.amazonaws.com/ossci-windows/magma_2.5.2_%CUDA_SUFFIX%_%BUILD_TYPE%.7z --output %TMP_DIR_WIN%\magma_2.5.2_%CUDA_SUFFIX%_%BUILD_TYPE%.7z
   ) else (
-    aws s3 cp s3://ossci-windows/magma_2.5.0_%CUDA_SUFFIX%_%BUILD_TYPE%.7z %TMP_DIR_WIN%\magma_2.5.0_%CUDA_SUFFIX%_%BUILD_TYPE%.7z --quiet
+    aws s3 cp s3://ossci-windows/magma_2.5.2_%CUDA_SUFFIX%_%BUILD_TYPE%.7z %TMP_DIR_WIN%\magma_2.5.2_%CUDA_SUFFIX%_%BUILD_TYPE%.7z --quiet
   )
-  7z x -aoa %TMP_DIR_WIN%\magma_2.5.0_%CUDA_SUFFIX%_%BUILD_TYPE%.7z -o%TMP_DIR_WIN%\magma
+  7z x -aoa %TMP_DIR_WIN%\magma_2.5.2_%CUDA_SUFFIX%_%BUILD_TYPE%.7z -o%TMP_DIR_WIN%\magma
 )
 set MAGMA_HOME=%TMP_DIR_WIN%\magma

.jenkins/pytorch/win-test-helpers/installation-helpers/install_miniconda3.bat

@@ -5,11 +5,11 @@ if "%BUILD_ENVIRONMENT%"=="" (
 )
 if "%REBUILD%"=="" (
   IF EXIST %CONDA_PARENT_DIR%\Miniconda3 ( rd /s /q %CONDA_PARENT_DIR%\Miniconda3 )
-  curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe --output %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe
+  curl --retry 3 -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe --output %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe
   %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /AddToPath=0 /D=%CONDA_PARENT_DIR%\Miniconda3
 )
 call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Miniconda3
 if "%REBUILD%"=="" (
-  :: We have to pin Python version to 3.6.7, until mkl supports Python 3.7
-  call conda install -y -q python=3.6.7 numpy cffi pyyaml boto3
+  call conda install -y -q python=%PYTHON_VERSION% numpy cffi pyyaml boto3
+  call conda install -y -q -c conda-forge cmake
 )

.jenkins/pytorch/win-test-helpers/installation-helpers/install_mkl.bat

@@ -1,8 +1,8 @@
 if "%REBUILD%"=="" (
   if "%BUILD_ENVIRONMENT%"=="" (
-    curl -k https://s3.amazonaws.com/ossci-windows/mkl_2019.4.245.7z --output %TMP_DIR_WIN%\mkl.7z
+    curl --retry 3 -k https://s3.amazonaws.com/ossci-windows/mkl_2020.0.166.7z --output %TMP_DIR_WIN%\mkl.7z
   ) else (
-    aws s3 cp s3://ossci-windows/mkl_2019.4.245.7z %TMP_DIR_WIN%\mkl.7z --quiet
+    aws s3 cp s3://ossci-windows/mkl_2020.0.166.7z %TMP_DIR_WIN%\mkl.7z --quiet
   )
   7z x -aoa %TMP_DIR_WIN%\mkl.7z -o%TMP_DIR_WIN%\mkl
 )

.jenkins/pytorch/win-test-helpers/installation-helpers/install_sccache.bat

@@ -4,11 +4,14 @@ if "%REBUILD%"=="" (
   :check_sccache
   %TMP_DIR_WIN%\bin\sccache.exe --show-stats || (
     taskkill /im sccache.exe /f /t || ver > nul
-    del %TMP_DIR_WIN%\bin\sccache.exe
+    del %TMP_DIR_WIN%\bin\sccache.exe || ver > nul
+    del %TMP_DIR_WIN%\bin\sccache-cl.exe || ver > nul
     if "%BUILD_ENVIRONMENT%"=="" (
-      curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %TMP_DIR_WIN%\bin\sccache.exe
+      curl --retry 3 -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %TMP_DIR_WIN%\bin\sccache.exe
+      curl --retry 3 -k https://s3.amazonaws.com/ossci-windows/sccache-cl.exe --output %TMP_DIR_WIN%\bin\sccache-cl.exe
     ) else (
       aws s3 cp s3://ossci-windows/sccache.exe %TMP_DIR_WIN%\bin\sccache.exe
+      aws s3 cp s3://ossci-windows/sccache-cl.exe %TMP_DIR_WIN%\bin\sccache-cl.exe
     )
     goto :check_sccache
   )

.jenkins/pytorch/win-test-helpers/setup_pytorch_env.bat

@@ -3,7 +3,7 @@ if exist "%TMP_DIR%/ci_scripts/pytorch_env_restore.bat" (
     exit /b 0
 )
 
-set PATH=C:\Program Files\CMake\bin;C:\Program Files\7-Zip;C:\ProgramData\chocolatey\bin;C:\Program Files\Git\cmd;C:\Program Files\Amazon\AWSCLI;%PATH%
+set PATH=C:\Program Files\CMake\bin;C:\Program Files\7-Zip;C:\ProgramData\chocolatey\bin;C:\Program Files\Git\cmd;C:\Program Files\Amazon\AWSCLI;C:\Program Files\Amazon\AWSCLI\bin;%PATH%
 
 :: Install Miniconda3
 if "%BUILD_ENVIRONMENT%"=="" (
@@ -13,7 +13,7 @@ if "%BUILD_ENVIRONMENT%"=="" (
 )
 if NOT "%BUILD_ENVIRONMENT%"=="" (
     IF EXIST %CONDA_PARENT_DIR%\Miniconda3 ( rd /s /q %CONDA_PARENT_DIR%\Miniconda3 )
-    curl https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe --output %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe
+    curl --retry 3 https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe --output %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe
     %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /AddToPath=0 /D=%CONDA_PARENT_DIR%\Miniconda3
 )
 call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Miniconda3
@@ -21,8 +21,10 @@ if NOT "%BUILD_ENVIRONMENT%"=="" (
     :: We have to pin Python version to 3.6.7, until mkl supports Python 3.7
     :: Numba is pinned to 0.44.0 to avoid https://github.com/numba/numba/issues/4352
     call conda install -y -q python=3.6.7 numpy mkl cffi pyyaml boto3 protobuf numba==0.44.0
+    call conda install -y -q -c conda-forge cmake
 )
-pip install -q ninja future "hypothesis==4.53.2" "librosa>=0.6.2" psutil pillow
+:: The version is fixed to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136
+pip install ninja future "hypothesis==4.53.2" "librosa>=0.6.2" psutil pillow
 :: No need to install faulthandler since we only test Python >= 3.6 on Windows
 :: faulthandler is builtin since Python 3.3
 
@@ -33,19 +35,27 @@ goto cuda_build_end
 :cuda_build_9
 
 pushd .
-call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" x64
+if "%VC_VERSION%" == "" (
+    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64
+) else (
+    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64 -vcvars_ver=%VC_VERSION%
+)
 @echo on
 popd
 
-set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0
-set CUDA_PATH_V9_0=%CUDA_PATH%
+set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.2
+set CUDA_PATH_V9_2=%CUDA_PATH%
 
 goto cuda_build_common
 
 :cuda_build_10
 
 pushd .
-call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" x64
+if "%VC_VERSION%" == "" (
+    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64
+) else (
+    call "C:\Program Files (x86)\Microsoft Visual Studio\%VC_YEAR%\%VC_PRODUCT%\VC\Auxiliary\Build\vcvarsall.bat" x64 -vcvars_ver=%VC_VERSION%
+)
 @echo on
 popd
 
@@ -56,6 +66,7 @@ goto cuda_build_common
 
 :cuda_build_common
 
+set DISTUTILS_USE_SDK=1
 set CUDNN_LIB_DIR=%CUDA_PATH%\lib\x64
 set CUDA_TOOLKIT_ROOT_DIR=%CUDA_PATH%
 set CUDNN_ROOT_DIR=%CUDA_PATH%

.jenkins/pytorch/win-test-helpers/test_python_all_except_nn.bat

@@ -1,3 +1,3 @@
 call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
-cd test && python run_test.py --exclude nn jit_simple jit_legacy jit_fuser_legacy --verbose && cd ..
+cd test && python run_test.py --exclude test_nn test_jit_simple test_jit_legacy test_jit_fuser_legacy --verbose --determine-from="%1" && cd ..
 if ERRORLEVEL 1 exit /b 1

.jenkins/pytorch/win-test-helpers/test_python_nn.bat

@@ -8,7 +8,7 @@ python %SCRIPT_HELPERS_DIR%\run_python_nn_smoketests.py
 if ERRORLEVEL 1 exit /b 1
 
 echo Run nn tests
-python run_test.py --include nn --verbose
+python run_test.py --include test_nn --verbose --determine-from="%1"
 if ERRORLEVEL 1 exit /b 1
 
 popd

.jenkins/pytorch/win-test.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -ex
 
 # shellcheck disable=SC2034
-COMPACT_JOB_NAME=pytorch-win-ws2016-cuda9-cudnn7-py3-test
+COMPACT_JOB_NAME=pytorch-win-ws2019-cuda10-cudnn7-py3-test
 
 SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 source "$SCRIPT_PARENT_DIR/common.sh"
@@ -30,18 +30,22 @@ fi
 
 export SCRIPT_HELPERS_DIR=$SCRIPT_PARENT_DIR/win-test-helpers
 
+if [ -n "$CIRCLE_PULL_REQUEST" ]; then
+  DETERMINE_FROM="${TMP_DIR}/determine_from"
+  file_diff_from_base "$DETERMINE_FROM"
+fi
 
 run_tests() {
     if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
-        $SCRIPT_HELPERS_DIR/test_python_nn.bat && \
-        $SCRIPT_HELPERS_DIR/test_python_all_except_nn.bat && \
+        $SCRIPT_HELPERS_DIR/test_python_nn.bat "$DETERMINE_FROM" && \
+        $SCRIPT_HELPERS_DIR/test_python_all_except_nn.bat "$DETERMINE_FROM" && \
         $SCRIPT_HELPERS_DIR/test_custom_script_ops.bat && \
         $SCRIPT_HELPERS_DIR/test_libtorch.bat
     else
         if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
-            $SCRIPT_HELPERS_DIR/test_python_nn.bat
+            $SCRIPT_HELPERS_DIR/test_python_nn.bat "$DETERMINE_FROM"
         elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
-            $SCRIPT_HELPERS_DIR/test_python_all_except_nn.bat && \
+            $SCRIPT_HELPERS_DIR/test_python_all_except_nn.bat "$DETERMINE_FROM" && \
             $SCRIPT_HELPERS_DIR/test_custom_script_ops.bat && \
             $SCRIPT_HELPERS_DIR/test_libtorch.bat
         fi

CITATION

@@ -1,6 +1,10 @@
-@inproceedings{paszke2017automatic,
-  title={Automatic Differentiation in {PyTorch}},
-  author={Paszke, Adam and Gross, Sam and Chintala, Soumith and Chanan, Gregory and Yang, Edward and DeVito, Zachary and Lin, Zeming and Desmaison, Alban and Antiga, Luca and Lerer, Adam},
-  booktitle={NIPS Autodiff Workshop},
-  year={2017}
+@incollection{NEURIPS2019_9015,
+title = {PyTorch: An Imperative Style, High-Performance Deep Learning Library},
+author = {Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and Desmaison, Alban and Kopf, Andreas and Yang, Edward and DeVito, Zachary and Raison, Martin and Tejani, Alykhan and Chilamkurthy, Sasank and Steiner, Benoit and Fang, Lu and Bai, Junjie and Chintala, Soumith},
+booktitle = {Advances in Neural Information Processing Systems 32},
+editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
+pages = {8024--8035},
+year = {2019},
+publisher = {Curran Associates, Inc.},
+url = {http://papers.neurips.cc/paper/9015-pytorch-an-imperative-style-high-performance-deep-learning-library.pdf}
 }

CMakeLists.txt

@@ -30,7 +30,7 @@ endif()
 
 set(CMAKE_INSTALL_MESSAGE NEVER)
 
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 if (NOT MSVC)
   set(CMAKE_C_STANDARD 11)
 endif()
@@ -81,12 +81,24 @@ if(APPLE)
   set(CMAKE_MACOSX_RPATH ON)
 endif()
 
-if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "(x86_64|i[3-6]+86)")
-  set(CPU_INTEL ON)
+if (WIN32)
+  # On Windows, CMAKE_HOST_SYSTEM_PROCESSOR is calculated through `PROCESSOR_ARCHITECTURE`,
+  # which only has the value of `x86` or `AMD64`. We cannot infer whether it's a Intel CPU
+  # or not. However, the environment variable `PROCESSOR_IDENTIFIER` could be used.
+  if ($ENV{PROCESSOR_IDENTIFIER} MATCHES "Intel")
+    set(CPU_INTEL ON)
+  else ()
+    set(CPU_INTEL OFF)
+  endif ()
 else ()
-  set(CPU_INTEL OFF)
+  if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "(x86_64|i[3-6]+86)")
+    set(CPU_INTEL ON)
+  else ()
+    set(CPU_INTEL OFF)
+  endif ()
 endif ()
 
+
 # For non-supported platforms, turn USE_DISTRIBUTED off by default.
 # It is not tested and likely won't work without additional changes.
 if(NOT LINUX)
@@ -107,11 +119,9 @@ option(BUILD_BINARY "Build C++ binaries" OFF)
 option(BUILD_DOCS "Build Caffe2 documentation" OFF)
 option(BUILD_CUSTOM_PROTOBUF "Build and use Caffe2's own protobuf under third_party" ON)
 option(BUILD_PYTHON "Build Python binaries" ON)
-cmake_dependent_option(BUILD_CAFFE2_OPS "Build Caffe2 operators" ON
-  "NOT MSVC" OFF)
+option(BUILD_CAFFE2_OPS "Build Caffe2 operators" ON)
 option(BUILD_SHARED_LIBS "Build libcaffe2.so" ON)
-option(BUILD_CAFFE2_MOBILE "Build libcaffe2 for mobile (deprecating)" ON)
-option(BUILD_NAMEDTENSOR "Experimental: compile with namedtensor support" OFF)
+option(BUILD_CAFFE2_MOBILE "Build libcaffe2 for mobile (deprecating)" OFF)
 option(USE_STATIC_DISPATCH "Use static dispatch for ATen operators" OFF)
 cmake_dependent_option(
     CAFFE2_LINK_LOCAL_PROTOBUF "If set, build protobuf inside libcaffe2.so." ON
@@ -175,6 +185,7 @@ option(USE_SNPE "Use Qualcomm's SNPE library" OFF)
 option(USE_SYSTEM_EIGEN_INSTALL
     "Use system Eigen instead of the one under third_party" OFF)
 option(USE_TENSORRT "Using Nvidia TensorRT library" OFF)
+option(USE_XNNPACK "Use XNNPACK" ON)
 option(USE_ZMQ "Use ZMQ" OFF)
 option(USE_ZSTD "Use ZSTD" OFF)
 cmake_dependent_option(
@@ -192,6 +203,7 @@ cmake_dependent_option(
     USE_GLOO "Use Gloo. Only available if USE_DISTRIBUTED is on." ON
     "USE_DISTRIBUTED" OFF)
 option(USE_TBB "Use TBB" OFF)
+option(ONNX_ML "Enable traditional ONNX ML API." ON)
 
 # Used when building Caffe2 through setup.py
 option(BUILDING_WITH_TORCH_LIBS "Tell cmake if Caffe2 is being built alongside torch libs" ON)
@@ -207,6 +219,8 @@ cmake_dependent_option(
 set(ONNX_NAMESPACE "onnx_torch" CACHE STRING "A namespace for ONNX; needed to build with other frameworks that share ONNX.")
 set(SELECTED_OP_LIST "" CACHE STRING
     "Path to the yaml file that contains the list of operators to include for custom build. Include all operators by default.")
+set(OP_DEPENDENCY "" CACHE STRING
+    "Path to the yaml file that contains the op dependency graph for custom build.")
 
 # This is a fix for a rare build issue on Ubuntu:
 # symbol lookup error: miniconda3/envs/pytorch-py3.7/lib/libmkl_intel_lp64.so: undefined symbol: mkl_blas_dsyrk
@@ -260,8 +274,16 @@ if (MSVC)
     endif()
 
     # /bigobj increases number of sections in .obj file, which is needed to link
-    # against libaries in Python 2.7 under Windows
-    set(${flag_var} "${${flag_var}} /MP /bigobj")
+    # against libraries in Python 2.7 under Windows
+    # For Visual Studio generators, if /MP is not added, then we may need
+    # to add /MP to the flags.
+    # For other generators like ninja, we don't need to add /MP because it is
+    # already handled by the generator itself.
+    if(CMAKE_GENERATOR MATCHES "Visual Studio" AND NOT ${flag_var} MATCHES "/MP")
+      set(${flag_var} "${${flag_var}} /MP /bigobj")
+    else()
+      set(${flag_var} "${${flag_var}} /bigobj")
+    endif()
   endforeach(flag_var)
 
   foreach(flag_var
@@ -283,17 +305,24 @@ if (MSVC)
   list(APPEND CUDA_NVCC_FLAGS "-Xcompiler /w -w")
 endif(MSVC)
 
+IF(NOT MSVC)
+  list(APPEND CUDA_NVCC_FLAGS_DEBUG "-g" "-lineinfo" "--source-in-ptx")
+  list(APPEND CUDA_NVCC_FLAGS_RELWITHDEBINFO "-g" "-lineinfo" "--source-in-ptx")
+ENDIF(NOT MSVC)
+
 # Set INTERN_BUILD_MOBILE for all mobile builds. Components that are not
 # applicable to mobile are disabled by this variable.
-if (ANDROID OR IOS)
+# Setting `BUILD_PYTORCH_MOBILE_WITH_HOST_TOOLCHAIN` environment variable can
+# force it to do mobile build with host toolchain - which is useful for testing
+# purpose.
+if (ANDROID OR IOS OR DEFINED ENV{BUILD_PYTORCH_MOBILE_WITH_HOST_TOOLCHAIN})
   set(INTERN_BUILD_MOBILE ON)
-endif()
 
-# Setting `PYTORCH_BUILD_MOBILE` environment variable can force it to do mobile
-# build with host toolchain.
-if (DEFINED ENV{PYTORCH_BUILD_MOBILE})
-  set(INTERN_BUILD_MOBILE ON)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DC10_MOBILE")
+  if (DEFINED ENV{BUILD_PYTORCH_MOBILE_WITH_HOST_TOOLCHAIN})
+    # C10_MOBILE is derived from Android/iOS toolchain macros in
+    # c10/macros/Macros.h, so it needs to be explicitly set here.
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DC10_MOBILE")
+  endif()
 endif()
 
 # INTERN_BUILD_ATEN_OPS is used to control whether to build ATen/TH operators.
@@ -318,11 +347,13 @@ if (INTERN_BUILD_MOBILE AND NOT BUILD_CAFFE2_MOBILE)
   set(FEATURE_TORCH_MOBILE ON)
   set(NO_API ON)
   set(USE_FBGEMM OFF)
-  set(USE_PYTORCH_QNNPACK ON)
   set(USE_QNNPACK OFF)
   set(INTERN_DISABLE_ONNX ON)
   set(INTERN_DISABLE_AUTOGRAD ON)
   set(INTERN_USE_EIGEN_BLAS ON)
+  # Disable developing mobile interpreter for actual mobile build.
+  # Enable it elsewhere to capture build error.
+  set(INTERN_DISABLE_MOBILE_INTERP ON)
 endif()
 
 # ---[ Utils
@@ -383,10 +414,6 @@ if(USE_FBGEMM)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_FBGEMM")
 endif()
 
-if(BUILD_NAMEDTENSOR)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBUILD_NAMEDTENSOR")
-endif()
-
 if(USE_QNNPACK)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_QNNPACK")
 endif()
@@ -395,6 +422,10 @@ if(USE_PYTORCH_QNNPACK)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_PYTORCH_QNNPACK")
 endif()
 
+if(USE_XNNPACK)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_XNNPACK -DUSE_INTERNAL_THREADPOOL_IMPL")
+endif()
+
 # ---[ Whitelist file if whitelist is specified
 include(cmake/Whitelist.cmake)
 
@@ -405,8 +436,8 @@ if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.8.0
 endif()
 
 # ---[ Build flags
-set(CMAKE_C_STANDARD 99)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_C_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 if(NOT MSVC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fPIC")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing")
@@ -414,6 +445,7 @@ if(NOT MSVC)
   # Details at http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1459
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=return-type")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-type-limits")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-array-bounds")
@@ -475,6 +507,10 @@ if(NOT MSVC)
   set (CMAKE_LINKER_FLAGS_DEBUG "${CMAKE_STATIC_LINKER_FLAGS_DEBUG} -fno-omit-frame-pointer -O0")
   set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-math-errno")
   set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-trapping-math")
+  check_cxx_compiler_flag("-Werror=format" HAS_WERROR_FORMAT)
+  if (HAS_WERROR_FORMAT)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=format")
+  endif()
 endif()
 
 if (USE_ASAN)

CODEOWNERS

@@ -10,13 +10,13 @@
 /test/test_c10d.py @pietern @mrshenli @zhaojuanmao
 /torch/utils/cpp_extension.py @goldsborough @fmassa @soumith @ezyang
 
-# Not there to stricly require the approval, but to be tagged as a reviewer
+# Not there to strictly require the approval, but to be tagged as a reviewer
 # on the PRs to push them into a high priority inbox.
 /torch/csrc/api/data/ @apaszke
-/torch/csrc/autograd/ @apaszke
+/torch/csrc/autograd/ @apaszke @albanD
 /torch/csrc/jit/ @apaszke
 /torch/nn/ @apaszke
-/torch/autograd/ @apaszke
+/torch/autograd/ @apaszke @albanD
 /torch/jit/ @apaszke
 /torch/utils/data/ @apaszke
 
@@ -25,3 +25,8 @@
 /torch/csrc/distributed/autograd @mrshenli @pritamdamania87 @zhaojuanmao
 /torch/distributed/rpc @mrshenli @pritamdamania87 @zhaojuanmao
 /torch/distributed/autograd @mrshenli @pritamdamania87 @zhaojuanmao
+/torch/distributed/optim @mrshenli @pritamdamania87 @zhaojuanmao @aazzolini
+
+# Distributed tests
+/test/distributed @mrshenli @pritamdamania87 @zhaojuanmao
+/torch/testing/_internal/distributed @mrshenli @pritamdamania87 @zhaojuanmao

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at <conduct@pytorch.org>. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -1,3 +1,37 @@
+- [Contributing to PyTorch](#contributing-to-pytorch)
+- [Developing PyTorch](#developing-pytorch)
+- [Codebase structure](#codebase-structure)
+- [Unit testing](#unit-testing)
+  * [Better local unit tests with pytest](#better-local-unit-tests-with-pytest)
+- [Writing documentation](#writing-documentation)
+  * [Building documentation](#building-documentation)
+    + [Tips](#tips)
+    + [Building C++ Documentation](#building-c---documentation)
+  * [Previewing changes](#previewing-changes)
+    + [Submitting changes for review](#submitting-changes-for-review)
+  * [Adding documentation tests](#adding-documentation-tests)
+- [Profiling with `py-spy`](#profiling-with-py-spy)
+- [Managing multiple build trees](#managing-multiple-build-trees)
+- [C++ development tips](#c---development-tips)
+  * [Build only what you need](#build-only-what-you-need)
+  * [Code completion and IDE support](#code-completion-and-ide-support)
+  * [Make no-op build fast](#make-no-op-build-fast)
+    + [Use Ninja](#use-ninja)
+    + [Use CCache](#use-ccache)
+    + [Use a faster linker](#use-a-faster-linker)
+  * [C++ frontend development tips](#c---frontend-development-tips)
+- [CUDA development tips](#cuda-development-tips)
+- [Windows development tips](#windows-development-tips)
+  * [Known MSVC (and MSVC with NVCC) bugs](#known-msvc--and-msvc-with-nvcc--bugs)
+  * [Running clang-tidy](#running-clang-tidy)
+  * [Pre-commit tidy/linting hook](#pre-commit-tidy-linting-hook)
+  * [Building PyTorch with ASAN](#building-pytorch-with-asan)
+    + [Getting `ccache` to work](#getting--ccache--to-work)
+    + [Why this stuff with `LD_PRELOAD` and `LIBASAN_RT`?](#why-this-stuff-with--ld-preload--and--libasan-rt--)
+    + [Why LD_PRELOAD in the build function?](#why-ld-preload-in-the-build-function-)
+    + [Why no leak detection?](#why-no-leak-detection-)
+- [Caffe2 notes](#caffe2-notes)
+
 ## Contributing to PyTorch
 
 If you are interested in contributing to PyTorch, your contributions will fall
@@ -95,7 +129,6 @@ and `python setup.py clean`. Then you can install in `develop` mode again.
   * [src](aten/src)
     * [TH](aten/src/TH)
       [THC](aten/src/THC)
-      [THNN](aten/src/THNN)
       [THCUNN](aten/src/THCUNN) - Legacy library code from the original
       Torch. Try not to add things here; we're slowly porting these to
       [native](aten/src/ATen/native).
@@ -185,26 +218,13 @@ pytest test/test_nn.py -k Loss -v
 The above is an example of testing a change to Loss functions: this command runs tests such as
 `TestNN.test_BCELoss` and `TestNN.test_MSELoss` and can be useful to save keystrokes.
 
-## Writing Documentation
+## Writing documentation
 
 PyTorch uses [Google style](http://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
 for formatting docstrings. Length of line inside docstrings block must be limited to 80 characters to
 fit into Jupyter documentation popups.
 
-For C++ documentation (https://pytorch.org/cppdocs), we use
-[Doxygen](http://www.doxygen.nl/) and then convert it to
-[Sphinx](http://www.sphinx-doc.org/) via
-[Breathe](https://github.com/michaeljones/breathe) and
-[Exhale](https://github.com/svenevs/exhale). Check the [Doxygen
-reference](http://www.stack.nl/~dimitri/doxygen/manual/index.html) for more
-information on the documentation syntax. To build the documentation locally,
-`cd` into `docs/cpp` and then `make html`.
-
-We run Doxygen in CI (Travis) to verify that you do not use invalid Doxygen
-commands. To run this check locally, run `./check-doxygen.sh` from inside
-`docs/cpp`.
-
-### Building Documentation
+### Building documentation
 
 To build the documentation:
 
@@ -229,28 +249,6 @@ cd docs
 make html
 ```
 
-4. To view HTML files, you must start an HTTP server. For example
-
-```bash
-# Start a server from the current directory (Python 3 only)
-cd docs/build/html
-python -m http.server
-```
-
-If you are developing on a remote machine, you can set up an SSH tunnel so that
-you can access the HTTP server on the remote machine on your local machine. To map
-remote port 8086 to local port 8086, use either of the following commands.
-
-```bash
-# For SSH
-ssh my_machine -L 8086:my_machine:8086
-
-# For Eternal Terminal
-et my_machine -t="8086:8086"
-```
-
-Then navigate to `localhost:8086` in your web browser.
-
 #### Tips
 
 The `.rst` source files live in [docs/source](docs/source). Some of the `.rst`
@@ -267,12 +265,84 @@ ls | grep rst | grep -v index | grep -v jit | xargs rm
 # Make your changes, build the docs, etc.
 
 # Don't commit the deletions!
-git add index.rst jit.rst 
+git add index.rst jit.rst
 ...
 ```
 
+#### Building C++ Documentation
+For C++ documentation (https://pytorch.org/cppdocs), we use
+[Doxygen](http://www.doxygen.nl/) and then convert it to
+[Sphinx](http://www.sphinx-doc.org/) via
+[Breathe](https://github.com/michaeljones/breathe) and
+[Exhale](https://github.com/svenevs/exhale). Check the [Doxygen
+reference](http://www.stack.nl/~dimitri/doxygen/manual/index.html) for more
+information on the documentation syntax.
 
-### Adding Documentation Tests
+We run Doxygen in CI (Travis) to verify that you do not use invalid Doxygen
+commands. To run this check locally, run `./check-doxygen.sh` from inside
+`docs/cpp`.
+
+To build the documentation, follow the same steps as above, but run them from
+`docs/cpp` instead of `docs`.
+
+### Previewing changes
+
+To view HTML files locally, you can open the files in your web browser. For example,
+navigate to `file:///your_pytorch_folder/docs/build/html/index.html` in a web
+browser.
+
+If you are developing on a remote machine, you can set up an SSH tunnel so that
+you can access the HTTP server on the remote machine from your local machine. To map
+remote port 8000 to local port 8000, use either of the following commands.
+
+```bash
+# For SSH
+ssh my_machine -L 8000:my_machine:8000
+
+# For Eternal Terminal
+et my_machine -t="8000:8000"
+```
+
+Then navigate to `localhost:8000` in your web browser.
+
+#### Submitting changes for review
+
+It is helpful when submitting a PR that changes the docs to provide a rendered
+version of the result. If your change is small, you can add a screenshot of the
+changed docs to your PR.
+
+If your change to the docs is large and affects multiple pages, you can host
+the docs yourself with the following steps, then add a link to the output in your
+PR. These instructions use GitHub pages to host the docs
+you have built. To do so, follow [these steps](https://guides.github.com/features/pages/)
+to make a repo to host your changed documentation.
+
+GitHub pages expects to be hosting a Jekyll generated website which does not work
+well with the static resource paths used in the PyTorch documentation. To get around
+this, you must add an empty file called `.nojekyll` to your repo.
+
+```bash
+cd your_github_pages_repo
+touch .nojekyll
+git add .
+git commit
+git push
+```
+
+Then, copy built documentation and push the changes:
+
+```bash
+cd your_github_pages_repo
+cp -r ~/my_pytorch_path/docs/build/html/* .
+git add .
+git commit
+git push
+```
+
+Then you should be able to see the changes at your_github_username.github.com/your_github_pages_repo.
+
+
+### Adding documentation tests
 
 It is easy for code snippets in docstrings and `.rst` files to get out of date. The docs
 build includes the [Sphinx Doctest Extension](https://www.sphinx-doc.org/en/master/usage/extensions/doctest.html),
@@ -337,7 +407,7 @@ privileges.
 tweaked to adjust the stack sampling rate, see the `py-spy` readme for more
 details.
 
-## Managing Multiple Build Trees
+## Managing multiple build trees
 
 One downside to using `python setup.py develop` is that your development
 version of PyTorch will be installed globally on your account (e.g., if
@@ -356,7 +426,7 @@ source activate pytorch-myfeature
 python setup.py develop
 ```
 
-## C++ Development tips
+## C++ development tips
 
 If you are working on the C++ code, there are a few important things that you
 will want to keep in mind:
@@ -364,7 +434,7 @@ will want to keep in mind:
 1. How to rebuild only the code you are working on.
 2. How to make rebuilds in the absence of changes go faster.
 
-### Build only what you need.
+### Build only what you need
 
 `python setup.py build` will build everything by default, but sometimes you are
 only interested in a specific component.
@@ -387,10 +457,11 @@ variables `DEBUG`, `USE_DISTRIBUTED`, `USE_MKLDNN`, `USE_CUDA`, `BUILD_TEST`, `U
 - `USE_FBGEMM=0` will disable using FBGEMM (quantized 8-bit server operators).
 - `USE_NNPACK=0` will disable compiling with NNPACK.
 - `USE_QNNPACK=0` will disable QNNPACK build (quantized 8-bit operators).
+- `USE_XNNPACK=0` will disable compiling with XNNPACK.
 
 For example:
 ```bash
-DEBUG=1 USE_DISTRIBUTED=0 USE_MKLDNN=0 USE_CUDA=0 BUILD_TEST=0 USE_FBGEMM=0 USE_NNPACK=0 USE_QNNPACK=0 python setup.py develop
+DEBUG=1 USE_DISTRIBUTED=0 USE_MKLDNN=0 USE_CUDA=0 BUILD_TEST=0 USE_FBGEMM=0 USE_NNPACK=0 USE_QNNPACK=0 USE_XNNPACK=0 python setup.py develop
 ```
 
 For subsequent builds (i.e., when `build/CMakeCache.txt` exists), the build
@@ -407,7 +478,7 @@ C++ code. You need to `pip install ninja` to generate accurate
 information for the code in `torch/csrc`. More information at:
 - https://sarcasm.github.io/notes/dev/compilation-database.html
 
-### Make no-op build fast.
+### Make no-op build fast
 
 #### Use Ninja
 
@@ -534,7 +605,17 @@ The easiest way to use `lld` this is download the
 ln -s /path/to/downloaded/ld.lld /usr/local/bin/ld
 ```
 
-## CUDA Development tips
+### C++ frontend development tips
+
+We have very extensive tests in the [test/cpp/api](test/cpp/api) folder. The
+tests are a great way to see how certain components are intended to be used.
+When compiling PyTorch from source, the test runner binary will be written to
+`build/bin/test_api`. The tests use the [GoogleTest](https://github.com/google/googletest/blob/master/googletest)
+framework, which you can read up about to learn how to configure the test runner. When
+submitting a new feature, we care very much that you write appropriate tests.
+Please follow the lead of the other tests to see how to write a new test case.
+
+## CUDA development tips
 
 If you are working on the CUDA code, here are some useful CUDA debugging tips:
 
@@ -543,7 +624,7 @@ If you are working on the CUDA code, here are some useful CUDA debugging tips:
     slow down the build process for about 50% (compared to only `DEBUG=1`), so use wisely.
 2. `cuda-gdb` and `cuda-memcheck` are your best CUDA debugging friends. Unlike`gdb`,
    `cuda-gdb` can display actual values in a CUDA tensor (rather than all zeros).
-3. CUDA supports a lot of C++11 features such as, `std::numeric_limits`, `std::nextafter`,
+3. CUDA supports a lot of C++11/14 features such as, `std::numeric_limits`, `std::nextafter`,
    `std::tuple` etc. in device code. Many of such features are possible because of the
    [--expt-relaxed-constexpr](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#constexpr-functions)
    nvcc flag. There is a known [issue](https://github.com/ROCm-Developer-Tools/HIP/issues/374)
@@ -620,7 +701,7 @@ two dynamic libraries, one linking with the other:
 
 ```CMake
 project(myproject CXX)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 add_library(foo SHARED foo.cpp)
 add_library(bar SHARED bar.cpp)
 # NB: don't forget to __declspec(dllexport) at least one symbol from foo,
@@ -694,7 +775,7 @@ static_assert(std::is_same(A*, decltype(A::singleton()))::value, "hmm");
   we have AliasAnalysisKind::PURE_FUNCTION and not AliasAnalysisKind::PURE.
   The same is likely true for other identifiers that we just didn't try to use yet.
 
-### Running Clang-Tidy
+## Running clang-tidy
 
 [Clang-Tidy](https://clang.llvm.org/extra/clang-tidy/index.html) is a C++
 linter and static analysis tool based on the clang compiler. We run clang-tidy
@@ -725,7 +806,7 @@ root folder if you used `setup.py build`. You can use `-c <clang-tidy-binary>`
 to change the clang-tidy this script uses. Make sure you have PyYaml installed,
 which is in PyTorch's `requirements.txt`.
 
-### Pre-commit Tidy/Linting Hook
+## Pre-commit tidy/linting hook
 
 We use clang-tidy and flake8 (installed with flake8-bugbear,
 flake8-comprehensions, flake8-mypy, and flake8-pyi) to perform additional
@@ -740,7 +821,7 @@ You'll need to install an appropriately configured flake8; see
 [Lint as you type](https://github.com/pytorch/pytorch/wiki/Lint-as-you-type)
 for documentation on how to do this.
 
-### Building PyTorch with ASAN
+## Building PyTorch with ASAN
 
 [ASAN](https://github.com/google/sanitizers/wiki/AddressSanitizer) is very
 useful for debugging memory errors in C++. We run it in CI, but here's how to
@@ -788,7 +869,7 @@ suo-devfair ~/pytorch ❯ build_with_asan
 suo-devfair ~/pytorch ❯ run_with_asan python test/test_jit.py
 ```
 
-#### Getting `ccache` to work
+### Getting `ccache` to work
 
 The scripts above specify the `clang` and `clang++` binaries directly, which
 bypasses `ccache`. Here's how to get `ccache` to work:
@@ -799,7 +880,7 @@ bypasses `ccache`. Here's how to get `ccache` to work:
 3. Change the `CC` and `CXX` variables in `build_with_asan()` to point
    directly to `clang` and `clang++`.
 
-#### Why this stuff with `LD_PRELOAD` and `LIBASAN_RT`?
+### Why this stuff with `LD_PRELOAD` and `LIBASAN_RT`?
 
 The “standard” workflow for ASAN assumes you have a standalone binary:
 
@@ -820,7 +901,7 @@ workaround for cases like this:
 More information can be found
 [here](https://github.com/google/sanitizers/wiki/AddressSanitizerAsDso).
 
-#### Why LD_PRELOAD in the build function?
+### Why LD_PRELOAD in the build function?
 
 We need `LD_PRELOAD` because there is a cmake check that ensures that a
 simple program builds and runs. If we are building with ASAN as a shared
@@ -829,7 +910,7 @@ dynamic linker errors and the check will fail.
 
 We don’t actually need either of these if we fix the cmake checks.
 
-#### Why no Leak detection?
+### Why no leak detection?
 
 Python leaks a lot of memory. Possibly we could configure a suppression file,
 but we haven’t gotten around to it.

Dockerfile

@@ -0,0 +1,73 @@
+# syntax = docker/dockerfile:experimental
+#
+# NOTE: To build this you will need a docker version > 18.06 with
+#       experimental enabled and DOCKER_BUILDKIT=1
+#
+#       If you do not use buildkit you are not going to have a good time
+#
+#       For reference: 
+#           https://docs.docker.com/develop/develop-images/build_enhancements/
+ARG BASE_IMAGE=ubuntu:18.04
+ARG PYTHON_VERSION=3.7
+
+FROM ${BASE_IMAGE} as dev-base
+RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
+    apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        ccache \
+        cmake \
+        curl \
+        git \
+        libjpeg-dev \
+        libpng-dev && \
+    rm -rf /var/lib/apt/lists/*
+RUN /usr/sbin/update-ccache-symlinks
+RUN mkdir /opt/ccache && ccache --set-config=cache_dir=/opt/ccache
+ENV PATH /opt/conda/bin:$PATH
+
+FROM dev-base as conda
+RUN curl -v -o ~/miniconda.sh -O  https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh  && \
+    chmod +x ~/miniconda.sh && \
+    ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh && \
+    /opt/conda/bin/conda install -y python=${PYTHON_VERSION} conda-build pyyaml numpy ipython&& \
+    /opt/conda/bin/conda clean -ya
+
+FROM dev-base as submodule-update
+WORKDIR /opt/pytorch
+COPY . .
+RUN git submodule update --init --recursive
+
+FROM conda as build
+WORKDIR /opt/pytorch
+COPY --from=conda /opt/conda /opt/conda
+COPY --from=submodule-update /opt/pytorch /opt/pytorch
+RUN --mount=type=cache,target=/opt/ccache \
+    TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \
+    CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \
+    python setup.py install
+
+FROM conda as conda-installs
+ARG INSTALL_CHANNEL=pytorch-nightly
+RUN /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -y pytorch torchvision && \
+    /opt/conda/bin/conda clean -ya
+
+FROM ${BASE_IMAGE} as official
+LABEL com.nvidia.volumes.needed="nvidia_driver"
+RUN --mount=type=cache,id=apt-final,target=/var/cache/apt \
+    apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        libjpeg-dev \
+        libpng-dev && \
+    rm -rf /var/lib/apt/lists/*
+COPY --from=conda-installs /opt/conda /opt/conda
+ENV PATH /opt/conda/bin:$PATH
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
+WORKDIR /workspace
+
+FROM official as dev
+# Should override the already installed version from the official-image stage
+COPY --from=build /opt/conda /opt/conda

README.md

@@ -25,8 +25,8 @@ You can reuse your favorite Python packages such as NumPy, SciPy and Cython to e
 | Linux CPU | [![Build Status](https://ci.pytorch.org/jenkins/job/pytorch-master/badge/icon)](https://ci.pytorch.org/jenkins/job/pytorch-master/) | [![Build Status](https://ci.pytorch.org/jenkins/job/pytorch-master/badge/icon)](https://ci.pytorch.org/jenkins/job/pytorch-master/) | <center>—</center> |
 | Linux GPU | [![Build Status](https://ci.pytorch.org/jenkins/job/pytorch-master/badge/icon)](https://ci.pytorch.org/jenkins/job/pytorch-master/) | [![Build Status](https://ci.pytorch.org/jenkins/job/pytorch-master/badge/icon)](https://ci.pytorch.org/jenkins/job/pytorch-master/) | <center>—</center> |
 | Windows CPU / GPU | <center>—</center> | [![Build Status](https://ci.pytorch.org/jenkins/job/pytorch-builds/job/pytorch-win-ws2016-cuda9-cudnn7-py3-trigger/badge/icon)](https://ci.pytorch.org/jenkins/job/pytorch-builds/job/pytorch-win-ws2016-cuda9-cudnn7-py3-trigger/) |  <center>—</center> |
-| Linux (ppc64le) CPU | [![Build Status](https://powerci.osuosl.org/job/pytorch-master-nightly-py2-linux-ppc64le/badge/icon)](https://powerci.osuosl.org/job/pytorch-master-nightly-py2-linux-ppc64le/) | — | [![Build Status](https://powerci.osuosl.org/job/pytorch-master-nightly-py3-linux-ppc64le/badge/icon)](https://powerci.osuosl.org/job/pytorch-master-nightly-py3-linux-ppc64le/) |
-| Linux (ppc64le) GPU | [![Build Status](https://powerci.osuosl.org/job/pytorch-linux-cuda9-cudnn7-py2-mpi-build-test-gpu/badge/icon)](https://powerci.osuosl.org/job/pytorch-linux-cuda9-cudnn7-py2-mpi-build-test-gpu/) | — | [![Build Status](https://powerci.osuosl.org/job/pytorch-linux-cuda92-cudnn7-py3-mpi-build-test-gpu/badge/icon)](https://powerci.osuosl.org/job/pytorch-linux-cuda92-cudnn7-py3-mpi-build-test-gpu/) |
+| Linux (ppc64le) CPU | <center>—</center> | <center>—</center> | [![Build Status](https://powerci.osuosl.org/job/pytorch-master-nightly-py3-linux-ppc64le/badge/icon)](https://powerci.osuosl.org/job/pytorch-master-nightly-py3-linux-ppc64le/) |
+| Linux (ppc64le) GPU | <center>—</center> | <center>—</center> | [![Build Status](https://powerci.osuosl.org/job/pytorch-linux-cuda92-cudnn7-py3-mpi-build-test-gpu/badge/icon)](https://powerci.osuosl.org/job/pytorch-linux-cuda92-cudnn7-py3-mpi-build-test-gpu/) |
 
 See also the [ci.pytorch.org HUD](https://ezyang.github.io/pytorch-ci-hud/build/pytorch-master).
 
@@ -53,7 +53,7 @@ Elaborating further:
 
 ### A GPU-Ready Tensor Library
 
-If you use NumPy, then you have used Tensors (a.k.a ndarray).
+If you use NumPy, then you have used Tensors (a.k.a. ndarray).
 
 ![Tensor illustration](./docs/source/_static/img/tensor_illustration.png)
 
@@ -200,6 +200,14 @@ export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
 MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ python setup.py install
 ```
 
+Each CUDA version only supports one particular XCode version. The following combinations have been reported to work with PyTorch.
+
+| CUDA version | XCode version |
+| ------------ | ------------- |
+| 10.0         | XCode 9.4     |
+| 10.1         | XCode 10.1    |
+
+
 On Windows
 
 At least Visual Studio 2017 Update 3 (version 15.3.3 with the toolset 14.11) and [NVTX](https://docs.nvidia.com/gameworks/content/gameworkslibrary/nvtx/nvidia_tools_extension_library_nvtx.htm) are needed.
@@ -272,20 +280,30 @@ ccmake build  # or cmake-gui build
 
 ### Docker Image
 
-Dockerfile is supplied to build images with cuda support and cudnn v7. You can pass `-e PYTHON_VERSION=x.y` flag to specify which Python version is to be used by Miniconda, or leave it unset to use the default. Build from pytorch repo directory as docker needs to copy git repo into docker filesystem while building the image.
-```
-docker build -t pytorch -f docker/pytorch/Dockerfile .  # [optional] --build-arg WITH_TORCHVISION=0
+#### Using pre-built images
+
+You can also pull a pre-built docker image from Docker Hub and run with docker v19.03+
+
+```bash
+docker run --gpus all --rm -ti --ipc=host pytorch/pytorch:latest
 ```
 
-You can also pull a pre-built docker image from Docker Hub and run with nvidia-docker,
-but this is not currently maintained and will pull PyTorch 0.2.
-```
-nvidia-docker run --rm -ti --ipc=host pytorch/pytorch:latest
-```
 Please note that PyTorch uses shared memory to share data between processes, so if torch multiprocessing is used (e.g.
 for multithreaded data loaders) the default shared memory segment size that container runs with is not enough, and you
 should increase shared memory size either with `--ipc=host` or `--shm-size` command line options to `nvidia-docker run`.
 
+#### Building the image yourself
+
+**NOTE:** Must be built with a docker version > 18.06
+
+The `Dockerfile` is supplied to build images with cuda support and cudnn v7.
+You can pass `PYTHON_VERSION=x.y` make variable to specify which Python version is to be used by Miniconda, or leave it
+unset to use the default.
+```bash
+make -f docker.Makefile
+# images are tagged as docker.io/${your_docker_username}/pytorch
+```
+
 ### Building the Documentation
 
 To build documentation in various formats, you will need [Sphinx](http://www.sphinx-doc.org) and the
@@ -316,6 +334,7 @@ Three pointers to get you started:
 * GitHub issues: bug reports, feature requests, install issues, RFCs, thoughts, etc.
 * Slack: The [PyTorch Slack](https://pytorch.slack.com/) hosts a primary audience of moderate to experienced PyTorch users and developers for general chat, online discussions, collaboration etc. If you are a beginner looking for help, the primary medium is [PyTorch Forums](https://discuss.pytorch.org). If you need a slack invite, please fill this form: https://goo.gl/forms/PP1AGvNHpSaJP8to1
 * newsletter: no-noise, one-way email newsletter with important announcements about pytorch. You can sign-up here: https://eepurl.com/cbG0rv
+* for brand guidelines, please visit our website at [pytorch.org](https://pytorch.org/)
 
 ## Releases and Contributing
 

android/README.md

@@ -34,12 +34,12 @@ repositories {
 
 dependencies {
     ...
-    implementation 'org.pytorch:pytorch_android:1.4.0-SNAPSHOT'
-    implementation 'org.pytorch:pytorch_android_torchvision:1.4.0-SNAPSHOT'
+    implementation 'org.pytorch:pytorch_android:1.5.0-SNAPSHOT'
+    implementation 'org.pytorch:pytorch_android_torchvision:1.5.0-SNAPSHOT'
     ...
 }
 ```
-The current nightly(snapshots) version is the value of `VERSION_NAME` in `gradle.properties` in current folder, at this moment it is `1.4.0-SNAPSHOT`.
+The current nightly(snapshots) version is the value of `VERSION_NAME` in `gradle.properties` in current folder, at this moment it is `1.5.0-SNAPSHOT`.
 
 ## Building PyTorch Android from Source
 
@@ -49,6 +49,7 @@ For this you can use `./scripts/build_pytorch_android.sh` script.
 ```
 git clone https://github.com/pytorch/pytorch.git
 cd pytorch
+git submodule update --init --recursive
 sh ./scripts/build_pytorch_android.sh
 ```
 
@@ -59,7 +60,7 @@ The workflow contains several steps:
 2\. Create symbolic links to the results of those builds:
 `android/pytorch_android/src/main/jniLibs/${abi}` to the directory with output libraries
 `android/pytorch_android/src/main/cpp/libtorch_include/${abi}` to the directory with headers. These directories are used to build `libpytorch.so` library that will be loaded on android device.
- 
+
 3\. And finally run `gradle` in `android/pytorch_android` directory with task `assembleRelease`
 
 Script requires that Android SDK, Android NDK and gradle are installed.
@@ -103,8 +104,15 @@ dependencies {
     implementation(name:'pytorch_android', ext:'aar')
     implementation(name:'pytorch_android_torchvision', ext:'aar')
     implementation(name:'pytorch_android_fbjni', ext:'aar')
+    ...
+    implementation 'com.android.support:appcompat-v7:28.0.0'
+    implementation 'com.facebook.soloader:nativeloader:0.8.0'
 }
 ```
+We also have to add all transitive dependencies of our aars.
+As `pytorch_android` [depends](https://github.com/pytorch/pytorch/blob/master/android/pytorch_android/build.gradle#L62-L63) on `'com.android.support:appcompat-v7:28.0.0'` and `'com.facebook.soloader:nativeloader:0.8.0'`, we need to add them.
+(In case of using maven dependencies they are added automatically from `pom.xml`).
+
 
 At the moment for the case of using aar files directly we need additional configuration due to packaging specific (`libfbjni.so` is packaged in both `pytorch_android_fbjni.aar` and `pytorch_android.aar`).
 ```

android/build.gradle

@@ -11,6 +11,10 @@ allprojects {
             runnerVersion = "1.2.0"
             rulesVersion = "1.2.0"
             junitVersion = "4.12"
+
+            androidSupportAppCompatV7Version = "28.0.0"
+            fbjniJavaOnlyVersion = "0.0.3"
+            soLoaderNativeLoaderVersion = "0.8.0"
         }
 
         repositories {
@@ -34,8 +38,6 @@ allprojects {
     }
 }
 
-ext.isPublishing = { ['uploadArchives', 'bintrayUpload'].any { gradle.startParameter.taskNames.contains(it) } }
-
 ext.deps = [
         jsr305: 'com.google.code.findbugs:jsr305:3.0.1',
 ]

android/build_test_app.sh

@@ -62,7 +62,7 @@ mkdir -p $OUT_DIR
 pushd $PYTORCH_DIR
 python $PYTORCH_DIR/setup.py clean
 
-ANDROID_ABI=$abi BUILD_PYTORCH_MOBILE=1 VERBOSE=1 ANDROID_DEBUG_SYMBOLS=1 $PYTORCH_DIR/scripts/build_android.sh -DANDROID_CCACHE=$(which ccache)
+ANDROID_ABI=$abi VERBOSE=1 ANDROID_DEBUG_SYMBOLS=1 $PYTORCH_DIR/scripts/build_android.sh -DANDROID_CCACHE=$(which ccache)
 
 cp -R $PYTORCH_DIR/build_android/install/lib $OUT_DIR/
 cp -R $PYTORCH_DIR/build_android/install/include $OUT_DIR/
@@ -97,4 +97,3 @@ find $PYTORCH_ANDROID_DIR -type f -name *apk
 find $PYTORCH_ANDROID_DIR -type f -name *apk | xargs echo "To install apk run: $ANDROID_HOME/platform-tools/adb install -r "
 
 popd
-

android/gradle.properties

@@ -1,6 +1,6 @@
 ABI_FILTERS=armeabi-v7a,arm64-v8a,x86,x86_64
 
-VERSION_NAME=1.4.0-SNAPSHOT
+VERSION_NAME=1.5.0-SNAPSHOT
 GROUP=org.pytorch
 MAVEN_GROUP=org.pytorch
 POM_URL=https://github.com/pytorch/pytorch/tree/master/android
@@ -27,3 +27,4 @@ android.useAndroidX=true
 android.enableJetifier=true
 
 nativeLibsDoNotStrip=false
+testAppAllVariantsEnabled=false

android/pytorch_android/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.4.1)
 project(pytorch_jni CXX)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_VERBOSE_MAKEFILE ON)
 
 set(TRACE_ENABLED OFF)
@@ -48,6 +48,14 @@ add_library(pytorch_jni SHARED
     ${pytorch_android_SOURCES}
 )
 
+if (APPLE)
+  # Need to add rpath so dlopen can find dependencies.
+  add_custom_command(TARGET pytorch_jni
+      POST_BUILD COMMAND
+      ${CMAKE_INSTALL_NAME_TOOL} -add_rpath "@loader_path"
+        $<TARGET_FILE:pytorch_jni>)
+endif()
+
 target_compile_options(pytorch_jni PRIVATE
   -fexceptions
 )
@@ -72,8 +80,10 @@ if (ANDROID_ABI)
   endfunction(import_static_lib)
 
   import_static_lib(libtorch)
+  import_static_lib(libtorch_cpu)
   import_static_lib(libc10)
   import_static_lib(libnnpack)
+  import_static_lib(libXNNPACK)
   import_static_lib(libpytorch_qnnpack)
   import_static_lib(libeigen_blas)
   import_static_lib(libcpuinfo)
@@ -85,9 +95,11 @@ if (ANDROID_ABI)
       -Wl,--gc-sections
       -Wl,--whole-archive
       libtorch
+      libtorch_cpu
       -Wl,--no-whole-archive
       libc10
       libnnpack
+      libXNNPACK
       libpytorch_qnnpack
       libeigen_blas
       libcpuinfo
@@ -100,8 +112,10 @@ else()
   target_link_libraries(pytorch_jni
       fbjni
       torch
+      torch_cpu
       c10
       nnpack
+      XNNPACK
       pytorch_qnnpack
       cpuinfo
       clog

android/pytorch_android/build.gradle

@@ -33,6 +33,11 @@ android {
             }
             jniLibs.srcDirs = ['src/main/jniLibs']
         }
+        androidTest {
+            java {
+                exclude 'org/pytorch/PytorchHostTests.java'
+            }
+        }
     }
     externalNativeBuild {
         cmake {
@@ -41,11 +46,6 @@ android {
     }
 
     packagingOptions {
-        if (rootProject.isPublishing()) {
-            exclude '**/libfbjni.so'
-        } else {
-            pickFirst '**/libfbjni.so'
-        }
         if (nativeLibsDoNotStrip.toBoolean()) {
             doNotStrip "**/*.so"
             logger.warn('WARNING: nativeLibsDoNotStrip==true; debug symbols included')
@@ -58,10 +58,9 @@ android {
 }
 
 dependencies {
-    api project(':fbjni')
-
-    implementation 'com.android.support:appcompat-v7:28.0.0'
-    implementation 'com.facebook.soloader:nativeloader:0.8.0'
+    implementation 'com.facebook.fbjni:fbjni-java-only:' + rootProject.fbjniJavaOnlyVersion
+    implementation 'com.android.support:appcompat-v7:' + rootProject.androidSupportAppCompatV7Version
+    implementation 'com.facebook.soloader:nativeloader:' + rootProject.soLoaderNativeLoaderVersion
 
     testImplementation 'junit:junit:' + rootProject.junitVersion
     testImplementation 'androidx.test:core:' + rootProject.coreVersion

android/pytorch_android/generate_test_asset.cpp

@@ -0,0 +1,20 @@
+#include <torch/jit.h>
+#include <torch/script.h>
+#include <torch/csrc/jit/api/module.h>
+
+#include <iostream>
+#include <fstream>
+#include <string>
+
+int main(int argc, char* argv[]) {
+  std::string input_file_path{argv[1]};
+  std::string output_file_path{argv[2]};
+
+  std::ifstream ifs(input_file_path);
+  std::stringstream buffer;
+  buffer << ifs.rdbuf();
+  torch::jit::Module m("TestModule");
+
+  m.define(buffer.str());
+  m.save(output_file_path);
+}

android/pytorch_android/host/build.gradle

@@ -14,7 +14,12 @@ repositories {
 
 sourceSets {
     main {
-        java.srcDir '../src/main/java'
+        java {
+            srcDir '../src/main/java'
+            exclude 'org/pytorch/PyTorchAndroid.java'
+            exclude 'org/pytorch/LiteModuleLoader.java'
+            exclude 'org/pytorch/LiteNativePeer.java'
+        }
     }
     test {
         java {