[1.7] Remove torch.vmap (#45571)

torch.vmap is a prototype feature and should not be in the stable
binary. This PR:
- Removes the `torch.vmap` API
- Removes the documentation entry for `torch.vmap`
- Changes the vmap tests to use an internal API instead of `torch.vmap`.

Test Plan:
- Tested locally (test_torch, test_type_hints, test_vmap), but also wait
for CI.

.circleci/README.md

@@ -178,8 +178,7 @@ CircleCI creates a  final yaml file by inlining every <<* segment, so if we were
 So, CircleCI has several executor types: macos, machine, and docker are the ones we use. The 'machine' executor gives you two cores on some linux vm. The 'docker' executor gives you considerably more cores (nproc was 32 instead of 2 back when I tried in February). Since the dockers are faster, we try to run everything that we can in dockers. Thus
 
 * linux build jobs use the docker executor. Running them on the docker executor was at least 2x faster than running them on the machine executor
-* linux test jobs use the machine executor and spin up their own docker. Why this nonsense? It's cause we run nvidia-docker for our GPU tests; any code that calls into the CUDA runtime needs to be run on nvidia-docker. To run a nvidia-docker you need to install some nvidia packages on the host machine and then call docker with the '—runtime nvidia' argument. CircleCI doesn't support this, so we have to do it ourself.
-    * This is not just a mere inconvenience. **This blocks all of our linux tests from using more than 2 cores.** But there is nothing that we can do about it, but wait for a fix on circleci's side. Right now, we only run some smoke tests (some simple imports) on the binaries, but this also affects non-binary test jobs.
+* linux test jobs use the machine executor in order for them to properly interface with GPUs since docker executors cannot execute with attached GPUs
 * linux upload jobs use the machine executor. The upload jobs are so short that it doesn't really matter what they use
 * linux smoke test jobs use the machine executor for the same reason as the linux test jobs
 
@@ -419,8 +418,6 @@ You can build Linux binaries locally easily using docker.
 #    in the docker container then you will see path/to/foo/baz on your local
 #    machine. You could also clone the pytorch and builder repos in the docker.
 #
-# If you're building a CUDA binary then use `nvidia-docker run` instead, see below.
-#
 # If you know how, add ccache as a volume too and speed up everything
 docker run \
     -v your/pytorch/repo:/pytorch \
@@ -444,9 +441,7 @@ export DESIRED_CUDA=cpu
 
 **Building CUDA binaries on docker**
 
-To build a CUDA binary you need to use `nvidia-docker run` instead of just `docker run` (or you can manually pass `--runtime=nvidia`). This adds some needed libraries and things to build CUDA stuff.
-
-You can build CUDA binaries on CPU only machines, but you can only run CUDA binaries on CUDA machines. This means that you can build a CUDA binary on a docker on your laptop if you so choose (though it’s gonna take a loong time).
+You can build CUDA binaries on CPU only machines, but you can only run CUDA binaries on CUDA machines. This means that you can build a CUDA binary on a docker on your laptop if you so choose (though it’s gonna take a long time).
 
 For Facebook employees, ask about beefy machines that have docker support and use those instead of your laptop; it will be 5x as fast.
 

.circleci/cimodel/data/binary_build_data.py

@@ -25,8 +25,10 @@ DEPS_INCLUSION_DIMENSIONS = [
 ]
 
 
-def get_processor_arch_name(cuda_version):
-    return "cpu" if not cuda_version else "cu" + cuda_version
+def get_processor_arch_name(gpu_version):
+    return "cpu" if not gpu_version else (
+        "cu" + gpu_version.strip("cuda") if gpu_version.startswith("cuda") else gpu_version
+    )
 
 
 LINUX_PACKAGE_VARIANTS = OrderedDict(
@@ -42,7 +44,7 @@ LINUX_PACKAGE_VARIANTS = OrderedDict(
 )
 
 CONFIG_TREE_DATA = OrderedDict(
-    linux=(dimensions.CUDA_VERSIONS, LINUX_PACKAGE_VARIANTS),
+    linux=(dimensions.GPU_VERSIONS, LINUX_PACKAGE_VARIANTS),
     macos=([None], OrderedDict(
         wheel=dimensions.STANDARD_PYTHON_VERSIONS,
         conda=dimensions.STANDARD_PYTHON_VERSIONS,
@@ -50,13 +52,17 @@ CONFIG_TREE_DATA = OrderedDict(
             "3.7",
         ],
     )),
-    windows=(dimensions.CUDA_VERSIONS, OrderedDict(
-        wheel=dimensions.STANDARD_PYTHON_VERSIONS,
-        conda=dimensions.STANDARD_PYTHON_VERSIONS,
-        libtorch=[
-            "3.7",
-        ],
-    )),
+    # Skip CUDA-9.2 builds on Windows
+    windows=(
+        [v for v in dimensions.GPU_VERSIONS if v not in ['cuda92'] + dimensions.ROCM_VERSION_LABELS],
+        OrderedDict(
+            wheel=dimensions.STANDARD_PYTHON_VERSIONS,
+            conda=dimensions.STANDARD_PYTHON_VERSIONS,
+            libtorch=[
+                "3.7",
+            ],
+        )
+    ),
 )
 
 # GCC config variants:
@@ -93,12 +99,12 @@ class TopLevelNode(ConfigNode):
 
 
 class OSConfigNode(ConfigNode):
-    def __init__(self, parent, os_name, cuda_versions, py_tree):
+    def __init__(self, parent, os_name, gpu_versions, py_tree):
         super(OSConfigNode, self).__init__(parent, os_name)
 
         self.py_tree = py_tree
         self.props["os_name"] = os_name
-        self.props["cuda_versions"] = cuda_versions
+        self.props["gpu_versions"] = gpu_versions
 
     def get_children(self):
         return [PackageFormatConfigNode(self, k, v) for k, v in self.py_tree.items()]
@@ -117,7 +123,7 @@ class PackageFormatConfigNode(ConfigNode):
         elif self.find_prop("os_name") == "windows" and self.find_prop("package_format") == "libtorch":
             return [WindowsLibtorchConfigNode(self, v) for v in WINDOWS_LIBTORCH_CONFIG_VARIANTS]
         else:
-            return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
+            return [ArchConfigNode(self, v) for v in self.find_prop("gpu_versions")]
 
 
 class LinuxGccConfigNode(ConfigNode):
@@ -127,14 +133,22 @@ class LinuxGccConfigNode(ConfigNode):
         self.props["gcc_config_variant"] = gcc_config_variant
 
     def get_children(self):
-        cuda_versions = self.find_prop("cuda_versions")
+        gpu_versions = self.find_prop("gpu_versions")
 
         # XXX devtoolset7 on CUDA 9.0 is temporarily disabled
         # see https://github.com/pytorch/pytorch/issues/20066
         if self.find_prop("gcc_config_variant") == 'devtoolset7':
-            cuda_versions = filter(lambda x: x != "90", cuda_versions)
+            gpu_versions = filter(lambda x: x != "cuda_90", gpu_versions)
 
-        return [ArchConfigNode(self, v) for v in cuda_versions]
+        # XXX disabling conda rocm build since docker images are not there
+        if self.find_prop("package_format") == 'conda':
+            gpu_versions = filter(lambda x: x not in dimensions.ROCM_VERSION_LABELS, gpu_versions)
+
+        # XXX libtorch rocm build  is temporarily disabled
+        if self.find_prop("package_format") == 'libtorch':
+            gpu_versions = filter(lambda x: x not in dimensions.ROCM_VERSION_LABELS, gpu_versions)
+
+        return [ArchConfigNode(self, v) for v in gpu_versions]
 
 
 class WindowsLibtorchConfigNode(ConfigNode):
@@ -144,14 +158,14 @@ class WindowsLibtorchConfigNode(ConfigNode):
         self.props["libtorch_config_variant"] = libtorch_config_variant
 
     def get_children(self):
-        return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
+        return [ArchConfigNode(self, v) for v in self.find_prop("gpu_versions")]
 
 
 class ArchConfigNode(ConfigNode):
-    def __init__(self, parent, cu):
-        super(ArchConfigNode, self).__init__(parent, get_processor_arch_name(cu))
+    def __init__(self, parent, gpu):
+        super(ArchConfigNode, self).__init__(parent, get_processor_arch_name(gpu))
 
-        self.props["cu"] = cu
+        self.props["gpu"] = gpu
 
     def get_children(self):
         return [PyVersionConfigNode(self, v) for v in self.find_prop("python_versions")]

.circleci/cimodel/data/binary_build_definitions.py

@@ -6,10 +6,10 @@ import cimodel.lib.conf_tree as conf_tree
 import cimodel.lib.miniutils as miniutils
 
 class Conf(object):
-    def __init__(self, os, cuda_version, pydistro, parms, smoke, libtorch_variant, gcc_config_variant, libtorch_config_variant):
+    def __init__(self, os, gpu_version, pydistro, parms, smoke, libtorch_variant, gcc_config_variant, libtorch_config_variant):
 
         self.os = os
-        self.cuda_version = cuda_version
+        self.gpu_version = gpu_version
         self.pydistro = pydistro
         self.parms = parms
         self.smoke = smoke
@@ -18,7 +18,7 @@ class Conf(object):
         self.libtorch_config_variant = libtorch_config_variant
 
     def gen_build_env_parms(self):
-        elems = [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)]
+        elems = [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.gpu_version)]
         if self.gcc_config_variant is not None:
             elems.append(str(self.gcc_config_variant))
         if self.libtorch_config_variant is not None:
@@ -37,9 +37,12 @@ class Conf(object):
         docker_distro_prefix = miniutils.override(self.pydistro, docker_word_substitution)
 
         # The cpu nightlies are built on the pytorch/manylinux-cuda102 docker image
-        alt_docker_suffix = self.cuda_version or "102"
-        docker_distro_suffix = "" if self.pydistro == "conda" else alt_docker_suffix
-        return miniutils.quote("pytorch/" + docker_distro_prefix + "-cuda" + docker_distro_suffix)
+        # TODO cuda images should consolidate into tag-base images similar to rocm
+        alt_docker_suffix = "cuda102" if not self.gpu_version else (
+            "rocm:" + self.gpu_version.strip("rocm") if self.gpu_version.startswith("rocm") else self.gpu_version)
+        docker_distro_suffix = alt_docker_suffix if self.pydistro != "conda" else (
+            "cuda" if alt_docker_suffix.startswith("cuda") else "rocm")
+        return miniutils.quote("pytorch/" + docker_distro_prefix + "-" + docker_distro_suffix)
 
     def get_name_prefix(self):
         return "smoke" if self.smoke else "binary"
@@ -69,14 +72,10 @@ class Conf(object):
                 "update_s3_htmls",
             ]
             job_def["filters"] = branch_filters.gen_filter_dict(
-                branches_list=["nightly"],
-                tags_list=[branch_filters.RC_PATTERN],
+                branches_list=["postnightly"],
             )
         else:
-            if phase in ["upload"]:
-                filter_branch = "nightly"
-            else:
-                filter_branch = r"/.*/"
+            filter_branch = r"/.*/"
             job_def["filters"] = branch_filters.gen_filter_dict(
                 branches_list=[filter_branch],
                 tags_list=[branch_filters.RC_PATTERN],
@@ -89,28 +88,61 @@ class Conf(object):
             if not (self.smoke and self.os == "macos") and self.os != "windows":
                 job_def["docker_image"] = self.gen_docker_image()
 
-            if self.os != "windows" and self.cuda_version:
+            # fix this. only works on cuda not rocm
+            if self.os != "windows" and self.gpu_version:
                 job_def["use_cuda_docker_runtime"] = miniutils.quote("1")
         else:
             if self.os == "linux" and phase != "upload":
                 job_def["docker_image"] = self.gen_docker_image()
 
         if phase == "test":
-            if self.cuda_version:
+            if self.gpu_version:
                 if self.os == "windows":
                     job_def["executor"] = "windows-with-nvidia-gpu"
                 else:
                     job_def["resource_class"] = "gpu.medium"
-        if phase == "upload":
-            job_def["context"] = "org-member"
-            job_def["requires"] = [
-                self.gen_build_name(upload_phase_dependency, nightly)
-            ]
 
         os_name = miniutils.override(self.os, {"macos": "mac"})
         job_name = "_".join([self.get_name_prefix(), os_name, phase])
         return {job_name : job_def}
 
+    def gen_upload_job(self, phase, requires_dependency):
+        """Generate binary_upload job for configuration
+
+        Output looks similar to:
+
+      - binary_upload:
+          name: binary_linux_manywheel_3_7m_cu92_devtoolset7_nightly_upload
+          context: org-member
+          requires: binary_linux_manywheel_3_7m_cu92_devtoolset7_nightly_test
+          filters:
+            branches:
+              only:
+                - nightly
+            tags:
+              only: /v[0-9]+(\\.[0-9]+)*-rc[0-9]+/
+          package_type: manywheel
+          upload_subfolder: cu92
+        """
+        return {
+            "binary_upload": OrderedDict({
+                "name": self.gen_build_name(phase, nightly=True),
+                "context": "org-member",
+                "requires": [self.gen_build_name(
+                    requires_dependency,
+                    nightly=True
+                )],
+                "filters": branch_filters.gen_filter_dict(
+                    branches_list=["nightly"],
+                    tags_list=[branch_filters.RC_PATTERN],
+                ),
+                "package_type": self.pydistro,
+                "upload_subfolder": binary_build_data.get_processor_arch_name(
+                    self.gpu_version,
+                ),
+            })
+        }
+
 def get_root(smoke, name):
 
     return binary_build_data.TopLevelNode(
@@ -129,7 +161,7 @@ def gen_build_env_list(smoke):
     for c in config_list:
         conf = Conf(
             c.find_prop("os_name"),
-            c.find_prop("cu"),
+            c.find_prop("gpu"),
             c.find_prop("package_format"),
             [c.find_prop("pyver")],
             c.find_prop("smoke"),
@@ -149,32 +181,19 @@ def get_nightly_uploads():
     mylist = []
     for conf in configs:
         phase_dependency = "test" if predicate_exclude_macos(conf) else "build"
-        mylist.append(conf.gen_workflow_job("upload", phase_dependency, nightly=True))
+        mylist.append(conf.gen_upload_job("upload", phase_dependency))
 
     return mylist
 
 def get_post_upload_jobs():
-    """Generate jobs to update HTML indices and report binary sizes"""
-    configs = gen_build_env_list(False)
-    common_job_def = {
-        "context": "org-member",
-        "filters": branch_filters.gen_filter_dict(
-            branches_list=["nightly"],
-            tags_list=[branch_filters.RC_PATTERN],
-        ),
-        "requires": [],
-    }
-    for conf in configs:
-        upload_job_name = conf.gen_build_name(
-            build_or_test="upload",
-            nightly=True
-        )
-        common_job_def["requires"].append(upload_job_name)
     return [
         {
             "update_s3_htmls": {
                 "name": "update_s3_htmls",
-                **common_job_def,
+                "context": "org-member",
+                "filters": branch_filters.gen_filter_dict(
+                    branches_list=["postnightly"],
+                ),
             },
         },
     ]

.circleci/cimodel/data/caffe2_build_data.py

@@ -1,91 +0,0 @@
-from cimodel.lib.conf_tree import ConfigNode, XImportant
-from cimodel.lib.conf_tree import Ver
-
-
-CONFIG_TREE_DATA = [
-    (Ver("ubuntu", "16.04"), [
-        ([Ver("clang", "7")], [XImportant("onnx_main_py3.6"),
-                               XImportant("onnx_ort1_py3.6"),
-                               XImportant("onnx_ort2_py3.6")]),
-    ]),
-]
-
-
-class TreeConfigNode(ConfigNode):
-    def __init__(self, parent, node_name, subtree):
-        super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name))
-        self.subtree = subtree
-        self.init2(node_name)
-
-    # noinspection PyMethodMayBeStatic
-    def modify_label(self, label):
-        return str(label)
-
-    def init2(self, node_name):
-        pass
-
-    def get_children(self):
-        return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
-
-    def is_build_only(self):
-        if str(self.find_prop("language_version")) == "onnx_main_py3.6" or \
-                str(self.find_prop("language_version")) == "onnx_ort1_py3.6" or \
-                str(self.find_prop("language_version")) == "onnx_ort2_py3.6":
-            return False
-        return set(str(c) for c in self.find_prop("compiler_version")).intersection({
-            "clang3.8",
-            "clang3.9",
-            "clang7",
-            "android",
-        }) or self.find_prop("distro_version").name == "macos"
-
-    def is_test_only(self):
-        if str(self.find_prop("language_version")) == "onnx_ort1_py3.6" or \
-                str(self.find_prop("language_version")) == "onnx_ort2_py3.6":
-            return True
-        return False
-
-
-class TopLevelNode(TreeConfigNode):
-    def __init__(self, node_name, subtree):
-        super(TopLevelNode, self).__init__(None, node_name, subtree)
-
-    # noinspection PyMethodMayBeStatic
-    def child_constructor(self):
-        return DistroConfigNode
-
-
-class DistroConfigNode(TreeConfigNode):
-    def init2(self, node_name):
-        self.props["distro_version"] = node_name
-
-    # noinspection PyMethodMayBeStatic
-    def child_constructor(self):
-        return CompilerConfigNode
-
-
-class CompilerConfigNode(TreeConfigNode):
-    def init2(self, node_name):
-        self.props["compiler_version"] = node_name
-
-    # noinspection PyMethodMayBeStatic
-    def child_constructor(self):
-        return LanguageConfigNode
-
-
-class LanguageConfigNode(TreeConfigNode):
-    def init2(self, node_name):
-        self.props["language_version"] = node_name
-        self.props["build_only"] = self.is_build_only()
-        self.props["test_only"] = self.is_test_only()
-
-    def child_constructor(self):
-        return ImportantConfigNode
-
-
-class ImportantConfigNode(TreeConfigNode):
-    def init2(self, node_name):
-        self.props["important"] = True
-
-    def get_children(self):
-        return []

.circleci/cimodel/data/caffe2_build_definitions.py

@@ -1,174 +0,0 @@
-from collections import OrderedDict
-
-import cimodel.data.dimensions as dimensions
-import cimodel.lib.conf_tree as conf_tree
-from cimodel.lib.conf_tree import Ver
-import cimodel.lib.miniutils as miniutils
-from cimodel.data.caffe2_build_data import CONFIG_TREE_DATA, TopLevelNode
-from cimodel.data.simple.util.branch_filters import gen_filter_dict
-
-from dataclasses import dataclass
-
-
-DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
-
-DOCKER_IMAGE_VERSION = "376"
-
-
-@dataclass
-class Conf:
-    language: str
-    distro: Ver
-    # There could be multiple compiler versions configured (e.g. nvcc
-    # for gpu files and host compiler (gcc/clang) for cpu files)
-    compilers: [Ver]
-    build_only: bool
-    test_only: bool
-    is_important: bool
-
-    @property
-    def compiler_names(self):
-        return [c.name for c in self.compilers]
-
-    # TODO: Eventually we can probably just remove the cudnn7 everywhere.
-    def get_cudnn_insertion(self):
-
-        omit = self.language == "onnx_main_py3.6" \
-            or self.language == "onnx_ort1_py3.6" \
-            or self.language == "onnx_ort2_py3.6" \
-            or set(self.compiler_names).intersection({"android", "mkl", "clang"}) \
-            or str(self.distro) in ["ubuntu14.04", "macos10.13"]
-
-        return [] if omit else ["cudnn7"]
-
-    def get_build_name_root_parts(self):
-        return [
-            "caffe2",
-            self.language,
-        ] + self.get_build_name_middle_parts()
-
-    def get_build_name_middle_parts(self):
-        return [str(c) for c in self.compilers] + self.get_cudnn_insertion() + [str(self.distro)]
-
-    def construct_phase_name(self, phase):
-        root_parts = self.get_build_name_root_parts()
-
-        build_name_substitutions = {
-            "onnx_ort1_py3.6": "onnx_main_py3.6",
-            "onnx_ort2_py3.6": "onnx_main_py3.6",
-        }
-        if phase == "build":
-            root_parts = [miniutils.override(r, build_name_substitutions) for r in root_parts]
-        return "_".join(root_parts + [phase]).replace(".", "_")
-
-    def get_platform(self):
-        platform = self.distro.name
-        if self.distro.name != "macos":
-            platform = "linux"
-        return platform
-
-    def gen_docker_image(self):
-
-        lang_substitutions = {
-            "onnx_main_py3.6": "py3.6",
-            "onnx_ort1_py3.6": "py3.6",
-            "onnx_ort2_py3.6": "py3.6",
-            "cmake": "py3",
-        }
-
-        lang = miniutils.override(self.language, lang_substitutions)
-        parts = [lang] + self.get_build_name_middle_parts()
-        return miniutils.quote(DOCKER_IMAGE_PATH_BASE + "-".join(parts) + ":" + str(DOCKER_IMAGE_VERSION))
-
-    def gen_workflow_params(self, phase):
-        parameters = OrderedDict()
-        lang_substitutions = {
-            "onnx_py3": "onnx-py3",
-            "onnx_main_py3.6": "onnx-main-py3.6",
-            "onnx_ort1_py3.6": "onnx-ort1-py3.6",
-            "onnx_ort2_py3.6": "onnx-ort2-py3.6",
-        }
-
-        lang = miniutils.override(self.language, lang_substitutions)
-
-        parts = [
-            "caffe2",
-            lang,
-        ] + self.get_build_name_middle_parts() + [phase]
-
-        build_env_name = "-".join(parts)
-        parameters["build_environment"] = miniutils.quote(build_env_name)
-        if "ios" in self.compiler_names:
-            parameters["build_ios"] = miniutils.quote("1")
-        if phase == "test":
-            # TODO cuda should not be considered a compiler
-            if "cuda" in self.compiler_names:
-                parameters["use_cuda_docker_runtime"] = miniutils.quote("1")
-
-        if self.distro.name != "macos":
-            parameters["docker_image"] = self.gen_docker_image()
-            if self.build_only:
-                parameters["build_only"] = miniutils.quote("1")
-        if phase == "test":
-            resource_class = "large" if "cuda" not in self.compiler_names else "gpu.medium"
-            parameters["resource_class"] = resource_class
-
-        return parameters
-
-    def gen_workflow_job(self, phase):
-        job_def = OrderedDict()
-        job_def["name"] = self.construct_phase_name(phase)
-
-        if phase == "test":
-            job_def["requires"] = [self.construct_phase_name("build")]
-            job_name = "caffe2_" + self.get_platform() + "_test"
-        else:
-            job_name = "caffe2_" + self.get_platform() + "_build"
-
-        if not self.is_important:
-            job_def["filters"] = gen_filter_dict()
-        job_def.update(self.gen_workflow_params(phase))
-        return {job_name : job_def}
-
-
-def get_root():
-    return TopLevelNode("Caffe2 Builds", CONFIG_TREE_DATA)
-
-
-def instantiate_configs():
-
-    config_list = []
-
-    root = get_root()
-    found_configs = conf_tree.dfs(root)
-    for fc in found_configs:
-        c = Conf(
-            language=fc.find_prop("language_version"),
-            distro=fc.find_prop("distro_version"),
-            compilers=fc.find_prop("compiler_version"),
-            build_only=fc.find_prop("build_only"),
-            test_only=fc.find_prop("test_only"),
-            is_important=fc.find_prop("important"),
-        )
-
-        config_list.append(c)
-
-    return config_list
-
-
-def get_workflow_jobs():
-
-    configs = instantiate_configs()
-
-    x = []
-    for conf_options in configs:
-        phases = ["build"]
-        if not conf_options.build_only:
-            phases = dimensions.PHASES
-        if conf_options.test_only:
-            phases = ["test"]
-
-        for phase in phases:
-            x.append(conf_options.gen_workflow_job(phase))
-
-    return x

.circleci/cimodel/data/dimensions.py

@@ -1,14 +1,23 @@
 PHASES = ["build", "test"]
 
 CUDA_VERSIONS = [
-    None,  # cpu build
     "92",
     "101",
     "102",
+    "110",
 ]
 
+ROCM_VERSIONS = [
+    "3.7",
+    "3.8",
+]
+
+ROCM_VERSION_LABELS = ["rocm" + v for v in ROCM_VERSIONS]
+
+GPU_VERSIONS = [None] + ["cuda" + v for v in CUDA_VERSIONS] + ROCM_VERSION_LABELS
+
 STANDARD_PYTHON_VERSIONS = [
     "3.6",
     "3.7",
-    "3.8"
+    "3.8",
 ]

.circleci/cimodel/data/pytorch_build_data.py

@@ -3,15 +3,13 @@ from cimodel.lib.conf_tree import ConfigNode, X, XImportant
 
 CONFIG_TREE_DATA = [
     ("xenial", [
-        (None, [
-            X("nightly"),
-        ]),
         ("gcc", [
             ("5.4", [  # All this subtree rebases to master and then build
-                XImportant("3.6"),
                 ("3.6", [
+                    ("important", [X(True)]),
                     ("parallel_tbb", [X(True)]),
                     ("parallel_native", [X(True)]),
+                    ("pure_torch", [X(True)]),
                 ]),
             ]),
             # TODO: bring back libtorch test
@@ -19,20 +17,41 @@ CONFIG_TREE_DATA = [
         ]),
         ("clang", [
             ("5", [
-                XImportant("3.6"),  # This is actually the ASAN build
+                ("3.6", [
+                    ("asan", [XImportant(True)]),
+                ]),
+            ]),
+            ("7", [
+                ("3.6", [
+                    ("onnx", [XImportant(True)]),
+                ]),
             ]),
         ]),
         ("cuda", [
             ("9.2", [
-                X("3.6"),
                 ("3.6", [
-                    ("cuda_gcc_override", [X("gcc5.4")])
+                    X(True),
+                    ("cuda_gcc_override", [
+                        ("gcc5.4", [
+                            ('build_only', [XImportant(True)]),
+                        ]),
+                    ]),
                 ])
             ]),
-            ("10.1", [X("3.6")]),
-            ("10.2", [
-                XImportant("3.6"),
+            ("10.1", [
                 ("3.6", [
+                    ('build_only', [X(True)]),
+                ]),
+            ]),
+            ("10.2", [
+                ("3.6", [
+                    ("important", [X(True)]),
+                    ("libtorch", [X(True)]),
+                ]),
+            ]),
+            ("11.0", [
+                ("3.8", [
+                    X(True),
                     ("libtorch", [XImportant(True)])
                 ]),
             ]),
@@ -46,11 +65,23 @@ CONFIG_TREE_DATA = [
             ("9", [
                 ("3.6", [
                     ("xla", [XImportant(True)]),
+                    ("vulkan", [XImportant(True)]),
                 ]),
             ]),
         ]),
         ("gcc", [
-            ("9", [XImportant("3.8")]),
+            ("9", [
+                ("3.8", [
+                    ("coverage", [XImportant(True)]),
+                ]),
+            ]),
+        ]),
+        ("rocm", [
+            ("3.7", [
+                ("3.6", [
+                    ('build_only', [XImportant(True)]),
+                ]),
+            ]),
         ]),
     ]),
 ]
@@ -118,17 +149,33 @@ class ExperimentalFeatureConfigNode(TreeConfigNode):
         experimental_feature = self.find_prop("experimental_feature")
 
         next_nodes = {
+            "asan": AsanConfigNode,
             "xla": XlaConfigNode,
+            "vulkan": VulkanConfigNode,
             "parallel_tbb": ParallelTBBConfigNode,
             "parallel_native": ParallelNativeConfigNode,
+            "onnx": ONNXConfigNode,
             "libtorch": LibTorchConfigNode,
             "important": ImportantConfigNode,
             "build_only": BuildOnlyConfigNode,
-            "cuda_gcc_override": CudaGccOverrideConfigNode
+            "cuda_gcc_override": CudaGccOverrideConfigNode,
+            "coverage": CoverageConfigNode,
+            "pure_torch": PureTorchConfigNode,
         }
         return next_nodes[experimental_feature]
 
 
+class PureTorchConfigNode(TreeConfigNode):
+    def modify_label(self, label):
+        return "PURE_TORCH=" + str(label)
+
+    def init2(self, node_name):
+        self.props["is_pure_torch"] = node_name
+
+    def child_constructor(self):
+        return ImportantConfigNode
+
+
 class XlaConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "XLA=" + str(label)
@@ -140,6 +187,39 @@ class XlaConfigNode(TreeConfigNode):
         return ImportantConfigNode
 
 
+class AsanConfigNode(TreeConfigNode):
+    def modify_label(self, label):
+        return "Asan=" + str(label)
+
+    def init2(self, node_name):
+        self.props["is_asan"] = node_name
+
+    def child_constructor(self):
+        return ImportantConfigNode
+
+
+class ONNXConfigNode(TreeConfigNode):
+    def modify_label(self, label):
+        return "Onnx=" + str(label)
+
+    def init2(self, node_name):
+        self.props["is_onnx"] = node_name
+
+    def child_constructor(self):
+        return ImportantConfigNode
+
+
+class VulkanConfigNode(TreeConfigNode):
+    def modify_label(self, label):
+        return "Vulkan=" + str(label)
+
+    def init2(self, node_name):
+        self.props["is_vulkan"] = node_name
+
+    def child_constructor(self):
+        return ImportantConfigNode
+
+
 class ParallelTBBConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "PARALLELTBB=" + str(label)
@@ -178,7 +258,7 @@ class CudaGccOverrideConfigNode(TreeConfigNode):
         self.props["cuda_gcc_override"] = node_name
 
     def child_constructor(self):
-        return ImportantConfigNode
+        return ExperimentalFeatureConfigNode
 
 class BuildOnlyConfigNode(TreeConfigNode):
 
@@ -186,7 +266,16 @@ class BuildOnlyConfigNode(TreeConfigNode):
         self.props["build_only"] = node_name
 
     def child_constructor(self):
-        return ImportantConfigNode
+        return ExperimentalFeatureConfigNode
+
+
+class CoverageConfigNode(TreeConfigNode):
+
+    def init2(self, node_name):
+        self.props["is_coverage"] = node_name
+
+    def child_constructor(self):
+        return ExperimentalFeatureConfigNode
 
 
 class ImportantConfigNode(TreeConfigNode):

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -1,14 +1,13 @@
 from collections import OrderedDict
+from dataclasses import dataclass, field
+from typing import List, Optional
 
-from cimodel.data.pytorch_build_data import TopLevelNode, CONFIG_TREE_DATA
 import cimodel.data.dimensions as dimensions
 import cimodel.lib.conf_tree as conf_tree
 import cimodel.lib.miniutils as miniutils
-from cimodel.data.simple.util.branch_filters import gen_filter_dict
-from cimodel.data.simple.util.docker_constants import gen_docker_image_path
-
-from dataclasses import dataclass, field
-from typing import List, Optional
+from cimodel.data.pytorch_build_data import CONFIG_TREE_DATA, TopLevelNode
+from cimodel.data.simple.util.branch_filters import gen_filter_dict, RC_PATTERN
+from cimodel.data.simple.util.docker_constants import gen_docker_image
 
 
 @dataclass
@@ -18,19 +17,25 @@ class Conf:
     parms_list_ignored_for_docker_image: Optional[List[str]] = None
     pyver: Optional[str] = None
     cuda_version: Optional[str] = None
+    rocm_version: Optional[str] = None
     # TODO expand this to cover all the USE_* that we want to test for
     #  tesnrorrt, leveldb, lmdb, redis, opencv, mkldnn, ideep, etc.
     # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453608)
     is_xla: bool = False
-    vulkan: bool = False
+    is_vulkan: bool = False
+    is_pure_torch: bool = False
     restrict_phases: Optional[List[str]] = None
     gpu_resource: Optional[str] = None
     dependent_tests: List = field(default_factory=list)
-    parent_build: Optional['Conf'] = None
+    parent_build: Optional["Conf"] = None
     is_libtorch: bool = False
     is_important: bool = False
     parallel_backend: Optional[str] = None
 
+    @staticmethod
+    def is_test_phase(phase):
+        return "test" in phase
+
     # TODO: Eliminate the special casing for docker paths
     # In the short term, we *will* need to support special casing as docker images are merged for caffe2 and pytorch
     def get_parms(self, for_docker):
@@ -42,31 +47,47 @@ class Conf:
         leading.append("pytorch")
         if self.is_xla and not for_docker:
             leading.append("xla")
+        if self.is_vulkan and not for_docker:
+            leading.append("vulkan")
         if self.is_libtorch and not for_docker:
             leading.append("libtorch")
+        if self.is_pure_torch and not for_docker:
+            leading.append("pure_torch")
         if self.parallel_backend is not None and not for_docker:
             leading.append(self.parallel_backend)
 
         cuda_parms = []
         if self.cuda_version:
-            cuda_parms.extend(["cuda" + self.cuda_version, "cudnn7"])
+            cudnn = "cudnn8" if self.cuda_version.startswith("11.") else "cudnn7"
+            cuda_parms.extend(["cuda" + self.cuda_version, cudnn])
+        if self.rocm_version:
+            cuda_parms.extend([f"rocm{self.rocm_version}"])
         result = leading + ["linux", self.distro] + cuda_parms + self.parms
         if not for_docker and self.parms_list_ignored_for_docker_image is not None:
             result = result + self.parms_list_ignored_for_docker_image
         return result
 
     def gen_docker_image_path(self):
-
         parms_source = self.parent_build or self
         base_build_env_name = "-".join(parms_source.get_parms(True))
+        image_name, _ = gen_docker_image(base_build_env_name)
+        return miniutils.quote(image_name)
 
-        return miniutils.quote(gen_docker_image_path(base_build_env_name))
+    def gen_docker_image_requires(self):
+        parms_source = self.parent_build or self
+        base_build_env_name = "-".join(parms_source.get_parms(True))
+        _, requires = gen_docker_image(base_build_env_name)
+        return miniutils.quote(requires)
 
     def get_build_job_name_pieces(self, build_or_test):
         return self.get_parms(False) + [build_or_test]
 
     def gen_build_name(self, build_or_test):
-        return ("_".join(map(str, self.get_build_job_name_pieces(build_or_test)))).replace(".", "_").replace("-", "_")
+        return (
+            ("_".join(map(str, self.get_build_job_name_pieces(build_or_test))))
+            .replace(".", "_")
+            .replace("-", "_")
+        )
 
     def get_dependents(self):
         return self.dependent_tests or []
@@ -78,20 +99,26 @@ class Conf:
         build_env_name = "-".join(map(str, build_job_name_pieces))
         parameters["build_environment"] = miniutils.quote(build_env_name)
         parameters["docker_image"] = self.gen_docker_image_path()
-        if phase == "test" and self.gpu_resource:
+        if Conf.is_test_phase(phase) and self.gpu_resource:
             parameters["use_cuda_docker_runtime"] = miniutils.quote("1")
-        if phase == "test":
+        if Conf.is_test_phase(phase):
             resource_class = "large"
             if self.gpu_resource:
                 resource_class = "gpu." + self.gpu_resource
+            if self.rocm_version is not None:
+                resource_class = "pytorch/amd-gpu"
             parameters["resource_class"] = resource_class
+        if phase == "build" and self.rocm_version is not None:
+            parameters["resource_class"] = "xlarge"
+        if hasattr(self, 'filters'):
+            parameters['filters'] = self.filters
         return parameters
 
     def gen_workflow_job(self, phase):
         job_def = OrderedDict()
         job_def["name"] = self.gen_build_name(phase)
 
-        if phase == "test":
+        if Conf.is_test_phase(phase):
 
             # TODO When merging the caffe2 and pytorch jobs, it might be convenient for a while to make a
             #  caffe2 test job dependent on a pytorch build job. This way we could quickly dedup the repeated
@@ -103,36 +130,59 @@ class Conf:
             job_name = "pytorch_linux_test"
         else:
             job_name = "pytorch_linux_build"
+            job_def["requires"] = [self.gen_docker_image_requires()]
 
         if not self.is_important:
             job_def["filters"] = gen_filter_dict()
         job_def.update(self.gen_workflow_params(phase))
 
-        return {job_name : job_def}
+        return {job_name: job_def}
 
 
 # TODO This is a hack to special case some configs just for the workflow list
 class HiddenConf(object):
-    def __init__(self, name, parent_build=None):
+    def __init__(self, name, parent_build=None, filters=None):
         self.name = name
         self.parent_build = parent_build
+        self.filters = filters
 
     def gen_workflow_job(self, phase):
-        return {self.gen_build_name(phase): {"requires": [self.parent_build.gen_build_name("build")]}}
+        return {
+            self.gen_build_name(phase): {
+                "requires": [self.parent_build.gen_build_name("build")],
+                "filters": self.filters,
+            }
+        }
 
     def gen_build_name(self, _):
         return self.name
 
+class DocPushConf(object):
+    def __init__(self, name, parent_build=None, branch="master"):
+        self.name = name
+        self.parent_build = parent_build
+        self.branch = branch
+
+    def gen_workflow_job(self, phase):
+        return {
+            "pytorch_doc_push": {
+                "name": self.name,
+                "branch": self.branch,
+                "requires": [self.parent_build],
+                "context": "org-member",
+                "filters": gen_filter_dict(branches_list=["nightly"],
+                                           tags_list=RC_PATTERN)
+            }
+        }
 
 # TODO Convert these to graph nodes
 def gen_dependent_configs(xenial_parent_config):
 
     extra_parms = [
         (["multigpu"], "large"),
-        (["NO_AVX2"], "medium"),
-        (["NO_AVX", "NO_AVX2"], "medium"),
+        (["nogpu", "NO_AVX2"], None),
+        (["nogpu", "NO_AVX"], None),
         (["slow"], "medium"),
-        (["nogpu"], None),
     ]
 
     configs = []
@@ -141,12 +191,12 @@ def gen_dependent_configs(xenial_parent_config):
         c = Conf(
             xenial_parent_config.distro,
             ["py3"] + parms,
-            pyver="3.6",
+            pyver=xenial_parent_config.pyver,
             cuda_version=xenial_parent_config.cuda_version,
             restrict_phases=["test"],
             gpu_resource=gpu,
             parent_build=xenial_parent_config,
-            is_important=xenial_parent_config.is_important,
+            is_important=False,
         )
 
         configs.append(c)
@@ -157,9 +207,44 @@ def gen_dependent_configs(xenial_parent_config):
 def gen_docs_configs(xenial_parent_config):
     configs = []
 
-    for x in ["pytorch_python_doc_push", "pytorch_cpp_doc_push", "pytorch_doc_test"]:
-        configs.append(HiddenConf(x, parent_build=xenial_parent_config))
+    configs.append(
+        HiddenConf(
+            "pytorch_python_doc_build",
+            parent_build=xenial_parent_config,
+            filters=gen_filter_dict(branches_list=r"/.*/",
+                                    tags_list=RC_PATTERN),
+        )
+    )
+    configs.append(
+        DocPushConf(
+            "pytorch_python_doc_push",
+            parent_build="pytorch_python_doc_build",
+            branch="site",
+        )
+    )
 
+    configs.append(
+        HiddenConf(
+            "pytorch_cpp_doc_build",
+            parent_build=xenial_parent_config,
+            filters=gen_filter_dict(branches_list=r"/.*/",
+                                    tags_list=RC_PATTERN),
+        )
+    )
+    configs.append(
+        DocPushConf(
+            "pytorch_cpp_doc_push",
+            parent_build="pytorch_cpp_doc_build",
+            branch="master",
+        )
+    )
+
+    configs.append(
+        HiddenConf(
+            "pytorch_doc_test",
+            parent_build=xenial_parent_config
+        )
+    )
     return configs
 
 
@@ -186,12 +271,12 @@ def instantiate_configs():
         compiler_name = fc.find_prop("compiler_name")
         compiler_version = fc.find_prop("compiler_version")
         is_xla = fc.find_prop("is_xla") or False
+        is_asan = fc.find_prop("is_asan") or False
+        is_onnx = fc.find_prop("is_onnx") or False
+        is_pure_torch = fc.find_prop("is_pure_torch") or False
+        is_vulkan = fc.find_prop("is_vulkan") or False
         parms_list_ignored_for_docker_image = []
 
-        vulkan = fc.find_prop("vulkan") or False
-        if vulkan:
-            parms_list_ignored_for_docker_image.append("vulkan")
-
         python_version = None
         if compiler_name == "cuda" or compiler_name == "android":
             python_version = fc.find_prop("pyver")
@@ -200,9 +285,14 @@ def instantiate_configs():
             parms_list = ["py" + fc.find_prop("pyver")]
 
         cuda_version = None
+        rocm_version = None
         if compiler_name == "cuda":
             cuda_version = fc.find_prop("compiler_version")
 
+        elif compiler_name == "rocm":
+            rocm_version = fc.find_prop("compiler_version")
+            restrict_phases = ["build", "test1", "test2", "caffe2_test"]
+
         elif compiler_name == "android":
             android_ndk_version = fc.find_prop("compiler_version")
             # TODO: do we need clang to compile host binaries like protoc?
@@ -216,14 +306,19 @@ def instantiate_configs():
             gcc_version = compiler_name + (fc.find_prop("compiler_version") or "")
             parms_list.append(gcc_version)
 
-            # TODO: This is a nasty special case
-            if gcc_version == 'clang5' and not is_xla:
-                parms_list.append("asan")
-                python_version = fc.find_prop("pyver")
-                parms_list[0] = fc.find_prop("abbreviated_pyver")
+        if is_asan:
+            parms_list.append("asan")
+            python_version = fc.find_prop("pyver")
+            parms_list[0] = fc.find_prop("abbreviated_pyver")
+            restrict_phases = ["build", "test1", "test2"]
 
-        if cuda_version in ["9.2", "10", "10.1", "10.2"]:
-            # TODO The gcc version is orthogonal to CUDA version?
+        if is_onnx:
+            parms_list.append("onnx")
+            python_version = fc.find_prop("pyver")
+            parms_list[0] = fc.find_prop("abbreviated_pyver")
+            restrict_phases = ["build", "ort_test1", "ort_test2"]
+
+        if cuda_version:
             cuda_gcc_version = fc.find_prop("cuda_gcc_override") or "gcc7"
             parms_list.append(cuda_gcc_version)
 
@@ -231,8 +326,13 @@ def instantiate_configs():
         is_important = fc.find_prop("is_important") or False
         parallel_backend = fc.find_prop("parallel_backend") or None
         build_only = fc.find_prop("build_only") or False
-        if build_only and restrict_phases is None:
+        is_coverage = fc.find_prop("is_coverage") or False
+        # TODO: fix pure_torch python test packaging issue.
+        if build_only or is_pure_torch:
             restrict_phases = ["build"]
+        if is_coverage and restrict_phases is None:
+            restrict_phases = ["build", "coverage_test"]
+
 
         gpu_resource = None
         if cuda_version and cuda_version != "10":
@@ -244,8 +344,10 @@ def instantiate_configs():
             parms_list_ignored_for_docker_image,
             python_version,
             cuda_version,
+            rocm_version,
             is_xla,
-            vulkan,
+            is_vulkan,
+            is_pure_torch,
             restrict_phases,
             gpu_resource,
             is_libtorch=is_libtorch,
@@ -255,20 +357,33 @@ def instantiate_configs():
 
         # run docs builds on "pytorch-linux-xenial-py3.6-gcc5.4". Docs builds
         # should run on a CPU-only build that runs on all PRs.
-        if distro_name == 'xenial' and fc.find_prop("pyver") == '3.6' \
-                and cuda_version is None \
-                and parallel_backend is None \
-                and compiler_name == 'gcc' \
-                and fc.find_prop('compiler_version') == '5.4':
+        # XXX should this be updated to a more modern build? Projects are
+        #     beginning to drop python3.6
+        if (
+            distro_name == "xenial"
+            and fc.find_prop("pyver") == "3.6"
+            and cuda_version is None
+            and parallel_backend is None
+            and not is_vulkan
+            and not is_pure_torch
+            and compiler_name == "gcc"
+            and fc.find_prop("compiler_version") == "5.4"
+        ):
+            c.filters = gen_filter_dict(branches_list=r"/.*/",
+                                        tags_list=RC_PATTERN)
             c.dependent_tests = gen_docs_configs(c)
 
-        if cuda_version == "10.1" and python_version == "3.6" and not is_libtorch:
+        if cuda_version == "10.2" and python_version == "3.6" and not is_libtorch:
             c.dependent_tests = gen_dependent_configs(c)
 
-        if (compiler_name == "gcc"
-                and compiler_version == "5.4"
-                and not is_libtorch
-                and parallel_backend is None):
+        if (
+            compiler_name == "gcc"
+            and compiler_version == "5.4"
+            and not is_libtorch
+            and not is_vulkan
+            and not is_pure_torch
+            and parallel_backend is None
+        ):
             bc_breaking_check = Conf(
                 "backward-compatibility-check",
                 [],
@@ -297,7 +412,7 @@ def get_workflow_jobs():
         for phase in phases:
 
             # TODO why does this not have a test?
-            if phase == "test" and conf_options.cuda_version == "10":
+            if Conf.is_test_phase(phase) and conf_options.cuda_version == "10":
                 continue
 
             x.append(conf_options.gen_workflow_job(phase))

.circleci/cimodel/data/simple/anaconda_prune_defintions.py

@@ -0,0 +1,28 @@
+from collections import OrderedDict
+
+from cimodel.data.simple.util.branch_filters import gen_filter_dict
+from cimodel.lib.miniutils import quote
+
+
+CHANNELS_TO_PRUNE = ["pytorch-nightly", "pytorch-test"]
+PACKAGES_TO_PRUNE = "pytorch torchvision torchaudio torchtext ignite torchcsprng"
+
+
+def gen_workflow_job(channel: str):
+    return OrderedDict(
+        {
+            "anaconda_prune": OrderedDict(
+                {
+                    "name": f"anaconda-prune-{channel}",
+                    "context": quote("org-member"),
+                    "packages": quote(PACKAGES_TO_PRUNE),
+                    "channel": channel,
+                    "filters": gen_filter_dict(branches_list=["postnightly"]),
+                }
+            )
+        }
+    )
+
+
+def get_workflow_jobs():
+    return [gen_workflow_job(channel) for channel in CHANNELS_TO_PRUNE]

.circleci/cimodel/data/simple/android_definitions.py

@@ -1,5 +1,7 @@
-import cimodel.data.simple.util.branch_filters
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_NDK
+import cimodel.data.simple.util.branch_filters as branch_filters
+from cimodel.data.simple.util.docker_constants import (
+    DOCKER_IMAGE_NDK, DOCKER_REQUIREMENT_NDK
+)
 
 
 class AndroidJob:
@@ -34,10 +36,11 @@ class AndroidJob:
             "name": full_job_name,
             "build_environment": "\"{}\"".format(build_env_name),
             "docker_image": "\"{}\"".format(DOCKER_IMAGE_NDK),
+            "requires": [DOCKER_REQUIREMENT_NDK]
         }
 
         if self.is_master_only:
-            props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
+            props_dict["filters"] = branch_filters.gen_filter_dict(branch_filters.NON_PR_BRANCH_LIST)
 
         return [{self.template_name: props_dict}]
 
@@ -47,12 +50,14 @@ class AndroidGradleJob:
                  job_name,
                  template_name,
                  dependencies,
-                 is_master_only=True):
+                 is_master_only=True,
+                 is_pr_only=False):
 
         self.job_name = job_name
         self.template_name = template_name
         self.dependencies = dependencies
         self.is_master_only = is_master_only
+        self.is_pr_only = is_pr_only
 
     def gen_tree(self):
 
@@ -62,7 +67,9 @@ class AndroidGradleJob:
         }
 
         if self.is_master_only:
-            props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
+            props_dict["filters"] = branch_filters.gen_filter_dict(branch_filters.NON_PR_BRANCH_LIST)
+        elif self.is_pr_only:
+            props_dict["filters"] = branch_filters.gen_filter_dict(branch_filters.PR_BRANCH_LIST)
 
         return [{self.template_name: props_dict}]
 
@@ -77,7 +84,14 @@ WORKFLOW_DATA = [
         "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32",
         "pytorch_android_gradle_build-x86_32",
         ["pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build"],
-        is_master_only=False),
+        is_master_only=False,
+        is_pr_only=True),
+    AndroidGradleJob(
+        "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-custom-build-single",
+        "pytorch_android_gradle_custom_build_single",
+        [DOCKER_REQUIREMENT_NDK],
+        is_master_only=False,
+        is_pr_only=True),
     AndroidGradleJob(
         "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build",
         "pytorch_android_gradle_build",

.circleci/cimodel/data/simple/bazel_definitions.py

@@ -1,4 +1,7 @@
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_GCC7
+from cimodel.data.simple.util.docker_constants import (
+    DOCKER_IMAGE_GCC7,
+    DOCKER_REQUIREMENT_GCC7
+)
 
 
 def gen_job_name(phase):
@@ -38,7 +41,10 @@ class BazelJob:
         full_job_name = gen_job_name(self.phase)
         build_env_name = "-".join(build_env_parts)
 
-        extra_requires = [gen_job_name("build")] if self.phase == "test" else []
+        extra_requires = (
+            [gen_job_name("build")] if self.phase == "test" else
+            [DOCKER_REQUIREMENT_GCC7]
+        )
 
         props_dict = {
             "build_environment": build_env_name,

.circleci/cimodel/data/simple/binary_smoketest.py

@@ -5,7 +5,7 @@ TODO: Refactor circleci/cimodel/data/binary_build_data.py to generate this file
 
  NB: If you modify this file, you need to also modify
  the binary_and_smoke_tests_on_pr variable in
- pytorch-ci-hud to adjust the list of whitelisted builds
+ pytorch-ci-hud to adjust the allowed build list
  at https://github.com/ezyang/pytorch-ci-hud/blob/master/src/BuildHistoryDisplay.js
 
  Note:

.circleci/cimodel/data/simple/docker_definitions.py

@@ -1,6 +1,7 @@
 from collections import OrderedDict
 
 from cimodel.lib.miniutils import quote
+from cimodel.data.simple.util.branch_filters import gen_filter_dict, RC_PATTERN
 
 
 # TODO: make this generated from a matrix rather than just a static list
@@ -11,6 +12,7 @@ IMAGE_NAMES = [
     "pytorch-linux-bionic-py3.6-clang9",
     "pytorch-linux-bionic-cuda10.2-cudnn7-py3.6-clang9",
     "pytorch-linux-bionic-py3.8-gcc9",
+    "pytorch-linux-bionic-rocm3.5.1-py3.6",
     "pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7",
     "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7",
     "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7",
@@ -19,26 +21,34 @@ IMAGE_NAMES = [
     "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7",
     "pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
     "pytorch-linux-xenial-py3-clang5-asan",
+    "pytorch-linux-xenial-py3-clang7-onnx",
     "pytorch-linux-xenial-py3.8",
     "pytorch-linux-xenial-py3.6-clang7",
     "pytorch-linux-xenial-py3.6-gcc4.8",
-    "pytorch-linux-xenial-py3.6-gcc5.4",
+    "pytorch-linux-xenial-py3.6-gcc5.4",  # this one is used in doc builds
     "pytorch-linux-xenial-py3.6-gcc7.2",
     "pytorch-linux-xenial-py3.6-gcc7",
-    "pytorch-linux-xenial-pynightly",
-    "pytorch-linux-xenial-rocm3.3-py3.6",
+    "pytorch-linux-bionic-rocm3.7-py3.6",
+    "pytorch-linux-bionic-rocm3.8-py3.6",
 ]
 
 
 def get_workflow_jobs():
     """Generates a list of docker image build definitions"""
-    return [
-        OrderedDict(
+    ret = []
+    for image_name in IMAGE_NAMES:
+        parameters = OrderedDict({
+            "name": quote(f"docker-{image_name}"),
+            "image_name": quote(image_name),
+        }) 
+        if image_name == "pytorch-linux-xenial-py3.6-gcc5.4":
+            # pushing documentation on tags requires CircleCI to also
+            # build all the dependencies on tags, including this docker image
+            parameters['filters'] = gen_filter_dict(branches_list=r"/.*/",
+                                                    tags_list=RC_PATTERN)
+        ret.append(OrderedDict(
             {
-                "docker_build_job": OrderedDict(
-                    {"name": quote(image_name), "image_name": quote(image_name)}
-                )
+                "docker_build_job": parameters
             }
-        )
-        for image_name in IMAGE_NAMES
-    ]
+        ))
+    return ret

.circleci/cimodel/data/simple/ios_definitions.py

@@ -62,8 +62,8 @@ class IOSJob:
 
 WORKFLOW_DATA = [
     IOSJob(IOS_VERSION, ArchVariant("x86_64"), is_org_member_context=False),
-    IOSJob(IOS_VERSION, ArchVariant("arm64")),
-    IOSJob(IOS_VERSION, ArchVariant("arm64", True), extra_props={"op_list": "mobilenetv2.yaml"}),
+    # IOSJob(IOS_VERSION, ArchVariant("arm64")),
+    # IOSJob(IOS_VERSION, ArchVariant("arm64", True), extra_props={"op_list": "mobilenetv2.yaml"}),
 ]
 
 

.circleci/cimodel/data/simple/mobile_definitions.py

@@ -4,12 +4,23 @@ PyTorch Mobile PR builds (use linux host toolchain + mobile build options)
 
 import cimodel.lib.miniutils as miniutils
 import cimodel.data.simple.util.branch_filters
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_ASAN, DOCKER_IMAGE_NDK
+from cimodel.data.simple.util.docker_constants import (
+    DOCKER_IMAGE_ASAN,
+    DOCKER_REQUIREMENT_ASAN,
+    DOCKER_IMAGE_NDK,
+    DOCKER_REQUIREMENT_NDK
+)
 
 
 class MobileJob:
-    def __init__(self, docker_image, variant_parts, is_master_only=False):
+    def __init__(
+            self,
+            docker_image,
+            docker_requires,
+            variant_parts,
+            is_master_only=False):
         self.docker_image = docker_image
+        self.docker_requires = docker_requires
         self.variant_parts = variant_parts
         self.is_master_only = is_master_only
 
@@ -30,6 +41,7 @@ class MobileJob:
             "build_environment": build_env_name,
             "build_only": miniutils.quote(str(int(True))),
             "docker_image": self.docker_image,
+            "requires": self.docker_requires,
             "name": full_job_name,
         }
 
@@ -40,15 +52,27 @@ class MobileJob:
 
 
 WORKFLOW_DATA = [
-    MobileJob(DOCKER_IMAGE_ASAN, ["build"]),
-    MobileJob(DOCKER_IMAGE_ASAN, ["custom", "build", "static"]),
+    MobileJob(
+        DOCKER_IMAGE_ASAN,
+        [DOCKER_REQUIREMENT_ASAN],
+        ["build"]
+    ),
 
     # Use LLVM-DEV toolchain in android-ndk-r19c docker image
-    MobileJob(DOCKER_IMAGE_NDK, ["custom", "build", "dynamic"]),
+    MobileJob(
+        DOCKER_IMAGE_NDK,
+        [DOCKER_REQUIREMENT_NDK],
+        ["custom", "build", "dynamic"]
+    ),
 
     # Use LLVM-DEV toolchain in android-ndk-r19c docker image
     # Most of this CI is already covered by "mobile-custom-build-dynamic" job
-    MobileJob(DOCKER_IMAGE_NDK, ["code", "analysis"], True),
+    MobileJob(
+        DOCKER_IMAGE_NDK,
+        [DOCKER_REQUIREMENT_NDK],
+        ["code", "analysis"],
+        True
+    ),
 ]
 
 

.circleci/cimodel/data/simple/nightly_android.py

@@ -1,4 +1,7 @@
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_NDK
+from cimodel.data.simple.util.docker_constants import (
+    DOCKER_IMAGE_NDK,
+    DOCKER_REQUIREMENT_NDK
+)
 
 
 class AndroidNightlyJob:
@@ -48,12 +51,13 @@ class AndroidNightlyJob:
 
         return [{self.template_name: props_dict}]
 
+BASE_REQUIRES = [DOCKER_REQUIREMENT_NDK]
 
 WORKFLOW_DATA = [
-    AndroidNightlyJob(["x86_32"], "pytorch_linux_build"),
-    AndroidNightlyJob(["x86_64"], "pytorch_linux_build"),
-    AndroidNightlyJob(["arm", "v7a"], "pytorch_linux_build"),
-    AndroidNightlyJob(["arm", "v8a"], "pytorch_linux_build"),
+    AndroidNightlyJob(["x86_32"], "pytorch_linux_build", requires=BASE_REQUIRES),
+    AndroidNightlyJob(["x86_64"], "pytorch_linux_build", requires=BASE_REQUIRES),
+    AndroidNightlyJob(["arm", "v7a"], "pytorch_linux_build", requires=BASE_REQUIRES),
+    AndroidNightlyJob(["arm", "v8a"], "pytorch_linux_build", requires=BASE_REQUIRES),
     AndroidNightlyJob(["android_gradle"], "pytorch_android_gradle_build",
                       with_docker=False,
                       requires=[

.circleci/cimodel/data/simple/nightly_ios.py

@@ -60,7 +60,7 @@ BUILD_CONFIGS = [
 
 
 WORKFLOW_DATA = BUILD_CONFIGS + [
-    IOSNightlyJob("binary", is_upload=True),
+    # IOSNightlyJob("binary", is_upload=True),
 ]
 
 

.circleci/cimodel/data/simple/util/branch_filters.py

@@ -4,6 +4,11 @@ NON_PR_BRANCH_LIST = [
     r"/release\/.*/",
 ]
 
+PR_BRANCH_LIST = [
+    r"/gh\/.*\/head/",
+    r"/pull\/.*/",
+]
+
 RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
 
 def gen_filter_dict(

.circleci/cimodel/data/simple/util/docker_constants.py

@@ -1,30 +1,33 @@
 AWS_DOCKER_HOST = "308535385114.dkr.ecr.us-east-1.amazonaws.com"
 
-# ARE YOU EDITING THIS NUMBER?  MAKE SURE YOU READ THE GUIDANCE AT THE
-# TOP OF .circleci/config.yml
-DOCKER_IMAGE_TAG = "209062ef-ab58-422a-b295-36c4eed6e906"
+def gen_docker_image(container_type):
+    return (
+        "/".join([AWS_DOCKER_HOST, "pytorch", container_type]),
+        f"docker-{container_type}",
+    )
+
+def gen_docker_image_requires(image_name):
+    return [f"docker-{image_name}"]
 
 
-def gen_docker_image_path(container_type):
-    return "/".join([
-        AWS_DOCKER_HOST,
-        "pytorch",
-        container_type + ":" + DOCKER_IMAGE_TAG,
-    ])
+DOCKER_IMAGE_BASIC, DOCKER_REQUIREMENT_BASE = gen_docker_image(
+    "pytorch-linux-xenial-py3.6-gcc5.4"
+)
+
+DOCKER_IMAGE_CUDA_10_2, DOCKER_REQUIREMENT_CUDA_10_2 = gen_docker_image(
+    "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
+)
+
+DOCKER_IMAGE_GCC7, DOCKER_REQUIREMENT_GCC7 = gen_docker_image(
+    "pytorch-linux-xenial-py3.6-gcc7"
+)
 
 
-DOCKER_IMAGE_BASIC = gen_docker_image_path("pytorch-linux-xenial-py3.6-gcc5.4")
-
-DOCKER_IMAGE_CUDA_10_2 = gen_docker_image_path("pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7")
-
-DOCKER_IMAGE_GCC7 = gen_docker_image_path("pytorch-linux-xenial-py3.6-gcc7")
-
-
-def gen_mobile_docker_name(specifier):
+def gen_mobile_docker(specifier):
     container_type = "pytorch-linux-xenial-py3-clang5-" + specifier
-    return gen_docker_image_path(container_type)
+    return gen_docker_image(container_type)
 
 
-DOCKER_IMAGE_ASAN = gen_mobile_docker_name("asan")
+DOCKER_IMAGE_ASAN, DOCKER_REQUIREMENT_ASAN = gen_mobile_docker("asan")
 
-DOCKER_IMAGE_NDK = gen_mobile_docker_name("android-ndk-r19c")
+DOCKER_IMAGE_NDK, DOCKER_REQUIREMENT_NDK = gen_mobile_docker("android-ndk-r19c")

.circleci/cimodel/data/windows_build_definitions.py

@@ -43,8 +43,11 @@ class WindowsJob:
         if base_phase == "test":
             prerequisite_jobs.append("_".join(base_name_parts + ["build"]))
 
+        if self.cuda_version:
+            self.cudnn_version = 8 if self.cuda_version.major == 11 else 7
+
         arch_env_elements = (
-            ["cuda" + str(self.cuda_version.major), "cudnn7"]
+            ["cuda" + str(self.cuda_version.major), "cudnn" + str(self.cudnn_version)]
             if self.cuda_version
             else ["cpu"]
         )
@@ -93,11 +96,14 @@ class WindowsJob:
 
 
 class VcSpec:
-    def __init__(self, year, version_elements=None):
+    def __init__(self, year, version_elements=None, hide_version=False):
         self.year = year
         self.version_elements = version_elements or []
+        self.hide_version = hide_version
 
     def get_elements(self):
+        if self.hide_version:
+            return [self.prefixed_year()]
         return [self.prefixed_year()] + self.version_elements
 
     def get_product(self):
@@ -110,7 +116,7 @@ class VcSpec:
         return "vs" + str(self.year)
 
     def render(self):
-        return "_".join(filter(None, [self.prefixed_year(), self.dotted_version()]))
+        return "_".join(self.get_elements())
 
 def FalsePred(_):
     return False
@@ -118,23 +124,22 @@ def FalsePred(_):
 def TruePred(_):
     return True
 
+_VC2019 = VcSpec(2019)
+
 WORKFLOW_DATA = [
-    # VS2017 CUDA-10.1
-    WindowsJob(None, VcSpec(2017, ["14", "11"]), CudaVersion(10, 1), master_only_pred=FalsePred),
-    WindowsJob(1, VcSpec(2017, ["14", "11"]), CudaVersion(10, 1)),
-    # VS2017 no-CUDA (builds only)
-    WindowsJob(None, VcSpec(2017, ["14", "16"]), CudaVersion(10, 1)),
-    WindowsJob(None, VcSpec(2017, ["14", "16"]), None),
     # VS2019 CUDA-10.1
-    WindowsJob(None, VcSpec(2019), CudaVersion(10, 1)),
-    WindowsJob(1, VcSpec(2019), CudaVersion(10, 1)),
-    WindowsJob(2, VcSpec(2019), CudaVersion(10, 1)),
+    WindowsJob(None, _VC2019, CudaVersion(10, 1)),
+    WindowsJob(1, _VC2019, CudaVersion(10, 1)),
+    WindowsJob(2, _VC2019, CudaVersion(10, 1)),
+    # VS2019 CUDA-11.0
+    WindowsJob(None, _VC2019, CudaVersion(11, 0)),
+    WindowsJob(1, _VC2019, CudaVersion(11, 0), master_only_pred=TruePred),
+    WindowsJob(2, _VC2019, CudaVersion(11, 0), master_only_pred=TruePred),
     # VS2019 CPU-only
-    WindowsJob(None, VcSpec(2019), None),
-    WindowsJob(1, VcSpec(2019), None),
-    WindowsJob(2, VcSpec(2019), None, master_only_pred=TruePred),
-    WindowsJob(1, VcSpec(2019), CudaVersion(10, 1), force_on_cpu=True),
-    WindowsJob(2, VcSpec(2019), CudaVersion(10, 1), force_on_cpu=True, master_only_pred=TruePred),
+    WindowsJob(None, _VC2019, None),
+    WindowsJob(1, _VC2019, None, master_only_pred=TruePred),
+    WindowsJob(2, _VC2019, None, master_only_pred=TruePred),
+    WindowsJob(1, _VC2019, CudaVersion(10, 1), force_on_cpu=True, master_only_pred=TruePred),
 ]
 
 

.circleci/docker/build.sh

@@ -10,14 +10,35 @@ if [ -z "${image}" ]; then
   exit 1
 fi
 
-# TODO: Generalize
-OS="ubuntu"
-DOCKERFILE="${OS}/Dockerfile"
-if [[ "$image" == *-cuda* ]]; then
-  DOCKERFILE="${OS}-cuda/Dockerfile"
-elif [[ "$image" == *-rocm* ]]; then
-  DOCKERFILE="${OS}-rocm/Dockerfile"
-fi
+function extract_version_from_image_name() {
+  eval export $2=$(echo "${image}" | perl -n -e"/$1(\d+(\.\d+)?(\.\d+)?)/ && print \$1")
+  if [ "x${!2}" = x ]; then
+    echo "variable '$2' not correctly parsed from image='$image'"
+    exit 1
+  fi
+}
+
+function extract_all_from_image_name() {
+  # parts $image into array, splitting on '-'
+  keep_IFS="$IFS"
+  IFS="-"
+  declare -a parts=($image)
+  IFS="$keep_IFS"
+  unset keep_IFS
+
+  for part in "${parts[@]}"; do
+    name=$(echo "${part}" | perl -n -e"/([a-zA-Z]+)\d+(\.\d+)?(\.\d+)?/ && print \$1")
+    vername="${name^^}_VERSION"
+    # "py" is the odd one out, needs this special case
+    if [ "x${name}" = xpy ]; then
+      vername=ANACONDA_PYTHON_VERSION
+    fi
+    # skip non-conforming fields such as "pytorch", "linux" or "xenial" without version string
+    if [ -n "${name}" ]; then
+      extract_version_from_image_name "${name}" "${vername}"
+    fi
+  done
+}
 
 if [[ "$image" == *-trusty* ]]; then
   UBUNTU_VERSION=14.04
@@ -29,6 +50,26 @@ elif [[ "$image" == *-bionic* ]]; then
   UBUNTU_VERSION=18.04
 elif [[ "$image" == *-focal* ]]; then
   UBUNTU_VERSION=20.04
+elif [[ "$image" == *ubuntu* ]]; then
+  extract_version_from_image_name ubuntu UBUNTU_VERSION
+elif [[ "$image" == *centos* ]]; then
+  extract_version_from_image_name centos CENTOS_VERSION
+fi
+
+if [ -n "${UBUNTU_VERSION}" ]; then
+  OS="ubuntu"
+elif [ -n "${CENTOS_VERSION}" ]; then
+  OS="centos"
+else
+  echo "Unable to derive operating system base..."
+  exit 1
+fi
+
+DOCKERFILE="${OS}/Dockerfile"
+if [[ "$image" == *cuda* ]]; then
+  DOCKERFILE="${OS}-cuda/Dockerfile"
+elif [[ "$image" == *rocm* ]]; then
+  DOCKERFILE="${OS}-rocm/Dockerfile"
 fi
 
 TRAVIS_DL_URL_PREFIX="https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/14.04/x86_64"
@@ -71,13 +112,6 @@ case "$image" in
     DB=yes
     VISION=yes
     ;;
-  pytorch-linux-xenial-pynightly)
-    TRAVIS_PYTHON_VERSION=nightly
-    GCC_VERSION=7
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ;;
   pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4)
     CUDA_VERSION=9.2
     CUDNN_VERSION=7
@@ -126,7 +160,6 @@ case "$image" in
     KATEX=yes
     ;;
   pytorch-linux-xenial-cuda11.0-cudnn8-py3-gcc7)
-    UBUNTU_VERSION=16.04-rc
     CUDA_VERSION=11.0
     CUDNN_VERSION=8
     ANACONDA_PYTHON_VERSION=3.6
@@ -143,6 +176,13 @@ case "$image" in
     DB=yes
     VISION=yes
     ;;
+  pytorch-linux-xenial-py3-clang7-onnx)
+    ANACONDA_PYTHON_VERSION=3.6
+    CLANG_VERSION=7
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    ;;
   pytorch-linux-xenial-py3-clang5-android-ndk-r19c)
     ANACONDA_PYTHON_VERSION=3.6
     CLANG_VERSION=5.0
@@ -167,6 +207,8 @@ case "$image" in
     PROTOBUF=yes
     DB=yes
     VISION=yes
+    VULKAN_SDK_VERSION=1.2.148.0
+    SWIFTSHADER=yes
     ;;
   pytorch-linux-bionic-py3.8-gcc9)
     ANACONDA_PYTHON_VERSION=3.8
@@ -194,7 +236,6 @@ case "$image" in
     VISION=yes
     ;;
   pytorch-linux-bionic-cuda11.0-cudnn8-py3.6-gcc9)
-    UBUNTU_VERSION=18.04-rc
     CUDA_VERSION=11.0
     CUDNN_VERSION=8
     ANACONDA_PYTHON_VERSION=3.6
@@ -205,7 +246,6 @@ case "$image" in
     KATEX=yes
     ;;
   pytorch-linux-bionic-cuda11.0-cudnn8-py3.8-gcc9)
-    UBUNTU_VERSION=18.04-rc
     CUDA_VERSION=11.0
     CUDNN_VERSION=8
     ANACONDA_PYTHON_VERSION=3.8
@@ -215,22 +255,52 @@ case "$image" in
     VISION=yes
     KATEX=yes
     ;;
-  pytorch-linux-xenial-rocm3.3-py3.6)
+  pytorch-linux-bionic-rocm3.7-py3.6)
     ANACONDA_PYTHON_VERSION=3.6
     PROTOBUF=yes
     DB=yes
     VISION=yes
-    ROCM_VERSION=3.3
-    # newer cmake version required
-    CMAKE_VERSION=3.6.3
+    ROCM_VERSION=3.7
     ;;
-  pytorch-linux-bionic-rocm3.3-py3.6)
+  pytorch-linux-bionic-rocm3.8-py3.6)
     ANACONDA_PYTHON_VERSION=3.6
     PROTOBUF=yes
     DB=yes
     VISION=yes
-    ROCM_VERSION=3.3
+    ROCM_VERSION=3.8
     ;;
+  *)
+    # Catch-all for builds that are not hardcoded.
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    echo "image '$image' did not match an existing build configuration"
+    if [[ "$image" == *py* ]]; then
+      extract_version_from_image_name py ANACONDA_PYTHON_VERSION
+    fi
+    if [[ "$image" == *cuda* ]]; then
+      extract_version_from_image_name cuda CUDA_VERSION
+      extract_version_from_image_name cudnn CUDNN_VERSION
+    fi
+    if [[ "$image" == *rocm* ]]; then
+      extract_version_from_image_name rocm ROCM_VERSION
+    fi
+    if [[ "$image" == *gcc* ]]; then
+      extract_version_from_image_name gcc GCC_VERSION
+    fi
+    if [[ "$image" == *clang* ]]; then
+      extract_version_from_image_name clang CLANG_VERSION
+    fi
+    if [[ "$image" == *devtoolset* ]]; then
+      extract_version_from_image_name devtoolset DEVTOOLSET_VERSION
+    fi
+    if [[ "$image" == *glibc* ]]; then
+      extract_version_from_image_name glibc GLIBC_VERSION
+    fi
+    if [[ "$image" == *cmake* ]]; then
+      extract_version_from_image_name cmake CMAKE_VERSION
+    fi
+  ;;
 esac
 
 # Set Jenkins UID and GID if running Jenkins
@@ -259,6 +329,9 @@ docker build \
        --build-arg "JENKINS_UID=${JENKINS_UID:-}" \
        --build-arg "JENKINS_GID=${JENKINS_GID:-}" \
        --build-arg "UBUNTU_VERSION=${UBUNTU_VERSION}" \
+       --build-arg "CENTOS_VERSION=${CENTOS_VERSION}" \
+       --build-arg "DEVTOOLSET_VERSION=${DEVTOOLSET_VERSION}" \
+       --build-arg "GLIBC_VERSION=${GLIBC_VERSION}" \
        --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
        --build-arg "ANACONDA_PYTHON_VERSION=${ANACONDA_PYTHON_VERSION}" \
        --build-arg "TRAVIS_PYTHON_VERSION=${TRAVIS_PYTHON_VERSION}" \
@@ -268,6 +341,8 @@ docker build \
        --build-arg "ANDROID=${ANDROID}" \
        --build-arg "ANDROID_NDK=${ANDROID_NDK_VERSION}" \
        --build-arg "GRADLE_VERSION=${GRADLE_VERSION}" \
+       --build-arg "VULKAN_SDK_VERSION=${VULKAN_SDK_VERSION}" \
+       --build-arg "SWIFTSHADER=${SWIFTSHADER}" \
        --build-arg "CMAKE_VERSION=${CMAKE_VERSION:-}" \
        --build-arg "NINJA_VERSION=${NINJA_VERSION:-}" \
        --build-arg "KATEX=${KATEX:-}" \
@@ -277,6 +352,14 @@ docker build \
        "$@" \
        .
 
+# NVIDIA dockers for RC releases use tag names like `11.0-cudnn8-devel-ubuntu18.04-rc`,
+# for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
+# find the correct image. As a result, here we have to replace the
+#   "$UBUNTU_VERSION" == "18.04-rc"
+# with
+#   "$UBUNTU_VERSION" == "18.04"
+UBUNTU_VERSION=$(echo ${UBUNTU_VERSION} | sed 's/-rc$//')
+
 function drun() {
   docker run --rm "$tmp_tag" $*
 }

.circleci/docker/build_docker.sh

@@ -13,7 +13,7 @@ retry () {
 
 #until we find a way to reliably reuse previous build, this last_tag is not in use
 # last_tag="$(( CIRCLE_BUILD_NUM - 1 ))"
-tag="${CIRCLE_WORKFLOW_ID}"
+tag="${DOCKER_TAG}"
 
 
 registry="308535385114.dkr.ecr.us-east-1.amazonaws.com"
@@ -45,9 +45,5 @@ trap "docker logout ${registry}" EXIT
 
 docker push "${image}:${tag}"
 
-# TODO: Get rid of duplicate tagging once ${DOCKER_TAG} becomes the default
-docker tag "${image}:${tag}" "${image}:${DOCKER_TAG}"
-docker push "${image}:${DOCKER_TAG}"
-
 docker save -o "${IMAGE_NAME}:${tag}.tar" "${image}:${tag}"
 aws s3 cp "${IMAGE_NAME}:${tag}.tar" "s3://ossci-linux-build/pytorch/base/${IMAGE_NAME}:${tag}.tar" --acl public-read

.circleci/docker/centos-rocm/Dockerfile

@@ -0,0 +1,93 @@
+ARG CENTOS_VERSION
+
+FROM centos:${CENTOS_VERSION}
+
+ARG CENTOS_VERSION
+
+# Install required packages to build Caffe2
+
+# Install common dependencies (so that this step can be cached separately)
+ARG EC2
+ADD ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install devtoolset
+ARG DEVTOOLSET_VERSION
+ADD ./common/install_devtoolset.sh install_devtoolset.sh
+RUN bash ./install_devtoolset.sh && rm install_devtoolset.sh
+ENV BASH_ENV "/etc/profile"
+
+# (optional) Install non-default glibc version
+ARG GLIBC_VERSION
+ADD ./common/install_glibc.sh install_glibc.sh
+RUN if [ -n "${GLIBC_VERSION}" ]; then bash ./install_glibc.sh; fi
+RUN rm install_glibc.sh
+
+# Install user
+ADD ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+# Install conda
+ENV PATH /opt/conda/bin:$PATH
+ARG ANACONDA_PYTHON_VERSION
+ADD ./common/install_conda.sh install_conda.sh
+RUN bash ./install_conda.sh && rm install_conda.sh
+
+# (optional) Install protobuf for ONNX
+ARG PROTOBUF
+ADD ./common/install_protobuf.sh install_protobuf.sh
+RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
+RUN rm install_protobuf.sh
+ENV INSTALLED_PROTOBUF ${PROTOBUF}
+
+# (optional) Install database packages like LMDB and LevelDB
+ARG DB
+ADD ./common/install_db.sh install_db.sh
+RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
+RUN rm install_db.sh
+ENV INSTALLED_DB ${DB}
+
+# (optional) Install vision packages like OpenCV and ffmpeg
+ARG VISION
+ADD ./common/install_vision.sh install_vision.sh
+RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
+RUN rm install_vision.sh
+ENV INSTALLED_VISION ${VISION}
+
+# Install rocm
+ARG ROCM_VERSION
+ADD ./common/install_rocm.sh install_rocm.sh
+RUN bash ./install_rocm.sh
+RUN rm install_rocm.sh
+ENV PATH /opt/rocm/bin:$PATH
+ENV PATH /opt/rocm/hcc/bin:$PATH
+ENV PATH /opt/rocm/hip/bin:$PATH
+ENV PATH /opt/rocm/opencl/bin:$PATH
+ENV PATH /opt/rocm/llvm/bin:$PATH
+ENV HIP_PLATFORM hcc
+ENV LANG en_US.utf8
+ENV LC_ALL en_US.utf8
+
+# (optional) Install non-default CMake version
+ARG CMAKE_VERSION
+ADD ./common/install_cmake.sh install_cmake.sh
+RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
+RUN rm install_cmake.sh
+
+# (optional) Install non-default Ninja version
+ARG NINJA_VERSION
+ADD ./common/install_ninja.sh install_ninja.sh
+RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
+RUN rm install_ninja.sh
+
+# Install ccache/sccache (do this last, so we get priority in PATH)
+ADD ./common/install_cache.sh install_cache.sh
+ENV PATH /opt/cache/bin:$PATH
+RUN bash ./install_cache.sh && rm install_cache.sh
+
+# Include BUILD_ENVIRONMENT environment variable in image
+ARG BUILD_ENVIRONMENT
+ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
+
+USER jenkins
+CMD ["bash"]

.circleci/docker/common/install_android.sh

@@ -4,13 +4,15 @@ set -ex
 
 [ -n "${ANDROID_NDK}" ]
 
+_https_amazon_aws=https://ossci-android.s3.amazonaws.com
+
 apt-get update
 apt-get install -y --no-install-recommends autotools-dev autoconf unzip
 apt-get autoclean && apt-get clean
 rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 pushd /tmp
-curl -Os --retry 3 https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK}-linux-x86_64.zip
+curl -Os --retry 3 $_https_amazon_aws/android-ndk-${ANDROID_NDK}-linux-x86_64.zip
 popd
 _ndk_dir=/opt/ndk
 mkdir -p "$_ndk_dir"
@@ -45,43 +47,22 @@ export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
 # Installing android sdk
 # https://github.com/circleci/circleci-images/blob/staging/android/Dockerfile.m4
 
-_sdk_version=sdk-tools-linux-3859397.zip
+_tmp_sdk_zip=/tmp/android-sdk-linux.zip
 _android_home=/opt/android/sdk
 
 rm -rf $_android_home
 sudo mkdir -p $_android_home
-curl --silent --show-error --location --fail --retry 3 --output /tmp/$_sdk_version https://dl.google.com/android/repository/$_sdk_version
-sudo unzip -q /tmp/$_sdk_version -d $_android_home
-rm /tmp/$_sdk_version
+curl --silent --show-error --location --fail --retry 3 --output /tmp/android-sdk-linux.zip $_https_amazon_aws/android-sdk-linux-tools3859397-build-tools2803-2902-platforms28-29.zip
+sudo unzip -q $_tmp_sdk_zip -d $_android_home
+rm $_tmp_sdk_zip
 
 sudo chmod -R 777 $_android_home
 
 export ANDROID_HOME=$_android_home
 export ADB_INSTALL_TIMEOUT=120
 
-export PATH="${ANDROID_HOME}/emulator:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:${PATH}"
+export PATH="${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:${PATH}"
 echo "PATH:${PATH}"
-alias sdkmanager="$ANDROID_HOME/tools/bin/sdkmanager"
-
-sudo mkdir ~/.android && sudo echo '### User Sources for Android SDK Manager' > ~/.android/repositories.cfg
-sudo chmod -R 777 ~/.android
-
-yes | sdkmanager --licenses
-yes | sdkmanager --update
-
-sdkmanager \
-  "tools" \
-  "platform-tools" \
-  "emulator"
-
-sdkmanager \
-  "build-tools;28.0.3" \
-  "build-tools;29.0.2"
-
-sdkmanager \
-  "platforms;android-28" \
-  "platforms;android-29"
-sdkmanager --list
 
 # Installing Gradle
 echo "GRADLE_VERSION:${GRADLE_VERSION}"
@@ -89,8 +70,7 @@ _gradle_home=/opt/gradle
 sudo rm -rf $gradle_home
 sudo mkdir -p $_gradle_home
 
-wget --no-verbose --output-document=/tmp/gradle.zip \
-"https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip"
+curl --silent --output /tmp/gradle.zip --retry 3 $_https_amazon_aws/gradle-${GRADLE_VERSION}-bin.zip
 
 sudo unzip -q /tmp/gradle.zip -d $_gradle_home
 rm /tmp/gradle.zip

.circleci/docker/common/install_base.sh

@@ -2,55 +2,123 @@
 
 set -ex
 
-# NVIDIA dockers for RC releases use tag names like `11.0-cudnn8-devel-ubuntu18.04-rc`,
-# for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
-# find the correct image. As a result, here we have to check for
-#   "$UBUNTU_VERSION" == "18.04"*
-# instead of
-#   "$UBUNTU_VERSION" == "18.04"
-if [[ "$UBUNTU_VERSION" == "18.04"* ]]; then
-  cmake3="cmake=3.10*"
-else
-  cmake3="cmake=3.5*"
-fi
+install_ubuntu() {
+  # NVIDIA dockers for RC releases use tag names like `11.0-cudnn8-devel-ubuntu18.04-rc`,
+  # for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
+  # find the correct image. As a result, here we have to check for
+  #   "$UBUNTU_VERSION" == "18.04"*
+  # instead of
+  #   "$UBUNTU_VERSION" == "18.04"
+  if [[ "$UBUNTU_VERSION" == "18.04"* ]]; then
+    cmake3="cmake=3.10*"
+  else
+    cmake3="cmake=3.5*"
+  fi
 
-# Install common dependencies
-apt-get update
-# TODO: Some of these may not be necessary
-# TODO: libiomp also gets installed by conda, aka there's a conflict
-ccache_deps="asciidoc docbook-xml docbook-xsl xsltproc"
-numpy_deps="gfortran"
-apt-get install -y --no-install-recommends \
-  $ccache_deps \
-  $numpy_deps \
-  ${cmake3} \
-  apt-transport-https \
-  autoconf \
-  automake \
-  build-essential \
-  ca-certificates \
-  curl \
-  git \
-  libatlas-base-dev \
-  libc6-dbg \
-  libiomp-dev \
-  libyaml-dev \
-  libz-dev \
-  libjpeg-dev \
-  libasound2-dev \
-  libsndfile-dev \
-  python \
-  python-dev \
-  python-setuptools \
-  python-wheel \
-  software-properties-common \
-  sudo \
-  wget \
-  vim
+  # Install common dependencies
+  apt-get update
+  # TODO: Some of these may not be necessary
+  # TODO: libiomp also gets installed by conda, aka there's a conflict
+  ccache_deps="asciidoc docbook-xml docbook-xsl xsltproc"
+  numpy_deps="gfortran"
+  apt-get install -y --no-install-recommends \
+    $ccache_deps \
+    $numpy_deps \
+    ${cmake3} \
+    apt-transport-https \
+    autoconf \
+    automake \
+    build-essential \
+    ca-certificates \
+    curl \
+    git \
+    libatlas-base-dev \
+    libc6-dbg \
+    libiomp-dev \
+    libyaml-dev \
+    libz-dev \
+    libjpeg-dev \
+    libasound2-dev \
+    libsndfile-dev \
+    python \
+    python-dev \
+    python-setuptools \
+    python-wheel \
+    software-properties-common \
+    sudo \
+    wget \
+    vim
+
+  # TODO: THIS IS A HACK!!!
+  # distributed nccl(2) tests are a bit busted, see https://github.com/pytorch/pytorch/issues/5877
+  if dpkg -s libnccl-dev; then
+    apt-get remove -y libnccl-dev libnccl2 --allow-change-held-packages
+  fi
+
+  # Cleanup package manager
+  apt-get autoclean && apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+install_centos() {
+  # Need EPEL for many packages we depend on.
+  # See http://fedoraproject.org/wiki/EPEL
+  yum --enablerepo=extras install -y epel-release
+
+  ccache_deps="asciidoc docbook-dtds docbook-style-xsl libxslt"
+  numpy_deps="gcc-gfortran"
+  # Note: protobuf-c-{compiler,devel} on CentOS are too old to be used
+  # for Caffe2. That said, we still install them to make sure the build
+  # system opts to build/use protoc and libprotobuf from third-party.
+  yum install -y \
+    $ccache_deps \
+    $numpy_deps \
+    autoconf \
+    automake \
+    bzip2 \
+    cmake \
+    cmake3 \
+    curl \
+    gcc \
+    gcc-c++ \
+    gflags-devel \
+    git \
+    glibc-devel \
+    glibc-headers \
+    glog-devel \
+    hiredis-devel \
+    libstdc++-devel \
+    make \
+    opencv-devel \
+    sudo \
+    wget \
+    vim
+
+  # Cleanup
+  yum clean all
+  rm -rf /var/cache/yum
+  rm -rf /var/lib/yum/yumdb
+  rm -rf /var/lib/yum/history
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu)
+    install_ubuntu
+    ;;
+  centos)
+    install_centos
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac
 
 # Install Valgrind separately since the apt-get version is too old.
 mkdir valgrind_build && cd valgrind_build
-VALGRIND_VERSION=3.15.0
+VALGRIND_VERSION=3.16.1
 if ! wget http://valgrind.org/downloads/valgrind-${VALGRIND_VERSION}.tar.bz2
 then
   wget https://sourceware.org/ftp/valgrind/valgrind-${VALGRIND_VERSION}.tar.bz2
@@ -64,12 +132,3 @@ cd ../../
 rm -rf valgrind_build
 alias valgrind="/usr/local/bin/valgrind"
 
-# TODO: THIS IS A HACK!!!
-# distributed nccl(2) tests are a bit busted, see https://github.com/pytorch/pytorch/issues/5877
-if dpkg -s libnccl-dev; then
-  apt-get remove -y libnccl-dev libnccl2 --allow-change-held-packages
-fi
-
-# Cleanup package manager
-apt-get autoclean && apt-get clean
-rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

.circleci/docker/common/install_cache.sh

@@ -8,7 +8,11 @@ sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment
 export PATH="/opt/cache/bin:$PATH"
 
 # Setup compiler cache
-curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
+if [ -n "$ROCM_VERSION" ]; then
+  curl --retry 3 http://repo.radeon.com/misc/.sccache_amd/sccache -o /opt/cache/bin/sccache
+else
+  curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
+fi
 chmod a+x /opt/cache/bin/sccache
 
 function write_sccache_stub() {
@@ -20,8 +24,12 @@ write_sccache_stub cc
 write_sccache_stub c++
 write_sccache_stub gcc
 write_sccache_stub g++
-write_sccache_stub clang
-write_sccache_stub clang++
+
+# NOTE: See specific ROCM_VERSION case below.
+if [ "x$ROCM_VERSION" = x ]; then
+  write_sccache_stub clang
+  write_sccache_stub clang++
+fi
 
 if [ -n "$CUDA_VERSION" ]; then
   # TODO: This is a workaround for the fact that PyTorch's FindCUDA
@@ -33,3 +41,47 @@ if [ -n "$CUDA_VERSION" ]; then
   printf "#!/bin/sh\nexec sccache $(which nvcc) \"\$@\"" > /opt/cache/lib/nvcc
   chmod a+x /opt/cache/lib/nvcc
 fi
+
+if [ -n "$ROCM_VERSION" ]; then
+  # ROCm compiler is hcc or clang. However, it is commonly invoked via hipcc wrapper.
+  # hipcc will call either hcc or clang using an absolute path starting with /opt/rocm,
+  # causing the /opt/cache/bin to be skipped. We must create the sccache wrappers
+  # directly under /opt/rocm while also preserving the original compiler names.
+  # Note symlinks will chain as follows: [hcc or clang++] -> clang -> clang-??
+  # Final link in symlink chain must point back to original directory.
+
+  # Original compiler is moved one directory deeper. Wrapper replaces it.
+  function write_sccache_stub_rocm() {
+    OLDCOMP=$1
+    COMPNAME=$(basename $OLDCOMP)
+    TOPDIR=$(dirname $OLDCOMP)
+    WRAPPED="$TOPDIR/original/$COMPNAME"
+    mv "$OLDCOMP" "$WRAPPED"
+    printf "#!/bin/sh\nexec sccache $WRAPPED \$*" > "$OLDCOMP"
+    chmod a+x "$1"
+  }
+
+  if [[ -e "/opt/rocm/hcc/bin/hcc" ]]; then
+    # ROCm 3.3 or earlier.
+    mkdir /opt/rocm/hcc/bin/original
+    write_sccache_stub_rocm /opt/rocm/hcc/bin/hcc
+    write_sccache_stub_rocm /opt/rocm/hcc/bin/clang
+    write_sccache_stub_rocm /opt/rocm/hcc/bin/clang++
+    # Fix last link in symlink chain, clang points to versioned clang in prior dir
+    pushd /opt/rocm/hcc/bin/original
+    ln -s ../$(readlink clang)
+    popd
+  elif [[ -e "/opt/rocm/llvm/bin/clang" ]]; then
+    # ROCm 3.5 and beyond.
+    mkdir /opt/rocm/llvm/bin/original
+    write_sccache_stub_rocm /opt/rocm/llvm/bin/clang
+    write_sccache_stub_rocm /opt/rocm/llvm/bin/clang++
+    # Fix last link in symlink chain, clang points to versioned clang in prior dir
+    pushd /opt/rocm/llvm/bin/original
+    ln -s ../$(readlink clang)
+    popd
+  else
+    echo "Cannot find ROCm compiler."
+    exit 1
+  fi
+fi

.circleci/docker/common/install_conda.sh

@@ -24,13 +24,20 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   mkdir /opt/conda
   chown jenkins:jenkins /opt/conda
 
+  # Work around bug where devtoolset replaces sudo and breaks it.
+  if [ -n "$DEVTOOLSET_VERSION" ]; then
+    SUDO=/bin/sudo
+  else
+    SUDO=sudo
+  fi
+
   as_jenkins() {
     # NB: unsetting the environment variables works around a conda bug
     # https://github.com/conda/conda/issues/6576
     # NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
     # NB: This must be run from a directory that jenkins has access to,
     # works around https://github.com/conda/conda-package-handling/pull/34
-    sudo -H -u jenkins env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" $*
+    $SUDO -H -u jenkins env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" $*
   }
 
   pushd /tmp
@@ -49,10 +56,10 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   pushd /opt/conda
 
   # Track latest conda update
-  as_jenkins conda update -n base conda
+  as_jenkins conda update -y -n base conda
 
   # Install correct Python version
-  as_jenkins conda install python="$ANACONDA_PYTHON_VERSION"
+  as_jenkins conda install -y python="$ANACONDA_PYTHON_VERSION"
 
   conda_install() {
     # Ensure that the install command don't upgrade/downgrade Python
@@ -67,9 +74,9 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   if [ "$ANACONDA_PYTHON_VERSION" = "3.8" ]; then
     # DO NOT install typing if installing python-3.8, since its part of python-3.8 core packages
     # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
-    conda_install numpy pyyaml mkl mkl-include setuptools cffi future six llvmdev=8.0.0
+    conda_install numpy=1.18.5 pyyaml mkl mkl-include setuptools cffi future six llvmdev=8.0.0 dataclasses
   else
-    conda_install numpy pyyaml mkl mkl-include setuptools cffi typing future six
+    conda_install numpy=1.18.5 pyyaml mkl mkl-include setuptools cffi typing future six dataclasses
   fi
   if [[ "$CUDA_VERSION" == 9.2* ]]; then
     conda_install magma-cuda92 -c pytorch
@@ -79,6 +86,8 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
     conda_install magma-cuda101 -c pytorch
   elif [[ "$CUDA_VERSION" == 10.2* ]]; then
     conda_install magma-cuda102 -c pytorch
+  elif [[ "$CUDA_VERSION" == 11.0* ]]; then
+    conda_install magma-cuda110 -c pytorch
   fi
 
   # TODO: This isn't working atm

.circleci/docker/common/install_db.sh

@@ -51,11 +51,16 @@ install_centos() {
 }
 
 # Install base packages depending on the base OS
-if [ -f /etc/lsb-release ]; then
-  install_ubuntu
-elif [ -f /etc/os-release ]; then
-  install_centos
-else
-  echo "Unable to determine OS..."
-  exit 1
-fi
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu)
+    install_ubuntu
+    ;;
+  centos)
+    install_centos
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac

.circleci/docker/common/install_devtoolset.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ex
+
+[ -n "$DEVTOOLSET_VERSION" ]
+
+yum install -y centos-release-scl
+yum install -y devtoolset-$DEVTOOLSET_VERSION
+
+echo "source scl_source enable devtoolset-$DEVTOOLSET_VERSION" > "/etc/profile.d/devtoolset-$DEVTOOLSET_VERSION.sh"

.circleci/docker/common/install_glibc.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -ex
+
+[ -n "$GLIBC_VERSION" ]
+if [[ -n "$CENTOS_VERSION" ]]; then
+  [ -n "$DEVTOOLSET_VERSION" ]
+fi
+
+yum install -y wget sed
+
+mkdir -p /packages && cd /packages
+wget -q http://ftp.gnu.org/gnu/glibc/glibc-$GLIBC_VERSION.tar.gz
+tar xzf glibc-$GLIBC_VERSION.tar.gz
+if [[ "$GLIBC_VERSION" == "2.26" ]]; then
+  cd glibc-$GLIBC_VERSION
+  sed -i 's/$name ne "nss_test1"/$name ne "nss_test1" \&\& $name ne "nss_test2"/' scripts/test-installation.pl
+  cd ..
+fi
+mkdir -p glibc-$GLIBC_VERSION-build && cd glibc-$GLIBC_VERSION-build
+
+if [[ -n "$CENTOS_VERSION" ]]; then
+  export PATH=/opt/rh/devtoolset-$DEVTOOLSET_VERSION/root/usr/bin:$PATH
+fi
+
+../glibc-$GLIBC_VERSION/configure --prefix=/usr CFLAGS='-Wno-stringop-truncation -Wno-format-overflow -Wno-restrict -Wno-format-truncation -g -O2'
+make -j$(nproc)
+make install
+
+# Cleanup
+rm -rf /packages
+rm -rf /var/cache/yum/*
+rm -rf /var/lib/rpm/__db.*
+yum clean all

.circleci/docker/common/install_llvm.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -ex
-llvm_url="https://github.com/llvm/llvm-project/releases/download/llvmorg-9.0.1/llvm-9.0.1.src.tar.xz"
-
-mkdir /opt/llvm
-
-pushd /tmp
-wget --no-verbose --output-document=llvm.tar.xz "$llvm_url"
-mkdir llvm
-tar -xf llvm.tar.xz -C llvm --strip-components 1
-rm -f llvm.tar.xz
-
-cd llvm
-mkdir build
-cd build
-cmake -G "Unix Makefiles" \
-  -DCMAKE_BUILD_TYPE=MinSizeRel \
-  -DLLVM_ENABLE_ASSERTIONS=ON \
-  -DCMAKE_INSTALL_PREFIX=/opt/llvm \
-  -DLLVM_TARGETS_TO_BUILD="host" \
-  -DLLVM_BUILD_TOOLS=OFF \
-  -DLLVM_BUILD_UTILS=OFF \
-  -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON \
-  ../
-
-make -j4
-sudo make install
-
-popd

.circleci/docker/common/install_protobuf.sh

@@ -46,11 +46,16 @@ install_centos() {
 }
 
 # Install base packages depending on the base OS
-if [ -f /etc/lsb-release ]; then
-  install_ubuntu
-elif [ -f /etc/os-release ]; then
-  install_centos
-else
-  echo "Unable to determine OS..."
-  exit 1
-fi
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu)
+    install_ubuntu
+    ;;
+  centos)
+    install_centos
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac

.circleci/docker/common/install_rocm.sh

@@ -8,6 +8,7 @@ install_ubuntu() {
       # gpg-agent is not available by default on 18.04
       apt-get install -y --no-install-recommends gpg-agent
     fi
+    apt-get install -y kmod
     apt-get install -y wget
     apt-get install -y libopenblas-dev
 
@@ -35,6 +36,15 @@ install_ubuntu() {
                    rocprofiler-dev \
                    roctracer-dev
 
+    # precompiled miopen kernels added in ROCm 3.5; search for all unversioned packages
+    # if search fails it will abort this script; use true to avoid case where search fails
+    MIOPENKERNELS=$(apt-cache search --names-only miopenkernels | awk '{print $1}' | grep -F -v . || true)
+    if [[ "x${MIOPENKERNELS}" = x ]]; then
+      echo "miopenkernels package not available"
+    else
+      DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated ${MIOPENKERNELS}
+    fi
+
   # Cleanup
   apt-get autoclean && apt-get clean
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
@@ -43,6 +53,7 @@ install_ubuntu() {
 install_centos() {
 
   yum update -y
+  yum install -y kmod
   yum install -y wget
   yum install -y openblas-devel
 
@@ -51,7 +62,7 @@ install_centos() {
 
   echo "[ROCm]" > /etc/yum.repos.d/rocm.repo
   echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo
-  echo "baseurl=http://repo.radeon.com/rocm/yum/rpm/" >> /etc/yum.repos.d/rocm.repo
+  echo "baseurl=http://repo.radeon.com/rocm/yum/${ROCM_VERSION}" >> /etc/yum.repos.d/rocm.repo
   echo "enabled=1" >> /etc/yum.repos.d/rocm.repo
   echo "gpgcheck=0" >> /etc/yum.repos.d/rocm.repo
 
@@ -79,11 +90,16 @@ install_centos() {
 }
 
 # Install Python packages depending on the base OS
-if [ -f /etc/lsb-release ]; then
-  install_ubuntu
-elif [ -f /etc/os-release ]; then
-  install_centos
-else
-  echo "Unable to determine OS..."
-  exit 1
-fi
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu)
+    install_ubuntu
+    ;;
+  centos)
+    install_centos
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac

.circleci/docker/common/install_swiftshader.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -ex
+
+[ -n "${SWIFTSHADER}" ]
+
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+_https_amazon_aws=https://ossci-android.s3.amazonaws.com
+
+# SwiftShader
+_swiftshader_dir=/var/lib/jenkins/swiftshader
+_swiftshader_file_targz=swiftshader-abe07b943-prebuilt.tar.gz
+mkdir -p $_swiftshader_dir
+_tmp_swiftshader_targz="/tmp/${_swiftshader_file_targz}"
+
+curl --silent --show-error --location --fail --retry 3 \
+  --output "${_tmp_swiftshader_targz}" "$_https_amazon_aws/${_swiftshader_file_targz}"
+
+tar -C "${_swiftshader_dir}" -xzf "${_tmp_swiftshader_targz}"
+
+export VK_ICD_FILENAMES="${_swiftshader_dir}/build/Linux/vk_swiftshader_icd.json"

.circleci/docker/common/install_travis_python.sh

@@ -49,26 +49,7 @@ if [ -n "$TRAVIS_PYTHON_VERSION" ]; then
 
   pip --version
 
-  if [[ "$TRAVIS_PYTHON_VERSION" == nightly ]]; then
-      # These two packages have broken Cythonizations uploaded
-      # to PyPi, see:
-      #
-      #  - https://github.com/numpy/numpy/issues/10500
-      #  - https://github.com/yaml/pyyaml/issues/117
-      #
-      # Furthermore, the released version of Cython does not
-      # have these issues fixed.
-      #
-      # While we are waiting on fixes for these, we build
-      # from Git for now.  Feel free to delete this conditional
-      # branch if things start working again (you may need
-      # to do this if these packages regress on Git HEAD.)
-      as_jenkins pip install git+https://github.com/cython/cython.git
-      as_jenkins pip install git+https://github.com/numpy/numpy.git
-      as_jenkins pip install git+https://github.com/yaml/pyyaml.git
-  else
-      as_jenkins pip install numpy pyyaml
-  fi
+  as_jenkins pip install numpy pyyaml
 
   as_jenkins pip install \
       future \
@@ -76,7 +57,8 @@ if [ -n "$TRAVIS_PYTHON_VERSION" ]; then
       protobuf \
       pytest \
       pillow \
-      typing
+      typing \
+      dataclasses
 
   as_jenkins pip install mkl mkl-devel
 

.circleci/docker/common/install_vision.sh

@@ -47,11 +47,16 @@ install_centos() {
 }
 
 # Install base packages depending on the base OS
-if [ -f /etc/lsb-release ]; then
-  install_ubuntu
-elif [ -f /etc/os-release ]; then
-  install_centos
-else
-  echo "Unable to determine OS..."
-  exit 1
-fi
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu)
+    install_ubuntu
+    ;;
+  centos)
+    install_centos
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac

.circleci/docker/common/install_vulkan_sdk.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -ex
+
+[ -n "${VULKAN_SDK_VERSION}" ]
+
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+_https_amazon_aws=https://ossci-android.s3.amazonaws.com
+
+_vulkansdk_dir=/var/lib/jenkins/vulkansdk
+mkdir -p $_vulkansdk_dir
+_tmp_vulkansdk_targz=/tmp/vulkansdk.tar.gz
+curl --silent --show-error --location --fail --retry 3 \
+  --output "$_tmp_vulkansdk_targz" "$_https_amazon_aws/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz"
+
+tar -C "$_vulkansdk_dir" -xzf "$_tmp_vulkansdk_targz" --strip-components 1
+
+export VULKAN_SDK="$_vulkansdk_dir/"
+
+rm "$_tmp_vulkansdk_targz"

.circleci/docker/ubuntu-cuda/Dockerfile

@@ -86,9 +86,8 @@ ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
 ENV TORCH_CUDA_ARCH_LIST Maxwell
 ENV TORCH_NVCC_FLAGS "-Xfatbin -compress-all"
 
-# Install LLVM dev version
-ADD ./common/install_llvm.sh install_llvm.sh
-RUN bash ./install_llvm.sh
+# Install LLVM dev version (Defined in the pytorch/builder github repository)
+COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
 
 USER jenkins
 CMD ["bash"]

.circleci/docker/ubuntu-rocm/Dockerfile

@@ -57,6 +57,7 @@ ENV PATH /opt/rocm/bin:$PATH
 ENV PATH /opt/rocm/hcc/bin:$PATH
 ENV PATH /opt/rocm/hip/bin:$PATH
 ENV PATH /opt/rocm/opencl/bin:$PATH
+ENV PATH /opt/rocm/llvm/bin:$PATH
 ENV HIP_PLATFORM hcc
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8

.circleci/docker/ubuntu/Dockerfile

@@ -85,6 +85,18 @@ RUN rm AndroidManifest.xml
 RUN rm build.gradle
 ENV INSTALLED_ANDROID ${ANDROID}
 
+# (optional) Install Vulkan SDK
+ARG VULKAN_SDK_VERSION
+ADD ./common/install_vulkan_sdk.sh install_vulkan_sdk.sh
+RUN if [ -n "${VULKAN_SDK_VERSION}" ]; then bash ./install_vulkan_sdk.sh; fi
+RUN rm install_vulkan_sdk.sh
+
+# (optional) Install swiftshader
+ARG SWIFTSHADER
+ADD ./common/install_swiftshader.sh install_swiftshader.sh
+RUN if [ -n "${SWIFTSHADER}" ]; then bash ./install_swiftshader.sh; fi
+RUN rm install_swiftshader.sh
+
 # (optional) Install non-default CMake version
 ARG CMAKE_VERSION
 ADD ./common/install_cmake.sh install_cmake.sh
@@ -111,9 +123,8 @@ RUN bash ./install_jni.sh && rm install_jni.sh
 ARG BUILD_ENVIRONMENT
 ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
 
-# Install LLVM dev version
-ADD ./common/install_llvm.sh install_llvm.sh
-RUN bash ./install_llvm.sh
+# Install LLVM dev version (Defined in the pytorch/builder github repository)
+COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
 
 USER jenkins
 CMD ["bash"]

.circleci/ecr_gc_docker/gc.py

@@ -88,6 +88,9 @@ parser = argparse.ArgumentParser(description="Delete old Docker tags from regist
 parser.add_argument(
     "--dry-run", action="store_true", help="Dry run; print tags that would be deleted"
 )
+parser.add_argument(
+    "--debug", action="store_true", help="Debug, print ignored / saved tags"
+)
 parser.add_argument(
     "--keep-stable-days",
     type=int,
@@ -164,51 +167,48 @@ for repo in repos(client):
 
     # Keep list of image digests to delete for this repository
     digest_to_delete = []
-    print(repositoryName)
 
     for image in images(client, repo):
         tags = image.get("imageTags")
         if not isinstance(tags, (list,)) or len(tags) == 0:
             continue
-
-        tag = tags[0]
         created = image["imagePushedAt"]
         age = now - created
-        if any([
-                looks_like_git_sha(tag),
-                tag.isdigit(),
-                tag.count("-") == 4,  # TODO: Remove, this no longer applies as tags are now built using a SHA1
-                tag in ignore_tags]):
-            window = stable_window
-            if tag in ignore_tags:
-                stable_window_tags.append((repositoryName, tag, "", age, created))
-            elif age < window:
-                stable_window_tags.append((repositoryName, tag, window, age, created))
-        else:
-            window = unstable_window
+        for tag in tags:
+            if any([
+                    looks_like_git_sha(tag),
+                    tag.isdigit(),
+                    tag.count("-") == 4,  # TODO: Remove, this no longer applies as tags are now built using a SHA1
+                    tag in ignore_tags]):
+                window = stable_window
+                if tag in ignore_tags:
+                    stable_window_tags.append((repositoryName, tag, "", age, created))
+                elif age < window:
+                    stable_window_tags.append((repositoryName, tag, window, age, created))
+            else:
+                window = unstable_window
 
-        if tag in ignore_tags:
-            print("Ignoring tag {}:{} (age: {})".format(repositoryName, tag, age))
-            continue
-        if age < window:
-            print("Not deleting manifest for tag {}:{} (age: {})".format(repositoryName, tag, age))
-            continue
-
-        if args.dry_run:
-            print("(dry run) Deleting manifest for tag {}:{} (age: {})".format(repositoryName, tag, age))
+            if tag in ignore_tags or age < window:
+                if args.debug:
+                    print("Ignoring {}:{} (age: {})".format(repositoryName, tag, age))
+                break
         else:
-            print("Deleting manifest for tag{}:{} (age: {})".format(repositoryName, tag, age))
+            for tag in tags:
+                print("{}Deleting {}:{} (age: {})".format("(dry run) " if args.dry_run else "", repositoryName, tag, age))
             digest_to_delete.append(image["imageDigest"])
+    if args.dry_run:
+        if args.debug:
+            print("Skipping actual deletion, moving on...")
+    else:
+        # Issue batch delete for all images to delete for this repository
+        # Note that as of 2018-07-25, the maximum number of images you can
+        # delete in a single batch is 100, so chunk our list into batches of
+        # 100
+        for c in chunks(digest_to_delete, 100):
+            client.batch_delete_image(
+                registryId="308535385114",
+                repositoryName=repositoryName,
+                imageIds=[{"imageDigest": digest} for digest in c],
+            )
 
-    # Issue batch delete for all images to delete for this repository
-    # Note that as of 2018-07-25, the maximum number of images you can
-    # delete in a single batch is 100, so chunk our list into batches of
-    # 100
-    for c in chunks(digest_to_delete, 100):
-        client.batch_delete_image(
-            registryId="308535385114",
-            repositoryName=repositoryName,
-            imageIds=[{"imageDigest": digest} for digest in c],
-        )
-
-    save_to_s3(args.filter_prefix, stable_window_tags)
+        save_to_s3(args.filter_prefix, stable_window_tags)

.circleci/generate_config_yml.py

@@ -8,10 +8,9 @@ Please see README.md in this directory for details.
 import os
 import shutil
 import sys
-from collections import OrderedDict, namedtuple
+from collections import namedtuple
 
 import cimodel.data.binary_build_definitions as binary_build_definitions
-import cimodel.data.caffe2_build_definitions as caffe2_build_definitions
 import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
 import cimodel.data.simple.android_definitions
 import cimodel.data.simple.bazel_definitions
@@ -23,6 +22,7 @@ import cimodel.data.simple.macos_definitions
 import cimodel.data.simple.mobile_definitions
 import cimodel.data.simple.nightly_android
 import cimodel.data.simple.nightly_ios
+import cimodel.data.simple.anaconda_prune_defintions
 import cimodel.data.windows_build_definitions as windows_build_definitions
 import cimodel.lib.miniutils as miniutils
 import cimodel.lib.miniyaml as miniyaml
@@ -83,6 +83,7 @@ class Header(object):
 
 def gen_build_workflows_tree():
     build_workflows_functions = [
+        cimodel.data.simple.docker_definitions.get_workflow_jobs,
         pytorch_build_definitions.get_workflow_jobs,
         cimodel.data.simple.macos_definitions.get_workflow_jobs,
         cimodel.data.simple.android_definitions.get_workflow_jobs,
@@ -90,23 +91,19 @@ def gen_build_workflows_tree():
         cimodel.data.simple.mobile_definitions.get_workflow_jobs,
         cimodel.data.simple.ge_config_tests.get_workflow_jobs,
         cimodel.data.simple.bazel_definitions.get_workflow_jobs,
-        caffe2_build_definitions.get_workflow_jobs,
         cimodel.data.simple.binary_smoketest.get_workflow_jobs,
         cimodel.data.simple.nightly_ios.get_workflow_jobs,
         cimodel.data.simple.nightly_android.get_workflow_jobs,
+        cimodel.data.simple.anaconda_prune_defintions.get_workflow_jobs,
         windows_build_definitions.get_windows_workflows,
+        binary_build_definitions.get_post_upload_jobs,
+        binary_build_definitions.get_binary_smoke_test_jobs,
     ]
 
     binary_build_functions = [
         binary_build_definitions.get_binary_build_jobs,
         binary_build_definitions.get_nightly_tests,
         binary_build_definitions.get_nightly_uploads,
-        binary_build_definitions.get_post_upload_jobs,
-        binary_build_definitions.get_binary_smoke_test_jobs,
-    ]
-
-    docker_builder_functions = [
-        cimodel.data.simple.docker_definitions.get_workflow_jobs
     ]
 
     return {
@@ -115,19 +112,6 @@ def gen_build_workflows_tree():
                 "when": r"<< pipeline.parameters.run_binary_tests >>",
                 "jobs": [f() for f in binary_build_functions],
             },
-            "docker_build": OrderedDict(
-                {
-                    "triggers": [
-                        {
-                            "schedule": {
-                                "cron": miniutils.quote("0 15 * * 0"),
-                                "filters": {"branches": {"only": ["master"]}},
-                            }
-                        }
-                    ],
-                    "jobs": [f() for f in docker_builder_functions],
-                }
-            ),
             "build": {"jobs": [f() for f in build_workflows_functions]},
         }
     }
@@ -140,12 +124,10 @@ YAML_SOURCES = [
     File("nightly-binary-build-defaults.yml"),
     Header("Build parameters"),
     File("build-parameters/pytorch-build-params.yml"),
-    File("build-parameters/caffe2-build-params.yml"),
     File("build-parameters/binary-build-params.yml"),
     File("build-parameters/promote-build-params.yml"),
     Header("Job specs"),
     File("job-specs/pytorch-job-specs.yml"),
-    File("job-specs/caffe2-job-specs.yml"),
     File("job-specs/binary-job-specs.yml"),
     File("job-specs/job-specs-custom.yml"),
     File("job-specs/job-specs-promote.yml"),

.circleci/scripts/binary_ios_upload.sh

@@ -14,7 +14,7 @@ mkdir -p ${ZIP_DIR}/src
 cp -R ${ARTIFACTS_DIR}/arm64/include ${ZIP_DIR}/install/
 # build a FAT bianry
 cd ${ZIP_DIR}/install/lib
-target_libs=(libc10.a libclog.a libcpuinfo.a libeigen_blas.a libpytorch_qnnpack.a libtorch_cpu.a libtorch.a libXNNPACK.a)
+target_libs=(libc10.a libclog.a libcpuinfo.a libeigen_blas.a libpthreadpool.a libpytorch_qnnpack.a libtorch_cpu.a libtorch.a libXNNPACK.a)
 for lib in ${target_libs[*]}
 do
     if [ -f "${ARTIFACTS_DIR}/x86_64/lib/${lib}" ] && [ -f "${ARTIFACTS_DIR}/arm64/lib/${lib}" ]; then

.circleci/scripts/binary_linux_build.sh

@@ -5,26 +5,18 @@ set -eux -o pipefail
 source /env
 
 # Defaults here so they can be changed in one place
-export MAX_JOBS=12
+export MAX_JOBS=${MAX_JOBS:-$(( $(nproc) - 2 ))}
 
 # Parse the parameters
 if [[ "$PACKAGE_TYPE" == 'conda' ]]; then
   build_script='conda/build_pytorch.sh'
 elif [[ "$DESIRED_CUDA" == cpu ]]; then
   build_script='manywheel/build_cpu.sh'
+elif [[ "$DESIRED_CUDA" == *"rocm"* ]]; then
+  build_script='manywheel/build_rocm.sh'
 else
   build_script='manywheel/build.sh'
 fi
 
-# We want to call unbuffer, which calls tclsh which finds the expect
-# package. The expect was installed by yum into /usr/bin so we want to
-# find /usr/bin/tclsh, but this is shadowed by /opt/conda/bin/tclsh in
-# the conda docker images, so we prepend it to the path here.
-if [[ "$PACKAGE_TYPE" == 'conda' ]]; then
-  mkdir /just_tclsh_bin
-  ln -s /usr/bin/tclsh /just_tclsh_bin/tclsh
-  export PATH=/just_tclsh_bin:$PATH
-fi
-
 # Build the package
-SKIP_ALL_TESTS=1 unbuffer "/builder/$build_script" | ts
+SKIP_ALL_TESTS=1 "/builder/$build_script"

.circleci/scripts/binary_linux_test.sh

@@ -40,7 +40,7 @@ if [[ "$PACKAGE_TYPE" == conda ]]; then
     else
       cu_ver="${DESIRED_CUDA:2:2}.${DESIRED_CUDA:4}"
     fi
-    retry conda install -yq -c pytorch "cudatoolkit=\${cu_ver}"
+    retry conda install -yq -c nvidia -c pytorch "cudatoolkit=\${cu_ver}"
   fi
 elif [[ "$PACKAGE_TYPE" != libtorch ]]; then
   pip install "\$pkg"

.circleci/scripts/binary_linux_upload.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-# Do NOT set -x
-source /home/circleci/project/env
-set -eu -o pipefail
-set +x
-declare -x "AWS_ACCESS_KEY_ID=${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
-declare -x "AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
-
-#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
-# DO NOT TURN -x ON BEFORE THIS LINE
-#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
-set -eux -o pipefail
-export PATH="$MINICONDA_ROOT/bin:$PATH"
-
-# This gets set in binary_populate_env.sh, but lets have a sane default just in case
-PIP_UPLOAD_FOLDER=${PIP_UPLOAD_FOLDER:-nightly}
-# TODO: Combine CONDA_UPLOAD_CHANNEL and PIP_UPLOAD_FOLDER into one variable
-#       The only difference is the trailing slash
-# Strip trailing slashes if there
-CONDA_UPLOAD_CHANNEL=$(echo "${PIP_UPLOAD_FOLDER}" | sed 's:/*$::')
-BACKUP_BUCKET="s3://pytorch-backup"
-
-# Upload the package to the final location
-pushd /home/circleci/project/final_pkgs
-if [[ "$PACKAGE_TYPE" == conda ]]; then
-  retry conda install -yq anaconda-client
-  retry anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload  "$(ls)" -u "pytorch-${CONDA_UPLOAD_CHANNEL}" --label main --no-progress --force
-  # Fetch  platform (eg. win-64, linux-64, etc.) from index file
-  # Because there's no actual conda command to read this
-  subdir=$(tar -xOf ./*.bz2 info/index.json | grep subdir  | cut -d ':' -f2 | sed -e 's/[[:space:]]//' -e 's/"//g' -e 's/,//')
-  BACKUP_DIR="conda/${subdir}"
-elif [[ "$PACKAGE_TYPE" == libtorch ]]; then
-  retry pip install -q awscli
-  s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-  for pkg in $(ls); do
-    retry aws s3 cp "$pkg" "$s3_dir" --acl public-read
-  done
-  BACKUP_DIR="libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-else
-  retry pip install -q awscli
-  s3_dir="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-  retry aws s3 cp "$(ls)" "$s3_dir" --acl public-read
-  BACKUP_DIR="whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-fi
-
-if [[ -n "${CIRCLE_TAG:-}" ]]; then
-  s3_dir="${BACKUP_BUCKET}/${CIRCLE_TAG}/${BACKUP_DIR}"
-  retry aws s3 cp . "$s3_dir"
-fi

.circleci/scripts/binary_macos_upload.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-# Do NOT set -x
-set -eu -o pipefail
-set +x
-export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
-export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
-
-#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
-# DO NOT TURN -x ON BEFORE THIS LINE
-#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
-set -eux -o pipefail
-
-source "/Users/distiller/project/env"
-export "PATH=$workdir/miniconda/bin:$PATH"
-
-# This gets set in binary_populate_env.sh, but lets have a sane default just in case
-PIP_UPLOAD_FOLDER=${PIP_UPLOAD_FOLDER:-nightly}
-# TODO: Combine CONDA_UPLOAD_CHANNEL and PIP_UPLOAD_FOLDER into one variable
-#       The only difference is the trailing slash
-# Strip trailing slashes if there
-CONDA_UPLOAD_CHANNEL=$(echo "${PIP_UPLOAD_FOLDER}" | sed 's:/*$::')
-BACKUP_BUCKET="s3://pytorch-backup"
-
-pushd "$workdir/final_pkgs"
-if [[ "$PACKAGE_TYPE" == conda ]]; then
-  retry conda install -yq anaconda-client
-  retry anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload "$(ls)" -u "pytorch-${CONDA_UPLOAD_CHANNEL}" --label main --no-progress --force
-  # Fetch  platform (eg. win-64, linux-64, etc.) from index file
-  # Because there's no actual conda command to read this
-  subdir=$(tar -xOf ./*.bz2 info/index.json | grep subdir  | cut -d ':' -f2 | sed -e 's/[[:space:]]//' -e 's/"//g' -e 's/,//')
-  BACKUP_DIR="conda/${subdir}"
-elif [[ "$PACKAGE_TYPE" == libtorch ]]; then
-  retry pip install -q awscli
-  s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-  for pkg in $(ls); do
-    retry aws s3 cp "$pkg" "$s3_dir" --acl public-read
-  done
-  BACKUP_DIR="libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-else
-  retry pip install -q awscli
-  s3_dir="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-  retry aws s3 cp "$(ls)" "$s3_dir" --acl public-read
-  BACKUP_DIR="whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-fi
-
-if [[ -n "${CIRCLE_TAG:-}" ]]; then
-  s3_dir="${BACKUP_BUCKET}/${CIRCLE_TAG}/${BACKUP_DIR}"
-  retry aws s3 cp . "$s3_dir"
-fi

.circleci/scripts/binary_populate_env.sh

@@ -73,7 +73,7 @@ PIP_UPLOAD_FOLDER='nightly/'
 # We put this here so that OVERRIDE_PACKAGE_VERSION below can read from it
 export DATE="$(date -u +%Y%m%d)"
 #TODO: We should be pulling semver version from the base version.txt
-BASE_BUILD_VERSION="1.6.0.dev$DATE"
+BASE_BUILD_VERSION="1.7.0.dev$DATE"
 # Change BASE_BUILD_VERSION to git tag when on a git tag
 # Use 'git -C' to make doubly sure we're in the correct directory for checking
 # the git tag
@@ -130,7 +130,7 @@ if [[ "${BUILD_FOR_SYSTEM:-}" == "windows" ]]; then
 fi
 
 export DATE="$DATE"
-export NIGHTLIES_DATE_PREAMBLE=1.6.0.dev
+export NIGHTLIES_DATE_PREAMBLE=1.7.0.dev
 export PYTORCH_BUILD_VERSION="$PYTORCH_BUILD_VERSION"
 export PYTORCH_BUILD_NUMBER="$PYTORCH_BUILD_NUMBER"
 export OVERRIDE_PACKAGE_VERSION="$PYTORCH_BUILD_VERSION"

.circleci/scripts/binary_run_in_docker.sh

@@ -19,7 +19,7 @@ chmod +x /home/circleci/project/ci_test_script.sh
 VOLUME_MOUNTS="-v /home/circleci/project/:/circleci_stuff -v /home/circleci/project/final_pkgs:/final_pkgs -v ${PYTORCH_ROOT}:/pytorch -v ${BUILDER_ROOT}:/builder"
 # Run the docker
 if [ -n "${USE_CUDA_DOCKER_RUNTIME:-}" ]; then
-  export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia ${VOLUME_MOUNTS} -t -d "${DOCKER_IMAGE}")
+  export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --gpus all ${VOLUME_MOUNTS} -t -d "${DOCKER_IMAGE}")
 else
   export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined ${VOLUME_MOUNTS} -t -d "${DOCKER_IMAGE}")
 fi

.circleci/scripts/binary_upload.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+PACKAGE_TYPE=${PACKAGE_TYPE:-conda}
+
+PKG_DIR=${PKG_DIR:-/tmp/workspace/final_pkgs}
+
+# Designates whether to submit as a release candidate or a nightly build
+# Value should be `test` when uploading release candidates
+# currently set within `designate_upload_channel`
+UPLOAD_CHANNEL=${UPLOAD_CHANNEL:-nightly}
+# Designates what subfolder to put packages into
+UPLOAD_SUBFOLDER=${UPLOAD_SUBFOLDER:-cpu}
+UPLOAD_BUCKET="s3://pytorch"
+BACKUP_BUCKET="s3://pytorch-backup"
+
+DRY_RUN=${DRY_RUN:-enabled}
+# Don't actually do work unless explicit
+ANACONDA="true anaconda"
+AWS_S3_CP="aws s3 cp --dryrun"
+if [[ "${DRY_RUN}" = "disabled" ]]; then
+  ANACONDA="anaconda"
+  AWS_S3_CP="aws s3 cp"
+fi
+
+do_backup() {
+  local backup_dir
+  backup_dir=$1
+  (
+    pushd /tmp/workspace
+    set -x
+    ${AWS_S3_CP} --recursive . "${BACKUP_BUCKET}/${CIRCLE_TAG}/${backup_dir}/"
+  )
+}
+
+conda_upload() {
+  (
+    set -x
+    ${ANACONDA} \
+      upload  \
+      ${PKG_DIR}/*.tar.bz2 \
+      -u "pytorch-${UPLOAD_CHANNEL}" \
+      --label main \
+      --no-progress \
+      --force
+  )
+}
+
+s3_upload() {
+  local extension
+  local pkg_type
+  extension="$1"
+  pkg_type="$2"
+  s3_dir="${UPLOAD_BUCKET}/${pkg_type}/${UPLOAD_CHANNEL}/${UPLOAD_SUBFOLDER}/"
+  (
+    for pkg in ${PKG_DIR}/*.${extension}; do
+      (
+        set -x
+        ${AWS_S3_CP} --no-progress --acl public-read "${pkg}" "${s3_dir}"
+      )
+    done
+  )
+}
+
+case "${PACKAGE_TYPE}" in
+  conda)
+    conda_upload
+    # Fetch  platform (eg. win-64, linux-64, etc.) from index file
+    # Because there's no actual conda command to read this
+    subdir=$(\
+      tar -xOf ${PKG_DIR}/*.bz2 info/index.json \
+        | grep subdir  \
+        | cut -d ':' -f2 \
+        | sed -e 's/[[:space:]]//' -e 's/"//g' -e 's/,//' \
+    )
+    BACKUP_DIR="conda/${subdir}"
+    ;;
+  libtorch)
+    s3_upload "zip" "libtorch"
+    BACKUP_DIR="libtorch/${UPLOAD_CHANNEL}/${UPLOAD_SUBFOLDER}"
+    ;;
+  # wheel can either refer to wheel/manywheel
+  *wheel)
+    s3_upload "whl" "whl"
+    BACKUP_DIR="whl/${UPLOAD_CHANNEL}/${UPLOAD_SUBFOLDER}"
+    ;;
+  *)
+    echo "ERROR: unknown package type: ${PACKAGE_TYPE}"
+    exit 1
+    ;;
+esac
+
+# CIRCLE_TAG is defined by upstream circleci,
+# this can be changed to recognize tagged versions
+if [[ -n "${CIRCLE_TAG:-}" ]]; then
+  do_backup "${BACKUP_DIR}"
+fi

.circleci/scripts/binary_windows_upload.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-set -eu -o pipefail
-set +x
-declare -x "AWS_ACCESS_KEY_ID=${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
-declare -x "AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
-
-#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
-# DO NOT TURN -x ON BEFORE THIS LINE
-#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
-set -eux -o pipefail
-
-source "/env"
-
-# This gets set in binary_populate_env.sh, but lets have a sane default just in case
-PIP_UPLOAD_FOLDER=${PIP_UPLOAD_FOLDER:-nightly/}
-# TODO: Combine CONDA_UPLOAD_CHANNEL and PIP_UPLOAD_FOLDER into one variable
-#       The only difference is the trailing slash
-# Strip trailing slashes if there
-CONDA_UPLOAD_CHANNEL=$(echo "${PIP_UPLOAD_FOLDER}" | sed 's:/*$::')
-BACKUP_BUCKET="s3://pytorch-backup"
-
-pushd /root/workspace/final_pkgs
-# Upload the package to the final location
-if [[ "$PACKAGE_TYPE" == conda ]]; then
-  retry conda install -yq anaconda-client
-  retry anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload  "$(ls)" -u "pytorch-${CONDA_UPLOAD_CHANNEL}" --label main --no-progress --force
-  # Fetch  platform (eg. win-64, linux-64, etc.) from index file
-  # Because there's no actual conda command to read this
-  subdir=$(tar -xOf ./*.bz2 info/index.json | grep subdir  | cut -d ':' -f2 | sed -e 's/[[:space:]]//' -e 's/"//g' -e 's/,//')
-  BACKUP_DIR="conda/${subdir}"
-elif [[ "$PACKAGE_TYPE" == libtorch ]]; then
-  retry conda install -c conda-forge -yq awscli
-  s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-  for pkg in $(ls); do
-    retry aws s3 cp "$pkg" "$s3_dir" --acl public-read
-  done
-  BACKUP_DIR="libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-else
-  retry conda install -c conda-forge -yq awscli
-  s3_dir="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-  retry aws s3 cp "$(ls)" "$s3_dir" --acl public-read
-  BACKUP_DIR="whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-fi
-
-if [[ -n "${CIRCLE_TAG:-}" ]]; then
-  s3_dir="${BACKUP_BUCKET}/${CIRCLE_TAG}/${BACKUP_DIR}"
-  retry aws s3 cp . "$s3_dir"
-fi

.circleci/scripts/build_android_gradle.sh

@@ -1,7 +1,11 @@
 #!/usr/bin/env bash
 set -eux -o pipefail
 
+env
+echo "BUILD_ENVIRONMENT:$BUILD_ENVIRONMENT"
+
 export ANDROID_NDK_HOME=/opt/ndk
+export ANDROID_NDK=/opt/ndk
 export ANDROID_HOME=/opt/android/sdk
 
 # Must be in sync with GRADLE_VERSION in docker image for android
@@ -10,6 +14,31 @@ export GRADLE_VERSION=4.10.3
 export GRADLE_HOME=/opt/gradle/gradle-$GRADLE_VERSION
 export GRADLE_PATH=$GRADLE_HOME/bin/gradle
 
+# touch gradle cache files to prevent expiration
+while IFS= read -r -d '' file
+do
+  touch "$file" || true
+done < <(find /var/lib/jenkins/.gradle -type f -print0)
+
+export GRADLE_LOCAL_PROPERTIES=~/workspace/android/local.properties
+rm -f $GRADLE_LOCAL_PROPERTIES
+echo "sdk.dir=/opt/android/sdk" >> $GRADLE_LOCAL_PROPERTIES
+echo "ndk.dir=/opt/ndk" >> $GRADLE_LOCAL_PROPERTIES
+echo "cmake.dir=/usr/local" >> $GRADLE_LOCAL_PROPERTIES
+
+retry () {
+  $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+# Run custom build script
+if [[ "${BUILD_ENVIRONMENT}" == *-gradle-custom-build* ]]; then
+  # Install torch & torchvision - used to download & dump used ops from test model.
+  retry pip install torch torchvision --progress-bar off
+
+  exec "$(dirname "${BASH_SOURCE[0]}")/../../android/build_test_app_custom.sh" armeabi-v7a
+fi
+
+# Run default build
 BUILD_ANDROID_INCLUDE_DIR_x86=~/workspace/build_android/install/include
 BUILD_ANDROID_LIB_DIR_x86=~/workspace/build_android/install/lib
 
@@ -44,9 +73,6 @@ ln -s ${BUILD_ANDROID_INCLUDE_DIR_arm_v8a} ${JNI_INCLUDE_DIR}/arm64-v8a
 ln -s ${BUILD_ANDROID_LIB_DIR_arm_v8a} ${JNI_LIBS_DIR}/arm64-v8a
 fi
 
-env
-echo "BUILD_ENVIRONMENT:$BUILD_ENVIRONMENT"
-
 GRADLE_PARAMS="-p android assembleRelease --debug --stacktrace"
 if [[ "${BUILD_ENVIRONMENT}" == *-gradle-build-only-x86_32* ]]; then
     GRADLE_PARAMS+=" -PABI_FILTERS=x86"
@@ -56,20 +82,6 @@ if [ -n "{GRADLE_OFFLINE:-}" ]; then
     GRADLE_PARAMS+=" --offline"
 fi
 
-# touch gradle cache files to prevent expiration
-while IFS= read -r -d '' file
-do
-  touch "$file" || true
-done < <(find /var/lib/jenkins/.gradle -type f -print0)
-
-env
-
-export GRADLE_LOCAL_PROPERTIES=~/workspace/android/local.properties
-rm -f $GRADLE_LOCAL_PROPERTIES
-echo "sdk.dir=/opt/android/sdk" >> $GRADLE_LOCAL_PROPERTIES
-echo "ndk.dir=/opt/ndk" >> $GRADLE_LOCAL_PROPERTIES
-echo "cmake.dir=/usr/local" >> $GRADLE_LOCAL_PROPERTIES
-
 $GRADLE_PATH $GRADLE_PARAMS
 
 find . -type f -name "*.a" -exec ls -lh {} \;

.circleci/scripts/cpp_doc_push_script.sh

@@ -30,13 +30,7 @@ if [ "$version" == "master" ]; then
   is_master_doc=true
 fi
 
-# Argument 3: (optional) If present, we will NOT do any pushing. Used for testing.
-dry_run=false
-if [ "$3" != "" ]; then
-  dry_run=true
-fi
-
-echo "install_path: $install_path  version: $version  dry_run: $dry_run"
+echo "install_path: $install_path  version: $version"
 
 # ======================== Building PyTorch C++ API Docs ========================
 
@@ -53,16 +47,11 @@ sudo apt-get -y install doxygen
 # Generate ATen files
 pushd "${pt_checkout}"
 pip install -r requirements.txt
-time python aten/src/ATen/gen.py \
+time python -m tools.codegen.gen \
   -s aten/src/ATen \
-  -d build/aten/src/ATen \
-  aten/src/ATen/Declarations.cwrap \
-  aten/src/THCUNN/generic/THCUNN.h \
-  aten/src/ATen/nn.yaml \
-  aten/src/ATen/native/native_functions.yaml
+  -d build/aten/src/ATen
 
 # Copy some required files
-cp aten/src/ATen/common_with_cwrap.py tools/shared/cwrap_common.py
 cp torch/_utils_internal.py tools/shared
 
 # Generate PyTorch files
@@ -72,12 +61,7 @@ time python tools/setup_helpers/generate_code.py \
 
 # Build the docs
 pushd docs/cpp
-pip install breathe==4.13.0 bs4 lxml six
-pip install --no-cache-dir -e "git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme"
-pip install exhale>=0.2.1
-pip install sphinx==2.4.4
-# Uncomment once it is fixed
-# pip install -r requirements.txt
+pip install -r requirements.txt
 time make VERBOSE=1 html -j
 
 popd
@@ -106,21 +90,5 @@ git config user.name "pytorchbot"
 git commit -m "Automatic sync on $(date)" || true
 git status
 
-if [ "$dry_run" = false ]; then
-  echo "Pushing to https://github.com/pytorch/cppdocs"
-  set +x
-/usr/bin/expect <<DONE
-  spawn git push -u origin master
-  expect "Username*"
-  send "pytorchbot\n"
-  expect "Password*"
-  send "$::env(GITHUB_PYTORCHBOT_TOKEN)\n"
-  expect eof
-DONE
-  set -x
-else
-  echo "Skipping push due to dry_run"
-fi
-
 popd
 # =================== The above code **should** be executed inside Docker container ===================

.circleci/scripts/driver_update.bat

@@ -0,0 +1,8 @@
+set "DRIVER_DOWNLOAD_LINK=https://s3.amazonaws.com/ossci-windows/451.82-tesla-desktop-winserver-2019-2016-international.exe"
+curl --retry 3 -kL %DRIVER_DOWNLOAD_LINK% --output 451.82-tesla-desktop-winserver-2019-2016-international.exe
+if errorlevel 1 exit /b 1
+
+start /wait 451.82-tesla-desktop-winserver-2019-2016-international.exe -s -noreboot
+if errorlevel 1 exit /b 1
+
+del 451.82-tesla-desktop-winserver-2019-2016-international.exe || ver > NUL

.circleci/scripts/python_doc_push_script.sh

@@ -7,6 +7,8 @@ sudo apt-get -y install expect-dev
 # This is where the local pytorch install in the docker image is located
 pt_checkout="/var/lib/jenkins/workspace"
 
+source "$pt_checkout/.jenkins/pytorch/common_utils.sh"
+
 echo "python_doc_push_script.sh: Invoked with $*"
 
 set -ex
@@ -38,13 +40,7 @@ echo "error: python_doc_push_script.sh: branch (arg3) not specified"
   exit 1
 fi
 
-# Argument 4: (optional) If present, we will NOT do any pushing. Used for testing.
-dry_run=false
-if [ "$4" != "" ]; then
-  dry_run=true
-fi
-
-echo "install_path: $install_path  version: $version  dry_run: $dry_run"
+echo "install_path: $install_path  version: $version"
 
 git clone https://github.com/pytorch/pytorch.github.io -b $branch
 pushd pytorch.github.io
@@ -54,25 +50,13 @@ export PATH=/opt/conda/bin:$PATH
 
 rm -rf pytorch || true
 
-# Install TensorBoard in python 3 so torch.utils.tensorboard classes render
-pip install -q https://s3.amazonaws.com/ossci-linux/wheels/tensorboard-1.14.0a0-py3-none-any.whl
-
 # Get all the documentation sources, put them in one place
 pushd "$pt_checkout"
-git clone https://github.com/pytorch/vision
-pushd vision
-conda install -q pillow
-time python setup.py install
-popd
 pushd docs
-rm -rf source/torchvision
-cp -a ../vision/docs/source source/torchvision
 
 # Build the docs
-pip -q install -r requirements.txt || true
+pip -q install -r requirements.txt
 if [ "$is_master_doc" = true ]; then
-  # TODO: fix gh-38011 then enable this which changes warnings into errors
-  # export SPHINXOPTS="-WT --keep-going"
   make html
   make coverage
   # Now we have the coverage report, we need to make sure it is empty.
@@ -126,21 +110,5 @@ git config user.name "pytorchbot"
 git commit -m "auto-generating sphinx docs" || true
 git status
 
-if [ "$dry_run" = false ]; then
-  echo "Pushing to pytorch.github.io:$branch"
-  set +x
-/usr/bin/expect <<DONE
-  spawn git push origin $branch
-  expect "Username*"
-  send "pytorchbot\n"
-  expect "Password*"
-  send "$::env(GITHUB_PYTORCHBOT_TOKEN)\n"
-  expect eof
-DONE
-  set -x
-else
-  echo "Skipping push due to dry_run"
-fi
-
 popd
 # =================== The above code **should** be executed inside Docker container ===================

.circleci/scripts/setup_ci_environment.sh

@@ -1,12 +1,6 @@
 #!/usr/bin/env bash
 set -ex -o pipefail
 
-# Set up NVIDIA docker repo
-curl -s -L --retry 3 https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
-echo "deb https://nvidia.github.io/libnvidia-container/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
-echo "deb https://nvidia.github.io/nvidia-container-runtime/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
-echo "deb https://nvidia.github.io/nvidia-docker/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
-
 # Remove unnecessary sources
 sudo rm -f /etc/apt/sources.list.d/google-chrome.list
 sudo rm -f /etc/apt/heroku.list
@@ -14,7 +8,7 @@ sudo rm -f /etc/apt/openjdk-r-ubuntu-ppa-xenial.list
 sudo rm -f /etc/apt/partner.list
 
 retry () {
-    $*  || $* || $* || $* || $*
+  $*  || $* || $* || $* || $*
 }
 
 # Method adapted from here: https://askubuntu.com/questions/875213/apt-get-to-retry-downloading
@@ -22,70 +16,75 @@ retry () {
 # This is better than retrying the whole apt-get command
 echo "APT::Acquire::Retries \"3\";" | sudo tee /etc/apt/apt.conf.d/80-retries
 
-sudo apt-get -y update
-sudo apt-get -y remove linux-image-generic linux-headers-generic linux-generic docker-ce
-# WARNING: Docker version is hardcoded here; you must update the
-# version number below for docker-ce and nvidia-docker2 to get newer
-# versions of Docker.  We hardcode these numbers because we kept
-# getting broken CI when Docker would update their docker version,
-# and nvidia-docker2 would be out of date for a day until they
-# released a newer version of their package.
-#
-# How to figure out what the correct versions of these packages are?
-# My preferred method is to start a Docker instance of the correct
-# Ubuntu version (e.g., docker run -it ubuntu:16.04) and then ask
-# apt what the packages you need are.  Note that the CircleCI image
-# comes with Docker.
-#
-# Using 'retry' here as belt-and-suspenders even though we are
-# presumably retrying at the single-package level via the
-# apt.conf.d/80-retries technique.
+retry sudo apt-get update -qq
 retry sudo apt-get -y install \
-  linux-headers-$(uname -r) \
-  linux-image-generic \
   moreutils \
-  docker-ce=5:18.09.4~3-0~ubuntu-xenial \
-  nvidia-container-runtime=2.0.0+docker18.09.4-1 \
-  nvidia-docker2=2.0.3+docker18.09.4-1 \
   expect-dev
 
-sudo pkill -SIGHUP dockerd
-
+echo "== DOCKER VERSION =="
+docker version
 
 retry sudo pip -q install awscli==1.16.35
 
 if [ -n "${USE_CUDA_DOCKER_RUNTIME:-}" ]; then
-  DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
+  DRIVER_FN="NVIDIA-Linux-x86_64-450.51.06.run"
   wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
   sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
   nvidia-smi
+
+  # Taken directly from https://github.com/NVIDIA/nvidia-docker
+  # Add the package repositories
+  distribution=$(. /etc/os-release;echo "$ID$VERSION_ID")
+  curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+  curl -s -L "https://nvidia.github.io/nvidia-docker/${distribution}/nvidia-docker.list" | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+
+  sudo apt-get update -qq
+  # Necessary to get the `--gpus` flag to function within docker
+  sudo apt-get install -y nvidia-container-toolkit
+  sudo systemctl restart docker
+else
+  # Explicitly remove nvidia docker apt repositories if not building for cuda
+  sudo rm -rf /etc/apt/sources.list.d/nvidia-docker.list
 fi
 
+add_to_env_file() {
+  local content
+  content=$1
+  # BASH_ENV should be set by CircleCI
+  echo "${content}" >> "${BASH_ENV:-/tmp/env}"
+}
+
+add_to_env_file "IN_CIRCLECI=1"
+add_to_env_file "COMMIT_SOURCE=${CIRCLE_BRANCH:-}"
+add_to_env_file "BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}"
+add_to_env_file "CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}"
+
+
 if [[ "${BUILD_ENVIRONMENT}" == *-build ]]; then
-  echo "declare -x IN_CIRCLECI=1" > /home/circleci/project/env
-  echo "declare -x COMMIT_SOURCE=${CIRCLE_BRANCH:-}" >> /home/circleci/project/env
-  echo "declare -x SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2" >> /home/circleci/project/env
+  add_to_env_file "SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2"
+
+  SCCACHE_MAX_JOBS=$(( $(nproc) - 1 ))
+  MEMORY_LIMIT_MAX_JOBS=8  # the "large" resource class on CircleCI has 32 CPU cores, if we use all of them we'll OOM
+  MAX_JOBS=$(( ${SCCACHE_MAX_JOBS} > ${MEMORY_LIMIT_MAX_JOBS} ? ${MEMORY_LIMIT_MAX_JOBS} : ${SCCACHE_MAX_JOBS} ))
+  add_to_env_file "MAX_JOBS=${MAX_JOBS}"
+
   if [ -n "${USE_CUDA_DOCKER_RUNTIME:-}" ]; then
-    echo "declare -x TORCH_CUDA_ARCH_LIST=5.2" >> /home/circleci/project/env
+    add_to_env_file "TORCH_CUDA_ARCH_LIST=5.2"
   fi
-  export SCCACHE_MAX_JOBS=`expr $(nproc) - 1`
-  export MEMORY_LIMIT_MAX_JOBS=8  # the "large" resource class on CircleCI has 32 CPU cores, if we use all of them we'll OOM
-  export MAX_JOBS=$(( ${SCCACHE_MAX_JOBS} > ${MEMORY_LIMIT_MAX_JOBS} ? ${MEMORY_LIMIT_MAX_JOBS} : ${SCCACHE_MAX_JOBS} ))
-  echo "declare -x MAX_JOBS=${MAX_JOBS}" >> /home/circleci/project/env
 
   if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
     # This IAM user allows write access to S3 bucket for sccache & bazels3cache
     set +x
-    echo "declare -x XLA_CLANG_CACHE_S3_BUCKET_NAME=${XLA_CLANG_CACHE_S3_BUCKET_NAME:-}" >> /home/circleci/project/env
-    echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_AND_XLA_BAZEL_S3_BUCKET_V2:-}" >> /home/circleci/project/env
-    echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_AND_XLA_BAZEL_S3_BUCKET_V2:-}" >> /home/circleci/project/env
+    add_to_env_file "XLA_CLANG_CACHE_S3_BUCKET_NAME=${XLA_CLANG_CACHE_S3_BUCKET_NAME:-}"
+    add_to_env_file "AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_AND_XLA_BAZEL_S3_BUCKET_V2:-}"
+    add_to_env_file "AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_AND_XLA_BAZEL_S3_BUCKET_V2:-}"
     set -x
   else
     # This IAM user allows write access to S3 bucket for sccache
     set +x
-    echo "declare -x XLA_CLANG_CACHE_S3_BUCKET_NAME=${XLA_CLANG_CACHE_S3_BUCKET_NAME:-}" >> /home/circleci/project/env
-    echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4:-}" >> /home/circleci/project/env
-    echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4:-}" >> /home/circleci/project/env
+    add_to_env_file "XLA_CLANG_CACHE_S3_BUCKET_NAME=${XLA_CLANG_CACHE_S3_BUCKET_NAME:-}"
+    add_to_env_file "AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4:-}"
+    add_to_env_file "AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4:-}"
     set -x
   fi
 fi
@@ -94,5 +93,5 @@ fi
 set +x
 export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_WRITE_V4:-}
 export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_WRITE_V4:-}
-eval $(aws ecr get-login --region us-east-1 --no-include-email)
+eval "$(aws ecr get-login --region us-east-1 --no-include-email)"
 set -x

.circleci/scripts/setup_linux_system_environment.sh

@@ -33,7 +33,7 @@ systemctl list-units --all | cat
 sudo pkill apt-get || true
 
 # For even better luck, purge unattended-upgrades
-sudo apt-get purge -y unattended-upgrades
+sudo apt-get purge -y unattended-upgrades || true
 
 cat /etc/apt/sources.list
 

.circleci/scripts/upload_binary_size_to_scuba.py

@@ -46,6 +46,7 @@ def build_message(size):
             "time": int(time.time()),
             "size": size,
             "commit_time": int(os.environ.get("COMMIT_TIME", "0")),
+            "run_duration": int(time.time() - os.path.getmtime(os.path.realpath(__file__))),
         },
     }
 
@@ -118,6 +119,7 @@ def report_android_sizes(file_dir):
                 "int": {
                     "time": int(time.time()),
                     "commit_time": int(os.environ.get("COMMIT_TIME", "0")),
+                    "run_duration": int(time.time() - os.path.getmtime(os.path.realpath(__file__))),
                     "size": comp_size,
                     "raw_size": uncomp_size,
                 },

.circleci/scripts/vs_install.ps1

@@ -1,7 +1,7 @@
 $VS_DOWNLOAD_LINK = "https://aka.ms/vs/15/release/vs_buildtools.exe"
 $COLLECT_DOWNLOAD_LINK = "https://aka.ms/vscollect.exe"
 $VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.14.11",
+                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.14.13",
                                                      "--add Microsoft.Component.MSBuild",
                                                      "--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
                                                      "--add Microsoft.VisualStudio.Component.TextTemplating",

.circleci/scripts/windows_cuda_install.sh

@@ -1,30 +1,50 @@
 #!/bin/bash
 set -eux -o pipefail
 
-curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe
-7z x cuda_10.1.243_426.00_win10.exe -ocuda_10.1.243_426.00_win10
-cd cuda_10.1.243_426.00_win10
+if [[ "$CUDA_VERSION" == "10" ]]; then
+    cuda_complete_version="10.1"
+    cuda_installer_name="cuda_10.1.243_426.00_win10"
+    msbuild_project_dir="CUDAVisualStudioIntegration/extras/visual_studio_integration/MSBuildExtensions"
+    cuda_install_packages="nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1"
+elif [[ "$CUDA_VERSION" == "11" ]]; then
+    cuda_complete_version="11.0"
+    cuda_installer_name="cuda_11.0.2_451.48_win10"
+    msbuild_project_dir="visual_studio_integration/CUDAVisualStudioIntegration/extras/visual_studio_integration/MSBuildExtensions"
+    cuda_install_packages="nvcc_11.0 cuobjdump_11.0 nvprune_11.0 nvprof_11.0 cupti_11.0 cublas_11.0 cublas_dev_11.0 cudart_11.0 cufft_11.0 cufft_dev_11.0 curand_11.0 curand_dev_11.0 cusolver_11.0 cusolver_dev_11.0 cusparse_11.0 cusparse_dev_11.0 npp_11.0 npp_dev_11.0 nvrtc_11.0 nvrtc_dev_11.0 nvml_dev_11.0"
+else
+    echo "CUDA_VERSION $CUDA_VERSION is not supported yet"
+    exit 1
+fi
+
+cuda_installer_link="https://ossci-windows.s3.amazonaws.com/${cuda_installer_name}.exe"
+
+curl --retry 3 -kLO $cuda_installer_link
+7z x ${cuda_installer_name}.exe -o${cuda_installer_name}
+cd ${cuda_installer_name}
 mkdir cuda_install_logs
 
 set +e
 
-./setup.exe -s nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
+./setup.exe -s ${cuda_install_packages} -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
 
 set -e
 
 if [[ "${VC_YEAR}" == "2017" ]]; then
-    cp -r CUDAVisualStudioIntegration/extras/visual_studio_integration/MSBuildExtensions/* "C:/Program Files (x86)/Microsoft Visual Studio/2017/${VC_PRODUCT}/Common7/IDE/VC/VCTargets/BuildCustomizations/"
+    cp -r ${msbuild_project_dir}/* "C:/Program Files (x86)/Microsoft Visual Studio/2017/${VC_PRODUCT}/Common7/IDE/VC/VCTargets/BuildCustomizations/"
 else
-    cp -r CUDAVisualStudioIntegration/extras/visual_studio_integration/MSBuildExtensions/* "C:/Program Files (x86)/Microsoft Visual Studio/2019/${VC_PRODUCT}/MSBuild/Microsoft/VC/v160/BuildCustomizations/"
+    cp -r ${msbuild_project_dir}/* "C:/Program Files (x86)/Microsoft Visual Studio/2019/${VC_PRODUCT}/MSBuild/Microsoft/VC/v160/BuildCustomizations/"
 fi
 
-curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/NvToolsExt.7z
-7z x NvToolsExt.7z -oNvToolsExt
-mkdir -p "C:/Program Files/NVIDIA Corporation/NvToolsExt"
-cp -r NvToolsExt/* "C:/Program Files/NVIDIA Corporation/NvToolsExt/"
-export NVTOOLSEXT_PATH="C:\\Program Files\\NVIDIA Corporation\\NvToolsExt\\"
+if ! ls "/c/Program Files/NVIDIA Corporation/NvToolsExt/bin/x64/nvToolsExt64_1.dll"
+then
+    curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/NvToolsExt.7z
+    7z x NvToolsExt.7z -oNvToolsExt
+    mkdir -p "C:/Program Files/NVIDIA Corporation/NvToolsExt"
+    cp -r NvToolsExt/* "C:/Program Files/NVIDIA Corporation/NvToolsExt/"
+    export NVTOOLSEXT_PATH="C:\\Program Files\\NVIDIA Corporation\\NvToolsExt\\"
+fi
 
-if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe"
+if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${cuda_complete_version}/bin/nvcc.exe"
 then
     echo "CUDA installation failed"
     mkdir -p /c/w/build-results
@@ -33,5 +53,5 @@ then
 fi
 
 cd ..
-rm -rf ./cuda_10.1.243_426.00_win10
-rm -f ./cuda_10.1.243_426.00_win10.exe
+rm -rf ./${cuda_installer_name}
+rm -f ./${cuda_installer_name}.exe

.circleci/scripts/windows_cudnn_install.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -eux -o pipefail
+
+if [[ "$CUDA_VERSION" == "10" ]]; then
+    cuda_complete_version="10.1"
+    cudnn_installer_name="cudnn-10.1-windows10-x64-v7.6.4.38"
+elif [[ "$CUDA_VERSION" == "11" ]]; then
+    cuda_complete_version="11.0"
+    cudnn_installer_name="cudnn-11.0-windows-x64-v8.0.2.39"
+else
+    echo "CUDNN for CUDA_VERSION $CUDA_VERSION is not supported yet"
+    exit 1
+fi
+
+cudnn_installer_link="https://ossci-windows.s3.amazonaws.com/${cudnn_installer_name}.zip"
+
+curl --retry 3 -O $cudnn_installer_link
+7z x ${cudnn_installer_name}.zip -ocudnn
+cp -r cudnn/cuda/* "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${cuda_complete_version}/"
+rm -rf cudnn
+rm -f ${cudnn_installer_name}.zip

.circleci/validate-docker-version.py

@@ -1,45 +0,0 @@
-#!/usr/bin/env python3
-import cimodel.data.caffe2_build_definitions as caffe2_build_definitions
-import cimodel.data.simple.util.docker_constants as pytorch_docker_constants
-
-from yaml import load
-
-try:
-    from yaml import CLoader as Loader
-except ImportError:
-    from yaml import Loader
-
-
-def load_config(filename=".circleci/config.yml"):
-    with open(filename, "r") as fh:
-        return load("".join(fh.readlines()), Loader)
-
-
-def load_tags_for_projects(workflow_config):
-    return {
-        v["ecr_gc_job"]["project"]: v["ecr_gc_job"]["tags_to_keep"]
-        for v in workflow_config["workflows"]["ecr_gc"]["jobs"]
-        if isinstance(v, dict) and "ecr_gc_job" in v
-    }
-
-
-def check_version(job, tags, expected_version):
-    valid_versions = tags[job].split(",")
-    if expected_version not in valid_versions:
-        raise RuntimeError(
-            "We configured {} to use Docker version {}; but this "
-            "version is not configured in job ecr_gc_job_for_{}.  Non-deployed versions will be "
-            "garbage collected two weeks after they are created.  DO NOT LAND "
-            "THIS TO MASTER without also updating ossci-job-dsl with this version."
-            "\n\nDeployed versions: {}".format(job, expected_version, job, tags[job])
-        )
-
-
-def validate_docker_version():
-    tags = load_tags_for_projects(load_config())
-    check_version("pytorch", tags, pytorch_docker_constants.DOCKER_IMAGE_TAG)
-    check_version("caffe2", tags, caffe2_build_definitions.DOCKER_IMAGE_VERSION)
-
-
-if __name__ == "__main__":
-    validate_docker_version()

.circleci/verbatim-sources/build-parameters/binary-build-params.yml

@@ -59,7 +59,7 @@ binary_windows_params: &binary_windows_params
       default: ""
     executor:
       type: string
-      default: "windows-cpu-with-nvidia-cuda"
+      default: "windows-xlarge-cpu-with-nvidia-cuda"
   environment:
     BUILD_ENVIRONMENT: << parameters.build_environment >>
     BUILD_FOR_SYSTEM: windows

.circleci/verbatim-sources/build-parameters/caffe2-build-params.yml

@@ -1,27 +0,0 @@
-caffe2_params: &caffe2_params
-  parameters:
-    build_environment:
-      type: string
-      default: ""
-    build_ios:
-      type: string
-      default: ""
-    docker_image:
-      type: string
-      default: ""
-    use_cuda_docker_runtime:
-      type: string
-      default: ""
-    build_only:
-      type: string
-      default: ""
-    resource_class:
-      type: string
-      default: "large"
-  environment:
-    BUILD_ENVIRONMENT: << parameters.build_environment >>
-    BUILD_IOS: << parameters.build_ios >>
-    USE_CUDA_DOCKER_RUNTIME: << parameters.use_cuda_docker_runtime >>
-    DOCKER_IMAGE: << parameters.docker_image >>
-    BUILD_ONLY: << parameters.build_only >>
-  resource_class: << parameters.resource_class >>

.circleci/verbatim-sources/build-parameters/pytorch-build-params.yml

@@ -46,7 +46,7 @@ pytorch_windows_params: &pytorch_windows_params
   parameters:
     executor:
       type: string
-      default: "windows-cpu-with-nvidia-cuda"
+      default: "windows-xlarge-cpu-with-nvidia-cuda"
     build_environment:
       type: string
       default: ""
@@ -61,10 +61,10 @@ pytorch_windows_params: &pytorch_windows_params
       default: "3.6"
     vc_version:
       type: string
-      default: "14.11"
+      default: "14.16"
     vc_year:
       type: string
-      default: "2017"
+      default: "2019"
     vc_product:
       type: string
       default: "BuildTools"

.circleci/verbatim-sources/commands.yml

@@ -1,23 +1,26 @@
 commands:
 
-  # Must be run after attaching workspace from previous steps
-  load_shared_env:
-    description: "Loads .circleci/shared/env_file into ${BASH_ENV}"
-    parameters:
-      # For some weird reason we decide to reattach our workspace to ~/workspace so
-      # in the vein of making it simple let's assume our share env_file is here
-      root:
-        type: string
-        default: "~/workspace"
+  calculate_docker_image_tag:
+    description: "Calculates the docker image tag"
     steps:
       - run:
-          name: "Load .circleci/shared/env_file into ${BASH_ENV}"
+          name: "Calculate docker image hash"
           command: |
-            if [[ -f  "<< parameters.root >>/.circleci/shared/env_file" ]]; then
-              cat << parameters.root >>/.circleci/shared/env_file >> ${BASH_ENV}
-            else
-              echo "We didn't have a shared env file, that's weird"
+            DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker)
+            echo "DOCKER_TAG=${DOCKER_TAG}" >> "${BASH_ENV}"
+
+  designate_upload_channel:
+    description: "inserts the correct upload channel into ${BASH_ENV}"
+    steps:
+      - run:
+          name: adding UPLOAD_CHANNEL to BASH_ENV
+          command: |
+            our_upload_channel=nightly
+            # On tags upload to test instead
+            if [[ -n "${CIRCLE_TAG}" ]]; then
+              our_upload_channel=test
             fi
+            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
 
   # This system setup script is meant to run before the CI-related scripts, e.g.,
   # installing Git client, checking out code, setting up CI env, and
@@ -130,4 +133,42 @@ commands:
               echo "This is not a pull request, skipping..."
             fi
 
-
+  upload_binary_size_for_android_build:
+    description: "Upload binary size data for Android build"
+    parameters:
+      build_type:
+        type: string
+        default: ""
+      artifacts:
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: "Binary Size - Install Dependencies"
+          no_output_timeout: "5m"
+          command: |
+            retry () {
+              $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+            }
+            retry pip3 install requests
+      - run:
+          name: "Binary Size - Untar Artifacts"
+          no_output_timeout: "5m"
+          command: |
+            # The artifact file is created inside docker container, which contains the result binaries.
+            # Now unpackage it into the project folder. The subsequent script will scan project folder
+            # to locate result binaries and report their sizes.
+            # If artifact file is not provided it assumes that the project folder has been mounted in
+            # the docker during build and already contains the result binaries, so this step can be skipped.
+            export ARTIFACTS="<< parameters.artifacts >>"
+            if [ -n "${ARTIFACTS}" ]; then
+              tar xf "${ARTIFACTS}" -C ~/project
+            fi
+      - run:
+          name: "Binary Size - Upload << parameters.build_type >>"
+          no_output_timeout: "5m"
+          command: |
+            cd ~/project
+            export ANDROID_BUILD_TYPE="<< parameters.build_type >>"
+            export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
+            python3 .circleci/scripts/upload_binary_size_to_scuba.py android

.circleci/verbatim-sources/header-section.yml

@@ -26,9 +26,14 @@ executors:
       image: windows-server-2019-nvidia:stable
       shell: bash.exe
 
-  windows-cpu-with-nvidia-cuda:
+  windows-xlarge-cpu-with-nvidia-cuda:
     machine:
-      # we will change to CPU host when it's ready
       resource_class: windows.xlarge
       image: windows-server-2019-vs2019:stable
       shell: bash.exe
+
+  windows-medium-cpu-with-nvidia-cuda:
+    machine:
+      resource_class: windows.medium
+      image: windows-server-2019-vs2019:stable
+      shell: bash.exe

.circleci/verbatim-sources/job-specs/binary-job-specs.yml

@@ -1,60 +1,42 @@
   binary_linux_build:
     <<: *binary_linux_build_params
     steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
     - checkout
+    - calculate_docker_image_tag
     - run:
         <<: *binary_checkout
     - run:
         <<: *binary_populate_env
-    - run:
-        name: Install unbuffer and ts
-        command: |
-            set -eux -o pipefail
-            source /env
-            OS_NAME=`awk -F= '/^NAME/{print $2}' /etc/os-release`
-            if [[ "$OS_NAME" == *"CentOS Linux"* ]]; then
-              retry yum -q -y install epel-release
-              retry yum -q -y install expect moreutils
-            elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then
-              retry apt-get update
-              retry apt-get -y install expect moreutils
-              retry conda install -y -c eumetsat expect
-              retry conda install -y cmake
-            fi
-    - run:
-        name: Update compiler to devtoolset7
-        command: |
-            set -eux -o pipefail
-            source /env
-            if [[ "$DESIRED_DEVTOOLSET" == 'devtoolset7' ]]; then
-              source "/builder/update_compiler.sh"
-
-              # Env variables are not persisted into the next step
-              echo "export PATH=$PATH" >> /env
-              echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> /env
-            else
-              echo "Not updating compiler"
-            fi
     - run:
         name: Build
         no_output_timeout: "1h"
         command: |
             source "/pytorch/.circleci/scripts/binary_linux_build.sh"
+            # Preserve build log
+            if [ -f /pytorch/build/.ninja_log ]; then
+              cp /pytorch/build/.ninja_log /final_pkgs
+            fi
+    - run:
+        name: Output binary sizes
+        no_output_timeout: "1m"
+        command: |
+            ls -lah /final_pkgs
     - run:
         name: save binary size
         no_output_timeout: "5m"
         command: |
             source /env
             cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
-            pip3 install requests && \
+            python3 -mpip install requests && \
             SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
             python3 /pytorch/.circleci/scripts/upload_binary_size_to_scuba.py || exit 0
-
     - persist_to_workspace:
         root: /
         paths: final_pkgs
 
+    - store_artifacts:
+        path: /final_pkgs
+
     # This should really just be another step of the binary_linux_build job above.
     # This isn't possible right now b/c the build job uses the docker executor
     # (otherwise they'd be really really slow) but this one uses the macine
@@ -63,11 +45,10 @@
   binary_linux_test:
     <<: *binary_linux_test_upload_params
     machine:
-        image: ubuntu-1604:201903-01
+        image: ubuntu-1604:202007-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
     - checkout
-    # TODO: We shouldn't attach the workspace multiple times
     - attach_workspace:
         at: /home/circleci/project
     - setup_linux_system_environment
@@ -83,25 +64,41 @@
     - run:
         <<: *binary_run_in_docker
 
-  binary_linux_upload:
-    <<: *binary_linux_test_upload_params
-    machine:
-        image: ubuntu-1604:201903-01
+  binary_upload:
+    parameters:
+      package_type:
+        type: string
+        description: "What type of package we are uploading (eg. wheel, libtorch, conda)"
+        default: "wheel"
+      upload_subfolder:
+        type: string
+        description: "What subfolder to put our package into (eg. cpu, cudaX.Y, etc.)"
+        default: "cpu"
+    docker:
+      - image: continuumio/miniconda3
+    environment:
+      - DRY_RUN: disabled
+      - PACKAGE_TYPE: "<< parameters.package_type >>"
+      - UPLOAD_SUBFOLDER: "<< parameters.upload_subfolder >>"
     steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
-    - setup_linux_system_environment
-    - setup_ci_environment
-    - attach_workspace:
-        at: /home/circleci/project
-    - run:
-        <<: *binary_populate_env
-    - run:
-        <<: *binary_install_miniconda
-    - run:
-        name: Upload
-        no_output_timeout: "1h"
-        command: .circleci/scripts/binary_linux_upload.sh
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - designate_upload_channel
+      - run:
+          name: Install dependencies
+          no_output_timeout: "1h"
+          command: |
+            conda install -yq anaconda-client
+            pip install -q awscli
+      - run:
+          name: Do upload
+          no_output_timeout: "1h"
+          command: |
+            AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" \
+              AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" \
+              ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN}" \
+              .circleci/scripts/binary_upload.sh
 
   # Nighlty build smoke tests defaults
   # These are the second-round smoke tests. These make sure that the binaries are
@@ -111,9 +108,10 @@
   smoke_linux_test:
     <<: *binary_linux_test_upload_params
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     - checkout
+    - calculate_docker_image_tag
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -137,7 +135,7 @@
   smoke_mac_test:
     <<: *binary_linux_test_upload_params
     macos:
-      xcode: "9.4.1"
+      xcode: "11.2.1"
     steps:
       - checkout
       - run:
@@ -162,7 +160,7 @@
   binary_mac_build:
     <<: *binary_mac_params
     macos:
-      xcode: "9.4.1"
+      xcode: "11.2.1"
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
     - checkout
@@ -200,30 +198,6 @@
         root: /Users/distiller/project
         paths: final_pkgs
 
-  binary_mac_upload: &binary_mac_upload
-    <<: *binary_mac_params
-    macos:
-      xcode: "9.4.1"
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
-    - run:
-        <<: *binary_checkout
-    - run:
-        <<: *binary_populate_env
-    - brew_update
-    - run:
-        <<: *binary_install_miniconda
-    - attach_workspace: # TODO - we can `cp` from ~/workspace
-        at: /Users/distiller/project
-    - run:
-        name: Upload
-        no_output_timeout: "10m"
-        command: |
-          script="/Users/distiller/project/pytorch/.circleci/scripts/binary_macos_upload.sh"
-          cat "$script"
-          source "$script"
-
   binary_ios_build:
     <<: *pytorch_ios_params
     macos:
@@ -276,7 +250,7 @@
         default: ""
       executor:
         type: string
-        default: "windows-cpu-with-nvidia-cuda"
+        default: "windows-xlarge-cpu-with-nvidia-cuda"
     executor: <<parameters.executor>>
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
@@ -305,7 +279,7 @@
         default: ""
       executor:
         type: string
-        default: "windows-cpu-with-nvidia-cuda"
+        default: "windows-medium-cpu-with-nvidia-cuda"
     executor: <<parameters.executor>>
     steps:
     - checkout
@@ -324,28 +298,6 @@
           cat "$script"
           source "$script"
 
-  binary_windows_upload:
-    <<: *binary_windows_params
-    docker:
-      - image: continuumio/miniconda
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
-    - attach_workspace:
-        at: /root/workspace
-    - run:
-        <<: *binary_checkout
-    - run:
-        <<: *binary_populate_env
-    - run:
-        name: Upload
-        no_output_timeout: "10m"
-        command: |
-          set -eux -o pipefail
-          script="/pytorch/.circleci/scripts/binary_windows_upload.sh"
-          cat "$script"
-          source "$script"
-
   smoke_windows_test:
     <<: *binary_windows_params
     parameters:
@@ -354,7 +306,7 @@
         default: ""
       executor:
         type: string
-        default: "windows-cpu-with-nvidia-cuda"
+        default: "windows-medium-cpu-with-nvidia-cuda"
     executor: <<parameters.executor>>
     steps:
     - checkout
@@ -372,3 +324,32 @@
           cat "$script"
           source "$script"
 
+  anaconda_prune:
+    parameters:
+      packages:
+        type: string
+        description: "What packages are we pruning? (quoted, space-separated string. eg. 'pytorch', 'torchvision torchaudio', etc.)"
+        default: "pytorch"
+      channel:
+        type: string
+        description: "What channel are we pruning? (eq. pytorch-nightly)"
+        default: "pytorch-nightly"
+    docker:
+      - image: continuumio/miniconda3
+    environment:
+      - PACKAGES: "<< parameters.packages >>"
+      - CHANNEL: "<< parameters.channel >>"
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          no_output_timeout: "1h"
+          command: |
+            conda install -yq anaconda-client
+      - run:
+          name: Prune packages
+          no_output_timeout: "1h"
+          command: |
+              ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN}" \
+              scripts/release/anaconda-prune/run.sh
+

.circleci/verbatim-sources/job-specs/binary_update_htmls.yml

@@ -8,7 +8,8 @@
   # then install the one with the most recent version.
   update_s3_htmls: &update_s3_htmls
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
+    resource_class: medium
     steps:
     - checkout
     - setup_linux_system_environment

.circleci/verbatim-sources/job-specs/caffe2-job-specs.yml

@@ -1,198 +0,0 @@
-  caffe2_linux_build:
-    <<: *caffe2_params
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
-    - setup_linux_system_environment
-    - setup_ci_environment
-    - run:
-        name: Build
-        no_output_timeout: "1h"
-        command: |
-          set -e
-          cat >/home/circleci/project/ci_build_script.sh \<<EOL
-          # =================== The following code will be executed inside Docker container ===================
-          set -ex
-          export BUILD_ENVIRONMENT="$BUILD_ENVIRONMENT"
-
-          # Reinitialize submodules
-          git submodule sync && git submodule update -q --init --recursive
-
-          # conda must be added to the path for Anaconda builds (this location must be
-          # the same as that in install_anaconda.sh used to build the docker image)
-          if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
-            export PATH=/opt/conda/bin:$PATH
-            sudo chown -R jenkins:jenkins '/opt/conda'
-          fi
-
-          # Build
-          ./.jenkins/caffe2/build.sh
-
-          # Show sccache stats if it is running
-          if pgrep sccache > /dev/null; then
-            sccache --show-stats
-          fi
-          # =================== The above code will be executed inside Docker container ===================
-          EOL
-          chmod +x /home/circleci/project/ci_build_script.sh
-
-          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
-          time docker pull ${DOCKER_IMAGE} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
-          docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
-
-          export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
-
-          # Push intermediate Docker image for next phase to use
-          if [ -z "${BUILD_ONLY}" ]; then
-            if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
-              export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-cmake-${CIRCLE_SHA1}
-            else
-              export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
-            fi
-            docker commit "$id" ${COMMIT_DOCKER_IMAGE}
-            time docker push ${COMMIT_DOCKER_IMAGE}
-          fi
-
-  caffe2_linux_test:
-    <<: *caffe2_params
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
-    - setup_linux_system_environment
-    - setup_ci_environment
-    - run:
-        name: Test
-        no_output_timeout: "1h"
-        command: |
-          set -e
-          # TODO: merge this into Caffe2 test.sh
-          cat >/home/circleci/project/ci_test_script.sh \<<EOL
-          # =================== The following code will be executed inside Docker container ===================
-          set -ex
-
-          export BUILD_ENVIRONMENT="$BUILD_ENVIRONMENT"
-
-          # libdc1394 (dependency of OpenCV) expects /dev/raw1394 to exist...
-          sudo ln /dev/null /dev/raw1394
-
-          # conda must be added to the path for Anaconda builds (this location must be
-          # the same as that in install_anaconda.sh used to build the docker image)
-          if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
-            export PATH=/opt/conda/bin:$PATH
-          fi
-
-          # Upgrade SSL module to avoid old SSL warnings
-          pip -q install --user --upgrade pyOpenSSL ndg-httpsclient pyasn1
-
-          pip -q install --user -b /tmp/pip_install_onnx "file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx"
-
-          # Build
-          ./.jenkins/caffe2/test.sh
-
-          # Remove benign core dumps.
-          # These are tests for signal handling (including SIGABRT).
-          rm -f ./crash/core.fatal_signal_as.*
-          rm -f ./crash/core.logging_test.*
-          # =================== The above code will be executed inside Docker container ===================
-          EOL
-          chmod +x /home/circleci/project/ci_test_script.sh
-
-          if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
-            export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-cmake-${CIRCLE_SHA1}
-          else
-            export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
-          fi
-          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
-          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
-          if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          else
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          fi
-          docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
-
-          export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_test_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
-
-  caffe2_macos_build:
-    <<: *caffe2_params
-    macos:
-      xcode: "9.4.1"
-    steps:
-      # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-      - checkout
-      - run_brew_for_macos_build
-      - run:
-          name: Build
-          no_output_timeout: "1h"
-          command: |
-            set -e
-
-            export IN_CIRCLECI=1
-
-            brew install cmake
-
-            # Reinitialize submodules
-            git submodule sync && git submodule update -q --init --recursive
-
-            # Reinitialize path (see man page for path_helper(8))
-            eval `/usr/libexec/path_helper -s`
-
-            export PATH=/usr/local/opt/python/libexec/bin:/usr/local/bin:$PATH
-
-            # Install Anaconda if we need to
-            if [ -n "${CAFFE2_USE_ANACONDA}" ]; then
-              rm -rf ${TMPDIR}/anaconda
-              curl --retry 3 -o ${TMPDIR}/conda.sh https://repo.anaconda.com/miniconda/Miniconda${ANACONDA_VERSION}-latest-MacOSX-x86_64.sh
-              chmod +x ${TMPDIR}/conda.sh
-              /bin/bash ${TMPDIR}/conda.sh -b -p ${TMPDIR}/anaconda
-              rm -f ${TMPDIR}/conda.sh
-              export PATH="${TMPDIR}/anaconda/bin:${PATH}"
-              source ${TMPDIR}/anaconda/bin/activate
-            fi
-
-            pip -q install numpy
-
-            # Install sccache
-            sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
-            sudo chmod +x /usr/local/bin/sccache
-            export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
-
-            # This IAM user allows write access to S3 bucket for sccache
-            set +x
-            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}
-            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}
-            set -x
-
-            export SCCACHE_BIN=${PWD}/sccache_bin
-            mkdir -p ${SCCACHE_BIN}
-            if which sccache > /dev/null; then
-              printf "#!/bin/sh\nexec sccache $(which clang++) \$*" > "${SCCACHE_BIN}/clang++"
-              chmod a+x "${SCCACHE_BIN}/clang++"
-
-              printf "#!/bin/sh\nexec sccache $(which clang) \$*" > "${SCCACHE_BIN}/clang"
-              chmod a+x "${SCCACHE_BIN}/clang"
-
-              export PATH="${SCCACHE_BIN}:$PATH"
-            fi
-
-            # Build
-            if [ "${BUILD_IOS:-0}" -eq 1 ]; then
-              unbuffer scripts/build_ios.sh 2>&1 | ts
-            elif [ -n "${CAFFE2_USE_ANACONDA}" ]; then
-              # All conda build logic should be in scripts/build_anaconda.sh
-              unbuffer scripts/build_anaconda.sh 2>&1 | ts
-            else
-              unbuffer scripts/build_local.sh 2>&1 | ts
-            fi
-
-            # Show sccache stats if it is running
-            if which sccache > /dev/null; then
-              sccache --show-stats
-            fi

.circleci/verbatim-sources/job-specs/docker_jobs.yml

@@ -4,7 +4,7 @@
           type: string
           default: ""
       machine:
-        image: ubuntu-1604:201903-01
+        image: ubuntu-1604:202007-01
       resource_class: large
       environment:
         IMAGE_NAME: << parameters.image_name >>
@@ -13,20 +13,7 @@
         DOCKER_BUILDKIT: 1
       steps:
         - checkout
-        - run:
-            name: Calculate docker tag
-            command: |
-              set -x
-              mkdir .circleci/shared
-              # git keeps a hash of all sub trees
-              echo "export DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker)" >> .circleci/shared/env_file
-        # Saves our calculated docker tag to our workpace for later use
-        - persist_to_workspace:
-            root: .
-            paths:
-              - .circleci/shared/
-        - load_shared_env:
-            root: .
+        - calculate_docker_image_tag
         - run:
             name: Check if image should be built
             command: |
@@ -35,7 +22,6 @@
               export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
               eval $(aws ecr get-login --no-include-email --region us-east-1)
               set -x
-              PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker")
               # Check if image already exists, if it does then skip building it
               if docker manifest inspect "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/${IMAGE_NAME}:${DOCKER_TAG}"; then
                 circleci-agent step halt
@@ -43,8 +29,15 @@
                 # explicitly exit the step here ourselves before it causes too much trouble
                 exit 0
               fi
+              # Covers the case where a previous tag doesn't exist for the tree
+              # this is only really applicable on trees that don't have `.circleci/docker` at its merge base, i.e. nightly
+              if ! git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker"; then
+                echo "Directory '.circleci/docker' not found in tree << pipeline.git.base_revision >>, you should probably rebase onto a more recent commit"
+                exit 1
+              fi
+              PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker")
               # If no image exists but the hash is the same as the previous hash then we should error out here
-              if [[ ${PREVIOUS_DOCKER_TAG} = ${DOCKER_TAG} ]]; then
+              if [[ "${PREVIOUS_DOCKER_TAG}" = "${DOCKER_TAG}" ]]; then
                 echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
                 echo "       contact the PyTorch team to restore the original images"
                 exit 1
@@ -60,7 +53,7 @@
               cd .circleci/docker && ./build_docker.sh
   docker_for_ecr_gc_build_job:
       machine:
-        image: ubuntu-1604:201903-01
+        image: ubuntu-1604:202007-01
       steps:
         - checkout
         - run:
@@ -113,23 +106,3 @@
               export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
               set -x
               /usr/bin/gc.py --filter-prefix ${PROJECT}  --ignore-tags "${IMAGE_TAG},${GENERATED_IMAGE_TAG}"
-
-  docker_hub_index_job:
-      docker:
-        - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
-          aws_auth:
-            aws_access_key_id: ${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
-            aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-
-      steps:
-        - run:
-            name: garbage collecting for ecr images
-            no_output_timeout: "1h"
-            command: |
-              set +x
-              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
-              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-              export DOCKER_HUB_USERNAME=${CIRCLECI_DOCKER_HUB_USERNAME}
-              export DOCKER_HUB_PASSWORD=${CIRCLECI_DOCKER_HUB_PASSWORD}
-              set -x
-              /usr/bin/docker_hub.py

.circleci/verbatim-sources/job-specs/job-specs-custom.yml

@@ -1,13 +1,39 @@
-  pytorch_python_doc_push:
+  pytorch_doc_push:
+    resource_class: medium
+    machine:
+      image: ubuntu-1604:202007-01
+    parameters:
+      branch:
+        type: string
+        default: "master"
+    steps:
+    - attach_workspace:
+        at: /tmp/workspace
+    - run:
+        name: Generate netrc
+        command: |
+          # set credentials for https pushing
+          cat > ~/.netrc \<<DONE
+            machine github.com
+            login pytorchbot
+            password ${GITHUB_PYTORCHBOT_TOKEN}
+          DONE
+    - run:
+        name: Docs push
+        command: |
+          pushd /tmp/workspace
+          git push -u origin "<< parameters.branch >>"
+
+  pytorch_python_doc_build:
     environment:
       BUILD_ENVIRONMENT: pytorch-python-doc-push
-      # TODO: stop hardcoding this
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
     resource_class: large
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     - checkout
+    - calculate_docker_image_tag
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -15,49 +41,44 @@
         no_output_timeout: "1h"
         command: |
           set -ex
-          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
           echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
+          tag=${CIRCLE_TAG:1:5}
+          target=${tag:-master}
+          echo "building for ${target}"
           time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
 
-          # master branch docs push
-          if [[ "${CIRCLE_BRANCH}" == "master" ]]; then
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && . ./.circleci/scripts/python_doc_push_script.sh docs/master master site") | docker exec -u jenkins -i "$id" bash) 2>&1'
-
-          # stable release docs push. Due to some circleci limitations, we keep
-          # an eternal PR open for merging v1.2.0 -> master for this job.
-          # XXX: The following code is only run on the v1.2.0 branch, which might
-          # not be exactly the same as what you see here.
-          elif [[ "${CIRCLE_BRANCH}" == "v1.2.0" ]]; then
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && . ./.circleci/scripts/python_doc_push_script.sh docs/stable 1.2.0 site dry_run") | docker exec -u jenkins -i "$id" bash) 2>&1'
-
-          # For open PRs: Do a dry_run of the docs build, don't push build
-          else
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && . ./.circleci/scripts/python_doc_push_script.sh docs/master master site dry_run") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          fi
+          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && . ./.circleci/scripts/python_doc_push_script.sh docs/'$target' master site") | docker exec -u jenkins -i "$id" bash) 2>&1'
 
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
           mkdir -p ~/workspace/build_artifacts
           docker cp $id:/var/lib/jenkins/workspace/pytorch.github.io/docs/master ~/workspace/build_artifacts
+          docker cp $id:/var/lib/jenkins/workspace/pytorch.github.io /tmp/workspace
 
           # Save the docs build so we can debug any problems
           export DEBUG_COMMIT_DOCKER_IMAGE=${COMMIT_DOCKER_IMAGE}-debug
           docker commit "$id" ${DEBUG_COMMIT_DOCKER_IMAGE}
           time docker push ${DEBUG_COMMIT_DOCKER_IMAGE}
+    - persist_to_workspace:
+        root: /tmp/workspace
+        paths:
+          - .
     - store_artifacts:
         path: ~/workspace/build_artifacts/master
         destination: docs
 
-  pytorch_cpp_doc_push:
+  pytorch_cpp_doc_build:
     environment:
       BUILD_ENVIRONMENT: pytorch-cpp-doc-push
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
     resource_class: large
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     - checkout
+    - calculate_docker_image_tag
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -65,39 +86,36 @@
         no_output_timeout: "1h"
         command: |
           set -ex
-          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
           echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
+          tag=${CIRCLE_TAG:1:5}
+          target=${tag:-master}
+          echo "building for ${target}"
           time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
 
-          # master branch docs push
-          if [[ "${CIRCLE_BRANCH}" == "master" ]]; then
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && . ./.circleci/scripts/cpp_doc_push_script.sh docs/master master") | docker exec -u jenkins -i "$id" bash) 2>&1'
-
-          # stable release docs push. Due to some circleci limitations, we keep
-          # an eternal PR open (#16502) for merging v1.0.1 -> master for this job.
-          # XXX: The following code is only run on the v1.0.1 branch, which might
-          # not be exactly the same as what you see here.
-          elif [[ "${CIRCLE_BRANCH}" == "v1.0.1" ]]; then
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && . ./.circleci/scripts/cpp_doc_push_script.sh docs/stable 1.0.1") | docker exec -u jenkins -i "$id" bash) 2>&1'
-
-          # For open PRs: Do a dry_run of the docs build, don't push build
-          else
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && . ./.circleci/scripts/cpp_doc_push_script.sh docs/master master dry_run") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          fi
+          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && . ./.circleci/scripts/cpp_doc_push_script.sh docs/"$target" master") | docker exec -u jenkins -i "$id" bash) 2>&1'
 
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
+          mkdir -p ~/workspace/build_artifacts
+          docker cp $id:/var/lib/jenkins/workspace/cppdocs/ /tmp/workspace
+
           # Save the docs build so we can debug any problems
           export DEBUG_COMMIT_DOCKER_IMAGE=${COMMIT_DOCKER_IMAGE}-debug
           docker commit "$id" ${DEBUG_COMMIT_DOCKER_IMAGE}
           time docker push ${DEBUG_COMMIT_DOCKER_IMAGE}
 
+    - persist_to_workspace:
+        root: /tmp/workspace
+        paths:
+          - .
+
   pytorch_macos_10_13_py3_build:
     environment:
       BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-build
     macos:
-      xcode: "9.4.1"
+      xcode: "11.2.1"
     steps:
       - checkout
       - run_brew_for_macos_build
@@ -131,7 +149,7 @@
     environment:
       BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-test
     macos:
-      xcode: "9.4.1"
+      xcode: "11.2.1"
     steps:
       - checkout
       - attach_workspace:
@@ -152,13 +170,14 @@
   pytorch_android_gradle_build:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
       PYTHON_VERSION: "3.6"
     resource_class: large
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     - checkout
+    - calculate_docker_image_tag
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -166,7 +185,7 @@
         no_output_timeout: "1h"
         command: |
           set -eux
-          docker_image_commit=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+          docker_image_commit=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
 
           docker_image_libtorch_android_x86_32=${docker_image_commit}-android-x86_32
           docker_image_libtorch_android_x86_64=${docker_image_commit}-android-x86_64
@@ -181,16 +200,16 @@
 
           # x86_32
           time docker pull ${docker_image_libtorch_android_x86_32} >/dev/null
-          export id_x86_32=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32})
+          export id_x86_32=$(docker run --env-file "${BASH_ENV}" -e GRADLE_OFFLINE=1 --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32})
 
-          export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
+          export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
           # arm-v7a
           time docker pull ${docker_image_libtorch_android_arm_v7a} >/dev/null
-          export id_arm_v7a=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_arm_v7a})
+          export id_arm_v7a=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_arm_v7a})
 
-          export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_arm_v7a" bash) 2>&1'
+          export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_arm_v7a" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
           mkdir -p ~/workspace/build_android_install_arm_v7a
@@ -198,9 +217,9 @@
 
           # x86_64
           time docker pull ${docker_image_libtorch_android_x86_64} >/dev/null
-          export id_x86_64=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_64})
+          export id_x86_64=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_64})
 
-          export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_x86_64" bash) 2>&1'
+          export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_x86_64" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
           mkdir -p ~/workspace/build_android_install_x86_64
@@ -208,9 +227,9 @@
 
           # arm-v8a
           time docker pull ${docker_image_libtorch_android_arm_v8a} >/dev/null
-          export id_arm_v8a=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_arm_v8a})
+          export id_arm_v8a=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_arm_v8a})
 
-          export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_arm_v8a" bash) 2>&1'
+          export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_arm_v8a" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
           mkdir -p ~/workspace/build_android_install_arm_v8a
@@ -221,7 +240,7 @@
           docker cp ~/workspace/build_android_install_arm_v8a $id_x86_32:/var/lib/jenkins/workspace/build_android_install_arm_v8a
 
           # run gradle buildRelease
-          export COMMAND='((echo "source ./workspace/env" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GRADLE_OFFLINE=1" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
+          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
           mkdir -p ~/workspace/build_android_artifacts
@@ -230,26 +249,9 @@
           output_image=$docker_image_libtorch_android_x86_32-gradle
           docker commit "$id_x86_32" ${output_image}
           time docker push ${output_image}
-    - run:
-        name: save binary size
-        no_output_timeout: "5m"
-        command: |
-          docker_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-x86_32-gradle
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image})
-          echo "docker-id: $id"
-          cat \<< EOL | docker exec -u jenkins -i "$id" bash
-          # ============================== Begin Docker ==============================
-          cd workspace
-          source ./env
-          export ANDROID_BUILD_TYPE="prebuild"
-          export COMMIT_TIME=\$(git log --max-count=1 --format=%ct || echo 0)
-          export CIRCLE_BUILD_NUM="${CIRCLE_BUILD_NUM}"
-          export CIRCLE_SHA1="${CIRCLE_SHA1}"
-          export CIRCLE_BRANCH="${CIRCLE_BRANCH}"
-          export SCRIBE_GRAPHQL_ACCESS_TOKEN="${SCRIBE_GRAPHQL_ACCESS_TOKEN}"
-          python .circleci/scripts/upload_binary_size_to_scuba.py android
-          # ==============================  End Docker  ==============================
-          EOL
+    - upload_binary_size_for_android_build:
+        build_type: prebuilt
+        artifacts: /home/circleci/workspace/build_android_artifacts/artifacts.tgz
     - store_artifacts:
         path: ~/workspace/build_android_artifacts/artifacts.tgz
         destination: artifacts.tgz
@@ -257,11 +259,11 @@
   pytorch_android_publish_snapshot:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-publish-snapshot
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:ab1632df-fa59-40e6-8c23-98e004f61148"
       PYTHON_VERSION: "3.6"
     resource_class: large
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     - checkout
     - setup_linux_system_environment
@@ -281,9 +283,9 @@
 
           # x86_32
           time docker pull ${docker_image_libtorch_android_x86_32_gradle} >/dev/null
-          export id_x86_32=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32_gradle})
+          export id_x86_32=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32_gradle})
 
-          export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export SONATYPE_NEXUS_USERNAME=${SONATYPE_NEXUS_USERNAME}" && echo "export SONATYPE_NEXUS_PASSWORD=${SONATYPE_NEXUS_PASSWORD}" && echo "export ANDROID_SIGN_KEY=${ANDROID_SIGN_KEY}" && echo "export ANDROID_SIGN_PASS=${ANDROID_SIGN_PASS}" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/publish_android_snapshot.sh") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
+          export COMMAND='((echo "sudo chown -R jenkins workspace" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export SONATYPE_NEXUS_USERNAME=${SONATYPE_NEXUS_USERNAME}" && echo "export SONATYPE_NEXUS_PASSWORD=${SONATYPE_NEXUS_PASSWORD}" && echo "export ANDROID_SIGN_KEY=${ANDROID_SIGN_KEY}" && echo "export ANDROID_SIGN_PASS=${ANDROID_SIGN_PASS}" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/publish_android_snapshot.sh") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
           output_image=${docker_image_libtorch_android_x86_32_gradle}-publish-snapshot
@@ -293,21 +295,14 @@
   pytorch_android_gradle_build-x86_32:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-only-x86_32
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
       PYTHON_VERSION: "3.6"
     resource_class: large
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     - checkout
-    - run:
-        name: filter out not PR runs
-        no_output_timeout: "5m"
-        command: |
-          echo "CIRCLE_PULL_REQUEST: ${CIRCLE_PULL_REQUEST:-}"
-          if [ -z "${CIRCLE_PULL_REQUEST:-}" ]; then
-            circleci step halt
-          fi
+    - calculate_docker_image_tag
     - setup_linux_system_environment
     - checkout
     - setup_ci_environment
@@ -316,14 +311,14 @@
         no_output_timeout: "1h"
         command: |
           set -e
-          docker_image_libtorch_android_x86_32=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-x86_32
+          docker_image_libtorch_android_x86_32=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}-android-x86_32
           echo "docker_image_libtorch_android_x86_32: "${docker_image_libtorch_android_x86_32}
 
           # x86
           time docker pull ${docker_image_libtorch_android_x86_32} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32})
+          export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32})
 
-          export COMMAND='((echo "source ./workspace/env" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GRADLE_OFFLINE=1" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GRADLE_OFFLINE=1" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
           mkdir -p ~/workspace/build_android_x86_32_artifacts
@@ -332,30 +327,53 @@
           output_image=${docker_image_libtorch_android_x86_32}-gradle
           docker commit "$id" ${output_image}
           time docker push ${output_image}
-    - run:
-        name: save binary size
-        no_output_timeout: "5m"
-        command: |
-          docker_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-x86_32-gradle
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image})
-          echo "docker-id: $id"
-          cat \<< EOL | docker exec -u jenkins -i "$id" bash
-          # ============================== Begin Docker ==============================
-          cd workspace
-          source ./env
-          export ANDROID_BUILD_TYPE="prebuild-single"
-          export COMMIT_TIME=\$(git log --max-count=1 --format=%ct || echo 0)
-          export CIRCLE_BUILD_NUM="${CIRCLE_BUILD_NUM}"
-          export CIRCLE_SHA1="${CIRCLE_SHA1}"
-          export CIRCLE_BRANCH="${CIRCLE_BRANCH}"
-          export SCRIBE_GRAPHQL_ACCESS_TOKEN="${SCRIBE_GRAPHQL_ACCESS_TOKEN}"
-          python .circleci/scripts/upload_binary_size_to_scuba.py android
-          # ==============================  End Docker  ==============================
-          EOL
+    - upload_binary_size_for_android_build:
+        build_type: prebuilt-single
+        artifacts: /home/circleci/workspace/build_android_x86_32_artifacts/artifacts.tgz
     - store_artifacts:
         path: ~/workspace/build_android_x86_32_artifacts/artifacts.tgz
         destination: artifacts.tgz
 
+  pytorch_android_gradle_custom_build_single:
+    environment:
+      BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-custom-build-single
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
+      PYTHON_VERSION: "3.6"
+    resource_class: large
+    machine:
+      image: ubuntu-1604:202007-01
+    steps:
+    - checkout
+    - calculate_docker_image_tag
+    - setup_linux_system_environment
+    - checkout
+    - calculate_docker_image_tag
+    - setup_ci_environment
+    - run:
+        name: pytorch android gradle custom build single architecture (for PR)
+        no_output_timeout: "1h"
+        command: |
+          set -e
+          # Unlike other gradle jobs, it's not worth building libtorch in a separate CI job and share via docker, because:
+          # 1) Not shareable: it's custom selective build, which is different from default libtorch mobile build;
+          # 2) Not parallelizable by architecture: it only builds libtorch for one architecture;
+
+          echo "DOCKER_IMAGE: ${DOCKER_IMAGE}:${DOCKER_TAG}"
+          time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
+
+          git submodule sync && git submodule update -q --init --recursive
+          VOLUME_MOUNTS="-v /home/circleci/project/:/var/lib/jenkins/workspace"
+          export id=$(docker run --env-file "${BASH_ENV}" ${VOLUME_MOUNTS} --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
+
+          export COMMAND='((echo "export GRADLE_OFFLINE=1" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+
+          # Skip docker push as this job is purely for size analysis purpose.
+          # Result binaries are already in `/home/circleci/project/` as it's mounted instead of copied.
+
+    - upload_binary_size_for_android_build:
+        build_type: custom-build-single
+
   pytorch_ios_build:
     <<: *pytorch_ios_params
     macos:
@@ -475,9 +493,10 @@
   pytorch_linux_bazel_build:
     <<: *pytorch_params
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     - checkout
+    - calculate_docker_image_tag
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -486,9 +505,9 @@
         command: |
           set -e
           # Pull Docker image and run build
-          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
-          time docker pull ${DOCKER_IMAGE} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}:${DOCKER_TAG}
+          time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
+          export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
 
           echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
 
@@ -496,14 +515,14 @@
 
           docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
 
-          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
 
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
           # Push intermediate Docker image for next phase to use
           if [ -z "${BUILD_ONLY}" ]; then
             # Augment our output image name with bazel to avoid collisions
-            output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
+            output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-bazel-${CIRCLE_SHA1}
             export COMMIT_DOCKER_IMAGE=$output_image
             docker commit "$id" ${COMMIT_DOCKER_IMAGE}
             time docker push ${COMMIT_DOCKER_IMAGE}
@@ -512,9 +531,10 @@
   pytorch_linux_bazel_test:
     <<: *pytorch_params
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     - checkout
+    - calculate_docker_image_tag
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -522,16 +542,16 @@
         no_output_timeout: "90m"
         command: |
           set -e
-          output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
+          output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-bazel-${CIRCLE_SHA1}
           export COMMIT_DOCKER_IMAGE=$output_image
           echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
 
           time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
 
           if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+            export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --gpus all -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
           else
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+            export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
           fi
 
           retrieve_test_reports() {
@@ -541,9 +561,9 @@
           trap "retrieve_test_reports" ERR
 
           if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+            export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
           else
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+            export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
           fi
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
@@ -555,13 +575,13 @@
   pytorch_doc_test:
     environment:
       BUILD_ENVIRONMENT: pytorch-doc-test
-      # TODO: stop hardcoding this
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
     resource_class: medium
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     - checkout
+    - calculate_docker_image_tag
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -569,9 +589,9 @@
         no_output_timeout: "30m"
         command: |
           set -ex
-          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
           echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
           time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
           export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && . ./.jenkins/pytorch/docs-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && . ./.jenkins/pytorch/docs-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml

@@ -2,12 +2,12 @@ jobs:
   pytorch_linux_build:
     <<: *pytorch_params
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
     - checkout
+    - calculate_docker_image_tag
     - setup_linux_system_environment
-    - checkout
     - optional_merge_target_branch
     - setup_ci_environment
     - run:
@@ -15,33 +15,46 @@ jobs:
         no_output_timeout: "1h"
         command: |
           set -e
+          # TODO: Remove this after we figure out why rocm tests are failing
+          if [[ "${DOCKER_IMAGE}" == *rocm3.5* ]]; then
+            export DOCKER_TAG="ab1632df-fa59-40e6-8c23-98e004f61148"
+          fi
+          if [[ "${DOCKER_IMAGE}" == *rocm3.7* ]]; then
+            export DOCKER_TAG="1045c7b891104cb4fd23399eab413b6213e48aeb"
+          fi
+          if [[ ${BUILD_ENVIRONMENT} == *"pure_torch"* ]]; then
+            echo 'BUILD_CAFFE2=OFF' >> "${BASH_ENV}"
+          fi
+          if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
+            echo 'ATEN_THREADING=TBB' >> "${BASH_ENV}"
+            echo 'USE_TBB=1' >> "${BASH_ENV}"
+          elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
+            echo 'ATEN_THREADING=NATIVE' >> "${BASH_ENV}"
+          fi
+          echo "Parallel backend flags: "${PARALLEL_FLAGS}
           # Pull Docker image and run build
-          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
-          time docker pull ${DOCKER_IMAGE} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}:${DOCKER_TAG}
+          time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
+          export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
 
           git submodule sync && git submodule update -q --init --recursive
 
           docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
 
-          if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
-            export PARALLEL_FLAGS="export ATEN_THREADING=TBB USE_TBB=1 "
-          elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
-            export PARALLEL_FLAGS="export ATEN_THREADING=NATIVE "
-          fi
-          echo "Parallel backend flags: "${PARALLEL_FLAGS}
-
-          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo '"$PARALLEL_FLAGS"' && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh && find ${BUILD_ROOT} -type f -name "*.a" -or -name "*.o" -delete") | docker exec -u jenkins -i "$id" bash) 2>&1'
 
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
+          # Copy dist folder back
+          docker cp $id:/var/lib/jenkins/workspace/dist /home/circleci/project/. || echo "Dist folder not found"
+
           # Push intermediate Docker image for next phase to use
           if [ -z "${BUILD_ONLY}" ]; then
             # Note [Special build images]
             # The xla build uses the same docker image as
             # pytorch-linux-trusty-py3.6-gcc5.4-build. In the push step, we have to
             # distinguish between them so the test can pick up the correct image.
-            output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+            output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
             if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then
               export COMMIT_DOCKER_IMAGE=$output_image-xla
             elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then
@@ -60,20 +73,25 @@ jobs:
               export COMMIT_DOCKER_IMAGE=$output_image-android-x86_32
             elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-vulkan-x86_32"* ]]; then
               export COMMIT_DOCKER_IMAGE=$output_image-android-vulkan-x86_32
+            elif [[ ${BUILD_ENVIRONMENT} == *"vulkan-linux"* ]]; then
+              export COMMIT_DOCKER_IMAGE=$output_image-vulkan
             else
               export COMMIT_DOCKER_IMAGE=$output_image
             fi
             docker commit "$id" ${COMMIT_DOCKER_IMAGE}
             time docker push ${COMMIT_DOCKER_IMAGE}
           fi
+    - store_artifacts:
+        path: /home/circleci/project/dist
 
   pytorch_linux_test:
     <<: *pytorch_params
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202007-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
     - checkout
+    - calculate_docker_image_tag
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -81,8 +99,16 @@ jobs:
         no_output_timeout: "90m"
         command: |
           set -e
+          export PYTHONUNBUFFERED=1
+          # TODO: Remove this after we figure out why rocm tests are failing
+          if [[ "${DOCKER_IMAGE}" == *rocm3.5* ]]; then
+            export DOCKER_TAG="ab1632df-fa59-40e6-8c23-98e004f61148"
+          fi
+          if [[ "${DOCKER_IMAGE}" == *rocm3.7* ]]; then
+            export DOCKER_TAG="1045c7b891104cb4fd23399eab413b6213e48aeb"
+          fi
           # See Note [Special build images]
-          output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+          output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
           if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then
             export COMMIT_DOCKER_IMAGE=$output_image-xla
           elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then
@@ -91,30 +117,34 @@ jobs:
             export COMMIT_DOCKER_IMAGE=$output_image-paralleltbb
           elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
             export COMMIT_DOCKER_IMAGE=$output_image-parallelnative
+          elif [[ ${BUILD_ENVIRONMENT} == *"vulkan-linux"* ]]; then
+            export COMMIT_DOCKER_IMAGE=$output_image-vulkan
           else
             export COMMIT_DOCKER_IMAGE=$output_image
           fi
           echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
 
           if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
-            export PARALLEL_FLAGS="export ATEN_THREADING=TBB USE_TBB=1 "
+            echo 'ATEN_THREADING=TBB' >> "${BASH_ENV}"
+            echo 'USE_TBB=1' >> "${BASH_ENV}"
           elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
-            export PARALLEL_FLAGS="export ATEN_THREADING=NATIVE "
+            echo 'ATEN_THREADING=NATIVE' >> "${BASH_ENV}"
           fi
           echo "Parallel backend flags: "${PARALLEL_FLAGS}
 
           time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
 
+          # TODO: Make this less painful
           if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+            export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --gpus all --shm-size=2g -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          elif [[ ${BUILD_ENVIRONMENT} == *"rocm"* ]]; then
+            hostname
+            export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size=8g --ipc=host --device /dev/kfd --device /dev/dri --group-add video -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
           else
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+            export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
           fi
+          echo "id=${id}" >> "${BASH_ENV}"
 
-          # Pass environment variables to the next step
-          # See https://circleci.com/docs/2.0/env-vars/#using-parameters-and-bash-environment
-          echo "export PARALLEL_FLAGS=\"${PARALLEL_FLAGS}\"" >> $BASH_ENV
-          echo "export id=$id" >> $BASH_ENV
     - run:
         name: Check for no AVX instruction by default
         no_output_timeout: "20m"
@@ -131,8 +161,8 @@ jobs:
           }
 
           if is_vanilla_build; then
-            echo "apt-get update && apt-get install -y qemu-user" | docker exec -u root -i "$id" bash
-            echo "cd workspace/build; qemu-x86_64 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU" | docker exec -u jenkins -i "$id" bash
+            echo "apt-get update && apt-get install -y qemu-user gdb" | docker exec -u root -i "$id" bash
+            echo "cd workspace/build; qemu-x86_64 -g 2345 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU & gdb ./bin/basic -ex 'set pagination off' -ex 'target remote :2345' -ex 'continue' -ex 'bt' -ex='set confirm off' -ex 'quit \$_isvoid(\$_exitcode)'" | docker exec -u jenkins -i "$id" bash
           else
             echo "Skipping for ${BUILD_ENVIRONMENT}"
           fi
@@ -142,21 +172,56 @@ jobs:
         command: |
           set -e
 
+          cat >docker_commands.sh \<<EOL
+          # =================== The following code will be executed inside Docker container ===================
+          set -ex
+          export SCRIBE_GRAPHQL_ACCESS_TOKEN="${SCRIBE_GRAPHQL_ACCESS_TOKEN}"
+          ${PARALLEL_FLAGS}
+          cd workspace
+          EOL
           if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "${PARALLEL_FLAGS}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+            echo ".jenkins/pytorch/multigpu-test.sh" >> docker_commands.sh
+          elif [[ ${BUILD_ENVIRONMENT} == *onnx* ]]; then
+            echo "pip install click mock tabulate networkx==2.0" >> docker_commands.sh
+            echo "pip -q install --user -b /tmp/pip_install_onnx \"file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx\"" >> docker_commands.sh
+            echo ".jenkins/caffe2/test.sh" >> docker_commands.sh
           else
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}" && echo "${PARALLEL_FLAGS}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+            echo ".jenkins/pytorch/test.sh" >> docker_commands.sh
           fi
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+          echo "(cat docker_commands.sh | docker exec -u jenkins -i "$id" bash) 2>&1" > command.sh
+          unbuffer bash command.sh | ts
     - run:
         name: Report results
         no_output_timeout: "5m"
         command: |
           set -e
           docker stats --all --no-stream
-          echo "cd workspace; python test/print_test_stats.py test" | docker exec -u jenkins -i "$id" bash
+
+          cat >docker_commands.sh \<<EOL
+          # =================== The following code will be executed inside Docker container ===================
+          set -ex
+          export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}
+          export SCRIBE_GRAPHQL_ACCESS_TOKEN="${SCRIBE_GRAPHQL_ACCESS_TOKEN}"
+          export CIRCLE_TAG="${CIRCLE_TAG:-}"
+          export CIRCLE_SHA1="$CIRCLE_SHA1"
+          export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-}"
+          export CIRCLE_BRANCH="$CIRCLE_BRANCH"
+          export CIRCLE_JOB="$CIRCLE_JOB"
+          cd workspace
+          python test/print_test_stats.py test
+          EOL
+          echo "(cat docker_commands.sh | docker exec -u jenkins -i "$id" bash) 2>&1" > command.sh
+          unbuffer bash command.sh | ts
+
           echo "Retrieving test reports"
           docker cp $id:/var/lib/jenkins/workspace/test/test-reports ./ || echo 'No test reports found!'
+          if [[ ${BUILD_ENVIRONMENT} == *"coverage"* ]]; then
+              echo "Retrieving coverage report"
+              docker cp $id:/var/lib/jenkins/workspace/test/.coverage ./test
+              docker cp $id:/var/lib/jenkins/workspace/test/coverage.xml ./test
+              python3 -mpip install codecov
+              python3 -mcodecov
+          fi
         when: always
     - store_test_results:
         path: test-reports
@@ -166,7 +231,7 @@ jobs:
     parameters:
       executor:
         type: string
-        default: "windows-cpu-with-nvidia-cuda"
+        default: "windows-xlarge-cpu-with-nvidia-cuda"
       build_environment:
         type: string
         default: ""
@@ -181,10 +246,10 @@ jobs:
         default: "3.6"
       vc_version:
         type: string
-        default: "14.11"
+        default: "14.16"
       vc_year:
         type: string
-        default: "2017"
+        default: "2019"
       vc_product:
         type: string
         default: "BuildTools"
@@ -194,12 +259,6 @@ jobs:
     executor: <<parameters.executor>>
     steps:
       - checkout
-      - run:
-          name: Install VS2017
-          command: |
-            if [[ "${VC_YEAR}" == "2017" ]]; then
-              powershell .circleci/scripts/vs_install.ps1
-            fi
       - run:
           name: Install Cuda
           no_output_timeout: 30m
@@ -211,10 +270,7 @@ jobs:
           name: Install Cudnn
           command : |
             if [[ "${USE_CUDA}" == "1" ]]; then
-              cd c:/
-              curl --retry 3 -O https://ossci-windows.s3.amazonaws.com/cudnn-10.1-windows10-x64-v7.6.4.38.zip
-              7z x cudnn-10.1-windows10-x64-v7.6.4.38.zip -ocudnn
-              cp -r cudnn/cuda/* "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/"
+              .circleci/scripts/windows_cudnn_install.sh
             fi
       - run:
           name: Build
@@ -237,7 +293,7 @@ jobs:
     parameters:
       executor:
         type: string
-        default: "windows-cpu-with-nvidia-cuda"
+        default: "windows-medium-cpu-with-nvidia-cuda"
       build_environment:
         type: string
         default: ""
@@ -252,10 +308,10 @@ jobs:
         default: "3.6"
       vc_version:
         type: string
-        default: "14.11"
+        default: "14.16"
       vc_year:
         type: string
-        default: "2017"
+        default: "2019"
       vc_product:
         type: string
         default: "BuildTools"
@@ -267,27 +323,23 @@ jobs:
       - checkout
       - attach_workspace:
           at: c:/users/circleci/workspace
-      - run:
-          name: Install VS2017
-          command: |
-            if [[ "${VC_YEAR}" == "2017" ]]; then
-              powershell .circleci/scripts/vs_install.ps1
-            fi
       - run:
           name: Install Cuda
           no_output_timeout: 30m
           command: |
-            if [[ "${CUDA_VERSION}" != "cpu" && "${JOB_EXECUTOR}" != "windows-with-nvidia-gpu" ]]; then
-              .circleci/scripts/windows_cuda_install.sh
+            if [[ "${CUDA_VERSION}" != "cpu" ]]; then
+              if [[ "${CUDA_VERSION}" != "10" || "${JOB_EXECUTOR}" != "windows-with-nvidia-gpu" ]]; then
+                .circleci/scripts/windows_cuda_install.sh
+              fi
+              if [[ "${CUDA_VERSION}" != "10" && "${JOB_EXECUTOR}" == "windows-with-nvidia-gpu" ]]; then
+                .circleci/scripts/driver_update.bat
+              fi
             fi
       - run:
           name: Install Cudnn
           command : |
-            if [[ "${CUDA_VERSION}" != "cpu" && "${JOB_EXECUTOR}" != "windows-with-nvidia-gpu" ]]; then
-              cd c:/
-              curl --retry 3 -O https://ossci-windows.s3.amazonaws.com/cudnn-10.1-windows10-x64-v7.6.4.38.zip
-              7z x cudnn-10.1-windows10-x64-v7.6.4.38.zip -ocudnn
-              cp -r cudnn/cuda/* "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/"
+            if [[ "${CUDA_VERSION}" != "cpu" ]]; then
+              .circleci/scripts/windows_cudnn_install.sh
             fi
       - run:
           name: Test

.circleci/verbatim-sources/workflows/workflows-ecr-gc.yml

@@ -11,7 +11,7 @@
       - ecr_gc_job:
             name: ecr_gc_job_for_pytorch
             project: pytorch
-            tags_to_keep: "271,262,256,278,282,291,300,323,327,347,389,401,402,403,405,a8006f9a-272d-4478-b137-d121c6f05c83,6e7b11da-a919-49e5-b2ba-da66e3d4bb0a,f990c76a-a798-42bb-852f-5be5006f8026,e43973a9-9d5a-4138-9181-a08a0fc55e2f,8fcf46ef-4a34-480b-a8ee-b0a30a4d3e59,9a3986fa-7ce7-4a36-a001-3c9bef9892e2,1bc00f11-e0f3-4e5c-859f-15937dd938cd,209062ef-ab58-422a-b295-36c4eed6e906,be76e8fd-44e2-484d-b090-07e0cc3a56f0"
+            tags_to_keep: "271,262,256,278,282,291,300,323,327,347,389,401,402,403,405,a8006f9a-272d-4478-b137-d121c6f05c83,6e7b11da-a919-49e5-b2ba-da66e3d4bb0a,f990c76a-a798-42bb-852f-5be5006f8026,e43973a9-9d5a-4138-9181-a08a0fc55e2f,8fcf46ef-4a34-480b-a8ee-b0a30a4d3e59,9a3986fa-7ce7-4a36-a001-3c9bef9892e2,1bc00f11-e0f3-4e5c-859f-15937dd938cd,209062ef-ab58-422a-b295-36c4eed6e906,be76e8fd-44e2-484d-b090-07e0cc3a56f0,fff7795428560442086f7b2bb6004b65245dc11a,ab1632df-fa59-40e6-8c23-98e004f61148"
             requires:
               - docker_for_ecr_gc_build_job
       - ecr_gc_job:
@@ -32,4 +32,3 @@
             tags_to_keep: "34"
             requires:
               - docker_for_ecr_gc_build_job
-      - docker_hub_index_job

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1 @@
+Fixes #{issue number}

.github/workflows/clang_format.yml

@@ -35,7 +35,7 @@ jobs:
           HEAD_SHA=${{ github.event.pull_request.head.sha }}
           MERGE_BASE=$(git merge-base $BASE_SHA $HEAD_SHA)
 
-          # only run clang-format on whitelisted files
+          # only run clang-format on allowlisted files
           echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
           echo "| clang-format failures found! Run: "
           echo "|    tools/clang_format_ci.sh ${MERGE_BASE} "

.github/workflows/jit_triage.yml

@@ -0,0 +1,78 @@
+name: jit-triage
+
+on:
+  issues:
+    types: [labeled]
+
+jobs:
+  welcome:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/github-script@v2
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            // Arguments available:
+            // - github: A pre-authenticated octokit/rest.js client
+            // - context: An object containing the context of the workflow run
+            // - core: A reference to the @actions/core package
+            // - io: A reference to the @actions/io package
+
+            // Check if issue has a JIT label.
+            const kJitLabel = "jit";
+
+            issue = await github.issues.get({
+              owner: context.issue.owner,
+              repo: context.issue.repo,
+              issue_number: context.issue.number,
+            })
+
+            const hasJitLabel = issue.data.labels.filter(label => label.name == kJitLabel).length > 0;
+
+            if (!hasJitLabel) {
+              core.debug("Issue " + issue.data.title + " does not have JIT label");
+              return;
+            }
+
+            // Get project column ID.
+            const kProjectName = "JIT Triage";
+            const kColumnName = "Need triage";
+
+            // Query all projects in the repository.
+            // TODO: Support pagination once there are > 30 projects.
+            const projects = await github.projects.listForRepo({
+              owner: context.issue.owner,
+              repo: context.issue.repo,
+            });
+
+            // Filter out unwanted projects and get the ID for the JIT Triage project.
+            const filteredProjects = projects.data.filter(project => project.name == kProjectName);
+
+            if (filteredProjects.length != 1) {
+              core.setFailed("Unable to find a project named " + kProjectName);
+              return;
+            }
+
+            const projectId = filteredProjects[0].id;
+            // First, query all columns in the project.
+            // TODO: Support pagination once there are > 30 columns.
+            const columns = await github.projects.listColumns({
+              project_id: projectId,
+            });
+
+            // Filter out unwanted projects and get the ID for the Need triage column.
+            const filteredColumns = columns.data.filter(column => column.name == kColumnName);
+
+            if (filteredColumns.length != 1) {
+              core.setFailed("Unable to find a column named " + kColumnName);
+              return;
+            }
+
+            const columnId = filteredColumns[0].id;
+
+            // Create a project card for this new issue.
+            await github.projects.createCard({
+              column_id: columnId,
+              content_id: issue.data.id,
+              content_type: "Issue",
+            })

.github/workflows/lint.yml

@@ -21,10 +21,6 @@ jobs:
         run: |
           pip install -r requirements.txt
           cd .circleci && ./ensure-consistency.py
-      - name: Ensure Docker version is correctly deployed
-        run: |
-          pip install pyyaml
-          .circleci/validate-docker-version.py
       - name: Shellcheck Jenkins scripts
         run: |
           sudo apt-get install -y shellcheck
@@ -135,13 +131,9 @@ jobs:
             time python setup.py --cmake-only build
 
             # Generate ATen files.
-            time python aten/src/ATen/gen.py \
+            time python -m tools.codegen.gen \
               -s aten/src/ATen \
-              -d build/aten/src/ATen \
-              aten/src/ATen/Declarations.cwrap \
-              aten/src/THCUNN/generic/THCUNN.h \
-              aten/src/ATen/nn.yaml \
-              aten/src/ATen/native/native_functions.yaml
+              -d build/aten/src/ATen
 
             # Generate PyTorch files.
             time python tools/setup_helpers/generate_code.py            \
@@ -152,16 +144,22 @@ jobs:
           # Run Clang-Tidy
           # The negative filters below are to exclude files that include onnx_pb.h or
           # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job.
+          # FunctionsManual.cpp is excluded to keep this diff clean. It will be fixed
+          # in a follow up PR.
           python tools/clang_tidy.py                               \
             --verbose                                              \
             --paths torch/csrc/                                    \
             --diff "$MERGE_BASE"                                   \
+            -g"-torch/csrc/jit/passes/onnx/helper.cpp"             \
+            -g"-torch/csrc/jit/passes/onnx/shape_type_inference.cpp"\
+            -g"-torch/csrc/jit/serialization/onnx.cpp"             \
             -g"-torch/csrc/jit/serialization/export.cpp"           \
             -g"-torch/csrc/jit/serialization/import.cpp"           \
             -g"-torch/csrc/jit/serialization/import_legacy.cpp"    \
             -g"-torch/csrc/onnx/init.cpp"                          \
             -g"-torch/csrc/cuda/nccl.*"                            \
             -g"-torch/csrc/cuda/python_nccl.cpp"                   \
+            -g"-torch/csrc/autograd/FunctionsManual.cpp"           \
             "$@" > ${GITHUB_WORKSPACE}/clang-tidy-output.txt
 
           cat ${GITHUB_WORKSPACE}/clang-tidy-output.txt

.gitignore

@@ -9,6 +9,7 @@
 ## PyTorch
 
 .coverage
+coverage.xml
 .gradle
 .hypothesis
 .mypy_cache
@@ -25,8 +26,10 @@
 aten/build/
 aten/src/ATen/Config.h
 aten/src/ATen/cuda/CUDAConfig.h
+benchmarks/.data
 caffe2/cpp_test/
 dist/
+docs/cpp/src
 docs/src/**/*
 docs/cpp/build
 docs/cpp/source/api
@@ -38,7 +41,7 @@ test/cpp/api/mnist
 test/custom_operator/model.pt
 test/data/legacy_modules.t7
 test/data/*.pt
-test/backward_compatibility/new_schemas.txt
+test/backward_compatibility/nightly_schemas.txt
 dropout_model.pt
 test/generated_type_hints_smoketest.py
 test/htmlcov
@@ -47,13 +50,15 @@ test/test-reports/
 third_party/build/
 tools/shared/_utils_internal.py
 torch.egg-info/
-torch/__init__.pyi
 torch/_C/__init__.pyi
 torch/_C/_nn.pyi
 torch/_C/_VariableFunctions.pyi
+torch/_VF.pyi
 torch/nn/functional.pyi
-torch/nn/modules/*.pyi
 torch/csrc/autograd/generated/*
+# Listed manually because some files in this directory are not generated
+torch/testing/_internal/generated/annotated_fn_args.py
+torch/testing/_internal/data/*.pt
 torch/csrc/cudnn/cuDNN.cpp
 torch/csrc/generated
 torch/csrc/generic/TensorMethods.cpp
@@ -105,9 +110,6 @@ env
 # macOS dir files
 .DS_Store
 
-# Symbolic files
-tools/shared/cwrap_common.py
-
 # Ninja files
 .ninja_deps
 .ninja_log
@@ -259,6 +261,7 @@ TAGS
 
 # clangd background index
 .clangd/
+.cache/
 
 # bazel symlinks
 bazel-*

.jenkins/caffe2/build.sh

@@ -248,6 +248,8 @@ else
     export MAX_JOBS=`expr $(nproc) - 1`
   fi
 
+  pip install --user dataclasses
+
   $PYTHON setup.py install --user
 
   report_compile_cache_stats
@@ -260,9 +262,4 @@ fi
 # Install ONNX into a local directory
 pip install --user -b /tmp/pip_install_onnx "file://${ROOT_DIR}/third_party/onnx#egg=onnx"
 
-if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
-  # runtime compilation of MIOpen kernels manages to crash sccache - hence undo the wrapping
-  bash tools/amd_build/unwrap_clang.sh
-fi
-
 report_compile_cache_stats

.jenkins/caffe2/common.sh

@@ -12,6 +12,18 @@ if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
   PYTHON=$(which "python${BASH_REMATCH[1]}")
 fi
 
+if [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
+    if which sccache > /dev/null; then
+        # Save sccache logs to file
+        sccache --stop-server || true
+        rm ~/sccache_error.log || true
+        SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=0 sccache --start-server
+
+        # Report sccache stats for easier debugging
+        sccache --zero-stats
+    fi
+fi
+
 # /usr/local/caffe2 is where the cpp bits are installed to in in cmake-only
 # builds. In +python builds the cpp tests are copied to /usr/local/caffe2 so
 # that the test code in .jenkins/test.sh is the same

.jenkins/caffe2/test.sh

@@ -12,30 +12,13 @@ if [[ "${BUILD_ENVIRONMENT}" == *-rocm* ]]; then
   export HSAKMT_DEBUG_LEVEL=4
 fi
 # These additional packages are needed for circleci ROCm builds.
-if [[ $BUILD_ENVIRONMENT == pytorch-linux-xenial-rocm* ]]; then
+if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
     # Need networkx 2.0 because bellmand_ford was moved in 2.1 . Scikit-image by
     # defaults installs the most recent networkx version, so we install this lower
     # version explicitly before scikit-image pulls it in as a dependency
     pip install networkx==2.0
     # click - onnx
     pip install --progress-bar off click protobuf tabulate virtualenv mock typing-extensions
-
-  # TODO: Remove this once ROCm CI images are >= ROCm 3.5
-  # ROCm 3.5 required a backwards-incompatible change; the kernel and thunk must match.
-  # Detect kernel version and upgrade thunk if this is a ROCm 3.3 container running on a 3.5 kernel.
-  ROCM_ASD_FW_VERSION=$(/opt/rocm/bin/rocm-smi --showfwinfo -d 1 | grep ASD |  awk '{print $6}')
-  if [[ $ROCM_ASD_FW_VERSION = 553648174 && "$BUILD_ENVIRONMENT" == *rocm3.3* ]]; then
-    # upgrade thunk to 3.5
-    mkdir rocm3.5-thunk
-    pushd rocm3.5-thunk
-    wget http://repo.radeon.com/rocm/apt/3.5/pool/main/h/hsakmt-roct3.5.0/hsakmt-roct3.5.0_1.0.9-347-gd4b224f_amd64.deb
-    wget http://repo.radeon.com/rocm/apt/3.5/pool/main/h/hsakmt-roct-dev3.5.0/hsakmt-roct-dev3.5.0_1.0.9-347-gd4b224f_amd64.deb
-    dpkg-deb -vx hsakmt-roct3.5.0_1.0.9-347-gd4b224f_amd64.deb .
-    dpkg-deb -vx hsakmt-roct-dev3.5.0_1.0.9-347-gd4b224f_amd64.deb .
-    sudo cp -r opt/rocm-3.5.0/* /opt/rocm-3.3.0/
-    popd
-    rm -rf rocm3.5-thunk
-  fi
 fi
 
 # Find where cpp tests and Caffe2 itself are installed
@@ -138,6 +121,10 @@ if [[ $BUILD_ENVIRONMENT == *-rocm* ]]; then
   # This test has been flaky in ROCm CI (but note the tests are
   # cpu-only so should be unrelated to ROCm)
   rocm_ignore_test+=("--ignore $caffe2_pypath/python/operator_test/blobs_queue_db_test.py")
+  # This test is skipped on Jenkins(compiled without MKL) and otherwise known flaky
+  rocm_ignore_test+=("--ignore $caffe2_pypath/python/ideep/convfusion_op_test.py")
+  # This test is skipped on Jenkins(compiled without MKL) and causing segfault on Circle
+  rocm_ignore_test+=("--ignore $caffe2_pypath/python/ideep/pool_op_test.py")
 fi
 
 # NB: Warnings are disabled because they make it harder to see what
@@ -183,8 +170,8 @@ if [[ "$BUILD_ENVIRONMENT" == *onnx* ]]; then
   if [[ "$BUILD_ENVIRONMENT" == *py3* ]]; then
     # default pip version is too old(9.0.2), unable to support tag `manylinux2010`.
     # Fix the pip error: Couldn't find a version that satisfies the requirement
-    sudo pip install --upgrade pip
-    pip install -q --user -i https://test.pypi.org/simple/ ort-nightly==1.3.0.dev202005123
+    pip install --upgrade pip
+    pip install -q --user -i https://test.pypi.org/simple/ ort-nightly==1.5.0.dev202009182
   fi
   "$ROOT_DIR/scripts/onnx/test.sh"
 fi

.jenkins/pytorch/build-asan.sh

@@ -14,7 +14,7 @@ clang --version
 
 # detect_leaks=0: Python is very leaky, so we need suppress it
 # symbolize=1: Gives us much better errors when things go wrong
-export ASAN_OPTIONS=detect_leaks=0:symbolize=1
+export ASAN_OPTIONS=detect_leaks=0:symbolize=1:detect_odr_violation=0
 
 # FIXME: Remove the hardcoded "-pthread" option.
 # With asan build, the cmake thread CMAKE_HAVE_LIBC_CREATE[1] checking will

.jenkins/pytorch/build-mobile-code-analysis.sh

@@ -15,11 +15,4 @@ clang --version
 export LLVM_DIR="$(llvm-config-5.0 --prefix)"
 echo "LLVM_DIR: ${LLVM_DIR}"
 
-# Run the following 2 steps together because they share the same (reusable) time
-# consuming process to build LibTorch into LLVM assembly.
-
-# 1. Run code analysis test first to fail fast
 time ANALYZE_TEST=1 CHECK_RESULT=1 tools/code_analyzer/build.sh
-
-# 2. Run code analysis on mobile LibTorch
-time ANALYZE_TORCH=1 tools/code_analyzer/build.sh

.jenkins/pytorch/build-mobile.sh

@@ -11,13 +11,18 @@ COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 # Install torch & torchvision - used to download & trace test model.
-retry pip install torch torchvision --progress-bar off
+# Ideally we should use the libtorch built on the PR so that backward
+# incompatible changes won't break this script - but it will significantly slow
+# down mobile CI jobs.
+# Here we install nightly instead of stable so that we have an option to
+# temporarily skip mobile CI jobs on BC-breaking PRs until they are in nightly.
+retry pip install --pre torch torchvision \
+  -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html \
+  --progress-bar off
 
 # Run end-to-end process of building mobile library, linking into the predictor
 # binary, and running forward pass with a real model.
-if [[ "$BUILD_ENVIRONMENT" == *-mobile-custom-build-static* ]]; then
-  TEST_CUSTOM_BUILD_STATIC=1 test/mobile/custom_build/build.sh
-elif [[ "$BUILD_ENVIRONMENT" == *-mobile-custom-build-dynamic* ]]; then
+if [[ "$BUILD_ENVIRONMENT" == *-mobile-custom-build-dynamic* ]]; then
   export LLVM_DIR="$(llvm-config-5.0 --prefix)"
   echo "LLVM_DIR: ${LLVM_DIR}"
   TEST_CUSTOM_BUILD_DYNAMIC=1 test/mobile/custom_build/build.sh

.jenkins/pytorch/build.sh

@@ -7,6 +7,13 @@
 # shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
 
+# Temp: use new sccache
+if [[ -n "$IN_CIRCLECI" && "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+  # Download customized sccache
+  sudo curl --retry 3 http://repo.radeon.com/misc/.sccache_amd/sccache -o /opt/cache/bin/sccache
+  sudo chmod 755 /opt/cache/bin/sccache
+fi
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 # For distributed, four environmental configs:
@@ -28,7 +35,6 @@ if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9*gcc7* ]] || [[ "$BUILD_ENVIRONMENT"
   else
     sudo apt-get -qq install --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
   fi
-  sudo apt-get -qq install --no-install-recommends openssh-client openssh-server
   sudo mkdir -p /var/run/sshd
 fi
 
@@ -115,6 +121,11 @@ if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
   exec ./scripts/build_android.sh "${build_args[@]}" "$@"
 fi
 
+if [[ "$BUILD_ENVIRONMENT" != *android* && "$BUILD_ENVIRONMENT" == *vulkan-linux* ]]; then
+  export USE_VULKAN=1
+  export VULKAN_SDK=/var/lib/jenkins/vulkansdk/
+fi
+
 if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
   # hcc used to run out of memory, silently exiting without stopping
   # the build process, leaving undefined symbols in the shared lib,
@@ -126,7 +137,7 @@ if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
 
   # ROCm CI is using Caffe2 docker images, which needs these wrapper
   # scripts to correctly use sccache.
-  if [ -n "${SCCACHE_BUCKET}" ]; then
+  if [[ -n "${SCCACHE_BUCKET}" && -z "$IN_CIRCLECI" ]]; then
     mkdir -p ./sccache
 
     SCCACHE="$(which sccache)"
@@ -150,12 +161,15 @@ if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
     export PATH="$CACHE_WRAPPER_DIR:$PATH"
   fi
 
+  if [[ -n "$IN_CIRCLECI" ]]; then
+      # Set ROCM_ARCH to gtx900 and gtx906 in CircleCI
+      echo "Limiting PYTORCH_ROCM_ARCH to gfx90[06] for CircleCI builds"
+      export PYTORCH_ROCM_ARCH="gfx900;gfx906"
+  fi
+
   python tools/amd_build/build_amd.py
   python setup.py install --user
 
-  # runtime compilation of MIOpen kernels manages to crash sccache - hence undo the wrapping
-  bash tools/amd_build/unwrap_clang.sh
-
   exit 0
 fi
 
@@ -204,9 +218,11 @@ else
     # set only when building other architectures
     # only use for "python setup.py install" line
     if [[ "$BUILD_ENVIRONMENT" != *ppc64le*  && "$BUILD_ENVIRONMENT" != *clang* ]]; then
-      WERROR=1 python setup.py install
+      WERROR=1 python setup.py bdist_wheel
+      python -mpip install dist/*.whl
     else
-      python setup.py install
+      python setup.py bdist_wheel
+      python -mpip install dist/*.whl
     fi
 
     # TODO: I'm not sure why, but somehow we lose verbose commands
@@ -218,6 +234,11 @@ else
     fi
 
     assert_git_not_dirty
+    # Copy ninja build logs to dist folder
+    mkdir -p dist
+    if [ -f build/.ninja_log ]; then
+      cp build/.ninja_log dist
+    fi
 
     # Build custom operator tests.
     CUSTOM_OP_BUILD="$PWD/../custom-op-build"
@@ -230,6 +251,17 @@ else
     make VERBOSE=1
     popd
     assert_git_not_dirty
+
+    # Build custom backend tests.
+    CUSTOM_BACKEND_BUILD="$PWD/../custom-backend-build"
+    CUSTOM_BACKEND_TEST="$PWD/test/custom_backend"
+    python --version
+    mkdir "$CUSTOM_BACKEND_BUILD"
+    pushd "$CUSTOM_BACKEND_BUILD"
+    cmake "$CUSTOM_BACKEND_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)"
+    make VERBOSE=1
+    popd
+    assert_git_not_dirty
   else
     # Test standalone c10 build
     if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda10.1-cudnn7-py3* ]]; then

.jenkins/pytorch/common.sh

@@ -1,20 +1,7 @@
 #!/bin/bash
 
 # Common setup for all Jenkins scripts
-
-# NB: define this function before set -x, so that we don't
-# pollute the log with a premature EXITED_USER_LAND ;)
-function cleanup {
-  # Note that if you've exited user land, then CI will conclude that
-  # any failure is the CI's fault.  So we MUST only output this
-  # string
-  retcode=$?
-  set +x
-  if [ $retcode -eq 0 ]; then
-    echo "EXITED_USER_LAND"
-  fi
-}
-
+source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
 set -ex
 
 # Save the SCRIPT_DIR absolute path in case later we chdir (as occurs in the gpu perf test)
@@ -77,28 +64,18 @@ declare -f -t trap_add
 
 trap_add cleanup EXIT
 
-function assert_git_not_dirty() {
-    # TODO: we should add an option to `build_amd.py` that reverts the repo to
-    #       an unmodified state.
-    if ([[ "$BUILD_ENVIRONMENT" != *rocm* ]] && [[ "$BUILD_ENVIRONMENT" != *xla* ]]) ; then
-        git_status=$(git status --porcelain)
-        if [[ $git_status ]]; then
-            echo "Build left local git repository checkout dirty"
-            echo "git status --porcelain:"
-            echo "${git_status}"
-            exit 1
-        fi
-    fi
-}
-
 if [[ "$BUILD_ENVIRONMENT" != *pytorch-win-* ]]; then
   if which sccache > /dev/null; then
     # Save sccache logs to file
     sccache --stop-server || true
     rm ~/sccache_error.log || true
-    # increasing SCCACHE_IDLE_TIMEOUT so that extension_backend_test.cpp can build after this PR:
-    # https://github.com/pytorch/pytorch/pull/16645
-    SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=1200 RUST_LOG=sccache::server=error sccache --start-server
+    if [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
+      SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=0 sccache --start-server
+    else
+      # increasing SCCACHE_IDLE_TIMEOUT so that extension_backend_test.cpp can build after this PR:
+      # https://github.com/pytorch/pytorch/pull/16645
+      SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=1200 RUST_LOG=sccache::server=error sccache --start-server
+    fi
 
     # Report sccache stats for easier debugging
     sccache --zero-stats
@@ -158,43 +135,7 @@ if [[ "$BUILD_ENVIRONMENT" == *pytorch-xla-linux-bionic* ]] || \
   fi
 fi
 
-function pip_install() {
-  # retry 3 times
-  # old versions of pip don't have the "--progress-bar" flag
-  pip install --progress-bar off "$@" || pip install --progress-bar off "$@" || pip install --progress-bar off "$@" ||\
-  pip install "$@" || pip install "$@" || pip install "$@"
-}
-
-function pip_uninstall() {
-  # uninstall 2 times
-  pip uninstall -y "$@" || pip uninstall -y "$@"
-}
-
 retry () {
   $*  || (sleep 1 && $*) || (sleep 2 && $*)
 }
 
-function get_exit_code() {
-  set +e
-  "$@"
-  retcode=$?
-  set -e
-  return $retcode
-}
-
-function file_diff_from_base() {
-  # The fetch may fail on Docker hosts, but it's not always necessary.
-  set +e
-  git fetch origin master --quiet
-  set -e
-  git diff --name-only "$(git merge-base origin/master HEAD)" > "$1"
-}
-
-function get_bazel() {
-  # download bazel version
-  wget https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-linux-x86_64 -O tools/bazel
-  # verify content
-  echo '753434f4fa730266cf5ce21d1fdd425e1e167dd9347ad3e8adc19e8c0d54edca  tools/bazel' | sha256sum --quiet -c
-
-  chmod +x tools/bazel
-}

.jenkins/pytorch/common_utils.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Common util **functions** that can be sourced in other scripts.
+
+# NB: define this function before set -x, so that we don't
+# pollute the log with a premature EXITED_USER_LAND ;)
+function cleanup {
+  # Note that if you've exited user land, then CI will conclude that
+  # any failure is the CI's fault.  So we MUST only output this
+  # string
+  retcode=$?
+  set +x
+  if [ $retcode -eq 0 ]; then
+    echo "EXITED_USER_LAND"
+  fi
+}
+
+function assert_git_not_dirty() {
+    # TODO: we should add an option to `build_amd.py` that reverts the repo to
+    #       an unmodified state.
+    if ([[ "$BUILD_ENVIRONMENT" != *rocm* ]] && [[ "$BUILD_ENVIRONMENT" != *xla* ]]) ; then
+        git_status=$(git status --porcelain)
+        if [[ $git_status ]]; then
+            echo "Build left local git repository checkout dirty"
+            echo "git status --porcelain:"
+            echo "${git_status}"
+            exit 1
+        fi
+    fi
+}
+
+function pip_install() {
+  # retry 3 times
+  # old versions of pip don't have the "--progress-bar" flag
+  pip install --progress-bar off "$@" || pip install --progress-bar off "$@" || pip install --progress-bar off "$@" ||\
+  pip install "$@" || pip install "$@" || pip install "$@"
+}
+
+function pip_uninstall() {
+  # uninstall 2 times
+  pip uninstall -y "$@" || pip uninstall -y "$@"
+}
+
+function get_exit_code() {
+  set +e
+  "$@"
+  retcode=$?
+  set -e
+  return $retcode
+}
+
+function file_diff_from_base() {
+  # The fetch may fail on Docker hosts, but it's not always necessary.
+  set +e
+  git fetch origin release/1.7 --quiet
+  set -e
+  git diff --name-only "$(git merge-base origin/release/1.7 HEAD)" > "$1"
+}
+
+function get_bazel() {
+  # download bazel version
+  wget https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-linux-x86_64 -O tools/bazel
+  # verify content
+  echo '753434f4fa730266cf5ce21d1fdd425e1e167dd9347ad3e8adc19e8c0d54edca  tools/bazel' | sha256sum --quiet -c
+
+  chmod +x tools/bazel
+}
+
+TORCHVISION_COMMIT=c2e8a00885e68ae1200eb6440f540e181d9125de
+
+function install_torchvision() {
+  # Check out torch/vision at Jun 11 2020 commit
+  # This hash must match one in .jenkins/caffe2/test.sh
+  pip_install --user "git+https://github.com/pytorch/vision.git@$TORCHVISION_COMMIT"
+}
+
+function checkout_install_torchvision() {
+  git clone https://github.com/pytorch/vision
+  pushd vision
+  git checkout "$TORCHVISION_COMMIT"
+  time python setup.py install
+  popd
+}

.jenkins/pytorch/dirty.sh

@@ -5,6 +5,5 @@ pr="$2"
 git diff --name-only "$upstream" "$pr"
 # Now that PyTorch build depends on Caffe2, unconditionally trigger
 # for any changes.
-# TODO: Replace this with a NEGATIVE regex that allows us to blacklist
-# files (letting us skip builds when they are unnecessary)
+# TODO: Replace this with a NEGATIVE regex that allows us to skip builds when they are unnecessary
 #git diff --name-only "$upstream" "$pr" | grep -Eq '^(aten/|caffe2/|.jenkins/pytorch|docs/(make.bat|Makefile|requirements.txt|source)|mypy|requirements.txt|setup.py|test/|third_party/|tools/|\.gitmodules|torch/)'

.jenkins/pytorch/macos-common.sh

@@ -15,12 +15,12 @@ mkdir -p ${WORKSPACE_DIR}
 # If a local installation of conda doesn't exist, we download and install conda
 if [ ! -d "${WORKSPACE_DIR}/miniconda3" ]; then
   mkdir -p ${WORKSPACE_DIR}
-  curl --retry 3 https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ${WORKSPACE_DIR}/miniconda3.sh
+  curl --retry 3 https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.3-MacOSX-x86_64.sh -o ${WORKSPACE_DIR}/miniconda3.sh
   retry bash ${WORKSPACE_DIR}/miniconda3.sh -b -p ${WORKSPACE_DIR}/miniconda3
 fi
 export PATH="${WORKSPACE_DIR}/miniconda3/bin:$PATH"
 source ${WORKSPACE_DIR}/miniconda3/bin/activate
-retry conda install -y mkl mkl-include numpy pyyaml=5.3 setuptools=46.0.0 cmake cffi ninja
+retry conda install -y mkl mkl-include numpy=1.18.5 pyyaml=5.3 setuptools=46.0.0 cmake cffi ninja typing_extensions dataclasses
 
 # The torch.hub tests make requests to GitHub.
 #

.jenkins/pytorch/macos-test.sh

@@ -63,7 +63,7 @@ test_python_all() {
   # Increase default limit on open file handles from 256 to 1024
   ulimit -n 1024
 
-  python test/run_test.py --verbose --exclude test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_te test_tensorexpr --determine-from="$DETERMINE_FROM"
+  python test/run_test.py --verbose --exclude test_jit_cuda_fuser_profiling test_jit_cuda_fuser_legacy test_jit_legacy test_jit_fuser_legacy --determine-from="$DETERMINE_FROM"
 
   assert_git_not_dirty
 }
@@ -99,6 +99,26 @@ test_libtorch() {
   fi
 }
 
+test_custom_backend() {
+  echo "Testing custom backends"
+  pushd test/custom_backend
+  rm -rf build && mkdir build
+  pushd build
+  SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
+  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" cmake ..
+  make VERBOSE=1
+  popd
+
+  # Run Python tests and export a lowered module.
+  python test_custom_backend.py -v
+  python backend.py --export-module-to=model.pt
+  # Run C++ tests using the exported module.
+  build/test_custom_backend ./model.pt
+  rm -f ./model.pt
+  popd
+  assert_git_not_dirty
+}
+
 test_custom_script_ops() {
   echo "Testing custom script operators"
   pushd test/custom_operator
@@ -124,11 +144,13 @@ if [ -z "${BUILD_ENVIRONMENT}" ] || [[ "${BUILD_ENVIRONMENT}" == *-test ]]; then
   test_python_all
   test_libtorch
   test_custom_script_ops
+  test_custom_backend
 else
   if [[ "${BUILD_ENVIRONMENT}" == *-test1 ]]; then
     test_python_all
   elif [[ "${BUILD_ENVIRONMENT}" == *-test2 ]]; then
     test_libtorch
     test_custom_script_ops
+    test_custom_backend
   fi
 fi

.jenkins/pytorch/multigpu-test.sh

@@ -24,14 +24,12 @@ if [ -n "${IN_CIRCLECI}" ]; then
     # TODO: move this to Docker
     sudo apt-get update
     sudo apt-get install -y --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
-    sudo apt-get install -y --no-install-recommends openssh-client openssh-server
-    sudo mkdir -p /var/run/sshd
   fi
 fi
 
 python tools/download_mnist.py --quiet -d test/cpp/api/mnist
 OMP_NUM_THREADS=2 TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" build/bin/test_api
-time python test/run_test.py --verbose -i distributed/test_distributed
+time python test/run_test.py --verbose -i distributed/test_distributed_fork
 time python test/run_test.py --verbose -i distributed/test_c10d
 time python test/run_test.py --verbose -i distributed/test_c10d_spawn
 assert_git_not_dirty

.jenkins/pytorch/print_sccache_log.py

@@ -6,6 +6,7 @@ with open(log_file_path) as f:
     lines = f.readlines()
 
 for line in lines:
-    # Ignore errors from CPU instruction set testing
-    if 'src.c' not in line:
+    # Ignore errors from CPU instruction set or symbol existing testing
+    keywords = ['src.c', 'CheckSymbolExists.c']
+    if all([keyword not in line for keyword in keywords]):
         print(line)

.jenkins/pytorch/test.sh

@@ -13,7 +13,7 @@ echo "Testing pytorch"
 
 if [ -n "${IN_CIRCLECI}" ]; then
   # TODO move this to docker
-  pip_install unittest-xml-reporting
+  pip_install unittest-xml-reporting coverage
 
   if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda10.1-* ]]; then
     # TODO: move this to Docker
@@ -25,47 +25,29 @@ if [ -n "${IN_CIRCLECI}" ]; then
     # TODO: move this to Docker
     sudo apt-get -qq update
     sudo apt-get -qq install --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
-    sudo apt-get -qq install --no-install-recommends openssh-client openssh-server
-    sudo mkdir -p /var/run/sshd
   fi
 
   if [[ "$BUILD_ENVIRONMENT" == *-slow-* ]]; then
     export PYTORCH_TEST_WITH_SLOW=1
     export PYTORCH_TEST_SKIP_FAST=1
   fi
+  if [[ "$BUILD_ENVIRONMENT" == *coverage* ]]; then
+    export PYTORCH_COLLECT_COVERAGE=1
+  fi
 fi
 
 if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
   # Print GPU info
   rocminfo | egrep 'Name:.*\sgfx|Marketing'
-  # TODO: Move this to Docker
-  sudo apt-get -qq update
-  sudo apt-get -qq install --no-install-recommends libsndfile1
-
-  # TODO: Remove this once ROCm CI images are >= ROCm 3.5
-  # ROCm 3.5 required a backwards-incompatible change; the kernel and thunk must match.
-  # Detect kernel version and upgrade thunk if this is a ROCm 3.3 container running on a 3.5 kernel.
-  ROCM_ASD_FW_VERSION=$(/opt/rocm/bin/rocm-smi --showfwinfo -d 1 | grep ASD |  awk '{print $6}')
-  if [[ $ROCM_ASD_FW_VERSION = 553648174 && "$BUILD_ENVIRONMENT" == *rocm3.3* ]]; then
-    # upgrade thunk to 3.5
-    mkdir rocm3.5-thunk
-    pushd rocm3.5-thunk
-    wget http://repo.radeon.com/rocm/apt/3.5/pool/main/h/hsakmt-roct3.5.0/hsakmt-roct3.5.0_1.0.9-347-gd4b224f_amd64.deb
-    wget http://repo.radeon.com/rocm/apt/3.5/pool/main/h/hsakmt-roct-dev3.5.0/hsakmt-roct-dev3.5.0_1.0.9-347-gd4b224f_amd64.deb
-    dpkg-deb -vx hsakmt-roct3.5.0_1.0.9-347-gd4b224f_amd64.deb .
-    dpkg-deb -vx hsakmt-roct-dev3.5.0_1.0.9-347-gd4b224f_amd64.deb .
-    sudo cp -r opt/rocm-3.5.0/* /opt/rocm-3.3.0/
-    popd
-    rm -rf rocm3.5-thunk
-  fi
 fi
 
 # --user breaks ppc64le builds and these packages are already in ppc64le docker
 if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]] && [[ "$BUILD_ENVIRONMENT" != *-bazel-* ]] ; then
   # JIT C++ extensions require ninja.
   pip_install --user ninja
-  # ninja is installed in /var/lib/jenkins/.local/bin
-  export PATH="/var/lib/jenkins/.local/bin:$PATH"
+  # ninja is installed in $HOME/.local/bin, e.g., /var/lib/jenkins/.local/bin for CI user jenkins
+  # but this script should be runnable by any user, including root
+  export PATH="$HOME/.local/bin:$PATH"
 
   # TODO: Please move this to Docker
   # The version is fixed to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136
@@ -85,7 +67,7 @@ fi
 # ASAN test is not working
 if [[ "$BUILD_ENVIRONMENT" == *asan* ]]; then
     # Suppress vptr violations arising from multiple copies of pybind11
-    export ASAN_OPTIONS=detect_leaks=0:symbolize=1:strict_init_order=true
+    export ASAN_OPTIONS=detect_leaks=0:symbolize=1:strict_init_order=true:detect_odr_violation=0
     export UBSAN_OPTIONS=print_stacktrace=1:suppressions=$PWD/ubsan.supp
     export PYTORCH_TEST_WITH_ASAN=1
     export PYTORCH_TEST_WITH_UBSAN=1
@@ -139,7 +121,7 @@ elif [[ "${BUILD_ENVIRONMENT}" == *-NO_AVX2-* ]]; then
   export ATEN_CPU_CAPABILITY=avx
 fi
 
-if [ -n "$CIRCLE_PULL_REQUEST" ]; then
+if ([ -n "$CIRCLE_PULL_REQUEST" ] && [[ "$BUILD_ENVIRONMENT" != *coverage* ]]); then
   DETERMINE_FROM=$(mktemp)
   file_diff_from_base "$DETERMINE_FROM"
 fi
@@ -150,17 +132,17 @@ test_python_nn() {
 }
 
 test_python_ge_config_profiling() {
-  time python test/run_test.py --include test_jit_profiling test_jit_fuser_te --verbose --determine-from="$DETERMINE_FROM"
+  time python test/run_test.py --include test_jit_cuda_fuser_profiling test_jit_profiling test_jit_fuser_te test_tensorexpr --verbose --determine-from="$DETERMINE_FROM"
   assert_git_not_dirty
 }
 
 test_python_ge_config_legacy() {
-  time python test/run_test.py --include test_jit_legacy test_jit_fuser_legacy --verbose --determine-from="$DETERMINE_FROM"
+  time python test/run_test.py --include test_jit_cuda_fuser_legacy test_jit_legacy test_jit_fuser_legacy --verbose --determine-from="$DETERMINE_FROM"
   assert_git_not_dirty
 }
 
 test_python_all_except_nn_and_cpp_extensions() {
-  time python test/run_test.py --exclude test_nn test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_te test_tensorexpr --verbose --determine-from="$DETERMINE_FROM"
+  time python test/run_test.py --exclude test_jit_cuda_fuser_profiling test_jit_cuda_fuser_legacy test_nn test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_te test_tensorexpr --verbose --determine-from="$DETERMINE_FROM"
   assert_git_not_dirty
 }
 
@@ -204,12 +186,6 @@ if [[ "${BUILD_ENVIRONMENT}" == *tbb* ]]; then
   sudo cp -r $PWD/third_party/tbb/include/tbb/* /usr/include/tbb
 fi
 
-test_torchvision() {
-  # Check out torch/vision at Jun 11 2020 commit
-  # This hash must match one in .jenkins/caffe2/test.sh
-  pip_install --user git+https://github.com/pytorch/vision.git@c2e8a00885e68ae1200eb6440f540e181d9125de
-}
-
 test_libtorch() {
   if [[ "$BUILD_ENVIRONMENT" != *rocm* ]]; then
     echo "Testing libtorch"
@@ -234,6 +210,53 @@ test_libtorch() {
   fi
 }
 
+test_vulkan() {
+  if [[ "$BUILD_ENVIRONMENT" == *vulkan-linux* ]]; then
+    export VK_ICD_FILENAMES=/var/lib/jenkins/swiftshader/build/Linux/vk_swiftshader_icd.json
+    mkdir -p test/test-reports/cpp-vulkan
+    build/bin/vulkan_test --gtest_output=xml:test/test-reports/cpp-vulkan/vulkan_test.xml
+  fi
+}
+
+test_distributed() {
+  if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
+    echo "Testing distributed C++ tests"
+    mkdir -p test/test-reports/cpp-distributed
+    build/bin/FileStoreTest --gtest_output=xml:test/test-reports/cpp-distributed/FileStoreTest.xml
+    build/bin/HashStoreTest --gtest_output=xml:test/test-reports/cpp-distributed/HashStoreTest.xml
+    build/bin/TCPStoreTest --gtest_output=xml:test/test-reports/cpp-distributed/TCPStoreTest.xml
+
+    build/bin/ProcessGroupGlooTest --gtest_output=xml:test/test-reports/cpp-distributed/ProcessGroupGlooTest.xml
+    build/bin/ProcessGroupNCCLTest --gtest_output=xml:test/test-reports/cpp-distributed/ProcessGroupNCCLTest.xml
+    build/bin/ProcessGroupNCCLErrorsTest --gtest_output=xml:test/test-reports/cpp-distributed/ProcessGroupNCCLErrorsTest.xml
+  fi
+}
+
+test_rpc() {
+  if [[ "$BUILD_ENVIRONMENT" != *rocm* ]]; then
+    echo "Testing RPC C++ tests"
+    mkdir -p test/test-reports/cpp-rpc
+    build/bin/test_cpp_rpc --gtest_output=xml:test/test-reports/cpp-rpc/test_cpp_rpc.xml
+  fi
+}
+
+test_custom_backend() {
+  if [[ "$BUILD_ENVIRONMENT" != *rocm* ]] && [[ "$BUILD_ENVIRONMENT" != *asan* ]] ; then
+    echo "Testing custom backends"
+    CUSTOM_BACKEND_BUILD="$PWD/../custom-backend-build"
+    pushd test/custom_backend
+    cp -a "$CUSTOM_BACKEND_BUILD" build
+    # Run tests Python-side and export a lowered module.
+    python test_custom_backend.py -v
+    python backend.py --export-module-to=model.pt
+    # Run tests C++-side and load the exported lowered module.
+    build/test_custom_backend ./model.pt
+    rm -f ./model.pt
+    popd
+    assert_git_not_dirty
+  fi
+}
+
 test_custom_script_ops() {
   if [[ "$BUILD_ENVIRONMENT" != *rocm* ]] && [[ "$BUILD_ENVIRONMENT" != *asan* ]] ; then
     echo "Testing custom script operators"
@@ -287,10 +310,15 @@ test_xla() {
 test_backward_compatibility() {
   set -x
   pushd test/backward_compatibility
-  python dump_all_function_schemas.py --filename new_schemas.txt
-  pip_uninstall torch
+  python -m venv venv
+  . venv/bin/activate
   pip_install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
-  python check_backward_compatibility.py --new-schemas new_schemas.txt
+  pip show torch
+  python dump_all_function_schemas.py --filename nightly_schemas.txt
+  deactivate
+  rm -r venv
+  pip show torch
+  python check_backward_compatibility.py --existing-schemas nightly_schemas.txt
   popd
   set +x
   assert_git_not_dirty
@@ -304,12 +332,44 @@ test_bazel() {
   tools/bazel test --test_timeout=480 --test_output=all --test_tag_filters=-gpu-required --test_filter=-*CUDA :all_tests
 }
 
+test_benchmarks() {
+  if [[ "$BUILD_ENVIRONMENT" == *cuda* && "$BUILD_ENVIRONMENT" != *nogpu* ]]; then
+    pip_install --user "pytest-benchmark==3.2.3"
+    pip_install --user "requests"
+    BENCHMARK_DATA="benchmarks/.data"
+    mkdir -p ${BENCHMARK_DATA}
+    pytest benchmarks/fastrnns/test_bench.py --benchmark-sort=Name --benchmark-json=${BENCHMARK_DATA}/fastrnns_default.json --fuser=default --executor=default
+    python benchmarks/upload_scribe.py --pytest_bench_json ${BENCHMARK_DATA}/fastrnns_default.json
+    pytest benchmarks/fastrnns/test_bench.py --benchmark-sort=Name --benchmark-json=${BENCHMARK_DATA}/fastrnns_legacy_old.json --fuser=old --executor=legacy
+    python benchmarks/upload_scribe.py --pytest_bench_json ${BENCHMARK_DATA}/fastrnns_legacy_old.json
+    pytest benchmarks/fastrnns/test_bench.py --benchmark-sort=Name --benchmark-json=${BENCHMARK_DATA}/fastrnns_profiling_te.json --fuser=te --executor=profiling
+    python benchmarks/upload_scribe.py --pytest_bench_json ${BENCHMARK_DATA}/fastrnns_profiling_te.json
+    assert_git_not_dirty
+  fi
+}
+
 test_cpp_extensions() {
   # This is to test whether cpp extension build is compatible with current env. No need to test both ninja and no-ninja build
   time python test/run_test.py --include test_cpp_extensions_aot_ninja --verbose --determine-from="$DETERMINE_FROM"
   assert_git_not_dirty
 }
 
+test_vec256() {
+  # This is to test vec256 instructions DEFAULT/AVX/AVX2 (platform dependent, some platforms might not support AVX/AVX2)
+  if [[ "$BUILD_ENVIRONMENT" != *asan* ]] && [[ "$BUILD_ENVIRONMENT" != *rocm* ]]; then
+    echo "Testing vec256 instructions"
+    mkdir -p test/test-reports/vec256
+    pushd build/bin
+    vec256_tests=$(find . -maxdepth 1 -executable -name 'vec256_test*')
+    for vec256_exec in $vec256_tests
+    do
+      $vec256_exec --gtest_output=xml:test/test-reports/vec256/$vec256_exec.xml
+    done
+    popd
+    assert_git_not_dirty
+  fi
+}
+
 if ! [[ "${BUILD_ENVIRONMENT}" == *libtorch* || "${BUILD_ENVIRONMENT}" == *-bazel-* ]]; then
   (cd test && python -c "import torch; print(torch.__config__.show())")
   (cd test && python -c "import torch; print(torch.__config__.parallel_info())")
@@ -319,7 +379,7 @@ if [[ "${BUILD_ENVIRONMENT}" == *backward* ]]; then
   test_backward_compatibility
   # Do NOT add tests after bc check tests, see its comment.
 elif [[ "${BUILD_ENVIRONMENT}" == *xla* || "${JOB_BASE_NAME}" == *xla* ]]; then
-  test_torchvision
+  install_torchvision
   test_xla
 elif [[ "${BUILD_ENVIRONMENT}" == *ge_config_legacy* || "${JOB_BASE_NAME}" == *ge_config_legacy* ]]; then
   test_python_ge_config_legacy
@@ -332,25 +392,39 @@ elif [[ "${BUILD_ENVIRONMENT}" == *-test1 || "${JOB_BASE_NAME}" == *-test1 ]]; t
   test_python_nn
   test_cpp_extensions
 elif [[ "${BUILD_ENVIRONMENT}" == *-test2 || "${JOB_BASE_NAME}" == *-test2 ]]; then
-  test_torchvision
+  install_torchvision
   test_python_all_except_nn_and_cpp_extensions
   test_aten
   test_libtorch
   test_custom_script_ops
+  test_custom_backend
   test_torch_function_benchmark
+elif [[ "${BUILD_ENVIRONMENT}" == *vulkan-linux* ]]; then
+  test_vulkan
 elif [[ "${BUILD_ENVIRONMENT}" == *-bazel-* ]]; then
   test_bazel
 elif [[ "${BUILD_ENVIRONMENT}" == pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4* ]]; then
-  # test cpp extension for xenial + cuda 9.2 + gcc 5.4 to make sure 
-  # cpp extension can be built correctly under this old env 
+  # test cpp extension for xenial + cuda 9.2 + gcc 5.4 to make sure
+  # cpp extension can be built correctly under this old env
   test_cpp_extensions
 else
-  test_torchvision
+  install_torchvision
   test_python_nn
   test_python_all_except_nn_and_cpp_extensions
   test_cpp_extensions
   test_aten
+  test_vec256
   test_libtorch
   test_custom_script_ops
+  test_custom_backend
   test_torch_function_benchmark
+  test_distributed
+  test_benchmarks
+  test_rpc
+  if [[ "$BUILD_ENVIRONMENT" == *coverage* ]]; then
+    pushd test
+    echo "Generating XML coverage report"
+    time python -mcoverage xml
+    popd
+  fi
 fi

.jenkins/pytorch/win-test-helpers/build_pytorch.bat

@@ -21,8 +21,8 @@ call %INSTALLER_DIR%\install_sccache.bat
 call %INSTALLER_DIR%\install_miniconda3.bat
 
 
-:: Install ninja
-if "%REBUILD%"=="" ( pip install -q "ninja==1.9.0" )
+:: Install ninja and other deps
+if "%REBUILD%"=="" ( pip install -q "ninja==1.9.0" dataclasses )
 
 git submodule sync --recursive
 git submodule update --init --recursive
@@ -39,6 +39,7 @@ popd
 
 if "%CUDA_VERSION%" == "9" goto cuda_build_9
 if "%CUDA_VERSION%" == "10" goto cuda_build_10
+if "%CUDA_VERSION%" == "11" goto cuda_build_11
 goto cuda_build_end
 
 :cuda_build_9
@@ -55,6 +56,13 @@ set CUDA_PATH_V10_1=%CUDA_PATH%
 
 goto cuda_build_common
 
+:cuda_build_11
+
+set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0
+set CUDA_PATH_V11_0=%CUDA_PATH%
+
+goto cuda_build_common
+
 :cuda_build_common
 
 set CUDNN_LIB_DIR=%CUDA_PATH%\lib\x64

.jenkins/pytorch/win-test-helpers/installation-helpers/install_magma.bat

@@ -1,17 +1,18 @@
 if "%CUDA_VERSION%" == "9" set CUDA_SUFFIX=cuda92
 if "%CUDA_VERSION%" == "10" set CUDA_SUFFIX=cuda101
+if "%CUDA_VERSION%" == "11" set CUDA_SUFFIX=cuda110
 
 if "%CUDA_SUFFIX%" == "" (
-  echo unknown CUDA version, please set `CUDA_VERSION` to 9 or 10.
+  echo unknown CUDA version, please set `CUDA_VERSION` to 9, 10 or 11.
   exit /b 1
 )
 
 if "%REBUILD%"=="" (
   if "%BUILD_ENVIRONMENT%"=="" (
-    curl --retry 3 -k https://s3.amazonaws.com/ossci-windows/magma_2.5.2_%CUDA_SUFFIX%_%BUILD_TYPE%.7z --output %TMP_DIR_WIN%\magma_2.5.2_%CUDA_SUFFIX%_%BUILD_TYPE%.7z
+    curl --retry 3 -k https://s3.amazonaws.com/ossci-windows/magma_2.5.3_%CUDA_SUFFIX%_%BUILD_TYPE%.7z --output %TMP_DIR_WIN%\magma_2.5.3_%CUDA_SUFFIX%_%BUILD_TYPE%.7z
   ) else (
-    aws s3 cp s3://ossci-windows/magma_2.5.2_%CUDA_SUFFIX%_%BUILD_TYPE%.7z %TMP_DIR_WIN%\magma_2.5.2_%CUDA_SUFFIX%_%BUILD_TYPE%.7z --quiet
+    aws s3 cp s3://ossci-windows/magma_2.5.3_%CUDA_SUFFIX%_%BUILD_TYPE%.7z %TMP_DIR_WIN%\magma_2.5.3_%CUDA_SUFFIX%_%BUILD_TYPE%.7z --quiet
   )
-  7z x -aoa %TMP_DIR_WIN%\magma_2.5.2_%CUDA_SUFFIX%_%BUILD_TYPE%.7z -o%TMP_DIR_WIN%\magma
+  7z x -aoa %TMP_DIR_WIN%\magma_2.5.3_%CUDA_SUFFIX%_%BUILD_TYPE%.7z -o%TMP_DIR_WIN%\magma
 )
 set MAGMA_HOME=%TMP_DIR_WIN%\magma

.jenkins/pytorch/win-test-helpers/installation-helpers/install_miniconda3.bat

@@ -12,4 +12,11 @@ call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Minic
 if "%REBUILD%"=="" (
   call conda install -y -q python=%PYTHON_VERSION% numpy cffi pyyaml boto3
   call conda install -y -q -c conda-forge cmake
+  call conda install -y -q -c rdonnelly libuv
 )
+
+:: Get installed libuv path
+@echo off
+set libuv_ROOT=%CONDA_PARENT_DIR%\Miniconda3\Library
+@echo on
+echo libuv_ROOT=%libuv_ROOT%

.jenkins/pytorch/win-test-helpers/run_python_nn_smoketests.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
-
 import subprocess
 import os
 

.jenkins/pytorch/win-test-helpers/setup_pytorch_env.bat

@@ -22,7 +22,7 @@ call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Minic
 if NOT "%BUILD_ENVIRONMENT%"=="" (
     :: We have to pin Python version to 3.6.7, until mkl supports Python 3.7
     :: Numba is pinned to 0.44.0 to avoid https://github.com/numba/numba/issues/4352
-    call conda install -y -q python=3.6.7 numpy mkl cffi pyyaml boto3 protobuf numba==0.44.0
+    call conda install -y -q python=3.6.7 numpy mkl cffi pyyaml boto3 protobuf numba==0.44.0 scipy==1.5.0 typing_extensions dataclasses
     if %errorlevel% neq 0 ( exit /b %errorlevel% )
     call conda install -y -q -c conda-forge cmake
     if %errorlevel% neq 0 ( exit /b %errorlevel% )
@@ -39,7 +39,7 @@ if %errorlevel% neq 0 ( exit /b %errorlevel% )
 popd
 
 :: The version is fixed to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136
-pip install "ninja==1.9.0" future "hypothesis==4.53.2" "librosa>=0.6.2" psutil pillow unittest-xml-reporting "scipy==1.4.1"
+pip install "ninja==1.10.0.post1" future "hypothesis==4.53.2" "librosa>=0.6.2" psutil pillow unittest-xml-reporting
 if %errorlevel% neq 0 ( exit /b %errorlevel% )
 :: No need to install faulthandler since we only test Python >= 3.6 on Windows
 :: faulthandler is builtin since Python 3.3
@@ -48,6 +48,7 @@ set DISTUTILS_USE_SDK=1
 
 if "%CUDA_VERSION%" == "9" goto cuda_build_9
 if "%CUDA_VERSION%" == "10" goto cuda_build_10
+if "%CUDA_VERSION%" == "11" goto cuda_build_11
 goto cuda_build_end
 
 :cuda_build_9
@@ -64,6 +65,13 @@ set CUDA_PATH_V10_1=%CUDA_PATH%
 
 goto cuda_build_common
 
+:cuda_build_11
+
+set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0
+set CUDA_PATH_V11_0=%CUDA_PATH%
+
+goto cuda_build_common
+
 :cuda_build_common
 
 set CUDNN_LIB_DIR=%CUDA_PATH%\lib\x64