Updating fbgemm

Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/31798

Differential Revision: D19272332

Pulled By: zou3519

fbshipit-source-id: 954d086a7b9f1a719e0dac303a4253bf7ec8e9f4

.bazelrc

@@ -1,3 +0,0 @@
-build --copt=--std=c++14
-build --copt=-I.
-build --copt=-isystem --copt bazel-out/k8-fastbuild/bin

.bazelversion

@@ -1 +0,0 @@
-3.1.0

.circleci/README.md

@@ -71,9 +71,9 @@ A **binary configuration** is a collection of
 * release or nightly
     * releases are stable, nightlies are beta and built every night
 * python version
-    * linux: 3.5m, 3.6m 3.7m (mu is wide unicode or something like that. It usually doesn't matter but you should know that it exists)
+    * linux: 2.7m, 2.7mu, 3.5m, 3.6m 3.7m (mu is wide unicode or something like that. It usually doesn't matter but you should know that it exists)
-    * macos: 3.6, 3.7, 3.8
+    * macos: 2.7, 3.5, 3.6, 3.7
-    * windows: 3.6, 3.7, 3.8
+    * windows: 3.5, 3.6, 3.7
 * cpu version
     * cpu, cuda 9.0, cuda 10.0
     * The supported cuda versions occasionally change
@@ -466,7 +466,7 @@ But if you want to try, then I’d recommend
 # Always install miniconda 3, even if building for Python <3
 new_conda="~/my_new_conda"
 conda_sh="$new_conda/install_miniconda.sh"
-curl -o "$conda_sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+curl -o "$conda_sh" https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
 chmod +x "$conda_sh"
 "$conda_sh" -b -p "$MINICONDA_ROOT"
 rm -f "$conda_sh"

.circleci/cimodel/data/binary_build_data.py

@@ -5,6 +5,9 @@ for "smoketest" builds.
 Each subclass of ConfigNode represents a layer of the configuration hierarchy.
 These tree nodes encapsulate the logic for whether a branch of the hierarchy
 should be "pruned".
+
+In addition to generating config.yml content, the tree is also traversed
+to produce a visualization of config dimensions.
 """
 
 from collections import OrderedDict
@@ -31,13 +34,15 @@ def get_processor_arch_name(cuda_version):
 
 LINUX_PACKAGE_VARIANTS = OrderedDict(
     manywheel=[
+        "2.7m",
+        "2.7mu",
+        "3.5m",
         "3.6m",
         "3.7m",
-        "3.8m",
     ],
     conda=dimensions.STANDARD_PYTHON_VERSIONS,
     libtorch=[
-        "3.7m",
+        "2.7m",
     ],
 )
 
@@ -47,14 +52,7 @@ CONFIG_TREE_DATA = OrderedDict(
         wheel=dimensions.STANDARD_PYTHON_VERSIONS,
         conda=dimensions.STANDARD_PYTHON_VERSIONS,
         libtorch=[
-            "3.7",
+            "2.7",
-        ],
-    )),
-    windows=(dimensions.CUDA_VERSIONS, OrderedDict(
-        wheel=dimensions.STANDARD_PYTHON_VERSIONS,
-        conda=dimensions.STANDARD_PYTHON_VERSIONS,
-        libtorch=[
-            "3.7",
         ],
     )),
 )
@@ -75,11 +73,6 @@ LINUX_GCC_CONFIG_VARIANTS = OrderedDict(
     ],
 )
 
-WINDOWS_LIBTORCH_CONFIG_VARIANTS = [
-    "debug",
-    "release",
-]
-
 
 class TopLevelNode(ConfigNode):
     def __init__(self, node_name, config_tree_data, smoke):
@@ -114,8 +107,6 @@ class PackageFormatConfigNode(ConfigNode):
     def get_children(self):
         if self.find_prop("os_name") == "linux":
             return [LinuxGccConfigNode(self, v) for v in LINUX_GCC_CONFIG_VARIANTS[self.find_prop("package_format")]]
-        elif self.find_prop("os_name") == "windows" and self.find_prop("package_format") == "libtorch":
-            return [WindowsLibtorchConfigNode(self, v) for v in WINDOWS_LIBTORCH_CONFIG_VARIANTS]
         else:
             return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
 
@@ -137,16 +128,6 @@ class LinuxGccConfigNode(ConfigNode):
         return [ArchConfigNode(self, v) for v in cuda_versions]
 
 
-class WindowsLibtorchConfigNode(ConfigNode):
-    def __init__(self, parent, libtorch_config_variant):
-        super(WindowsLibtorchConfigNode, self).__init__(parent, "LIBTORCH_CONFIG_VARIANT=" + str(libtorch_config_variant))
-
-        self.props["libtorch_config_variant"] = libtorch_config_variant
-
-    def get_children(self):
-        return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
-
-
 class ArchConfigNode(ConfigNode):
     def __init__(self, parent, cu):
         super(ArchConfigNode, self).__init__(parent, get_processor_arch_name(cu))
@@ -164,6 +145,8 @@ class PyVersionConfigNode(ConfigNode):
         self.props["pyver"] = pyver
 
     def get_children(self):
+
+        smoke = self.find_prop("smoke")
         package_format = self.find_prop("package_format")
         os_name = self.find_prop("os_name")
 

.circleci/cimodel/data/binary_build_definitions.py

@@ -1,12 +1,12 @@
 from collections import OrderedDict
 
-import cimodel.data.simple.util.branch_filters as branch_filters
 import cimodel.data.binary_build_data as binary_build_data
 import cimodel.lib.conf_tree as conf_tree
 import cimodel.lib.miniutils as miniutils
 
+
 class Conf(object):
-    def __init__(self, os, cuda_version, pydistro, parms, smoke, libtorch_variant, gcc_config_variant, libtorch_config_variant):
+    def __init__(self, os, cuda_version, pydistro, parms, smoke, libtorch_variant, gcc_config_variant):
 
         self.os = os
         self.cuda_version = cuda_version
@@ -15,19 +15,16 @@ class Conf(object):
         self.smoke = smoke
         self.libtorch_variant = libtorch_variant
         self.gcc_config_variant = gcc_config_variant
-        self.libtorch_config_variant = libtorch_config_variant
 
     def gen_build_env_parms(self):
         elems = [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)]
         if self.gcc_config_variant is not None:
             elems.append(str(self.gcc_config_variant))
-        if self.libtorch_config_variant is not None:
-            elems.append(str(self.libtorch_config_variant))
         return elems
 
     def gen_docker_image(self):
         if self.gcc_config_variant == 'gcc5.4_cxx11-abi':
-            return miniutils.quote("pytorch/pytorch-binary-docker-image-ubuntu16.04:latest")
+            return miniutils.quote("pytorch/conda-cuda-cxx11-ubuntu1604:latest")
 
         docker_word_substitution = {
             "manywheel": "manylinux",
@@ -36,9 +33,11 @@ class Conf(object):
 
         docker_distro_prefix = miniutils.override(self.pydistro, docker_word_substitution)
 
-        # The cpu nightlies are built on the pytorch/manylinux-cuda102 docker image
+        # The cpu nightlies are built on the pytorch/manylinux-cuda100 docker image
-        alt_docker_suffix = self.cuda_version or "102"
+        alt_docker_suffix = self.cuda_version or "100"
         docker_distro_suffix = "" if self.pydistro == "conda" else alt_docker_suffix
+        if self.cuda_version == "101":
+            return "soumith/manylinux-cuda101@sha256:5d62be90d5b7777121180e6137c7eed73d37aaf9f669c51b783611e37e0b4916"
         return miniutils.quote("pytorch/" + docker_distro_prefix + "-cuda" + docker_distro_suffix)
 
     def get_name_prefix(self):
@@ -64,32 +63,22 @@ class Conf(object):
         job_def = OrderedDict()
         job_def["name"] = self.gen_build_name(phase, nightly)
         job_def["build_environment"] = miniutils.quote(" ".join(self.gen_build_env_parms()))
+        job_def["requires"] = ["setup"]
         if self.smoke:
-            job_def["requires"] = [
+            job_def["requires"].append("update_s3_htmls_for_nightlies")
-                "update_s3_htmls",
+            job_def["requires"].append("update_s3_htmls_for_nightlies_devtoolset7")
-            ]
+            job_def["filters"] = {"branches": {"only": "postnightly"}}
-            job_def["filters"] = branch_filters.gen_filter_dict(
-                branches_list=["nightly"],
-                tags_list=[branch_filters.RC_PATTERN],
-            )
         else:
-            if phase in ["upload"]:
+            job_def["filters"] = {"branches": {"only": "nightly"}}
-                filter_branch = "nightly"
-            else:
-                filter_branch = r"/.*/"
-            job_def["filters"] = branch_filters.gen_filter_dict(
-                branches_list=[filter_branch],
-                tags_list=[branch_filters.RC_PATTERN],
-            )
         if self.libtorch_variant:
             job_def["libtorch_variant"] = miniutils.quote(self.libtorch_variant)
         if phase == "test":
             if not self.smoke:
-                job_def["requires"] = [self.gen_build_name("build", nightly)]
+                job_def["requires"].append(self.gen_build_name("build", nightly))
-            if not (self.smoke and self.os == "macos") and self.os != "windows":
+            if not (self.smoke and self.os == "macos"):
                 job_def["docker_image"] = self.gen_docker_image()
 
-            if self.os != "windows" and self.cuda_version:
+            if self.cuda_version:
                 job_def["use_cuda_docker_runtime"] = miniutils.quote("1")
         else:
             if self.os == "linux" and phase != "upload":
@@ -97,15 +86,10 @@ class Conf(object):
 
         if phase == "test":
             if self.cuda_version:
-                if self.os == "windows":
+                job_def["resource_class"] = "gpu.medium"
-                    job_def["executor"] = "windows-with-nvidia-gpu"
-                else:
-                    job_def["resource_class"] = "gpu.medium"
         if phase == "upload":
             job_def["context"] = "org-member"
-            job_def["requires"] = [
+            job_def["requires"] = ["setup", self.gen_build_name(upload_phase_dependency, nightly)]
-                self.gen_build_name(upload_phase_dependency, nightly)
-            ]
 
         os_name = miniutils.override(self.os, {"macos": "mac"})
         job_name = "_".join([self.get_name_prefix(), os_name, phase])
@@ -135,54 +119,29 @@ def gen_build_env_list(smoke):
             c.find_prop("smoke"),
             c.find_prop("libtorch_variant"),
             c.find_prop("gcc_config_variant"),
-            c.find_prop("libtorch_config_variant"),
         )
         newlist.append(conf)
 
     return newlist
 
-def predicate_exclude_macos(config):
+
-    return config.os == "linux" or config.os == "windows"
+def predicate_exclude_nonlinux_and_libtorch(config):
+    return config.os == "linux"
+
 
 def get_nightly_uploads():
     configs = gen_build_env_list(False)
     mylist = []
     for conf in configs:
-        phase_dependency = "test" if predicate_exclude_macos(conf) else "build"
+        phase_dependency = "test" if predicate_exclude_nonlinux_and_libtorch(conf) else "build"
         mylist.append(conf.gen_workflow_job("upload", phase_dependency, nightly=True))
 
     return mylist
 
-def get_post_upload_jobs():
-    """Generate jobs to update HTML indices and report binary sizes"""
-    configs = gen_build_env_list(False)
-    common_job_def = {
-        "context": "org-member",
-        "filters": branch_filters.gen_filter_dict(
-            branches_list=["nightly"],
-            tags_list=[branch_filters.RC_PATTERN],
-        ),
-        "requires": [],
-    }
-    for conf in configs:
-        upload_job_name = conf.gen_build_name(
-            build_or_test="upload",
-            nightly=True
-        )
-        common_job_def["requires"].append(upload_job_name)
-    return [
-        {
-            "update_s3_htmls": {
-                "name": "update_s3_htmls",
-                **common_job_def,
-            },
-        },
-    ]
-
 def get_nightly_tests():
 
     configs = gen_build_env_list(False)
-    filtered_configs = filter(predicate_exclude_macos, configs)
+    filtered_configs = filter(predicate_exclude_nonlinux_and_libtorch, configs)
 
     tests = []
     for conf_options in filtered_configs:

.circleci/cimodel/data/caffe2_build_data.py

@@ -4,9 +4,8 @@ from cimodel.lib.conf_tree import Ver
 
 CONFIG_TREE_DATA = [
     (Ver("ubuntu", "16.04"), [
-        ([Ver("clang", "7")], [XImportant("onnx_main_py3.6"),
+        ([Ver("gcc", "5")], [XImportant("onnx_py2")]),
-                               XImportant("onnx_ort1_py3.6"),
+        ([Ver("clang", "7")], [XImportant("onnx_py3.6")]),
-                               XImportant("onnx_ort2_py3.6")]),
     ]),
 ]
 
@@ -28,9 +27,7 @@ class TreeConfigNode(ConfigNode):
         return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
 
     def is_build_only(self):
-        if str(self.find_prop("language_version")) == "onnx_main_py3.6" or \
+        if str(self.find_prop("language_version")) == "onnx_py3.6":
-                str(self.find_prop("language_version")) == "onnx_ort1_py3.6" or \
-                str(self.find_prop("language_version")) == "onnx_ort2_py3.6":
             return False
         return set(str(c) for c in self.find_prop("compiler_version")).intersection({
             "clang3.8",
@@ -39,12 +36,6 @@ class TreeConfigNode(ConfigNode):
             "android",
         }) or self.find_prop("distro_version").name == "macos"
 
-    def is_test_only(self):
-        if str(self.find_prop("language_version")) == "onnx_ort1_py3.6" or \
-                str(self.find_prop("language_version")) == "onnx_ort2_py3.6":
-            return True
-        return False
-
 
 class TopLevelNode(TreeConfigNode):
     def __init__(self, node_name, subtree):
@@ -77,7 +68,6 @@ class LanguageConfigNode(TreeConfigNode):
     def init2(self, node_name):
         self.props["language_version"] = node_name
         self.props["build_only"] = self.is_build_only()
-        self.props["test_only"] = self.is_test_only()
 
     def child_constructor(self):
         return ImportantConfigNode

.circleci/cimodel/data/caffe2_build_definitions.py

@@ -5,14 +5,14 @@ import cimodel.lib.conf_tree as conf_tree
 from cimodel.lib.conf_tree import Ver
 import cimodel.lib.miniutils as miniutils
 from cimodel.data.caffe2_build_data import CONFIG_TREE_DATA, TopLevelNode
-from cimodel.data.simple.util.branch_filters import gen_filter_dict
+
 
 from dataclasses import dataclass
 
 
 DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
 
-DOCKER_IMAGE_VERSION = "376"
+DOCKER_IMAGE_VERSION = 345
 
 
 @dataclass
@@ -23,7 +23,6 @@ class Conf:
     # for gpu files and host compiler (gcc/clang) for cpu files)
     compilers: [Ver]
     build_only: bool
-    test_only: bool
     is_important: bool
 
     @property
@@ -33,9 +32,8 @@ class Conf:
     # TODO: Eventually we can probably just remove the cudnn7 everywhere.
     def get_cudnn_insertion(self):
 
-        omit = self.language == "onnx_main_py3.6" \
+        omit = self.language == "onnx_py2" \
-            or self.language == "onnx_ort1_py3.6" \
+            or self.language == "onnx_py3.6" \
-            or self.language == "onnx_ort2_py3.6" \
             or set(self.compiler_names).intersection({"android", "mkl", "clang"}) \
             or str(self.distro) in ["ubuntu14.04", "macos10.13"]
 
@@ -52,13 +50,6 @@ class Conf:
 
     def construct_phase_name(self, phase):
         root_parts = self.get_build_name_root_parts()
-
-        build_name_substitutions = {
-            "onnx_ort1_py3.6": "onnx_main_py3.6",
-            "onnx_ort2_py3.6": "onnx_main_py3.6",
-        }
-        if phase == "build":
-            root_parts = [miniutils.override(r, build_name_substitutions) for r in root_parts]
         return "_".join(root_parts + [phase]).replace(".", "_")
 
     def get_platform(self):
@@ -70,10 +61,9 @@ class Conf:
     def gen_docker_image(self):
 
         lang_substitutions = {
-            "onnx_main_py3.6": "py3.6",
+            "onnx_py2": "py2",
-            "onnx_ort1_py3.6": "py3.6",
+            "onnx_py3.6": "py3.6",
-            "onnx_ort2_py3.6": "py3.6",
+            "cmake": "py2",
-            "cmake": "py3",
         }
 
         lang = miniutils.override(self.language, lang_substitutions)
@@ -83,10 +73,8 @@ class Conf:
     def gen_workflow_params(self, phase):
         parameters = OrderedDict()
         lang_substitutions = {
-            "onnx_py3": "onnx-py3",
+            "onnx_py2": "onnx-py2",
-            "onnx_main_py3.6": "onnx-main-py3.6",
+            "onnx_py3.6": "onnx-py3.6",
-            "onnx_ort1_py3.6": "onnx-ort1-py3.6",
-            "onnx_ort2_py3.6": "onnx-ort2-py3.6",
         }
 
         lang = miniutils.override(self.language, lang_substitutions)
@@ -118,15 +106,16 @@ class Conf:
     def gen_workflow_job(self, phase):
         job_def = OrderedDict()
         job_def["name"] = self.construct_phase_name(phase)
+        job_def["requires"] = ["setup"]
 
         if phase == "test":
-            job_def["requires"] = [self.construct_phase_name("build")]
+            job_def["requires"].append(self.construct_phase_name("build"))
             job_name = "caffe2_" + self.get_platform() + "_test"
         else:
             job_name = "caffe2_" + self.get_platform() + "_build"
 
         if not self.is_important:
-            job_def["filters"] = gen_filter_dict()
+            job_def["filters"] = {"branches": {"only": ["master", r"/ci-all\/.*/"]}}
         job_def.update(self.gen_workflow_params(phase))
         return {job_name : job_def}
 
@@ -147,7 +136,6 @@ def instantiate_configs():
             distro=fc.find_prop("distro_version"),
             compilers=fc.find_prop("compiler_version"),
             build_only=fc.find_prop("build_only"),
-            test_only=fc.find_prop("test_only"),
             is_important=fc.find_prop("important"),
         )
 
@@ -162,11 +150,10 @@ def get_workflow_jobs():
 
     x = []
     for conf_options in configs:
+
         phases = ["build"]
         if not conf_options.build_only:
             phases = dimensions.PHASES
-        if conf_options.test_only:
-            phases = ["test"]
 
         for phase in phases:
             x.append(conf_options.gen_workflow_job(phase))

.circleci/cimodel/data/dimensions.py

@@ -3,12 +3,13 @@ PHASES = ["build", "test"]
 CUDA_VERSIONS = [
     None,  # cpu build
     "92",
+    "100",
     "101",
-    "102",
 ]
 
 STANDARD_PYTHON_VERSIONS = [
+    "2.7",
+    "3.5",
     "3.6",
     "3.7",
-    "3.8"
 ]

.circleci/cimodel/data/pytorch_build_data.py

@@ -4,14 +4,17 @@ from cimodel.lib.conf_tree import ConfigNode, X, XImportant
 CONFIG_TREE_DATA = [
     ("xenial", [
         (None, [
+            XImportant("2.7.9"),
+            X("2.7"),
+            XImportant("3.5"),  # Not run on all PRs, but should be included on [test all]
             X("nightly"),
         ]),
         ("gcc", [
             ("5.4", [  # All this subtree rebases to master and then build
                 XImportant("3.6"),
                 ("3.6", [
-                    ("parallel_tbb", [X(True)]),
+                    ("parallel_tbb", [XImportant(True)]),
-                    ("parallel_native", [X(True)]),
+                    ("parallel_native", [XImportant(True)]),
                 ]),
             ]),
             # TODO: bring back libtorch test
@@ -21,43 +24,40 @@ CONFIG_TREE_DATA = [
             ("5", [
                 XImportant("3.6"),  # This is actually the ASAN build
             ]),
+            # ("7", [
+            #     ("3.6", [
+            #         ("xla", [XImportant(True)]),
+            #     ]),
+            # ]),
         ]),
         ("cuda", [
-            ("9.2", [
+            ("9", [
-                X("3.6"),
+                # Note there are magic strings here
-                ("3.6", [
+                # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L21
-                    ("cuda_gcc_override", [X("gcc5.4")])
+                # and
-                ])
+                # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L143
-            ]),
+                # and
-            ("10.1", [X("3.6")]),
+                # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L153
-            ("10.2", [
+                # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453144)
                 XImportant("3.6"),
                 ("3.6", [
                     ("libtorch", [XImportant(True)])
                 ]),
             ]),
-            ("11.0", [
+            ("9.2", [X("3.6")]),
-                X("3.8"),
+            ("10", [X("3.6")]),
-                ("3.8", [
+            ("10.1", [X("3.6")]),
-                    ("libtorch", [X(True)])
-                ]),
-            ]),
         ]),
-    ]),
+        ("android", [
-    ("bionic", [
+            ("r19c", [
-        ("clang", [
-            ("9", [
-                XImportant("3.6"),
-            ]),
-            ("9", [
                 ("3.6", [
-                    ("xla", [XImportant(True)]),
+                    ("android_abi", [XImportant("x86_32")]),
-                ]),
+                    ("android_abi", [X("x86_64")]),
+                    ("android_abi", [X("arm-v7a")]),
+                    ("android_abi", [X("arm-v8a")]),
+                ])
             ]),
         ]),
-        ("gcc", [
-            ("9", [XImportant("3.8")]),
-        ]),
     ]),
 ]
 
@@ -101,7 +101,6 @@ class DistroConfigNode(TreeConfigNode):
 
         next_nodes = {
             "xenial": XenialCompilerConfigNode,
-            "bionic": BionicCompilerConfigNode,
         }
         return next_nodes[distro]
 
@@ -129,8 +128,7 @@ class ExperimentalFeatureConfigNode(TreeConfigNode):
             "parallel_native": ParallelNativeConfigNode,
             "libtorch": LibTorchConfigNode,
             "important": ImportantConfigNode,
-            "build_only": BuildOnlyConfigNode,
+            "android_abi": AndroidAbiConfigNode,
-            "cuda_gcc_override": CudaGccOverrideConfigNode
         }
         return next_nodes[experimental_feature]
 
@@ -145,7 +143,6 @@ class XlaConfigNode(TreeConfigNode):
     def child_constructor(self):
         return ImportantConfigNode
 
-
 class ParallelTBBConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "PARALLELTBB=" + str(label)
@@ -156,7 +153,6 @@ class ParallelTBBConfigNode(TreeConfigNode):
     def child_constructor(self):
         return ImportantConfigNode
 
-
 class ParallelNativeConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "PARALLELNATIVE=" + str(label)
@@ -167,7 +163,6 @@ class ParallelNativeConfigNode(TreeConfigNode):
     def child_constructor(self):
         return ImportantConfigNode
 
-
 class LibTorchConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "BUILD_TEST_LIBTORCH=" + str(label)
@@ -178,23 +173,14 @@ class LibTorchConfigNode(TreeConfigNode):
     def child_constructor(self):
         return ImportantConfigNode
 
+class AndroidAbiConfigNode(TreeConfigNode):
 
-class CudaGccOverrideConfigNode(TreeConfigNode):
     def init2(self, node_name):
-        self.props["cuda_gcc_override"] = node_name
+        self.props["android_abi"] = node_name
 
     def child_constructor(self):
         return ImportantConfigNode
 
-class BuildOnlyConfigNode(TreeConfigNode):
-
-    def init2(self, node_name):
-        self.props["build_only"] = node_name
-
-    def child_constructor(self):
-        return ImportantConfigNode
-
-
 class ImportantConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "IMPORTANT=" + str(label)
@@ -220,20 +206,6 @@ class XenialCompilerConfigNode(TreeConfigNode):
         return XenialCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode
 
 
-class BionicCompilerConfigNode(TreeConfigNode):
-
-    def modify_label(self, label):
-        return label or "<unspecified>"
-
-    def init2(self, node_name):
-        self.props["compiler_name"] = node_name
-
-    # noinspection PyMethodMayBeStatic
-    def child_constructor(self):
-
-        return BionicCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode
-
-
 class XenialCompilerVersionConfigNode(TreeConfigNode):
     def init2(self, node_name):
         self.props["compiler_version"] = node_name
@@ -241,12 +213,3 @@ class XenialCompilerVersionConfigNode(TreeConfigNode):
     # noinspection PyMethodMayBeStatic
     def child_constructor(self):
         return PyVerConfigNode
-
-
-class BionicCompilerVersionConfigNode(TreeConfigNode):
-    def init2(self, node_name):
-        self.props["compiler_version"] = node_name
-
-    # noinspection PyMethodMayBeStatic
-    def child_constructor(self):
-        return PyVerConfigNode

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -4,13 +4,18 @@ from cimodel.data.pytorch_build_data import TopLevelNode, CONFIG_TREE_DATA
 import cimodel.data.dimensions as dimensions
 import cimodel.lib.conf_tree as conf_tree
 import cimodel.lib.miniutils as miniutils
-from cimodel.data.simple.util.branch_filters import gen_filter_dict
-from cimodel.data.simple.util.docker_constants import gen_docker_image_path
 
 from dataclasses import dataclass, field
 from typing import List, Optional
 
 
+DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/"
+
+# ARE YOU EDITING THIS NUMBER?  MAKE SURE YOU READ THE GUIDANCE AT THE
+# TOP OF .circleci/config.yml
+DOCKER_IMAGE_VERSION = 405
+
+
 @dataclass
 class Conf:
     distro: str
@@ -22,7 +27,6 @@ class Conf:
     #  tesnrorrt, leveldb, lmdb, redis, opencv, mkldnn, ideep, etc.
     # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453608)
     is_xla: bool = False
-    vulkan: bool = False
     restrict_phases: Optional[List[str]] = None
     gpu_resource: Optional[str] = None
     dependent_tests: List = field(default_factory=list)
@@ -49,10 +53,9 @@ class Conf:
 
         cuda_parms = []
         if self.cuda_version:
-            cudnn = "cudnn8" if self.cuda_version.startswith("11.") else "cudnn7"
+            cuda_parms.extend(["cuda" + self.cuda_version, "cudnn7"])
-            cuda_parms.extend(["cuda" + self.cuda_version, cudnn])
         result = leading + ["linux", self.distro] + cuda_parms + self.parms
-        if not for_docker and self.parms_list_ignored_for_docker_image is not None:
+        if (not for_docker and self.parms_list_ignored_for_docker_image is not None):
             result = result + self.parms_list_ignored_for_docker_image
         return result
 
@@ -61,7 +64,7 @@ class Conf:
         parms_source = self.parent_build or self
         base_build_env_name = "-".join(parms_source.get_parms(True))
 
-        return miniutils.quote(gen_docker_image_path(base_build_env_name))
+        return miniutils.quote(DOCKER_IMAGE_PATH_BASE + base_build_env_name + ":" + str(DOCKER_IMAGE_VERSION))
 
     def get_build_job_name_pieces(self, build_or_test):
         return self.get_parms(False) + [build_or_test]
@@ -89,8 +92,10 @@ class Conf:
         return parameters
 
     def gen_workflow_job(self, phase):
+        # All jobs require the setup job
         job_def = OrderedDict()
         job_def["name"] = self.gen_build_name(phase)
+        job_def["requires"] = ["setup"]
 
         if phase == "test":
 
@@ -100,13 +105,16 @@ class Conf:
             #  pytorch build job (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259452641)
 
             dependency_build = self.parent_build or self
-            job_def["requires"] = [dependency_build.gen_build_name("build")]
+            job_def["requires"].append(dependency_build.gen_build_name("build"))
             job_name = "pytorch_linux_test"
         else:
             job_name = "pytorch_linux_build"
 
+
         if not self.is_important:
-            job_def["filters"] = gen_filter_dict()
+            # If you update this, update
+            # caffe2_build_definitions.py too
+            job_def["filters"] = {"branches": {"only": ["master", r"/ci-all\/.*/"]}}
         job_def.update(self.gen_workflow_params(phase))
 
         return {job_name : job_def}
@@ -152,13 +160,7 @@ def gen_dependent_configs(xenial_parent_config):
 
         configs.append(c)
 
-    return configs
+    for x in ["pytorch_python_doc_push", "pytorch_cpp_doc_push"]:
-
-
-def gen_docs_configs(xenial_parent_config):
-    configs = []
-
-    for x in ["pytorch_python_doc_push", "pytorch_cpp_doc_push", "pytorch_doc_test"]:
         configs.append(HiddenConf(x, parent_build=xenial_parent_config))
 
     return configs
@@ -180,19 +182,15 @@ def instantiate_configs():
 
     root = get_root()
     found_configs = conf_tree.dfs(root)
+    restrict_phases = None
     for fc in found_configs:
 
-        restrict_phases = None
         distro_name = fc.find_prop("distro_name")
         compiler_name = fc.find_prop("compiler_name")
         compiler_version = fc.find_prop("compiler_version")
         is_xla = fc.find_prop("is_xla") or False
         parms_list_ignored_for_docker_image = []
 
-        vulkan = fc.find_prop("vulkan") or False
-        if vulkan:
-            parms_list_ignored_for_docker_image.append("vulkan")
-
         python_version = None
         if compiler_name == "cuda" or compiler_name == "android":
             python_version = fc.find_prop("pyver")
@@ -212,27 +210,25 @@ def instantiate_configs():
             android_abi = fc.find_prop("android_abi")
             parms_list_ignored_for_docker_image.append(android_abi)
             restrict_phases = ["build"]
+            fc.props["is_important"] = True
 
         elif compiler_name:
             gcc_version = compiler_name + (fc.find_prop("compiler_version") or "")
             parms_list.append(gcc_version)
 
             # TODO: This is a nasty special case
-            if gcc_version == 'clang5' and not is_xla:
+            if compiler_name == "clang" and not is_xla:
                 parms_list.append("asan")
                 python_version = fc.find_prop("pyver")
                 parms_list[0] = fc.find_prop("abbreviated_pyver")
 
-        if cuda_version:
+        if cuda_version in ["9.2", "10", "10.1"]:
-            cuda_gcc_version = fc.find_prop("cuda_gcc_override") or "gcc7"
+            # TODO The gcc version is orthogonal to CUDA version?
-            parms_list.append(cuda_gcc_version)
+            parms_list.append("gcc7")
 
         is_libtorch = fc.find_prop("is_libtorch") or False
         is_important = fc.find_prop("is_important") or False
         parallel_backend = fc.find_prop("parallel_backend") or None
-        build_only = fc.find_prop("build_only") or False
-        if build_only and restrict_phases is None:
-            restrict_phases = ["build"]
 
         gpu_resource = None
         if cuda_version and cuda_version != "10":
@@ -245,7 +241,6 @@ def instantiate_configs():
             python_version,
             cuda_version,
             is_xla,
-            vulkan,
             restrict_phases,
             gpu_resource,
             is_libtorch=is_libtorch,
@@ -253,16 +248,7 @@ def instantiate_configs():
             parallel_backend=parallel_backend,
         )
 
-        # run docs builds on "pytorch-linux-xenial-py3.6-gcc5.4". Docs builds
+        if cuda_version == "9" and python_version == "3.6" and not is_libtorch:
-        # should run on a CPU-only build that runs on all PRs.
-        if distro_name == 'xenial' and fc.find_prop("pyver") == '3.6' \
-                and cuda_version is None \
-                and parallel_backend is None \
-                and compiler_name == 'gcc' \
-                and fc.find_prop('compiler_version') == '5.4':
-            c.dependent_tests = gen_docs_configs(c)
-
-        if cuda_version == "10.1" and python_version == "3.6" and not is_libtorch:
             c.dependent_tests = gen_dependent_configs(c)
 
         if (compiler_name == "gcc"
@@ -289,7 +275,7 @@ def get_workflow_jobs():
 
     config_list = instantiate_configs()
 
-    x = []
+    x = ["setup"]
     for conf_options in config_list:
 
         phases = conf_options.restrict_phases or dimensions.PHASES

.circleci/cimodel/data/simple/android_definitions.py

@@ -1,92 +0,0 @@
-import cimodel.data.simple.util.branch_filters
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_NDK
-
-
-class AndroidJob:
-    def __init__(self,
-                 variant,
-                 template_name,
-                 is_master_only=True):
-
-        self.variant = variant
-        self.template_name = template_name
-        self.is_master_only = is_master_only
-
-    def gen_tree(self):
-
-        base_name_parts = [
-            "pytorch",
-            "linux",
-            "xenial",
-            "py3",
-            "clang5",
-            "android",
-            "ndk",
-            "r19c",
-        ] + self.variant + [
-            "build",
-        ]
-
-        full_job_name = "_".join(base_name_parts)
-        build_env_name = "-".join(base_name_parts)
-
-        props_dict = {
-            "name": full_job_name,
-            "build_environment": "\"{}\"".format(build_env_name),
-            "docker_image": "\"{}\"".format(DOCKER_IMAGE_NDK),
-        }
-
-        if self.is_master_only:
-            props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
-
-        return [{self.template_name: props_dict}]
-
-
-class AndroidGradleJob:
-    def __init__(self,
-                 job_name,
-                 template_name,
-                 dependencies,
-                 is_master_only=True):
-
-        self.job_name = job_name
-        self.template_name = template_name
-        self.dependencies = dependencies
-        self.is_master_only = is_master_only
-
-    def gen_tree(self):
-
-        props_dict = {
-            "name": self.job_name,
-            "requires": self.dependencies,
-        }
-
-        if self.is_master_only:
-            props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
-
-        return [{self.template_name: props_dict}]
-
-
-WORKFLOW_DATA = [
-    AndroidJob(["x86_32"], "pytorch_linux_build", is_master_only=False),
-    AndroidJob(["x86_64"], "pytorch_linux_build"),
-    AndroidJob(["arm", "v7a"], "pytorch_linux_build"),
-    AndroidJob(["arm", "v8a"], "pytorch_linux_build"),
-    AndroidJob(["vulkan", "x86_32"], "pytorch_linux_build", is_master_only=False),
-    AndroidGradleJob(
-        "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32",
-        "pytorch_android_gradle_build-x86_32",
-        ["pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build"],
-        is_master_only=False),
-    AndroidGradleJob(
-        "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build",
-        "pytorch_android_gradle_build",
-        ["pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build",
-         "pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build",
-         "pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build",
-         "pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build"]),
-]
-
-
-def get_workflow_jobs():
-    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/bazel_definitions.py

@@ -1,63 +0,0 @@
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_GCC7
-
-
-def gen_job_name(phase):
-    job_name_parts = [
-        "pytorch",
-        "bazel",
-        phase,
-    ]
-
-    return "_".join(job_name_parts)
-
-
-class BazelJob:
-    def __init__(self, phase, extra_props=None):
-        self.phase = phase
-        self.extra_props = extra_props or {}
-
-    def gen_tree(self):
-
-        template_parts = [
-            "pytorch",
-            "linux",
-            "bazel",
-            self.phase,
-        ]
-
-        build_env_parts = [
-            "pytorch",
-            "linux",
-            "xenial",
-            "py3.6",
-            "gcc7",
-            "bazel",
-            self.phase,
-        ]
-
-        full_job_name = gen_job_name(self.phase)
-        build_env_name = "-".join(build_env_parts)
-
-        extra_requires = [gen_job_name("build")] if self.phase == "test" else []
-
-        props_dict = {
-            "build_environment": build_env_name,
-            "docker_image": DOCKER_IMAGE_GCC7,
-            "name": full_job_name,
-            "requires": extra_requires,
-        }
-
-        props_dict.update(self.extra_props)
-
-        template_name = "_".join(template_parts)
-        return [{template_name: props_dict}]
-
-
-WORKFLOW_DATA = [
-    BazelJob("build", {"resource_class": "large"}),
-    BazelJob("test"),
-]
-
-
-def get_workflow_jobs():
-    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/binary_smoketest.py

@@ -1,193 +0,0 @@
-"""
-TODO: Refactor circleci/cimodel/data/binary_build_data.py to generate this file
-       instead of doing one offs here
- Binary builds (subset, to smoke test that they'll work)
-
- NB: If you modify this file, you need to also modify
- the binary_and_smoke_tests_on_pr variable in
- pytorch-ci-hud to adjust the list of whitelisted builds
- at https://github.com/ezyang/pytorch-ci-hud/blob/master/src/BuildHistoryDisplay.js
-
- Note:
- This binary build is currently broken, see https://github_com/pytorch/pytorch/issues/16710
- - binary_linux_conda_3_6_cu90_devtoolset7_build
- - binary_linux_conda_3_6_cu90_devtoolset7_test
-
- TODO
- we should test a libtorch cuda build, but they take too long
- - binary_linux_libtorch_3_6m_cu90_devtoolset7_static-without-deps_build
-"""
-
-import cimodel.lib.miniutils as miniutils
-import cimodel.data.simple.util.branch_filters
-
-
-class SmoketestJob:
-    def __init__(self,
-                 template_name,
-                 build_env_parts,
-                 docker_image,
-                 job_name,
-                 is_master_only=False,
-                 requires=None,
-                 has_libtorch_variant=False,
-                 extra_props=None):
-
-        self.template_name = template_name
-        self.build_env_parts = build_env_parts
-        self.docker_image = docker_image
-        self.job_name = job_name
-        self.is_master_only = is_master_only
-        self.requires = requires or []
-        self.has_libtorch_variant = has_libtorch_variant
-        self.extra_props = extra_props or {}
-
-    def gen_tree(self):
-
-        props_dict = {
-            "build_environment": " ".join(self.build_env_parts),
-            "name": self.job_name,
-            "requires": self.requires,
-        }
-
-        if self.docker_image:
-            props_dict["docker_image"] = self.docker_image
-
-        if self.is_master_only:
-            props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
-
-        if self.has_libtorch_variant:
-            props_dict["libtorch_variant"] = "shared-with-deps"
-
-        props_dict.update(self.extra_props)
-
-        return [{self.template_name: props_dict}]
-
-
-WORKFLOW_DATA = [
-    SmoketestJob(
-        "binary_linux_build",
-        ["manywheel", "3.7m", "cu102", "devtoolset7"],
-        "pytorch/manylinux-cuda102",
-        "binary_linux_manywheel_3_7m_cu102_devtoolset7_build",
-        is_master_only=True,
-    ),
-    SmoketestJob(
-        "binary_linux_build",
-        ["libtorch", "3.7m", "cpu", "devtoolset7"],
-        "pytorch/manylinux-cuda102",
-        "binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build",
-        is_master_only=False,
-        has_libtorch_variant=True,
-    ),
-    SmoketestJob(
-        "binary_linux_build",
-        ["libtorch", "3.7m", "cpu", "gcc5.4_cxx11-abi"],
-        "pytorch/pytorch-binary-docker-image-ubuntu16.04:latest",
-        "binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build",
-        is_master_only=False,
-        has_libtorch_variant=True,
-    ),
-    SmoketestJob(
-        "binary_mac_build",
-        ["wheel", "3.7", "cpu"],
-        None,
-        "binary_macos_wheel_3_7_cpu_build",
-        is_master_only=True,
-    ),
-    # This job has an average run time of 3 hours o.O
-    # Now only running this on master to reduce overhead
-    SmoketestJob(
-        "binary_mac_build",
-        ["libtorch", "3.7", "cpu"],
-        None,
-        "binary_macos_libtorch_3_7_cpu_build",
-        is_master_only=True,
-    ),
-    SmoketestJob(
-        "binary_windows_build",
-        ["libtorch", "3.7", "cpu", "debug"],
-        None,
-        "binary_windows_libtorch_3_7_cpu_debug_build",
-        is_master_only=False,
-    ),
-    SmoketestJob(
-        "binary_windows_build",
-        ["libtorch", "3.7", "cpu", "release"],
-        None,
-        "binary_windows_libtorch_3_7_cpu_release_build",
-        is_master_only=False,
-    ),
-    SmoketestJob(
-        "binary_windows_build",
-        ["wheel", "3.7", "cu102"],
-        None,
-        "binary_windows_wheel_3_7_cu102_build",
-        is_master_only=True,
-    ),
-
-    SmoketestJob(
-        "binary_windows_test",
-        ["libtorch", "3.7", "cpu", "debug"],
-        None,
-        "binary_windows_libtorch_3_7_cpu_debug_test",
-        is_master_only=False,
-        requires=["binary_windows_libtorch_3_7_cpu_debug_build"],
-    ),
-    SmoketestJob(
-        "binary_windows_test",
-        ["libtorch", "3.7", "cpu", "release"],
-        None,
-        "binary_windows_libtorch_3_7_cpu_release_test",
-        is_master_only=False,
-        requires=["binary_windows_libtorch_3_7_cpu_release_build"],
-    ),
-    SmoketestJob(
-        "binary_windows_test",
-        ["wheel", "3.7", "cu102"],
-        None,
-        "binary_windows_wheel_3_7_cu102_test",
-        is_master_only=True,
-        requires=["binary_windows_wheel_3_7_cu102_build"],
-        extra_props={
-            "executor": "windows-with-nvidia-gpu",
-        },
-    ),
-
-
-
-    SmoketestJob(
-        "binary_linux_test",
-        ["manywheel", "3.7m", "cu102", "devtoolset7"],
-        "pytorch/manylinux-cuda102",
-        "binary_linux_manywheel_3_7m_cu102_devtoolset7_test",
-        is_master_only=True,
-        requires=["binary_linux_manywheel_3_7m_cu102_devtoolset7_build"],
-        extra_props={
-            "resource_class": "gpu.medium",
-            "use_cuda_docker_runtime": miniutils.quote((str(1))),
-        },
-    ),
-    SmoketestJob(
-        "binary_linux_test",
-        ["libtorch", "3.7m", "cpu", "devtoolset7"],
-        "pytorch/manylinux-cuda102",
-        "binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_test",
-        is_master_only=False,
-        requires=["binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build"],
-        has_libtorch_variant=True,
-    ),
-    SmoketestJob(
-        "binary_linux_test",
-        ["libtorch", "3.7m", "cpu", "gcc5.4_cxx11-abi"],
-        "pytorch/pytorch-binary-docker-image-ubuntu16.04:latest",
-        "binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_test",
-        is_master_only=False,
-        requires=["binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build"],
-        has_libtorch_variant=True,
-    ),
-]
-
-
-def get_workflow_jobs():
-    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/docker_definitions.py

@@ -1,44 +0,0 @@
-from collections import OrderedDict
-
-from cimodel.lib.miniutils import quote
-
-
-# TODO: make this generated from a matrix rather than just a static list
-IMAGE_NAMES = [
-    "pytorch-linux-bionic-cuda11.0-cudnn8-py3.6-gcc9",
-    "pytorch-linux-bionic-cuda11.0-cudnn8-py3.8-gcc9",
-    "pytorch-linux-bionic-cuda10.2-cudnn7-py3.8-gcc9",
-    "pytorch-linux-bionic-py3.6-clang9",
-    "pytorch-linux-bionic-cuda10.2-cudnn7-py3.6-clang9",
-    "pytorch-linux-bionic-py3.8-gcc9",
-    "pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7",
-    "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7",
-    "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7",
-    "pytorch-linux-xenial-cuda11.0-cudnn8-py3-gcc7",
-    "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4",
-    "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7",
-    "pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
-    "pytorch-linux-xenial-py3-clang5-asan",
-    "pytorch-linux-xenial-py3.8",
-    "pytorch-linux-xenial-py3.6-clang7",
-    "pytorch-linux-xenial-py3.6-gcc4.8",
-    "pytorch-linux-xenial-py3.6-gcc5.4",
-    "pytorch-linux-xenial-py3.6-gcc7.2",
-    "pytorch-linux-xenial-py3.6-gcc7",
-    "pytorch-linux-xenial-pynightly",
-    "pytorch-linux-xenial-rocm3.3-py3.6",
-]
-
-
-def get_workflow_jobs():
-    """Generates a list of docker image build definitions"""
-    return [
-        OrderedDict(
-            {
-                "docker_build_job": OrderedDict(
-                    {"name": quote(image_name), "image_name": quote(image_name)}
-                )
-            }
-        )
-        for image_name in IMAGE_NAMES
-    ]

.circleci/cimodel/data/simple/ge_config_tests.py

@@ -1,103 +0,0 @@
-import cimodel.lib.miniutils as miniutils
-from cimodel.data.simple.util.versions import MultiPartVersion, CudaVersion
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_BASIC, DOCKER_IMAGE_CUDA_10_2
-
-
-class GeConfigTestJob:
-    def __init__(self,
-                 py_version,
-                 gcc_version,
-                 cuda_version,
-                 variant_parts,
-                 extra_requires,
-                 use_cuda_docker=False,
-                 build_env_override=None):
-
-        self.py_version = py_version
-        self.gcc_version = gcc_version
-        self.cuda_version = cuda_version
-        self.variant_parts = variant_parts
-        self.extra_requires = extra_requires
-        self.use_cuda_docker = use_cuda_docker
-        self.build_env_override = build_env_override
-
-    def get_all_parts(self, with_dots):
-
-        maybe_py_version = self.py_version.render_dots_or_parts(with_dots) if self.py_version else []
-        maybe_gcc_version = self.gcc_version.render_dots_or_parts(with_dots) if self.gcc_version else []
-        maybe_cuda_version = self.cuda_version.render_dots_or_parts(with_dots) if self.cuda_version else []
-
-        common_parts = [
-            "pytorch",
-            "linux",
-            "xenial",
-        ] + maybe_cuda_version + maybe_py_version + maybe_gcc_version
-
-        return common_parts + self.variant_parts
-
-    def gen_tree(self):
-
-        resource_class = "gpu.medium" if self.use_cuda_docker else "large"
-        docker_image = DOCKER_IMAGE_CUDA_10_2 if self.use_cuda_docker else DOCKER_IMAGE_BASIC
-        full_name = "_".join(self.get_all_parts(False))
-        build_env = self.build_env_override or "-".join(self.get_all_parts(True))
-
-        props_dict = {
-            "name": full_name,
-            "build_environment": build_env,
-            "requires": self.extra_requires,
-            "resource_class": resource_class,
-            "docker_image": docker_image,
-        }
-
-        if self.use_cuda_docker:
-            props_dict["use_cuda_docker_runtime"] = miniutils.quote(str(1))
-
-        return [{"pytorch_linux_test": props_dict}]
-
-
-WORKFLOW_DATA = [
-    GeConfigTestJob(
-        MultiPartVersion([3, 6], "py"),
-        MultiPartVersion([5, 4], "gcc"),
-        None,
-        ["ge_config_legacy", "test"],
-        ["pytorch_linux_xenial_py3_6_gcc5_4_build"]),
-    GeConfigTestJob(
-        MultiPartVersion([3, 6], "py"),
-        MultiPartVersion([5, 4], "gcc"),
-        None,
-        ["ge_config_profiling", "test"],
-        ["pytorch_linux_xenial_py3_6_gcc5_4_build"]),
-    GeConfigTestJob(
-        MultiPartVersion([3, 6], "py"),
-        MultiPartVersion([5, 4], "gcc"),
-        None,
-        ["ge_config_simple", "test"],
-        ["pytorch_linux_xenial_py3_6_gcc5_4_build"],
-    ),
-    GeConfigTestJob(
-        None,
-        None,
-        CudaVersion(10, 2),
-        ["cudnn7", "py3", "ge_config_legacy", "test"],
-        ["pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build"],
-        use_cuda_docker=True,
-        # TODO Why does the build environment specify cuda10.1, while the
-        # job name is cuda10_2?
-        build_env_override="pytorch-linux-xenial-cuda10.1-cudnn7-ge_config_legacy-test"),
-    GeConfigTestJob(
-        None,
-        None,
-        CudaVersion(10, 2),
-        ["cudnn7", "py3", "ge_config_profiling", "test"],
-        ["pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build"],
-        use_cuda_docker=True,
-        # TODO Why does the build environment specify cuda10.1, while the
-        # job name is cuda10_2?
-        build_env_override="pytorch-linux-xenial-cuda10.1-cudnn7-ge_config_profiling-test"),
-]
-
-
-def get_workflow_jobs():
-    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/ios_definitions.py

@@ -1,71 +0,0 @@
-from cimodel.data.simple.util.versions import MultiPartVersion
-
-
-IOS_VERSION = MultiPartVersion([11, 2, 1])
-
-
-class ArchVariant:
-    def __init__(self, name, is_custom=False):
-        self.name = name
-        self.is_custom = is_custom
-
-    def render(self):
-        extra_parts = ["custom"] if self.is_custom else []
-        return "_".join([self.name] + extra_parts)
-
-
-def get_platform(arch_variant_name):
-    return "SIMULATOR" if arch_variant_name == "x86_64" else "OS"
-
-
-class IOSJob:
-    def __init__(self, ios_version, arch_variant, is_org_member_context=True, extra_props=None):
-        self.ios_version = ios_version
-        self.arch_variant = arch_variant
-        self.is_org_member_context = is_org_member_context
-        self.extra_props = extra_props
-
-    def gen_name_parts(self, with_version_dots):
-
-        version_parts = self.ios_version.render_dots_or_parts(with_version_dots)
-        build_variant_suffix = "_".join([self.arch_variant.render(), "build"])
-
-        return [
-            "pytorch",
-            "ios",
-        ] + version_parts + [
-            build_variant_suffix,
-        ]
-
-    def gen_job_name(self):
-        return "_".join(self.gen_name_parts(False))
-
-    def gen_tree(self):
-
-        platform_name = get_platform(self.arch_variant.name)
-
-        props_dict = {
-            "build_environment": "-".join(self.gen_name_parts(True)),
-            "ios_arch": self.arch_variant.name,
-            "ios_platform": platform_name,
-            "name": self.gen_job_name(),
-        }
-
-        if self.is_org_member_context:
-            props_dict["context"] = "org-member"
-
-        if self.extra_props:
-            props_dict.update(self.extra_props)
-
-        return [{"pytorch_ios_build": props_dict}]
-
-
-WORKFLOW_DATA = [
-    IOSJob(IOS_VERSION, ArchVariant("x86_64"), is_org_member_context=False),
-    IOSJob(IOS_VERSION, ArchVariant("arm64")),
-    IOSJob(IOS_VERSION, ArchVariant("arm64", True), extra_props={"op_list": "mobilenetv2.yaml"}),
-]
-
-
-def get_workflow_jobs():
-    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/macos_definitions.py

@@ -1,28 +0,0 @@
-class MacOsJob:
-    def __init__(self, os_version, is_test=False):
-        self.os_version = os_version
-        self.is_test = is_test
-
-    def gen_tree(self):
-        non_phase_parts = ["pytorch", "macos", self.os_version, "py3"]
-
-        phase_name = "test" if self.is_test else "build"
-
-        full_job_name = "_".join(non_phase_parts + [phase_name])
-
-        test_build_dependency = "_".join(non_phase_parts + ["build"])
-        extra_dependencies = [test_build_dependency] if self.is_test else []
-        job_dependencies = extra_dependencies
-
-        # Yes we name the job after itself, it needs a non-empty value in here
-        # for the YAML output to work.
-        props_dict = {"requires": job_dependencies, "name": full_job_name}
-
-        return [{full_job_name: props_dict}]
-
-
-WORKFLOW_DATA = [MacOsJob("10_13"), MacOsJob("10_13", True)]
-
-
-def get_workflow_jobs():
-    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/mobile_definitions.py

@@ -1,56 +0,0 @@
-"""
-PyTorch Mobile PR builds (use linux host toolchain + mobile build options)
-"""
-
-import cimodel.lib.miniutils as miniutils
-import cimodel.data.simple.util.branch_filters
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_ASAN, DOCKER_IMAGE_NDK
-
-
-class MobileJob:
-    def __init__(self, docker_image, variant_parts, is_master_only=False):
-        self.docker_image = docker_image
-        self.variant_parts = variant_parts
-        self.is_master_only = is_master_only
-
-    def gen_tree(self):
-        non_phase_parts = [
-            "pytorch",
-            "linux",
-            "xenial",
-            "py3",
-            "clang5",
-            "mobile",
-        ] + self.variant_parts
-
-        full_job_name = "_".join(non_phase_parts)
-        build_env_name = "-".join(non_phase_parts)
-
-        props_dict = {
-            "build_environment": build_env_name,
-            "build_only": miniutils.quote(str(int(True))),
-            "docker_image": self.docker_image,
-            "name": full_job_name,
-        }
-
-        if self.is_master_only:
-            props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
-
-        return [{"pytorch_linux_build": props_dict}]
-
-
-WORKFLOW_DATA = [
-    MobileJob(DOCKER_IMAGE_ASAN, ["build"]),
-    MobileJob(DOCKER_IMAGE_ASAN, ["custom", "build", "static"]),
-
-    # Use LLVM-DEV toolchain in android-ndk-r19c docker image
-    MobileJob(DOCKER_IMAGE_NDK, ["custom", "build", "dynamic"]),
-
-    # Use LLVM-DEV toolchain in android-ndk-r19c docker image
-    # Most of this CI is already covered by "mobile-custom-build-dynamic" job
-    MobileJob(DOCKER_IMAGE_NDK, ["code", "analysis"], True),
-]
-
-
-def get_workflow_jobs():
-    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/nightly_android.py

@@ -1,73 +0,0 @@
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_NDK
-
-
-class AndroidNightlyJob:
-    def __init__(self,
-                 variant,
-                 template_name,
-                 extra_props=None,
-                 with_docker=True,
-                 requires=None,
-                 no_build_suffix=False):
-
-        self.variant = variant
-        self.template_name = template_name
-        self.extra_props = extra_props or {}
-        self.with_docker = with_docker
-        self.requires = requires
-        self.no_build_suffix = no_build_suffix
-
-    def gen_tree(self):
-
-        base_name_parts = [
-            "pytorch",
-            "linux",
-            "xenial",
-            "py3",
-            "clang5",
-            "android",
-            "ndk",
-            "r19c",
-        ] + self.variant
-
-        build_suffix = [] if self.no_build_suffix else ["build"]
-        full_job_name = "_".join(["nightly"] + base_name_parts + build_suffix)
-        build_env_name = "-".join(base_name_parts)
-
-        props_dict = {
-            "name": full_job_name,
-            "requires": self.requires,
-            "filters": {"branches": {"only": "nightly"}},
-        }
-
-        props_dict.update(self.extra_props)
-
-        if self.with_docker:
-            props_dict["docker_image"] = DOCKER_IMAGE_NDK
-            props_dict["build_environment"] = build_env_name
-
-        return [{self.template_name: props_dict}]
-
-
-WORKFLOW_DATA = [
-    AndroidNightlyJob(["x86_32"], "pytorch_linux_build"),
-    AndroidNightlyJob(["x86_64"], "pytorch_linux_build"),
-    AndroidNightlyJob(["arm", "v7a"], "pytorch_linux_build"),
-    AndroidNightlyJob(["arm", "v8a"], "pytorch_linux_build"),
-    AndroidNightlyJob(["android_gradle"], "pytorch_android_gradle_build",
-                      with_docker=False,
-                      requires=[
-                          "nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build",
-                          "nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build",
-                          "nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build",
-                          "nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build"]),
-    AndroidNightlyJob(["x86_32_android_publish_snapshot"], "pytorch_android_publish_snapshot",
-                      extra_props={"context": "org-member"},
-                      with_docker=False,
-                      requires=["nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build"],
-                      no_build_suffix=True),
-]
-
-
-def get_workflow_jobs():
-    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/nightly_ios.py

@@ -1,68 +0,0 @@
-import cimodel.data.simple.ios_definitions as ios_definitions
-
-
-class IOSNightlyJob:
-    def __init__(self,
-                 variant,
-                 is_upload=False):
-
-        self.variant = variant
-        self.is_upload = is_upload
-
-    def get_phase_name(self):
-        return "upload" if self.is_upload else "build"
-
-    def get_common_name_pieces(self, with_version_dots):
-
-        extra_name_suffix = [self.get_phase_name()] if self.is_upload else []
-
-        common_name_pieces = [
-            "ios",
-        ] + ios_definitions.IOS_VERSION.render_dots_or_parts(with_version_dots) + [
-            "nightly",
-            self.variant,
-            "build",
-        ] + extra_name_suffix
-
-        return common_name_pieces
-
-    def gen_job_name(self):
-        return "_".join(["pytorch"] + self.get_common_name_pieces(False))
-
-    def gen_tree(self):
-        extra_requires = [x.gen_job_name() for x in BUILD_CONFIGS] if self.is_upload else []
-
-        props_dict = {
-            "build_environment": "-".join(["libtorch"] + self.get_common_name_pieces(True)),
-            "requires": extra_requires,
-            "context": "org-member",
-            "filters": {"branches": {"only": "nightly"}},
-        }
-
-        if not self.is_upload:
-            props_dict["ios_arch"] = self.variant
-            props_dict["ios_platform"] = ios_definitions.get_platform(self.variant)
-            props_dict["name"] = self.gen_job_name()
-
-        template_name = "_".join([
-            "binary",
-            "ios",
-            self.get_phase_name(),
-        ])
-
-        return [{template_name: props_dict}]
-
-
-BUILD_CONFIGS = [
-    IOSNightlyJob("x86_64"),
-    IOSNightlyJob("arm64"),
-]
-
-
-WORKFLOW_DATA = BUILD_CONFIGS + [
-    IOSNightlyJob("binary", is_upload=True),
-]
-
-
-def get_workflow_jobs():
-    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/data/simple/util/branch_filters.py

@@ -1,22 +0,0 @@
-NON_PR_BRANCH_LIST = [
-    "master",
-    r"/ci-all\/.*/",
-    r"/release\/.*/",
-]
-
-RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
-
-def gen_filter_dict(
-        branches_list=NON_PR_BRANCH_LIST,
-        tags_list=None
-):
-    """Generates a filter dictionary for use with CircleCI's job filter"""
-    filter_dict = {
-        "branches": {
-            "only": branches_list,
-        },
-    }
-
-    if tags_list is not None:
-        filter_dict["tags"] = {"only": tags_list}
-    return filter_dict

.circleci/cimodel/data/simple/util/docker_constants.py

@@ -1,30 +0,0 @@
-AWS_DOCKER_HOST = "308535385114.dkr.ecr.us-east-1.amazonaws.com"
-
-# ARE YOU EDITING THIS NUMBER?  MAKE SURE YOU READ THE GUIDANCE AT THE
-# TOP OF .circleci/config.yml
-DOCKER_IMAGE_TAG = "209062ef-ab58-422a-b295-36c4eed6e906"
-
-
-def gen_docker_image_path(container_type):
-    return "/".join([
-        AWS_DOCKER_HOST,
-        "pytorch",
-        container_type + ":" + DOCKER_IMAGE_TAG,
-    ])
-
-
-DOCKER_IMAGE_BASIC = gen_docker_image_path("pytorch-linux-xenial-py3.6-gcc5.4")
-
-DOCKER_IMAGE_CUDA_10_2 = gen_docker_image_path("pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7")
-
-DOCKER_IMAGE_GCC7 = gen_docker_image_path("pytorch-linux-xenial-py3.6-gcc7")
-
-
-def gen_mobile_docker_name(specifier):
-    container_type = "pytorch-linux-xenial-py3-clang5-" + specifier
-    return gen_docker_image_path(container_type)
-
-
-DOCKER_IMAGE_ASAN = gen_mobile_docker_name("asan")
-
-DOCKER_IMAGE_NDK = gen_mobile_docker_name("android-ndk-r19c")

.circleci/cimodel/data/simple/util/versions.py

@@ -1,31 +0,0 @@
-class MultiPartVersion:
-    def __init__(self, parts, prefix=""):
-        self.parts = parts
-        self.prefix = prefix
-
-    def prefixed_parts(self):
-        """
-        Prepends the first element of the version list
-        with the prefix string.
-        """
-        if self.parts:
-            return [self.prefix + str(self.parts[0])] + list(map(str, self.parts[1:]))
-        else:
-            return [self.prefix]
-
-    def render_dots(self):
-        return ".".join(self.prefixed_parts())
-
-    def render_dots_or_parts(self, with_dots):
-        if with_dots:
-            return [self.render_dots()]
-        else:
-            return self.prefixed_parts()
-
-
-class CudaVersion(MultiPartVersion):
-    def __init__(self, major, minor):
-        self.major = major
-        self.minor = minor
-
-        super().__init__([self.major, self.minor], "cuda")

.circleci/cimodel/data/windows_build_definitions.py

@@ -1,142 +0,0 @@
-import cimodel.data.simple.util.branch_filters
-import cimodel.lib.miniutils as miniutils
-from cimodel.data.simple.util.versions import CudaVersion
-
-
-class WindowsJob:
-    def __init__(
-        self,
-        test_index,
-        vscode_spec,
-        cuda_version,
-        force_on_cpu=False,
-        master_only_pred=lambda job: job.vscode_spec.year != 2019,
-    ):
-        self.test_index = test_index
-        self.vscode_spec = vscode_spec
-        self.cuda_version = cuda_version
-        self.force_on_cpu = force_on_cpu
-        self.master_only_pred = master_only_pred
-
-    def gen_tree(self):
-
-        base_phase = "build" if self.test_index is None else "test"
-        numbered_phase = (
-            base_phase if self.test_index is None else base_phase + str(self.test_index)
-        )
-
-        key_name = "_".join(["pytorch", "windows", base_phase])
-
-        cpu_forcing_name_parts = ["on", "cpu"] if self.force_on_cpu else []
-
-        target_arch = self.cuda_version.render_dots() if self.cuda_version else "cpu"
-
-        base_name_parts = [
-            "pytorch",
-            "windows",
-            self.vscode_spec.render(),
-            "py36",
-            target_arch,
-        ]
-
-        prerequisite_jobs = []
-        if base_phase == "test":
-            prerequisite_jobs.append("_".join(base_name_parts + ["build"]))
-
-        arch_env_elements = (
-            ["cuda" + str(self.cuda_version.major), "cudnn7"]
-            if self.cuda_version
-            else ["cpu"]
-        )
-
-        build_environment_string = "-".join(
-            ["pytorch", "win"]
-            + self.vscode_spec.get_elements()
-            + arch_env_elements
-            + ["py3"]
-        )
-
-        is_running_on_cuda = bool(self.cuda_version) and not self.force_on_cpu
-
-        props_dict = {
-            "build_environment": build_environment_string,
-            "python_version": miniutils.quote("3.6"),
-            "vc_version": miniutils.quote(self.vscode_spec.dotted_version()),
-            "vc_year": miniutils.quote(str(self.vscode_spec.year)),
-            "vc_product": self.vscode_spec.get_product(),
-            "use_cuda": miniutils.quote(str(int(is_running_on_cuda))),
-            "requires": prerequisite_jobs,
-        }
-
-        if self.master_only_pred(self):
-            props_dict[
-                "filters"
-            ] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
-
-        name_parts = base_name_parts + cpu_forcing_name_parts + [numbered_phase]
-
-        if base_phase == "test":
-            test_name = "-".join(["pytorch", "windows", numbered_phase])
-            props_dict["test_name"] = test_name
-
-            if is_running_on_cuda:
-                props_dict["executor"] = "windows-with-nvidia-gpu"
-
-        props_dict["cuda_version"] = (
-            miniutils.quote(str(self.cuda_version.major))
-            if self.cuda_version
-            else "cpu"
-        )
-        props_dict["name"] = "_".join(name_parts)
-
-        return [{key_name: props_dict}]
-
-
-class VcSpec:
-    def __init__(self, year, version_elements=None):
-        self.year = year
-        self.version_elements = version_elements or []
-
-    def get_elements(self):
-        return [self.prefixed_year()] + self.version_elements
-
-    def get_product(self):
-        return "Community" if self.year == 2019 else "BuildTools"
-
-    def dotted_version(self):
-        return ".".join(self.version_elements)
-
-    def prefixed_year(self):
-        return "vs" + str(self.year)
-
-    def render(self):
-        return "_".join(filter(None, [self.prefixed_year(), self.dotted_version()]))
-
-def FalsePred(_):
-    return False
-
-def TruePred(_):
-    return True
-
-WORKFLOW_DATA = [
-    # VS2017 CUDA-10.1
-    WindowsJob(None, VcSpec(2017, ["14", "11"]), CudaVersion(10, 1), master_only_pred=FalsePred),
-    WindowsJob(1, VcSpec(2017, ["14", "11"]), CudaVersion(10, 1)),
-    # VS2017 no-CUDA (builds only)
-    WindowsJob(None, VcSpec(2017, ["14", "16"]), CudaVersion(10, 1)),
-    WindowsJob(None, VcSpec(2017, ["14", "16"]), None),
-    # VS2019 CUDA-10.1
-    WindowsJob(None, VcSpec(2019), CudaVersion(10, 1)),
-    WindowsJob(1, VcSpec(2019), CudaVersion(10, 1)),
-    WindowsJob(2, VcSpec(2019), CudaVersion(10, 1)),
-    # VS2019 CPU-only
-    WindowsJob(None, VcSpec(2019), None),
-    WindowsJob(1, VcSpec(2019), None),
-    WindowsJob(2, VcSpec(2019), None, master_only_pred=TruePred),
-    WindowsJob(1, VcSpec(2019), CudaVersion(10, 1), force_on_cpu=True),
-    WindowsJob(2, VcSpec(2019), CudaVersion(10, 1), force_on_cpu=True, master_only_pred=TruePred),
-]
-
-
-def get_windows_workflows():
-    return [item.gen_tree() for item in WORKFLOW_DATA]

.circleci/cimodel/lib/miniyaml.py

@@ -1,7 +1,5 @@
 from collections import OrderedDict
 
-import cimodel.lib.miniutils as miniutils
-
 
 LIST_MARKER = "- "
 INDENTATION_WIDTH = 2
@@ -31,8 +29,7 @@ def render(fh, data, depth, is_list_member=False):
             tuples.sort()
 
         for i, (k, v) in enumerate(tuples):
-            if not v:
+
-                continue
             # If this dict is itself a list member, the first key gets prefixed with a list marker
             list_marker_prefix = LIST_MARKER if is_list_member and not i else ""
 
@@ -46,7 +43,5 @@ def render(fh, data, depth, is_list_member=False):
             render(fh, v, depth, True)
 
     else:
-        # use empty quotes to denote an empty string value instead of blank space
-        modified_data = miniutils.quote(data) if data == "" else data
         list_member_prefix = indentation + LIST_MARKER if is_list_member else ""
-        fh.write(list_member_prefix + str(modified_data) + "\n")
+        fh.write(list_member_prefix + str(data) + "\n")

.circleci/cimodel/lib/visualization.py

@@ -0,0 +1,84 @@
+"""
+This module encapsulates dependencies on pygraphviz
+"""
+
+import colorsys
+
+import cimodel.lib.conf_tree as conf_tree
+
+
+def rgb2hex(rgb_tuple):
+    def to_hex(f):
+        return "%02x" % int(f * 255)
+
+    return "#" + "".join(map(to_hex, list(rgb_tuple)))
+
+
+def handle_missing_graphviz(f):
+    """
+    If the user has not installed pygraphviz, this causes
+    calls to the draw() method of the returned object to do nothing.
+    """
+    try:
+        import pygraphviz  # noqa: F401
+        return f
+
+    except ModuleNotFoundError:
+
+        class FakeGraph:
+            def draw(self, *args, **kwargs):
+                pass
+
+        return lambda _: FakeGraph()
+
+
+@handle_missing_graphviz
+def generate_graph(toplevel_config_node):
+    """
+    Traverses the graph once first just to find the max depth
+    """
+
+    config_list = conf_tree.dfs(toplevel_config_node)
+
+    max_depth = 0
+    for config in config_list:
+        max_depth = max(max_depth, config.get_depth())
+
+    # color the nodes using the max depth
+
+    from pygraphviz import AGraph
+    dot = AGraph()
+
+    def node_discovery_callback(node, sibling_index, sibling_count):
+        depth = node.get_depth()
+
+        sat_min, sat_max = 0.1, 0.6
+        sat_range = sat_max - sat_min
+
+        saturation_fraction = sibling_index / float(sibling_count - 1) if sibling_count > 1 else 1
+        saturation = sat_min + sat_range * saturation_fraction
+
+        # TODO Use a hash of the node label to determine the color
+        hue = depth / float(max_depth + 1)
+
+        rgb_tuple = colorsys.hsv_to_rgb(hue, saturation, 1)
+
+        this_node_key = node.get_node_key()
+
+        dot.add_node(
+            this_node_key,
+            label=node.get_label(),
+            style="filled",
+            # fillcolor=hex_color + ":orange",
+            fillcolor=rgb2hex(rgb_tuple),
+            penwidth=3,
+            color=rgb2hex(colorsys.hsv_to_rgb(hue, saturation, 0.9))
+        )
+
+    def child_callback(node, child):
+        this_node_key = node.get_node_key()
+        child_node_key = child.get_node_key()
+        dot.add_edge((this_node_key, child_node_key))
+
+    conf_tree.dfs_recurse(toplevel_config_node, lambda x: None, node_discovery_callback, child_callback)
+    return dot

.circleci/codegen_validation/compare_normalized_yaml.sh

@@ -1,17 +0,0 @@
-#!/bin/bash -xe
-
-
-YAML_FILENAME=verbatim-sources/workflows-pytorch-ge-config-tests.yml
-DIFF_TOOL=meld
-
-
-# Allows this script to be invoked from any directory:
-cd $(dirname "$0")
-
-pushd ..
-
-
-$DIFF_TOOL $YAML_FILENAME <(./codegen_validation/normalize_yaml_fragment.py < $YAML_FILENAME)
-
-
-popd

.circleci/codegen_validation/normalize_yaml_fragment.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import yaml
-
-# Need to import modules that lie on an upward-relative path
-sys.path.append(os.path.join(sys.path[0], '..'))
-
-import cimodel.lib.miniyaml as miniyaml
-
-
-def regurgitate(depth, use_pyyaml_formatter=False):
-    data = yaml.safe_load(sys.stdin)
-
-    if use_pyyaml_formatter:
-        output = yaml.dump(data, sort_keys=True)
-        sys.stdout.write(output)
-    else:
-        miniyaml.render(sys.stdout, data, depth)
-
-
-if __name__ == "__main__":
-    regurgitate(3)

.circleci/codegen_validation/overwrite_with_normalized.sh

@@ -1,15 +0,0 @@
-#!/bin/bash -xe
-
-YAML_FILENAME=$1
-
-# Allows this script to be invoked from any directory:
-cd $(dirname "$0")
-
-pushd ..
-
-TEMP_FILENAME=$(mktemp)
-
-cat $YAML_FILENAME | ./codegen_validation/normalize_yaml_fragment.py > $TEMP_FILENAME
-mv $TEMP_FILENAME $YAML_FILENAME
-
-popd

.circleci/docker/build.sh

@@ -15,8 +15,6 @@ OS="ubuntu"
 DOCKERFILE="${OS}/Dockerfile"
 if [[ "$image" == *-cuda* ]]; then
   DOCKERFILE="${OS}-cuda/Dockerfile"
-elif [[ "$image" == *-rocm* ]]; then
-  DOCKERFILE="${OS}-rocm/Dockerfile"
 fi
 
 if [[ "$image" == *-trusty* ]]; then
@@ -27,20 +25,32 @@ elif [[ "$image" == *-artful* ]]; then
   UBUNTU_VERSION=17.10
 elif [[ "$image" == *-bionic* ]]; then
   UBUNTU_VERSION=18.04
-elif [[ "$image" == *-focal* ]]; then
-  UBUNTU_VERSION=20.04
 fi
 
-TRAVIS_DL_URL_PREFIX="https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/14.04/x86_64"
-
 # It's annoying to rename jobs every time you want to rewrite a
 # configuration, so we hardcode everything here rather than do it
 # from scratch
 case "$image" in
-  pytorch-linux-xenial-py3.8)
+  pytorch-linux-bionic-clang9-thrift-llvmdev)
-    # TODO: This is a hack, get rid of this as soon as you get rid of the travis downloads
+    CLANG_VERSION=9
-    TRAVIS_DL_URL_PREFIX="https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/16.04/x86_64"
+    THRIFT=yes
-    TRAVIS_PYTHON_VERSION=3.8
+    LLVMDEV=yes
+    PROTOBUF=yes
+    ;;
+  pytorch-linux-xenial-py2.7.9)
+    TRAVIS_PYTHON_VERSION=2.7.9
+    GCC_VERSION=7
+    # Do not install PROTOBUF, DB, and VISION as a test
+    ;;
+  pytorch-linux-xenial-py2.7)
+    TRAVIS_PYTHON_VERSION=2.7
+    GCC_VERSION=7
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    ;;
+  pytorch-linux-xenial-py3.5)
+    TRAVIS_PYTHON_VERSION=3.5
     GCC_VERSION=7
     # Do not install PROTOBUF, DB, and VISION as a test
     ;;
@@ -57,7 +67,6 @@ case "$image" in
     PROTOBUF=yes
     DB=yes
     VISION=yes
-    KATEX=yes
     ;;
   pytorch-linux-xenial-py3.6-gcc7.2)
     ANACONDA_PYTHON_VERSION=3.6
@@ -78,15 +87,39 @@ case "$image" in
     DB=yes
     VISION=yes
     ;;
-  pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4)
+  pytorch-linux-xenial-cuda8-cudnn7-py2)
-    CUDA_VERSION=9.2
+    CUDA_VERSION=8.0
     CUDNN_VERSION=7
-    ANACONDA_PYTHON_VERSION=3.6
+    ANACONDA_PYTHON_VERSION=2.7
-    GCC_VERSION=5
     PROTOBUF=yes
     DB=yes
     VISION=yes
     ;;
+  pytorch-linux-xenial-cuda8-cudnn7-py3)
+    CUDA_VERSION=8.0
+    CUDNN_VERSION=7
+    ANACONDA_PYTHON_VERSION=3.6
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    ;;
+  pytorch-linux-xenial-cuda9-cudnn7-py2)
+    CUDA_VERSION=9.0
+    CUDNN_VERSION=7
+    ANACONDA_PYTHON_VERSION=2.7
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    ;;
+  pytorch-linux-xenial-cuda9-cudnn7-py3)
+    CUDA_VERSION=9.0
+    CUDNN_VERSION=7
+    ANACONDA_PYTHON_VERSION=3.6
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    KATEX=yes
+    ;;
   pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7)
     CUDA_VERSION=9.2
     CUDNN_VERSION=7
@@ -113,28 +146,6 @@ case "$image" in
     PROTOBUF=yes
     DB=yes
     VISION=yes
-    KATEX=yes
-    ;;
-  pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7)
-    CUDA_VERSION=10.2
-    CUDNN_VERSION=7
-    ANACONDA_PYTHON_VERSION=3.6
-    GCC_VERSION=7
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    ;;
-  pytorch-linux-xenial-cuda11.0-cudnn8-py3-gcc7)
-    UBUNTU_VERSION=16.04-rc
-    CUDA_VERSION=11.0
-    CUDNN_VERSION=8
-    ANACONDA_PYTHON_VERSION=3.6
-    GCC_VERSION=7
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
     ;;
   pytorch-linux-xenial-py3-clang5-asan)
     ANACONDA_PYTHON_VERSION=3.6
@@ -146,7 +157,6 @@ case "$image" in
   pytorch-linux-xenial-py3-clang5-android-ndk-r19c)
     ANACONDA_PYTHON_VERSION=3.6
     CLANG_VERSION=5.0
-    LLVMDEV=yes
     PROTOBUF=yes
     ANDROID=yes
     ANDROID_NDK_VERSION=r19c
@@ -161,76 +171,6 @@ case "$image" in
     DB=yes
     VISION=yes
     ;;
-  pytorch-linux-bionic-py3.6-clang9)
-    ANACONDA_PYTHON_VERSION=3.6
-    CLANG_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ;;
-  pytorch-linux-bionic-py3.8-gcc9)
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ;;
-  pytorch-linux-bionic-cuda10.2-cudnn7-py3.6-clang9)
-    CUDA_VERSION=10.2
-    CUDNN_VERSION=7
-    ANACONDA_PYTHON_VERSION=3.6
-    CLANG_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ;;
-  pytorch-linux-bionic-cuda10.2-cudnn7-py3.8-gcc9)
-    CUDA_VERSION=10.2
-    CUDNN_VERSION=7
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ;;
-  pytorch-linux-bionic-cuda11.0-cudnn8-py3.6-gcc9)
-    UBUNTU_VERSION=18.04-rc
-    CUDA_VERSION=11.0
-    CUDNN_VERSION=8
-    ANACONDA_PYTHON_VERSION=3.6
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    ;;
-  pytorch-linux-bionic-cuda11.0-cudnn8-py3.8-gcc9)
-    UBUNTU_VERSION=18.04-rc
-    CUDA_VERSION=11.0
-    CUDNN_VERSION=8
-    ANACONDA_PYTHON_VERSION=3.8
-    GCC_VERSION=9
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    KATEX=yes
-    ;;
-  pytorch-linux-xenial-rocm3.3-py3.6)
-    ANACONDA_PYTHON_VERSION=3.6
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ROCM_VERSION=3.3
-    # newer cmake version required
-    CMAKE_VERSION=3.6.3
-    ;;
-  pytorch-linux-bionic-rocm3.3-py3.6)
-    ANACONDA_PYTHON_VERSION=3.6
-    PROTOBUF=yes
-    DB=yes
-    VISION=yes
-    ROCM_VERSION=3.3
-    ;;
 esac
 
 # Set Jenkins UID and GID if running Jenkins
@@ -242,12 +182,8 @@ fi
 tmp_tag="tmp-$(cat /dev/urandom | tr -dc 'a-z' | fold -w 32 | head -n 1)"
 
 # Build image
-# TODO: build-arg THRIFT is not turned on for any image, remove it once we confirm
-# it's no longer needed.
 docker build \
        --no-cache \
-       --progress=plain \
-       --build-arg "TRAVIS_DL_URL_PREFIX=${TRAVIS_DL_URL_PREFIX}" \
        --build-arg "BUILD_ENVIRONMENT=${image}" \
        --build-arg "PROTOBUF=${PROTOBUF:-}" \
        --build-arg "THRIFT=${THRIFT:-}" \
@@ -271,7 +207,6 @@ docker build \
        --build-arg "CMAKE_VERSION=${CMAKE_VERSION:-}" \
        --build-arg "NINJA_VERSION=${NINJA_VERSION:-}" \
        --build-arg "KATEX=${KATEX:-}" \
-       --build-arg "ROCM_VERSION=${ROCM_VERSION:-}" \
        -f $(dirname ${DOCKERFILE})/Dockerfile \
        -t "$tmp_tag" \
        "$@" \

.circleci/docker/build_docker.sh

@@ -45,9 +45,5 @@ trap "docker logout ${registry}" EXIT
 
 docker push "${image}:${tag}"
 
-# TODO: Get rid of duplicate tagging once ${DOCKER_TAG} becomes the default
-docker tag "${image}:${tag}" "${image}:${DOCKER_TAG}"
-docker push "${image}:${DOCKER_TAG}"
-
 docker save -o "${IMAGE_NAME}:${tag}.tar" "${image}:${tag}"
 aws s3 cp "${IMAGE_NAME}:${tag}.tar" "s3://ossci-linux-build/pytorch/base/${IMAGE_NAME}:${tag}.tar" --acl public-read

.circleci/docker/common/install_android.sh

@@ -10,7 +10,7 @@ apt-get autoclean && apt-get clean
 rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 pushd /tmp
-curl -Os --retry 3 https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK}-linux-x86_64.zip
+curl -Os https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK}-linux-x86_64.zip
 popd
 _ndk_dir=/opt/ndk
 mkdir -p "$_ndk_dir"

.circleci/docker/common/install_base.sh

@@ -2,16 +2,17 @@
 
 set -ex
 
-# NVIDIA dockers for RC releases use tag names like `11.0-cudnn8-devel-ubuntu18.04-rc`,
+if [[ "$UBUNTU_VERSION" == "14.04" ]]; then
-# for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
+  # cmake 2 is too old
-# find the correct image. As a result, here we have to check for
+  cmake3=cmake3
-#   "$UBUNTU_VERSION" == "18.04"*
+else
-# instead of
+  cmake3=cmake
-#   "$UBUNTU_VERSION" == "18.04"
+fi
-if [[ "$UBUNTU_VERSION" == "18.04"* ]]; then
+
+if [[ "$UBUNTU_VERSION" == "18.04" ]]; then
   cmake3="cmake=3.10*"
 else
-  cmake3="cmake=3.5*"
+  cmake3="${cmake3}=3.5*"
 fi
 
 # Install common dependencies
@@ -50,15 +51,14 @@ apt-get install -y --no-install-recommends \
 
 # Install Valgrind separately since the apt-get version is too old.
 mkdir valgrind_build && cd valgrind_build
-VALGRIND_VERSION=3.15.0
+if ! wget http://valgrind.org/downloads/valgrind-3.14.0.tar.bz2
-if ! wget http://valgrind.org/downloads/valgrind-${VALGRIND_VERSION}.tar.bz2
 then
-  wget https://sourceware.org/ftp/valgrind/valgrind-${VALGRIND_VERSION}.tar.bz2
+  wget https://sourceware.org/ftp/valgrind/valgrind-3.14.0.tar.bz2
 fi
-tar -xjf valgrind-${VALGRIND_VERSION}.tar.bz2
+tar -xjf valgrind-3.14.0.tar.bz2
-cd valgrind-${VALGRIND_VERSION}
+cd valgrind-3.14.0
 ./configure --prefix=/usr/local
-make -j 4
+make
 sudo make install
 cd ../../
 rm -rf valgrind_build

.circleci/docker/common/install_cache.sh

@@ -8,7 +8,7 @@ sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment
 export PATH="/opt/cache/bin:$PATH"
 
 # Setup compiler cache
-curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
+curl https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
 chmod a+x /opt/cache/bin/sccache
 
 function write_sccache_stub() {

.circleci/docker/common/install_cmake.sh

@@ -10,7 +10,7 @@ file="cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz"
 
 # Download and install specific CMake version in /usr/local
 pushd /tmp
-curl -Os --retry 3 "https://cmake.org/files/${path}/${file}"
+curl -Os "https://cmake.org/files/${path}/${file}"
 tar -C /usr/local --strip-components 1 --no-same-owner -zxf cmake-*.tar.gz
 rm -f cmake-*.tar.gz
 popd

.circleci/docker/common/install_conda.sh

@@ -4,7 +4,7 @@ set -ex
 
 # Optionally install conda
 if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
-  BASE_URL="https://repo.anaconda.com/miniconda"
+  BASE_URL="https://repo.continuum.io/miniconda"
 
   MAJOR_PYTHON_VERSION=$(echo "$ANACONDA_PYTHON_VERSION" | cut -d . -f 1)
 
@@ -64,21 +64,19 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   # Install PyTorch conda deps, as per https://github.com/pytorch/pytorch README
   # DO NOT install cmake here as it would install a version newer than 3.5, but
   # we want to pin to version 3.5.
-  if [ "$ANACONDA_PYTHON_VERSION" = "3.8" ]; then
+  conda_install numpy pyyaml mkl mkl-include setuptools cffi typing future six
-    # DO NOT install typing if installing python-3.8, since its part of python-3.8 core packages
+  if [[ "$CUDA_VERSION" == 8.0* ]]; then
-    # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
+    conda_install magma-cuda80 -c pytorch
-    conda_install numpy pyyaml mkl mkl-include setuptools cffi future six llvmdev=8.0.0
+  elif [[ "$CUDA_VERSION" == 9.0* ]]; then
-  else
+    conda_install magma-cuda90 -c pytorch
-    conda_install numpy pyyaml mkl mkl-include setuptools cffi typing future six
+  elif [[ "$CUDA_VERSION" == 9.1* ]]; then
-  fi
+    conda_install magma-cuda91 -c pytorch
-  if [[ "$CUDA_VERSION" == 9.2* ]]; then
+  elif [[ "$CUDA_VERSION" == 9.2* ]]; then
     conda_install magma-cuda92 -c pytorch
   elif [[ "$CUDA_VERSION" == 10.0* ]]; then
     conda_install magma-cuda100 -c pytorch
   elif [[ "$CUDA_VERSION" == 10.1* ]]; then
     conda_install magma-cuda101 -c pytorch
-  elif [[ "$CUDA_VERSION" == 10.2* ]]; then
-    conda_install magma-cuda102 -c pytorch
   fi
 
   # TODO: This isn't working atm
@@ -90,7 +88,7 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   # scikit-learn is pinned because of
   # https://github.com/scikit-learn/scikit-learn/issues/14485 (affects gcc 5.5
   # only)
-  as_jenkins pip install --progress-bar off pytest scipy==1.1.0 scikit-learn==0.20.3 scikit-image librosa>=0.6.2 psutil numba==0.46.0 llvmlite==0.30.0
+  as_jenkins pip install --progress-bar off pytest scipy==1.1.0 scikit-learn==0.20.3 scikit-image librosa>=0.6.2 psutil numba==0.43.1 llvmlite==0.28.0
 
   popd
 fi

.circleci/docker/common/install_gcc.sh

@@ -7,11 +7,7 @@ if [ -n "$GCC_VERSION" ]; then
   # Need the official toolchain repo to get alternate packages
   add-apt-repository ppa:ubuntu-toolchain-r/test
   apt-get update
-  if [ "$UBUNTU_VERSION" = "16.04" -a "$GCC_VERSION" = "5" ]; then
+  apt-get install -y g++-$GCC_VERSION
-    apt-get install -y g++-5=5.4.0-6ubuntu1~16.04.12
-  else
-    apt-get install -y g++-$GCC_VERSION
-  fi
 
   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-"$GCC_VERSION" 50
   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-"$GCC_VERSION" 50

.circleci/docker/common/install_llvm.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -ex
-llvm_url="https://github.com/llvm/llvm-project/releases/download/llvmorg-9.0.1/llvm-9.0.1.src.tar.xz"
-
-mkdir /opt/llvm
-
-pushd /tmp
-wget --no-verbose --output-document=llvm.tar.xz "$llvm_url"
-mkdir llvm
-tar -xf llvm.tar.xz -C llvm --strip-components 1
-rm -f llvm.tar.xz
-
-cd llvm
-mkdir build
-cd build
-cmake -G "Unix Makefiles" \
-  -DCMAKE_BUILD_TYPE=MinSizeRel \
-  -DLLVM_ENABLE_ASSERTIONS=ON \
-  -DCMAKE_INSTALL_PREFIX=/opt/llvm \
-  -DLLVM_TARGETS_TO_BUILD="host" \
-  -DLLVM_BUILD_TOOLS=OFF \
-  -DLLVM_BUILD_UTILS=OFF \
-  -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON \
-  ../
-
-make -j4
-sudo make install
-
-popd

.circleci/docker/common/install_rocm.sh

@@ -1,89 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-install_ubuntu() {
-    apt-get update
-    if [[ $UBUNTU_VERSION == 18.04 ]]; then
-      # gpg-agent is not available by default on 18.04
-      apt-get install -y --no-install-recommends gpg-agent
-    fi
-    apt-get install -y wget
-    apt-get install -y libopenblas-dev
-
-    # Need the libc++1 and libc++abi1 libraries to allow torch._C to load at runtime
-    apt-get install -y libc++1
-    apt-get install -y libc++abi1
-
-    DEB_ROCM_REPO=http://repo.radeon.com/rocm/apt/${ROCM_VERSION}
-    # Add rocm repository
-    wget -qO - $DEB_ROCM_REPO/rocm.gpg.key | apt-key add -
-    echo "deb [arch=amd64] $DEB_ROCM_REPO xenial main" > /etc/apt/sources.list.d/rocm.list
-    apt-get update --allow-insecure-repositories
-
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
-                   rocm-dev \
-                   rocm-utils \
-                   rocfft \
-                   miopen-hip \
-                   rocblas \
-                   hipsparse \
-                   rocrand \
-                   hipcub \
-                   rocthrust \
-                   rccl \
-                   rocprofiler-dev \
-                   roctracer-dev
-
-  # Cleanup
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-}
-
-install_centos() {
-
-  yum update -y
-  yum install -y wget
-  yum install -y openblas-devel
-
-  yum install -y epel-release
-  yum install -y dkms kernel-headers-`uname -r` kernel-devel-`uname -r`
-
-  echo "[ROCm]" > /etc/yum.repos.d/rocm.repo
-  echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo
-  echo "baseurl=http://repo.radeon.com/rocm/yum/rpm/" >> /etc/yum.repos.d/rocm.repo
-  echo "enabled=1" >> /etc/yum.repos.d/rocm.repo
-  echo "gpgcheck=0" >> /etc/yum.repos.d/rocm.repo
-
-  yum update -y
-
-  yum install -y \
-                   rocm-dev \
-                   rocm-utils \
-                   rocfft \
-                   miopen-hip \
-                   rocblas \
-                   hipsparse \
-                   rocrand \
-                   rccl \
-                   hipcub \
-                   rocthrust \
-                   rocprofiler-dev \
-                   roctracer-dev
-
-  # Cleanup
-  yum clean all
-  rm -rf /var/cache/yum
-  rm -rf /var/lib/yum/yumdb
-  rm -rf /var/lib/yum/history
-}
-
-# Install Python packages depending on the base OS
-if [ -f /etc/lsb-release ]; then
-  install_ubuntu
-elif [ -f /etc/os-release ]; then
-  install_centos
-else
-  echo "Unable to determine OS..."
-  exit 1
-fi

.circleci/docker/common/install_travis_python.sh

@@ -14,7 +14,7 @@ if [ -n "$TRAVIS_PYTHON_VERSION" ]; then
 
   # Download Python binary from Travis
   pushd tmp
-  as_jenkins wget --quiet ${TRAVIS_DL_URL_PREFIX}/python-$TRAVIS_PYTHON_VERSION.tar.bz2
+  as_jenkins wget --quiet https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/14.04/x86_64/python-$TRAVIS_PYTHON_VERSION.tar.bz2
   # NB: The tarball also comes with /home/travis virtualenv that we
   # don't care about.  (Maybe we should, but we've worked around the
   # "how do I install to python" issue by making this entire directory
@@ -88,9 +88,6 @@ if [ -n "$TRAVIS_PYTHON_VERSION" ]; then
   # Install psutil for dataloader tests
   as_jenkins pip install psutil
 
-  # Install dill for serialization tests
-  as_jenkins pip install "dill>=0.3.1"
-
   # Cleanup package manager
   apt-get autoclean && apt-get clean
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

.circleci/docker/ubuntu-cuda/Dockerfile

@@ -35,11 +35,6 @@ ARG GCC_VERSION
 ADD ./common/install_gcc.sh install_gcc.sh
 RUN bash ./install_gcc.sh && rm install_gcc.sh
 
-# Install clang
-ARG CLANG_VERSION
-ADD ./common/install_clang.sh install_clang.sh
-RUN bash ./install_clang.sh && rm install_clang.sh
-
 # Install non-standard Python versions (via Travis binaries)
 ARG TRAVIS_PYTHON_VERSION
 ENV PATH /opt/python/$TRAVIS_PYTHON_VERSION/bin:$PATH
@@ -86,9 +81,5 @@ ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
 ENV TORCH_CUDA_ARCH_LIST Maxwell
 ENV TORCH_NVCC_FLAGS "-Xfatbin -compress-all"
 
-# Install LLVM dev version
-ADD ./common/install_llvm.sh install_llvm.sh
-RUN bash ./install_llvm.sh
-
 USER jenkins
 CMD ["bash"]

.circleci/docker/ubuntu-rocm/.gitignore

@@ -1 +0,0 @@
-*.sh

.circleci/docker/ubuntu-rocm/Dockerfile

@@ -1,86 +0,0 @@
-ARG UBUNTU_VERSION
-
-FROM ubuntu:${UBUNTU_VERSION}
-
-ARG UBUNTU_VERSION
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install common dependencies (so that this step can be cached separately)
-ARG EC2
-ADD ./common/install_base.sh install_base.sh
-RUN bash ./install_base.sh && rm install_base.sh
-
-# Install clang
-ARG LLVMDEV
-ARG CLANG_VERSION
-ADD ./common/install_clang.sh install_clang.sh
-RUN bash ./install_clang.sh && rm install_clang.sh
-
-# Install user
-ADD ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
-# Install conda
-ENV PATH /opt/conda/bin:$PATH
-ARG ANACONDA_PYTHON_VERSION
-ADD ./common/install_conda.sh install_conda.sh
-RUN bash ./install_conda.sh && rm install_conda.sh
-
-# (optional) Install protobuf for ONNX
-ARG PROTOBUF
-ADD ./common/install_protobuf.sh install_protobuf.sh
-RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
-RUN rm install_protobuf.sh
-ENV INSTALLED_PROTOBUF ${PROTOBUF}
-
-# (optional) Install database packages like LMDB and LevelDB
-ARG DB
-ADD ./common/install_db.sh install_db.sh
-RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
-RUN rm install_db.sh
-ENV INSTALLED_DB ${DB}
-
-# (optional) Install vision packages like OpenCV and ffmpeg
-ARG VISION
-ADD ./common/install_vision.sh install_vision.sh
-RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
-RUN rm install_vision.sh
-ENV INSTALLED_VISION ${VISION}
-
-# Install rocm
-ARG ROCM_VERSION
-ADD ./common/install_rocm.sh install_rocm.sh
-RUN bash ./install_rocm.sh
-RUN rm install_rocm.sh
-ENV PATH /opt/rocm/bin:$PATH
-ENV PATH /opt/rocm/hcc/bin:$PATH
-ENV PATH /opt/rocm/hip/bin:$PATH
-ENV PATH /opt/rocm/opencl/bin:$PATH
-ENV HIP_PLATFORM hcc
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-
-# (optional) Install non-default CMake version
-ARG CMAKE_VERSION
-ADD ./common/install_cmake.sh install_cmake.sh
-RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
-RUN rm install_cmake.sh
-
-# (optional) Install non-default Ninja version
-ARG NINJA_VERSION
-ADD ./common/install_ninja.sh install_ninja.sh
-RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
-RUN rm install_ninja.sh
-
-# Install ccache/sccache (do this last, so we get priority in PATH)
-ADD ./common/install_cache.sh install_cache.sh
-ENV PATH /opt/cache/bin:$PATH
-RUN bash ./install_cache.sh && rm install_cache.sh
-
-# Include BUILD_ENVIRONMENT environment variable in image
-ARG BUILD_ENVIRONMENT
-ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
-
-USER jenkins
-CMD ["bash"]

.circleci/docker/ubuntu/Dockerfile

@@ -46,7 +46,6 @@ RUN bash ./install_gcc.sh && rm install_gcc.sh
 
 # Install non-standard Python versions (via Travis binaries)
 ARG TRAVIS_PYTHON_VERSION
-ARG TRAVIS_DL_URL_PREFIX
 ENV PATH /opt/python/$TRAVIS_PYTHON_VERSION/bin:$PATH
 ADD ./common/install_travis_python.sh install_travis_python.sh
 RUN bash ./install_travis_python.sh && rm install_travis_python.sh
@@ -111,9 +110,5 @@ RUN bash ./install_jni.sh && rm install_jni.sh
 ARG BUILD_ENVIRONMENT
 ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
 
-# Install LLVM dev version
-ADD ./common/install_llvm.sh install_llvm.sh
-RUN bash ./install_llvm.sh
-
 USER jenkins
 CMD ["bash"]

.circleci/ecr_gc_docker/Dockerfile

@@ -1,13 +0,0 @@
-FROM ubuntu:16.04
-
-RUN apt-get update && apt-get install -y python-pip git && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
-
-ADD requirements.txt /requirements.txt
-
-RUN pip install -r /requirements.txt
-
-ADD gc.py /usr/bin/gc.py
-
-ADD docker_hub.py /usr/bin/docker_hub.py
-
-ENTRYPOINT ["/usr/bin/gc.py"]

.circleci/ecr_gc_docker/docker_hub.py

@@ -1,125 +0,0 @@
-#!/usr/bin/env python
-
-from collections import namedtuple
-
-import boto3
-import requests
-import os
-
-
-IMAGE_INFO = namedtuple(
-    "IMAGE_INFO", ("repo", "tag", "size", "last_updated_at", "last_updated_by")
-)
-
-
-def build_access_token(username, passwordtr):
-    r = requests.post(
-        "https://hub.docker.com/v2/users/login/",
-        data={"username": username, "password": password},
-    )
-    r.raise_for_status()
-    token = r.json().get("token")
-    return {"Authorization": "JWT " + token}
-
-
-def list_repos(user, token):
-    r = requests.get("https://hub.docker.com/v2/repositories/" + user, headers=token)
-    r.raise_for_status()
-    ret = sorted(
-        repo["user"] + "/" + repo["name"] for repo in r.json().get("results", [])
-    )
-    if ret:
-        print("repos found:")
-        print("".join("\n\t" + r for r in ret))
-    return ret
-
-
-def list_tags(repo, token):
-    r = requests.get(
-        "https://hub.docker.com/v2/repositories/" + repo + "/tags", headers=token
-    )
-    r.raise_for_status()
-    return [
-        IMAGE_INFO(
-            repo=repo,
-            tag=t["name"],
-            size=t["full_size"],
-            last_updated_at=t["last_updated"],
-            last_updated_by=t["last_updater_username"],
-        )
-        for t in r.json().get("results", [])
-    ]
-
-
-def save_to_s3(tags):
-    table_content = ""
-    client = boto3.client("s3")
-    for t in tags:
-        table_content += (
-            "<tr><td>{repo}</td><td>{tag}</td><td>{size}</td>"
-            "<td>{last_updated_at}</td><td>{last_updated_by}</td></tr>"
-        ).format(
-            repo=t.repo,
-            tag=t.tag,
-            size=t.size,
-            last_updated_at=t.last_updated_at,
-            last_updated_by=t.last_updated_by,
-        )
-    html_body = """
-    <html>
-        <head>
-            <link rel="stylesheet"
-                href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css"
-                integrity="sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh"
-                crossorigin="anonymous">
-            <link rel="stylesheet" type="text/css"
-                href="https://cdn.datatables.net/1.10.20/css/jquery.dataTables.css">
-            <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js">
-            </script>
-            <script type="text/javascript" charset="utf8"
-                src="https://cdn.datatables.net/1.10.20/js/jquery.dataTables.js"></script>
-            <title> docker image info</title>
-        </head>
-        <body>
-            <table class="table table-striped table-hover" id="docker">
-            <caption>Docker images on docker hub</caption>
-            <thead class="thead-dark">
-                <tr>
-                <th scope="col">repo</th>
-                <th scope="col">tag</th>
-                <th scope="col">size</th>
-                <th scope="col">last_updated_at</th>
-                <th scope="col">last_updated_by</th>
-                </tr>
-            </thead>
-            <tbody>
-                {table_content}
-            </tbody>
-            </table>
-        </body>
-        <script>
-            $(document).ready( function () {{
-                $('#docker').DataTable({{paging: false}});
-            }} );py
-        </script>
-    </html>
-    """.format(
-        table_content=table_content
-    )
-    client.put_object(
-        Bucket="docker.pytorch.org",
-        ACL="public-read",
-        Key="docker_hub.html",
-        Body=html_body,
-        ContentType="text/html",
-    )
-
-
-if __name__ == "__main__":
-    username = os.environ.get("DOCKER_HUB_USERNAME")
-    password = os.environ.get("DOCKER_HUB_PASSWORD")
-    token = build_access_token(username, password)
-    tags = []
-    for repo in list_repos("pytorch", token):
-        tags.extend(list_tags(repo, token))
-    save_to_s3(tags)

.circleci/ecr_gc_docker/gc.py

@@ -1,214 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import datetime
-import boto3
-import pytz
-import sys
-import re
-
-
-def save_to_s3(project, data):
-    table_content = ""
-    client = boto3.client("s3")
-    for repo, tag, window, age, pushed in data:
-        table_content += "<tr><td>{repo}</td><td>{tag}</td><td>{window}</td><td>{age}</td><td>{pushed}</td></tr>".format(
-            repo=repo, tag=tag, window=window, age=age, pushed=pushed
-        )
-    html_body = """
-    <html>
-        <head>
-            <link rel="stylesheet"
-                href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css"
-                integrity="sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh"
-                crossorigin="anonymous">
-            <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.20/css/jquery.dataTables.css">
-            <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>
-            <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.20/js/jquery.dataTables.js"></script>
-            <title>{project} nightly and permanent docker image info</title>
-        </head>
-        <body>
-            <table class="table table-striped table-hover" id="docker">
-            <thead class="thead-dark">
-                <tr>
-                <th scope="col">repo</th>
-                <th scope="col">tag</th>
-                <th scope="col">keep window</th>
-                <th scope="col">age</th>
-                <th scope="col">pushed at</th>
-                </tr>
-            </thead>
-            <tbody>
-                {table_content}
-            </tbody>
-            </table>
-        </body>
-        <script>
-            $(document).ready( function () {{
-                $('#docker').DataTable({{paging: false}});
-            }} );
-        </script>
-    </html>
-    """.format(
-        project=project, table_content=table_content
-    )
-
-    # for pytorch, file can be found at
-    # http://ossci-docker.s3-website.us-east-1.amazonaws.com/pytorch.html
-    # and later one we can config docker.pytorch.org to point to the location
-
-    client.put_object(
-        Bucket="docker.pytorch.org",
-        ACL="public-read",
-        Key="{project}.html".format(project=project),
-        Body=html_body,
-        ContentType="text/html",
-    )
-
-
-def repos(client):
-    paginator = client.get_paginator("describe_repositories")
-    pages = paginator.paginate(registryId="308535385114")
-    for page in pages:
-        for repo in page["repositories"]:
-            yield repo
-
-
-def images(client, repository):
-    paginator = client.get_paginator("describe_images")
-    pages = paginator.paginate(
-        registryId="308535385114", repositoryName=repository["repositoryName"]
-    )
-    for page in pages:
-        for image in page["imageDetails"]:
-            yield image
-
-
-parser = argparse.ArgumentParser(description="Delete old Docker tags from registry")
-parser.add_argument(
-    "--dry-run", action="store_true", help="Dry run; print tags that would be deleted"
-)
-parser.add_argument(
-    "--keep-stable-days",
-    type=int,
-    default=14,
-    help="Days of stable Docker tags to keep (non per-build images)",
-)
-parser.add_argument(
-    "--keep-unstable-days",
-    type=int,
-    default=1,
-    help="Days of unstable Docker tags to keep (per-build images)",
-)
-parser.add_argument(
-    "--filter-prefix",
-    type=str,
-    default="",
-    help="Only run cleanup for repositories with this prefix",
-)
-parser.add_argument(
-    "--ignore-tags",
-    type=str,
-    default="",
-    help="Never cleanup these tags (comma separated)",
-)
-args = parser.parse_args()
-
-if not args.ignore_tags or not args.filter_prefix:
-    print(
-        """
-Missing required arguments --ignore-tags and --filter-prefix
-
-You must specify --ignore-tags and --filter-prefix to avoid accidentally
-pruning a stable Docker tag which is being actively used.  This will
-make you VERY SAD.  So pay attention.
-
-First, which filter-prefix do you want?  The list of valid prefixes
-is in jobs/private.groovy under the 'docker-registry-cleanup' job.
-You probably want either pytorch or caffe2.
-
-Second, which ignore-tags do you want?  It should be whatever the most
-up-to-date DockerVersion for the repository in question is.  Follow
-the imports of jobs/pytorch.groovy to find them.
-"""
-    )
-    sys.exit(1)
-
-client = boto3.client("ecr", region_name="us-east-1")
-stable_window = datetime.timedelta(days=args.keep_stable_days)
-unstable_window = datetime.timedelta(days=args.keep_unstable_days)
-now = datetime.datetime.now(pytz.UTC)
-ignore_tags = args.ignore_tags.split(",")
-
-
-def chunks(chunkable, n):
-    """ Yield successive n-sized chunks from l.
-    """
-    for i in range(0, len(chunkable), n):
-        yield chunkable[i : i + n]
-
-SHA_PATTERN = re.compile(r'^[0-9a-f]{40}$')
-def looks_like_git_sha(tag):
-    """Returns a boolean to check if a tag looks like a git sha
-
-    For reference a sha1 is 40 characters with only 0-9a-f and contains no
-    "-" characters
-    """
-    return re.match(SHA_PATTERN, tag) is not None
-
-stable_window_tags = []
-for repo in repos(client):
-    repositoryName = repo["repositoryName"]
-    if not repositoryName.startswith(args.filter_prefix):
-        continue
-
-    # Keep list of image digests to delete for this repository
-    digest_to_delete = []
-    print(repositoryName)
-
-    for image in images(client, repo):
-        tags = image.get("imageTags")
-        if not isinstance(tags, (list,)) or len(tags) == 0:
-            continue
-
-        tag = tags[0]
-        created = image["imagePushedAt"]
-        age = now - created
-        if any([
-                looks_like_git_sha(tag),
-                tag.isdigit(),
-                tag.count("-") == 4,  # TODO: Remove, this no longer applies as tags are now built using a SHA1
-                tag in ignore_tags]):
-            window = stable_window
-            if tag in ignore_tags:
-                stable_window_tags.append((repositoryName, tag, "", age, created))
-            elif age < window:
-                stable_window_tags.append((repositoryName, tag, window, age, created))
-        else:
-            window = unstable_window
-
-        if tag in ignore_tags:
-            print("Ignoring tag {}:{} (age: {})".format(repositoryName, tag, age))
-            continue
-        if age < window:
-            print("Not deleting manifest for tag {}:{} (age: {})".format(repositoryName, tag, age))
-            continue
-
-        if args.dry_run:
-            print("(dry run) Deleting manifest for tag {}:{} (age: {})".format(repositoryName, tag, age))
-        else:
-            print("Deleting manifest for tag{}:{} (age: {})".format(repositoryName, tag, age))
-            digest_to_delete.append(image["imageDigest"])
-
-    # Issue batch delete for all images to delete for this repository
-    # Note that as of 2018-07-25, the maximum number of images you can
-    # delete in a single batch is 100, so chunk our list into batches of
-    # 100
-    for c in chunks(digest_to_delete, 100):
-        client.batch_delete_image(
-            registryId="308535385114",
-            repositoryName=repositoryName,
-            imageIds=[{"imageDigest": digest} for digest in c],
-        )
-
-    save_to_s3(args.filter_prefix, stable_window_tags)

.circleci/ecr_gc_docker/requirements.txt

@@ -1,3 +0,0 @@
-boto3
-pytz
-requests

.circleci/generate_config_yml.py

@@ -6,24 +6,13 @@ Please see README.md in this directory for details.
 """
 
 import os
-import shutil
 import sys
-from collections import OrderedDict, namedtuple
+import shutil
+from collections import namedtuple, OrderedDict
 
+import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
 import cimodel.data.binary_build_definitions as binary_build_definitions
 import cimodel.data.caffe2_build_definitions as caffe2_build_definitions
-import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
-import cimodel.data.simple.android_definitions
-import cimodel.data.simple.bazel_definitions
-import cimodel.data.simple.binary_smoketest
-import cimodel.data.simple.docker_definitions
-import cimodel.data.simple.ge_config_tests
-import cimodel.data.simple.ios_definitions
-import cimodel.data.simple.macos_definitions
-import cimodel.data.simple.mobile_definitions
-import cimodel.data.simple.nightly_android
-import cimodel.data.simple.nightly_ios
-import cimodel.data.windows_build_definitions as windows_build_definitions
 import cimodel.lib.miniutils as miniutils
 import cimodel.lib.miniyaml as miniyaml
 
@@ -32,7 +21,6 @@ class File(object):
     """
     Verbatim copy the contents of a file into config.yml
     """
-
     def __init__(self, filename):
         self.filename = filename
 
@@ -41,7 +29,7 @@ class File(object):
             shutil.copyfileobj(fh, output_filehandle)
 
 
-class FunctionGen(namedtuple("FunctionGen", "function depth")):
+class FunctionGen(namedtuple('FunctionGen', 'function depth')):
     __slots__ = ()
 
 
@@ -51,14 +39,15 @@ class Treegen(FunctionGen):
     """
 
     def write(self, output_filehandle):
-        miniyaml.render(output_filehandle, self.function(), self.depth)
+        build_dict = OrderedDict()
+        self.function(build_dict)
+        miniyaml.render(output_filehandle, build_dict, self.depth)
 
 
 class Listgen(FunctionGen):
     """
     Insert the content of a YAML list into config.yml
     """
-
     def write(self, output_filehandle):
         miniyaml.render(output_filehandle, self.function(), self.depth)
 
@@ -68,6 +57,7 @@ def horizontal_rule():
 
 
 class Header(object):
+
     def __init__(self, title, summary=None):
         self.title = title
         self.summary_lines = summary or []
@@ -81,81 +71,43 @@ class Header(object):
             output_filehandle.write(line + "\n")
 
 
-def gen_build_workflows_tree():
-    build_workflows_functions = [
-        pytorch_build_definitions.get_workflow_jobs,
-        cimodel.data.simple.macos_definitions.get_workflow_jobs,
-        cimodel.data.simple.android_definitions.get_workflow_jobs,
-        cimodel.data.simple.ios_definitions.get_workflow_jobs,
-        cimodel.data.simple.mobile_definitions.get_workflow_jobs,
-        cimodel.data.simple.ge_config_tests.get_workflow_jobs,
-        cimodel.data.simple.bazel_definitions.get_workflow_jobs,
-        caffe2_build_definitions.get_workflow_jobs,
-        cimodel.data.simple.binary_smoketest.get_workflow_jobs,
-        cimodel.data.simple.nightly_ios.get_workflow_jobs,
-        cimodel.data.simple.nightly_android.get_workflow_jobs,
-        windows_build_definitions.get_windows_workflows,
-    ]
-
-    binary_build_functions = [
-        binary_build_definitions.get_binary_build_jobs,
-        binary_build_definitions.get_nightly_tests,
-        binary_build_definitions.get_nightly_uploads,
-        binary_build_definitions.get_post_upload_jobs,
-        binary_build_definitions.get_binary_smoke_test_jobs,
-    ]
-
-    docker_builder_functions = [
-        cimodel.data.simple.docker_definitions.get_workflow_jobs
-    ]
-
-    return {
-        "workflows": {
-            "binary_builds": {
-                "when": r"<< pipeline.parameters.run_binary_tests >>",
-                "jobs": [f() for f in binary_build_functions],
-            },
-            "docker_build": OrderedDict(
-                {
-                    "triggers": [
-                        {
-                            "schedule": {
-                                "cron": miniutils.quote("0 15 * * 0"),
-                                "filters": {"branches": {"only": ["master"]}},
-                            }
-                        }
-                    ],
-                    "jobs": [f() for f in docker_builder_functions],
-                }
-            ),
-            "build": {"jobs": [f() for f in build_workflows_functions]},
-        }
-    }
-
-
 # Order of this list matters to the generated config.yml.
 YAML_SOURCES = [
     File("header-section.yml"),
     File("commands.yml"),
     File("nightly-binary-build-defaults.yml"),
     Header("Build parameters"),
-    File("build-parameters/pytorch-build-params.yml"),
+    File("pytorch-build-params.yml"),
-    File("build-parameters/caffe2-build-params.yml"),
+    File("caffe2-build-params.yml"),
-    File("build-parameters/binary-build-params.yml"),
+    File("binary-build-params.yml"),
-    File("build-parameters/promote-build-params.yml"),
     Header("Job specs"),
-    File("job-specs/pytorch-job-specs.yml"),
+    File("pytorch-job-specs.yml"),
-    File("job-specs/caffe2-job-specs.yml"),
+    File("caffe2-job-specs.yml"),
-    File("job-specs/binary-job-specs.yml"),
+    File("binary-job-specs.yml"),
-    File("job-specs/job-specs-custom.yml"),
+    File("job-specs-setup.yml"),
-    File("job-specs/job-specs-promote.yml"),
+    File("job-specs-custom.yml"),
-    File("job-specs/binary_update_htmls.yml"),
+    File("binary_update_htmls.yml"),
-    File("job-specs/binary-build-tests.yml"),
+    File("binary-build-tests.yml"),
-    File("job-specs/docker_jobs.yml"),
+    File("docker_build_job.yml"),
-    Header("Workflows"),
+    File("workflows.yml"),
-    Treegen(gen_build_workflows_tree, 0),
+    Listgen(pytorch_build_definitions.get_workflow_jobs, 3),
-    File("workflows/workflows-ecr-gc.yml"),
+    File("workflows-pytorch-macos-builds.yml"),
-    File("workflows/workflows-promote.yml"),
+    File("workflows-pytorch-android-gradle-build.yml"),
+    File("workflows-pytorch-ios-builds.yml"),
+    File("workflows-pytorch-mobile-builds.yml"),
+    File("workflows-pytorch-ge-config-tests.yml"),
+    Listgen(caffe2_build_definitions.get_workflow_jobs, 3),
+    File("workflows-binary-builds-smoke-subset.yml"),
+    Listgen(binary_build_definitions.get_binary_smoke_test_jobs, 3),
+    Listgen(binary_build_definitions.get_binary_build_jobs, 3),
+    File("workflows-nightly-ios-binary-builds.yml"),
+    File("workflows-nightly-android-binary-builds.yml"),
+    Header("Nightly tests"),
+    Listgen(binary_build_definitions.get_nightly_tests, 3),
+    File("workflows-nightly-uploads-header.yml"),
+    Listgen(binary_build_definitions.get_nightly_uploads, 3),
+    File("workflows-s3-html.yml"),
+    File("workflows-docker-builder.yml")
 ]
 
 

.circleci/scripts/binary_checkout.sh

@@ -1,20 +1,9 @@
 #!/bin/bash
 set -eux -o pipefail
-
-retry () {
-    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
-}
-
-
 # This step runs on multiple executors with different envfile locations
 if [[ "$(uname)" == Darwin ]]; then
   # macos executor (builds and tests)
   workdir="/Users/distiller/project"
-elif [[ "$OSTYPE" == "msys" ]]; then
-  # windows executor (builds and tests)
-  rm -rf /c/w
-  ln -s "/c/Users/circleci/project" /c/w
-  workdir="/c/w"
 elif [[ -d "/home/circleci/project" ]]; then
   # machine executor (binary tests)
   workdir="/home/circleci/project"
@@ -24,17 +13,11 @@ else
 fi
 
 # It is very important that this stays in sync with binary_populate_env.sh
-if [[ "$OSTYPE" == "msys" ]]; then
+export PYTORCH_ROOT="$workdir/pytorch"
-  # We need to make the paths as short as possible on Windows
+export BUILDER_ROOT="$workdir/builder"
-  export PYTORCH_ROOT="$workdir/p"
-  export BUILDER_ROOT="$workdir/b"
-else
-  export PYTORCH_ROOT="$workdir/pytorch"
-  export BUILDER_ROOT="$workdir/builder"
-fi
 
 # Clone the Pytorch branch
-retry git clone https://github.com/pytorch/pytorch.git "$PYTORCH_ROOT"
+git clone https://github.com/pytorch/pytorch.git "$PYTORCH_ROOT"
 pushd "$PYTORCH_ROOT"
 if [[ -n "${CIRCLE_PR_NUMBER:-}" ]]; then
   # "smoke" binary build on PRs
@@ -50,13 +33,13 @@ else
   echo "Can't tell what to checkout"
   exit 1
 fi
-retry git submodule update --init --recursive
+git submodule update --init --recursive --quiet
 echo "Using Pytorch from "
 git --no-pager log --max-count 1
 popd
 
 # Clone the Builder master repo
-retry git clone -q https://github.com/pytorch/builder.git "$BUILDER_ROOT"
+git clone -q https://github.com/pytorch/builder.git "$BUILDER_ROOT"
 pushd "$BUILDER_ROOT"
 echo "Using builder from "
 git --no-pager log --max-count 1

.circleci/scripts/binary_install_miniconda.sh

@@ -31,9 +31,9 @@ fi
 
 conda_sh="$workdir/install_miniconda.sh"
 if [[ "$(uname)" == Darwin ]]; then
-  curl --retry 3 -o "$conda_sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+  retry curl -o "$conda_sh" https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
 else
-  curl --retry 3 -o "$conda_sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+  retry curl -o "$conda_sh" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
 fi
 chmod +x "$conda_sh"
 "$conda_sh" -b -p "$MINICONDA_ROOT"

.circleci/scripts/binary_ios_build.sh

@@ -5,24 +5,20 @@ echo ""
 echo "DIR: $(pwd)"
 WORKSPACE=/Users/distiller/workspace
 PROJ_ROOT=/Users/distiller/project
-export TCLLIBPATH="/usr/local/lib"
+export TCLLIBPATH="/usr/local/lib" 
-
 # Install conda
-curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+curl -o ~/Downloads/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
-chmod +x ~/conda.sh
+chmod +x ~/Downloads/conda.sh
-/bin/bash ~/conda.sh -b -p ~/anaconda
+/bin/bash ~/Downloads/conda.sh -b -p ~/anaconda
 export PATH="~/anaconda/bin:${PATH}"
 source ~/anaconda/bin/activate
-
 # Install dependencies
 conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing requests --yes
 export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
-
 # sync submodules
 cd ${PROJ_ROOT}
 git submodule sync
 git submodule update --init --recursive
-
 # run build script
 chmod a+x ${PROJ_ROOT}/scripts/build_ios.sh
 echo "########################################################"
@@ -30,13 +26,13 @@ cat ${PROJ_ROOT}/scripts/build_ios.sh
 echo "########################################################"
 echo "IOS_ARCH: ${IOS_ARCH}"
 echo "IOS_PLATFORM: ${IOS_PLATFORM}"
+export BUILD_PYTORCH_MOBILE=1
 export IOS_ARCH=${IOS_ARCH}
 export IOS_PLATFORM=${IOS_PLATFORM}
 unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts
-
 #store the binary
 cd ${WORKSPACE}
 DEST_DIR=${WORKSPACE}/ios
 mkdir -p ${DEST_DIR}
 cp -R ${PROJ_ROOT}/build_ios/install ${DEST_DIR}
-mv ${DEST_DIR}/install ${DEST_DIR}/${IOS_ARCH}
+mv ${DEST_DIR}/install ${DEST_DIR}/${IOS_ARCH}

.circleci/scripts/binary_ios_upload.sh

@@ -14,14 +14,14 @@ mkdir -p ${ZIP_DIR}/src
 cp -R ${ARTIFACTS_DIR}/arm64/include ${ZIP_DIR}/install/
 # build a FAT bianry
 cd ${ZIP_DIR}/install/lib
-target_libs=(libc10.a libclog.a libcpuinfo.a libeigen_blas.a libpthreadpool.a libpytorch_qnnpack.a libtorch_cpu.a libtorch.a libXNNPACK.a)
+target_libs=(libc10.a libclog.a libcpuinfo.a libeigen_blas.a libpytorch_qnnpack.a libtorch.a)
 for lib in ${target_libs[*]}
 do
-    if [ -f "${ARTIFACTS_DIR}/x86_64/lib/${lib}" ] && [ -f "${ARTIFACTS_DIR}/arm64/lib/${lib}" ]; then
+    libs=(${ARTIFACTS_DIR}/x86_64/lib/${lib} ${ARTIFACTS_DIR}/arm64/lib/${lib})
-        libs=("${ARTIFACTS_DIR}/x86_64/lib/${lib}" "${ARTIFACTS_DIR}/arm64/lib/${lib}")
+    lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/${lib}
-        lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/${lib}
-    fi
 done
+# for nnpack, we only support arm64 build
+cp ${ARTIFACTS_DIR}/arm64/lib/libnnpack.a ./
 lipo -i ${ZIP_DIR}/install/lib/*.a
 # copy the umbrella header and license
 cp ${PROJ_ROOT}/ios/LibTorch.h ${ZIP_DIR}/src/

.circleci/scripts/binary_linux_test.sh

@@ -9,15 +9,13 @@ set -eux -o pipefail
 if [[ "$PACKAGE_TYPE" == conda ]]; then
   retry conda create -qyn testenv python="$DESIRED_PYTHON"
   source activate testenv >/dev/null
+elif [[ "$DESIRED_PYTHON" == 2.7mu ]]; then
+  export PATH="/opt/python/cp27-cp27mu/bin:\$PATH"
+elif [[ "$DESIRED_PYTHON" == 3.8m ]]; then
+  export PATH="/opt/python/cp38-cp38/bin:\$PATH"
 elif [[ "$PACKAGE_TYPE" != libtorch ]]; then
   python_nodot="\$(echo $DESIRED_PYTHON | tr -d m.u)"
-  python_path="/opt/python/cp\$python_nodot-cp\${python_nodot}"
+  export PATH="/opt/python/cp\$python_nodot-cp\${python_nodot}m/bin:\$PATH"
-  # Prior to Python 3.8 paths were suffixed with an 'm'
-  if [[ -d  "\${python_path}/bin" ]]; then
-    export PATH="\${python_path}/bin:\$PATH"
-  elif [[ -d "\${python_path}m/bin" ]]; then
-    export PATH="\${python_path}m/bin:\$PATH"
-  fi
 fi
 
 # Install the package
@@ -30,11 +28,11 @@ pkg="/final_pkgs/\$(ls /final_pkgs)"
 if [[ "$PACKAGE_TYPE" == conda ]]; then
   conda install -y "\$pkg" --offline
   if [[ "$DESIRED_CUDA" == 'cpu' ]]; then
-    retry conda install -y cpuonly -c pytorch
+    conda install -y cpuonly -c pytorch
   fi
   retry conda install -yq future numpy protobuf six
   if [[ "$DESIRED_CUDA" != 'cpu' ]]; then
-    # DESIRED_CUDA is in format cu90 or cu102
+    # DESIRED_CUDA is in format cu90 or cu100
     if [[ "${#DESIRED_CUDA}" == 4 ]]; then
       cu_ver="${DESIRED_CUDA:2:1}.${DESIRED_CUDA:3}"
     else
@@ -54,7 +52,6 @@ fi
 
 # Test the package
 /builder/check_binary.sh
-
 # =================== The above code will be executed inside Docker container ===================
 EOL
 echo

.circleci/scripts/binary_linux_upload.sh

@@ -5,6 +5,15 @@ set -eu -o pipefail
 set +x
 declare -x "AWS_ACCESS_KEY_ID=${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
 declare -x "AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
+cat >/home/circleci/project/login_to_anaconda.sh <<EOL
+set +x
+echo "Trying to login to Anaconda"
+yes | anaconda login \
+    --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
+    --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
+set -x
+EOL
+chmod +x /home/circleci/project/login_to_anaconda.sh
 
 #!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
 # DO NOT TURN -x ON BEFORE THIS LINE
@@ -12,37 +21,20 @@ declare -x "AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
 set -eux -o pipefail
 export PATH="$MINICONDA_ROOT/bin:$PATH"
 
-# This gets set in binary_populate_env.sh, but lets have a sane default just in case
-PIP_UPLOAD_FOLDER=${PIP_UPLOAD_FOLDER:-nightly}
-# TODO: Combine CONDA_UPLOAD_CHANNEL and PIP_UPLOAD_FOLDER into one variable
-#       The only difference is the trailing slash
-# Strip trailing slashes if there
-CONDA_UPLOAD_CHANNEL=$(echo "${PIP_UPLOAD_FOLDER}" | sed 's:/*$::')
-BACKUP_BUCKET="s3://pytorch-backup"
-
-retry pip install -q awscli
 # Upload the package to the final location
 pushd /home/circleci/project/final_pkgs
 if [[ "$PACKAGE_TYPE" == conda ]]; then
   retry conda install -yq anaconda-client
-  retry anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload  "$(ls)" -u "pytorch-${CONDA_UPLOAD_CHANNEL}" --label main --no-progress --force
+  retry timeout 30 /home/circleci/project/login_to_anaconda.sh
-  # Fetch  platform (eg. win-64, linux-64, etc.) from index file
+  anaconda upload "$(ls)" -u pytorch-nightly --label main --no-progress --force
-  # Because there's no actual conda command to read this
-  subdir=$(tar -xOf ./*.bz2 info/index.json | grep subdir  | cut -d ':' -f2 | sed -e 's/[[:space:]]//' -e 's/"//g' -e 's/,//')
-  BACKUP_DIR="conda/${subdir}"
 elif [[ "$PACKAGE_TYPE" == libtorch ]]; then
+  retry pip install -q awscli
   s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
   for pkg in $(ls); do
     retry aws s3 cp "$pkg" "$s3_dir" --acl public-read
   done
-  BACKUP_DIR="libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
 else
+  retry pip install -q awscli
   s3_dir="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
   retry aws s3 cp "$(ls)" "$s3_dir" --acl public-read
-  BACKUP_DIR="whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-fi
-
-if [[ -n "${CIRCLE_TAG:-}" ]]; then
-  s3_dir="${BACKUP_BUCKET}/${CIRCLE_TAG}/${BACKUP_DIR}"
-  retry aws s3 cp --recursive . "$s3_dir"
 fi

.circleci/scripts/binary_macos_upload.sh

@@ -4,6 +4,15 @@ set -eu -o pipefail
 set +x
 export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
 export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
+cat >/Users/distiller/project/login_to_anaconda.sh <<EOL
+set +x
+echo "Trying to login to Anaconda"
+yes | anaconda login \
+    --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
+    --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
+set -x
+EOL
+chmod +x /Users/distiller/project/login_to_anaconda.sh
 
 #!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
 # DO NOT TURN -x ON BEFORE THIS LINE
@@ -13,36 +22,19 @@ set -eux -o pipefail
 source "/Users/distiller/project/env"
 export "PATH=$workdir/miniconda/bin:$PATH"
 
-# This gets set in binary_populate_env.sh, but lets have a sane default just in case
-PIP_UPLOAD_FOLDER=${PIP_UPLOAD_FOLDER:-nightly}
-# TODO: Combine CONDA_UPLOAD_CHANNEL and PIP_UPLOAD_FOLDER into one variable
-#       The only difference is the trailing slash
-# Strip trailing slashes if there
-CONDA_UPLOAD_CHANNEL=$(echo "${PIP_UPLOAD_FOLDER}" | sed 's:/*$::')
-BACKUP_BUCKET="s3://pytorch-backup"
-
-retry pip install -q awscli
 pushd "$workdir/final_pkgs"
 if [[ "$PACKAGE_TYPE" == conda ]]; then
   retry conda install -yq anaconda-client
-  retry anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload "$(ls)" -u "pytorch-${CONDA_UPLOAD_CHANNEL}" --label main --no-progress --force
+  retry /Users/distiller/project/login_to_anaconda.sh
-  # Fetch  platform (eg. win-64, linux-64, etc.) from index file
+  retry anaconda upload "$(ls)" -u pytorch-nightly --label main --no-progress --force
-  # Because there's no actual conda command to read this
-  subdir=$(tar -xOf ./*.bz2 info/index.json | grep subdir  | cut -d ':' -f2 | sed -e 's/[[:space:]]//' -e 's/"//g' -e 's/,//')
-  BACKUP_DIR="conda/${subdir}"
 elif [[ "$PACKAGE_TYPE" == libtorch ]]; then
+  retry pip install -q awscli
   s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
   for pkg in $(ls); do
     retry aws s3 cp "$pkg" "$s3_dir" --acl public-read
   done
-  BACKUP_DIR="libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
 else
+  retry pip install -q awscli
   s3_dir="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
   retry aws s3 cp "$(ls)" "$s3_dir" --acl public-read
-  BACKUP_DIR="whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-fi
-
-if [[ -n "${CIRCLE_TAG:-}" ]]; then
-  s3_dir="${BACKUP_BUCKET}/${CIRCLE_TAG}/${BACKUP_DIR}"
-  retry aws s3 cp --recursive . "$s3_dir"
 fi

.circleci/scripts/binary_populate_env.sh

@@ -2,31 +2,11 @@
 set -eux -o pipefail
 export TZ=UTC
 
-tagged_version() {
-  # Grabs version from either the env variable CIRCLE_TAG
-  # or the pytorch git described version
-  if [[ "$OSTYPE" == "msys" ]]; then
-    GIT_DESCRIBE="git --git-dir ${workdir}/p/.git describe"
-  else
-    GIT_DESCRIBE="git --git-dir ${workdir}/pytorch/.git describe"
-  fi
-  if [[ -n "${CIRCLE_TAG:-}" ]]; then
-    echo "${CIRCLE_TAG}"
-  elif ${GIT_DESCRIBE} --exact --tags >/dev/null; then
-    ${GIT_DESCRIBE} --tags
-  else
-    return 1
-  fi
-}
-
 # We need to write an envfile to persist these variables to following
 # steps, but the location of the envfile depends on the circleci executor
 if [[ "$(uname)" == Darwin ]]; then
   # macos executor (builds and tests)
   workdir="/Users/distiller/project"
-elif [[ "$OSTYPE" == "msys" ]]; then
-  # windows executor (builds and tests)
-  workdir="/c/w"
 elif [[ -d "/home/circleci/project" ]]; then
   # machine executor (binary tests)
   workdir="/home/circleci/project"
@@ -43,15 +23,7 @@ configs=($BUILD_ENVIRONMENT)
 export PACKAGE_TYPE="${configs[0]}"
 export DESIRED_PYTHON="${configs[1]}"
 export DESIRED_CUDA="${configs[2]}"
-if [[ "${BUILD_FOR_SYSTEM:-}" == "windows" ]]; then
+export DESIRED_DEVTOOLSET="${configs[3]:-}"
-  export DESIRED_DEVTOOLSET=""
-  export LIBTORCH_CONFIG="${configs[3]:-}"
-  if [[ "$LIBTORCH_CONFIG" == 'debug' ]]; then
-    export DEBUG=1
-  fi
-else
-  export DESIRED_DEVTOOLSET="${configs[3]:-}"
-fi
 if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
   export BUILD_PYTHONLESS=1
 fi
@@ -68,27 +40,25 @@ if [[ -z "$DOCKER_IMAGE" ]]; then
   fi
 fi
 
-# Default to nightly, since that's where this normally uploads to
+# Upload to parallel folder for devtoolsets
-PIP_UPLOAD_FOLDER='nightly/'
+# All nightlies used to be devtoolset3, then devtoolset7 was added as a build
+# option, so the upload was redirected to nightly/devtoolset7 to avoid
+# conflicts with other binaries (there shouldn't be any conflicts). Now we are
+# making devtoolset7 the default.
+if [[ "$DESIRED_DEVTOOLSET" == 'devtoolset7' || "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* || "$(uname)" == 'Darwin' ]]; then
+  export PIP_UPLOAD_FOLDER='nightly/'
+else
+  # On linux machines, this shouldn't actually be called anymore. This is just
+  # here for extra safety.
+  export PIP_UPLOAD_FOLDER='nightly/devtoolset3/'
+fi
+
 # We put this here so that OVERRIDE_PACKAGE_VERSION below can read from it
 export DATE="$(date -u +%Y%m%d)"
-#TODO: We should be pulling semver version from the base version.txt
+if [[ "$(uname)" == 'Darwin' ]] || [[ "$DESIRED_CUDA" == "cu101" ]] || [[ "$PACKAGE_TYPE" == conda ]]; then
-BASE_BUILD_VERSION="1.6.0.dev$DATE"
+  export PYTORCH_BUILD_VERSION="1.4.0.dev$DATE"
-# Change BASE_BUILD_VERSION to git tag when on a git tag
-# Use 'git -C' to make doubly sure we're in the correct directory for checking
-# the git tag
-if tagged_version >/dev/null; then
-  # Switch upload folder to 'test/' if we are on a tag
-  PIP_UPLOAD_FOLDER='test/'
-  # Grab git tag, remove prefixed v and remove everything after -
-  # Used to clean up tags that are for release candidates like v1.6.0-rc1
-  # Turns tag v1.6.0-rc1 -> v1.6.0
-  BASE_BUILD_VERSION="$(tagged_version | sed -e 's/^v//' -e 's/-.*$//')"
-fi
-if [[ "$(uname)" == 'Darwin' ]] || [[ "$DESIRED_CUDA" == "cu102" ]] || [[ "$PACKAGE_TYPE" == conda ]]; then
-  export PYTORCH_BUILD_VERSION="${BASE_BUILD_VERSION}"
 else
-  export PYTORCH_BUILD_VERSION="${BASE_BUILD_VERSION}+$DESIRED_CUDA"
+  export PYTORCH_BUILD_VERSION="1.4.0.dev$DATE+$DESIRED_CUDA"
 fi
 export PYTORCH_BUILD_NUMBER=1
 
@@ -124,13 +94,9 @@ export DESIRED_CUDA="$DESIRED_CUDA"
 export LIBTORCH_VARIANT="${LIBTORCH_VARIANT:-}"
 export BUILD_PYTHONLESS="${BUILD_PYTHONLESS:-}"
 export DESIRED_DEVTOOLSET="$DESIRED_DEVTOOLSET"
-if [[ "${BUILD_FOR_SYSTEM:-}" == "windows" ]]; then
-  export LIBTORCH_CONFIG="${LIBTORCH_CONFIG:-}"
-  export DEBUG="${DEBUG:-}"
-fi
 
 export DATE="$DATE"
-export NIGHTLIES_DATE_PREAMBLE=1.6.0.dev
+export NIGHTLIES_DATE_PREAMBLE=1.4.0.dev
 export PYTORCH_BUILD_VERSION="$PYTORCH_BUILD_VERSION"
 export PYTORCH_BUILD_NUMBER="$PYTORCH_BUILD_NUMBER"
 export OVERRIDE_PACKAGE_VERSION="$PYTORCH_BUILD_VERSION"
@@ -147,13 +113,8 @@ export DOCKER_IMAGE="$DOCKER_IMAGE"
 
 export workdir="$workdir"
 export MAC_PACKAGE_WORK_DIR="$workdir"
-if [[ "$OSTYPE" == "msys" ]]; then
+export PYTORCH_ROOT="$workdir/pytorch"
-  export PYTORCH_ROOT="$workdir/p"
+export BUILDER_ROOT="$workdir/builder"
-  export BUILDER_ROOT="$workdir/b"
-else
-  export PYTORCH_ROOT="$workdir/pytorch"
-  export BUILDER_ROOT="$workdir/builder"
-fi
 export MINICONDA_ROOT="$workdir/miniconda"
 export PYTORCH_FINAL_PACKAGE_DIR="$workdir/final_pkgs"
 

.circleci/scripts/binary_run_in_docker.sh

@@ -16,12 +16,31 @@ set -eux -o pipefail
 # Expect actual code to be written to this file
 chmod +x /home/circleci/project/ci_test_script.sh
 
-VOLUME_MOUNTS="-v /home/circleci/project/:/circleci_stuff -v /home/circleci/project/final_pkgs:/final_pkgs -v ${PYTORCH_ROOT}:/pytorch -v ${BUILDER_ROOT}:/builder"
 # Run the docker
 if [ -n "${USE_CUDA_DOCKER_RUNTIME:-}" ]; then
-  export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia ${VOLUME_MOUNTS} -t -d "${DOCKER_IMAGE}")
+  export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d "${DOCKER_IMAGE}")
 else
-  export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined ${VOLUME_MOUNTS} -t -d "${DOCKER_IMAGE}")
+  export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d "${DOCKER_IMAGE}")
+fi
+
+# Copy the envfile and script with all the code to run into the docker.
+docker cp /home/circleci/project/. "$id:/circleci_stuff"
+
+# Copy built packages into the docker to test. This should only exist on the
+# binary test jobs. The package should've been created from a binary build job,
+# whhich persisted the package to a CircleCI workspace, which this job then
+# copies into a GPU enabled docker for testing
+if [[ -d "/home/circleci/project/final_pkgs" ]]; then
+  docker cp /home/circleci/project/final_pkgs "$id:/final_pkgs"
+fi
+
+# Copy the needed repos into the docker. These do not exist in the smoke test
+# jobs, since the smoke test jobs do not need the Pytorch source code.
+if [[ -d "$PYTORCH_ROOT" ]]; then
+  docker cp "$PYTORCH_ROOT" "$id:/pytorch"
+fi
+if [[ -d "$BUILDER_ROOT" ]]; then
+  docker cp "$BUILDER_ROOT" "$id:/builder"
 fi
 
 # Execute the test script that was populated by an earlier section

.circleci/scripts/binary_windows_build.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-set -eux -o pipefail
-
-source "/c/w/env"
-mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR"
-
-export CUDA_VERSION="${DESIRED_CUDA/cu/}"
-export USE_SCCACHE=1
-export SCCACHE_BUCKET=ossci-compiler-cache-windows
-export NIGHTLIES_PYTORCH_ROOT="$PYTORCH_ROOT"
-
-if [[ "$CUDA_VERSION" == "92" || "$CUDA_VERSION" == "100" ]]; then
-  export VC_YEAR=2017
-else
-  export VC_YEAR=2019
-fi
-
-set +x
-export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4:-}
-export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4:-}
-set -x
-
-if [[ "$CIRCLECI" == 'true' && -d "C:\\ProgramData\\Microsoft\\VisualStudio\\Packages\\_Instances" ]]; then
-  mv "C:\\ProgramData\\Microsoft\\VisualStudio\\Packages\\_Instances" .
-  rm -rf "C:\\ProgramData\\Microsoft\\VisualStudio\\Packages"
-  mkdir -p "C:\\ProgramData\\Microsoft\\VisualStudio\\Packages"
-  mv _Instances "C:\\ProgramData\\Microsoft\\VisualStudio\\Packages"
-fi
-
-echo "Free space on filesystem before build:"
-df -h
-
-pushd "$BUILDER_ROOT"
-if [[ "$PACKAGE_TYPE" == 'conda' ]]; then
-  ./windows/internal/build_conda.bat
-elif [[ "$PACKAGE_TYPE" == 'wheel' || "$PACKAGE_TYPE" == 'libtorch' ]]; then
-  ./windows/internal/build_wheels.bat
-fi
-
-echo "Free space on filesystem after build:"
-df -h

.circleci/scripts/binary_windows_test.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-set -eux -o pipefail
-
-source "/c/w/env"
-
-export CUDA_VERSION="${DESIRED_CUDA/cu/}"
-export VC_YEAR=2017
-
-if [[ "$CUDA_VERSION" == "92" || "$CUDA_VERSION" == "100" ]]; then
-  export VC_YEAR=2017
-else
-  export VC_YEAR=2019
-fi
-
-pushd "$BUILDER_ROOT"
-
-./windows/internal/smoke_test.bat
-
-popd

.circleci/scripts/binary_windows_upload.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-set -eu -o pipefail
-set +x
-declare -x "AWS_ACCESS_KEY_ID=${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
-declare -x "AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
-
-#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
-# DO NOT TURN -x ON BEFORE THIS LINE
-#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
-set -eux -o pipefail
-
-source "/env"
-
-# This gets set in binary_populate_env.sh, but lets have a sane default just in case
-PIP_UPLOAD_FOLDER=${PIP_UPLOAD_FOLDER:-nightly/}
-# TODO: Combine CONDA_UPLOAD_CHANNEL and PIP_UPLOAD_FOLDER into one variable
-#       The only difference is the trailing slash
-# Strip trailing slashes if there
-CONDA_UPLOAD_CHANNEL=$(echo "${PIP_UPLOAD_FOLDER}" | sed 's:/*$::')
-BACKUP_BUCKET="s3://pytorch-backup"
-
-retry pip install -q awscli
-pushd /root/workspace/final_pkgs
-# Upload the package to the final location
-if [[ "$PACKAGE_TYPE" == conda ]]; then
-  retry conda install -yq anaconda-client
-  retry anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload  "$(ls)" -u "pytorch-${CONDA_UPLOAD_CHANNEL}" --label main --no-progress --force
-  # Fetch  platform (eg. win-64, linux-64, etc.) from index file
-  # Because there's no actual conda command to read this
-  subdir=$(tar -xOf ./*.bz2 info/index.json | grep subdir  | cut -d ':' -f2 | sed -e 's/[[:space:]]//' -e 's/"//g' -e 's/,//')
-  BACKUP_DIR="conda/${subdir}"
-elif [[ "$PACKAGE_TYPE" == libtorch ]]; then
-  s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-  for pkg in $(ls); do
-    retry aws s3 cp "$pkg" "$s3_dir" --acl public-read
-  done
-  BACKUP_DIR="libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-else
-  s3_dir="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-  retry aws s3 cp "$(ls)" "$s3_dir" --acl public-read
-  BACKUP_DIR="whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
-fi
-
-if [[ -n "${CIRCLE_TAG:-}" ]]; then
-  s3_dir="${BACKUP_BUCKET}/${CIRCLE_TAG}/${BACKUP_DIR}"
-  retry aws s3 cp --recursive . "$s3_dir"
-fi

.circleci/scripts/cpp_doc_push_script.sh

@@ -57,6 +57,7 @@ time python aten/src/ATen/gen.py \
   -s aten/src/ATen \
   -d build/aten/src/ATen \
   aten/src/ATen/Declarations.cwrap \
+  aten/src/THNN/generic/THNN.h \
   aten/src/THCUNN/generic/THCUNN.h \
   aten/src/ATen/nn.yaml \
   aten/src/ATen/native/native_functions.yaml
@@ -72,10 +73,10 @@ time python tools/setup_helpers/generate_code.py \
 
 # Build the docs
 pushd docs/cpp
-pip install breathe==4.13.0 bs4 lxml six
+pip install breathe==4.11.1 bs4 lxml six
 pip install --no-cache-dir -e "git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme"
 pip install exhale>=0.2.1
-pip install sphinx==2.4.4
+pip install sphinx==1.8.5
 # Uncomment once it is fixed
 # pip install -r requirements.txt
 time make VERBOSE=1 html -j

.circleci/scripts/python_doc_push_script.sh

@@ -71,30 +71,8 @@ cp -a ../vision/docs/source source/torchvision
 # Build the docs
 pip -q install -r requirements.txt || true
 if [ "$is_master_doc" = true ]; then
-  # TODO: fix gh-38011 then enable this which changes warnings into errors
-  # export SPHINXOPTS="-WT --keep-going"
   make html
-  make coverage
-  # Now we have the coverage report, we need to make sure it is empty.
-  # Count the number of lines in the file and turn that number into a variable
-  # $lines. The `cut -f1 ...` is to only parse the number, not the filename
-  # Skip the report header by subtracting 2: the header will be output even if
-  # there are no undocumented items.
-  #
-  # Also: see docs/source/conf.py for "coverage_ignore*" items, which should
-  # be documented then removed from there.
-  lines=$(wc -l build/coverage/python.txt 2>/dev/null |cut -f1 -d' ')
-  undocumented=$(($lines - 2))
-  if [ $undocumented -lt 0 ]; then
-    echo coverage output not found
-    exit 1
-  elif [ $undocumented -gt 0 ]; then
-    echo undocumented objects found:
-    cat build/coverage/python.txt
-    exit 1
-  fi
 else
-  # Don't fail the build on coverage problems
   make html-stable
 fi
 
@@ -112,12 +90,6 @@ else
   find "$install_path" -name "*.html" -print0 | xargs -0 perl -pi -w -e "s@master\s+\((\d\.\d\.[A-Fa-f0-9]+\+[A-Fa-f0-9]+)\s+\)@<a href='http://pytorch.org/docs/versions.html'>$version \&#x25BC</a>@g"
 fi
 
-# Prevent Google from indexing $install_path/_modules. This folder contains
-# generated source files.
-# NB: the following only works on gnu sed. The sed shipped with mac os is different.
-# One can `brew install gnu-sed` on a mac and then use "gsed" instead of "sed".
-find "$install_path/_modules" -name "*.html" -print0 | xargs -0 sed -i '/<head>/a \ \ <meta name="robots" content="noindex">'
-
 git add "$install_path" || true
 git status
 git config user.email "soumith+bot@pytorch.org"

.circleci/scripts/setup_ci_environment.sh

@@ -2,7 +2,7 @@
 set -ex -o pipefail
 
 # Set up NVIDIA docker repo
-curl -s -L --retry 3 https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+curl -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
 echo "deb https://nvidia.github.io/libnvidia-container/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
 echo "deb https://nvidia.github.io/nvidia-container-runtime/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
 echo "deb https://nvidia.github.io/nvidia-docker/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
@@ -13,15 +13,6 @@ sudo rm -f /etc/apt/heroku.list
 sudo rm -f /etc/apt/openjdk-r-ubuntu-ppa-xenial.list
 sudo rm -f /etc/apt/partner.list
 
-retry () {
-    $*  || $* || $* || $* || $*
-}
-
-# Method adapted from here: https://askubuntu.com/questions/875213/apt-get-to-retry-downloading
-# (with use of tee to avoid permissions problems)
-# This is better than retrying the whole apt-get command
-echo "APT::Acquire::Retries \"3\";" | sudo tee /etc/apt/apt.conf.d/80-retries
-
 sudo apt-get -y update
 sudo apt-get -y remove linux-image-generic linux-headers-generic linux-generic docker-ce
 # WARNING: Docker version is hardcoded here; you must update the
@@ -36,11 +27,7 @@ sudo apt-get -y remove linux-image-generic linux-headers-generic linux-generic d
 # Ubuntu version (e.g., docker run -it ubuntu:16.04) and then ask
 # apt what the packages you need are.  Note that the CircleCI image
 # comes with Docker.
-#
+sudo apt-get -y install \
-# Using 'retry' here as belt-and-suspenders even though we are
-# presumably retrying at the single-package level via the
-# apt.conf.d/80-retries technique.
-retry sudo apt-get -y install \
   linux-headers-$(uname -r) \
   linux-image-generic \
   moreutils \
@@ -51,11 +38,14 @@ retry sudo apt-get -y install \
 
 sudo pkill -SIGHUP dockerd
 
+retry () {
+    $*  || $* || $* || $* || $*
+}
 
 retry sudo pip -q install awscli==1.16.35
 
 if [ -n "${USE_CUDA_DOCKER_RUNTIME:-}" ]; then
-  DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
+  DRIVER_FN="NVIDIA-Linux-x86_64-430.40.run"
   wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
   sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
   nvidia-smi

.circleci/scripts/setup_linux_system_environment.sh

@@ -2,7 +2,7 @@
 set -eux -o pipefail
 
 # Set up CircleCI GPG keys for apt, if needed
-curl --retry 3 -s -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
+curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
 
 # Stop background apt updates.  Hypothetically, the kill should not
 # be necessary, because stop is supposed to send a kill signal to

.circleci/scripts/should_run_job.py

@@ -0,0 +1,140 @@
+import argparse
+import re
+import sys
+
+# Modify this variable if you want to change the set of default jobs
+# which are run on all pull requests.
+#
+# WARNING: Actually, this is a lie; we're currently also controlling
+# the set of jobs to run via the Workflows filters in CircleCI config.
+
+default_set = set([
+    # PyTorch CPU
+    # Selected oldest Python 2 version to ensure Python 2 coverage
+    'pytorch-linux-xenial-py2.7.9',
+    # PyTorch CUDA
+    'pytorch-linux-xenial-cuda9-cudnn7-py3',
+    # PyTorch ASAN
+    'pytorch-linux-xenial-py3-clang5-asan',
+    # PyTorch DEBUG
+    'pytorch-linux-xenial-py3.6-gcc5.4',
+    # LibTorch
+    'pytorch-libtorch-linux-xenial-cuda9-cudnn7-py3',
+
+    # Caffe2 CPU
+    'caffe2-py2-mkl-ubuntu16.04',
+    # Caffe2 CUDA
+    'caffe2-py3.5-cuda10.1-cudnn7-ubuntu16.04',
+    # Caffe2 ONNX
+    'caffe2-onnx-py2-gcc5-ubuntu16.04',
+    'caffe2-onnx-py3.6-clang7-ubuntu16.04',
+    # Caffe2 Clang
+    'caffe2-py2-clang7-ubuntu16.04',
+    # Caffe2 CMake
+    'caffe2-cmake-cuda9.0-cudnn7-ubuntu16.04',
+    # Caffe2 CentOS
+    'caffe2-py3.6-devtoolset7-cuda9.0-cudnn7-centos7',
+
+    # Binaries
+    'manywheel 2.7mu cpu devtoolset7',
+    'libtorch 2.7m cpu devtoolset7',
+    'libtorch 2.7m cpu gcc5.4_cxx11-abi',
+    'libtorch 2.7 cpu',
+    'libtorch-ios-11.2.1-nightly-x86_64-build',
+    'libtorch-ios-11.2.1-nightly-arm64-build',
+    'libtorch-ios-11.2.1-nightly-binary-build-upload',
+
+    # Caffe2 Android
+    'caffe2-py2-android-ubuntu16.04',
+    # Caffe2 OSX
+    'caffe2-py2-system-macos10.13',
+    # PyTorch OSX
+    'pytorch-macos-10.13-py3',
+    'pytorch-macos-10.13-cuda9.2-cudnn7-py3',
+    # PyTorch Android
+    'pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32-build',
+    'pytorch-linux-xenial-py3-clang5-android-ndk-r19',
+    # PyTorch Android gradle
+    'pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-only-x86_32',
+
+    # Pytorch iOS builds
+    'pytorch-ios-11.2.1-x86_64_build',
+    'pytorch-ios-11.2.1-arm64_build',
+    # PyTorch Mobile builds
+    'pytorch-linux-xenial-py3-clang5-mobile-build',
+
+    # Pytorch backward compatibility check
+    'pytorch-linux-backward-compatibility-check-test',
+
+    # XLA
+    'pytorch-xla-linux-xenial-py3.6-clang7',
+
+    # GraphExecutor config jobs
+    'pytorch-linux-xenial-py3.6-gcc5.4-ge_config_simple-test',
+    'pytorch-linux-xenial-py3.6-gcc5.4-ge_config_legacy-test',
+
+    # Other checks
+    'pytorch-short-perf-test-gpu',
+    'pytorch-python-doc-push',
+    'pytorch-cpp-doc-push',
+])
+
+# Collection of jobs that are *temporarily* excluded from running on PRs.
+# Use this if there is a long-running job breakage that we can't fix with a
+# single revert.
+skip_override = {
+    # example entry:
+    # 'pytorch-cpp-doc-push': "https://github.com/pytorch/pytorch/issues/<related issue>"
+}
+
+# Takes in commit message to analyze via stdin
+#
+# This script will query Git and attempt to determine if we should
+# run the current CI job under question
+#
+# NB: Try to avoid hard-coding names here, so there's less place to update when jobs
+# are updated/renamed
+#
+# Semantics in the presence of multiple tags:
+#   - Let D be the set of default builds
+#   - Let S be the set of explicitly specified builds
+#   - Let O be the set of temporarily skipped builds
+#   - Run S \/ (D - O)
+
+parser = argparse.ArgumentParser()
+parser.add_argument('build_environment')
+args = parser.parse_args()
+
+commit_msg = sys.stdin.read()
+
+# Matches anything that looks like [foo ci] or [ci foo] or [foo test]
+# or [test foo]
+RE_MARKER = re.compile(r'\[(?:([^ \[\]]+) )?(?:ci|test)(?: ([^ \[\]]+))?\]')
+
+markers = RE_MARKER.finditer(commit_msg)
+
+for m in markers:
+    if m.group(1) and m.group(2):
+        print("Unrecognized marker: {}".format(m.group(0)))
+        continue
+    spec = m.group(1) or m.group(2)
+    if spec is None:
+        print("Unrecognized marker: {}".format(m.group(0)))
+        continue
+    if spec in args.build_environment or spec == 'all':
+        print("Accepting {} due to commit marker {}".format(args.build_environment, m.group(0)))
+        sys.exit(0)
+
+skip_override_set = set(skip_override.keys())
+should_run_set = default_set - skip_override_set
+for spec in should_run_set:
+    if spec in args.build_environment:
+        print("Accepting {} as part of default set".format(args.build_environment))
+        sys.exit(0)
+
+print("Rejecting {}".format(args.build_environment))
+for spec, issue in skip_override.items():
+    if spec in args.build_environment:
+        print("This job is temporarily excluded from running on PRs. Reason: {}".format(issue))
+        break
+sys.exit(1)

.circleci/scripts/should_run_job.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -exu -o pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# Check if we should actually run
+echo "BUILD_ENVIRONMENT: ${BUILD_ENVIRONMENT:-}"
+echo "CIRCLE_PULL_REQUEST: ${CIRCLE_PULL_REQUEST:-}"
+if [ -z "${BUILD_ENVIRONMENT:-}" ]; then
+  echo "Cannot run should_run_job.sh if BUILD_ENVIRONMENT is not defined!"
+  echo "CircleCI scripts are probably misconfigured."
+  exit 1
+fi
+if ! [ -e "$SCRIPT_DIR/COMMIT_MSG" ]; then
+  echo "Cannot run should_run_job.sh if you don't have COMMIT_MSG"
+  echo "written out.  Are you perhaps running the wrong copy of this script?"
+  echo "You should be running the copy in ~/workspace; SCRIPT_DIR=$SCRIPT_DIR"
+  exit 1
+fi
+if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then
+  if [[ $CIRCLE_BRANCH != "ci-all/"* ]] && [[ $CIRCLE_BRANCH != "nightly" ]] &&  [[ $CIRCLE_BRANCH != "postnightly" ]] ; then
+    # Don't swallow "script doesn't exist
+    [ -e "$SCRIPT_DIR/should_run_job.py"  ]
+    if ! python "$SCRIPT_DIR/should_run_job.py" "${BUILD_ENVIRONMENT:-}" < "$SCRIPT_DIR/COMMIT_MSG" ; then
+      circleci step halt
+      exit
+    fi
+  fi
+fi

.circleci/scripts/upload_binary_size_to_scuba.py

@@ -1,145 +0,0 @@
-import glob
-import json
-import logging
-import os
-import os.path
-import pathlib
-import re
-import sys
-import time
-import zipfile
-
-import requests
-
-
-def get_size(file_dir):
-    try:
-        # we should only expect one file, if no, something is wrong
-        file_name = glob.glob(os.path.join(file_dir, "*"))[0]
-        return os.stat(file_name).st_size
-    except:
-        logging.exception(f"error getting file from: {file_dir}")
-        return 0
-
-
-def build_message(size):
-    pkg_type, py_ver, cu_ver, *_ = os.environ.get("BUILD_ENVIRONMENT", "").split() + [
-        None,
-        None,
-        None,
-    ]
-    os_name = os.uname()[0].lower()
-    if os_name == "darwin":
-        os_name = "macos"
-    return {
-        "normal": {
-            "os": os_name,
-            "pkg_type": pkg_type,
-            "py_ver": py_ver,
-            "cu_ver": cu_ver,
-            "pr": os.environ.get("CIRCLE_PR_NUMBER"),
-            "build_num": os.environ.get("CIRCLE_BUILD_NUM"),
-            "sha1": os.environ.get("CIRCLE_SHA1"),
-            "branch": os.environ.get("CIRCLE_BRANCH"),
-        },
-        "int": {
-            "time": int(time.time()),
-            "size": size,
-            "commit_time": int(os.environ.get("COMMIT_TIME", "0")),
-        },
-    }
-
-
-def send_message(messages):
-    access_token = os.environ.get("SCRIBE_GRAPHQL_ACCESS_TOKEN")
-    if not access_token:
-        raise ValueError("Can't find access token from environment variable")
-    url = "https://graph.facebook.com/scribe_logs"
-    r = requests.post(
-        url,
-        data={
-            "access_token": access_token,
-            "logs": json.dumps(
-                [
-                    {
-                        "category": "perfpipe_pytorch_binary_size",
-                        "message": json.dumps(message),
-                        "line_escape": False,
-                    }
-                    for message in messages
-                ]
-            ),
-        },
-    )
-    print(r.text)
-    r.raise_for_status()
-
-
-def report_android_sizes(file_dir):
-    def gen_sizes():
-        # we should only expect one file, if no, something is wrong
-        aar_files = list(pathlib.Path(file_dir).rglob("pytorch_android-*.aar"))
-        if len(aar_files) != 1:
-            logging.exception(f"error getting aar files from: {file_dir} / {aar_files}")
-            return
-
-        aar_file = aar_files[0]
-        zf = zipfile.ZipFile(aar_file)
-        for info in zf.infolist():
-            # Scan ".so" libs in `jni` folder. Examples:
-            # jni/arm64-v8a/libfbjni.so
-            # jni/arm64-v8a/libpytorch_jni.so
-            m = re.match(r"^jni/([^/]+)/(.*\.so)$", info.filename)
-            if not m:
-                continue
-            arch, lib = m.groups()
-            # report per architecture library size
-            yield [arch, lib, info.compress_size, info.file_size]
-
-        # report whole package size
-        yield ["aar", aar_file.name, os.stat(aar_file).st_size, 0]
-
-    def gen_messages():
-        android_build_type = os.environ.get("ANDROID_BUILD_TYPE")
-        for arch, lib, comp_size, uncomp_size in gen_sizes():
-            print(android_build_type, arch, lib, comp_size, uncomp_size)
-            yield {
-                "normal": {
-                    "os": "android",
-                    # TODO: create dedicated columns
-                    "pkg_type": "{}/{}/{}".format(android_build_type, arch, lib),
-                    "cu_ver": "",  # dummy value for derived field `build_name`
-                    "py_ver": "",  # dummy value for derived field `build_name`
-                    "pr": os.environ.get("CIRCLE_PR_NUMBER"),
-                    "build_num": os.environ.get("CIRCLE_BUILD_NUM"),
-                    "sha1": os.environ.get("CIRCLE_SHA1"),
-                    "branch": os.environ.get("CIRCLE_BRANCH"),
-                },
-                "int": {
-                    "time": int(time.time()),
-                    "commit_time": int(os.environ.get("COMMIT_TIME", "0")),
-                    "size": comp_size,
-                    "raw_size": uncomp_size,
-                },
-            }
-
-    send_message(list(gen_messages()))
-
-
-if __name__ == "__main__":
-    file_dir = os.environ.get(
-        "PYTORCH_FINAL_PACKAGE_DIR", "/home/circleci/project/final_pkgs"
-    )
-    if len(sys.argv) == 2:
-        file_dir = sys.argv[1]
-    print("checking dir: " + file_dir)
-
-    if "-android" in os.environ.get("BUILD_ENVIRONMENT", ""):
-        report_android_sizes(file_dir)
-    else:
-        size = get_size(file_dir)
-        if size != 0:
-            try:
-                send_message([build_message(size)])
-            except:
-                logging.exception("can't send message")

.circleci/scripts/vs_install.ps1

@@ -1,34 +0,0 @@
-$VS_DOWNLOAD_LINK = "https://aka.ms/vs/15/release/vs_buildtools.exe"
-$COLLECT_DOWNLOAD_LINK = "https://aka.ms/vscollect.exe"
-$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.14.11",
-                                                     "--add Microsoft.Component.MSBuild",
-                                                     "--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
-                                                     "--add Microsoft.VisualStudio.Component.TextTemplating",
-                                                     "--add Microsoft.VisualStudio.Component.VC.CoreIde",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
-                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
-                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81")
-
-curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
-if ($LASTEXITCODE -ne 0) {
-    echo "Download of the VS 2017 installer failed"
-    exit 1
-}
-
-$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
-Remove-Item -Path vs_installer.exe -Force
-$exitCode = $process.ExitCode
-if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
-    echo "VS 2017 installer exited with code $exitCode, which should be one of [0, 3010]."
-    curl.exe --retry 3 -kL $COLLECT_DOWNLOAD_LINK --output Collect.exe
-    if ($LASTEXITCODE -ne 0) {
-        echo "Download of the VS Collect tool failed."
-        exit 1
-    }
-    Start-Process "${PWD}\Collect.exe" -NoNewWindow -Wait -PassThru
-    New-Item -Path "C:\w\build-results" -ItemType "directory" -Force
-    Copy-Item -Path "C:\Users\circleci\AppData\Local\Temp\vslogs.zip" -Destination "C:\w\build-results\"
-    exit 1
-}

.circleci/scripts/windows_cuda_install.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-set -eux -o pipefail
-
-curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe
-7z x cuda_10.1.243_426.00_win10.exe -ocuda_10.1.243_426.00_win10
-cd cuda_10.1.243_426.00_win10
-mkdir cuda_install_logs
-
-set +e
-
-./setup.exe -s nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
-
-set -e
-
-if [[ "${VC_YEAR}" == "2017" ]]; then
-    cp -r CUDAVisualStudioIntegration/extras/visual_studio_integration/MSBuildExtensions/* "C:/Program Files (x86)/Microsoft Visual Studio/2017/${VC_PRODUCT}/Common7/IDE/VC/VCTargets/BuildCustomizations/"
-else
-    cp -r CUDAVisualStudioIntegration/extras/visual_studio_integration/MSBuildExtensions/* "C:/Program Files (x86)/Microsoft Visual Studio/2019/${VC_PRODUCT}/MSBuild/Microsoft/VC/v160/BuildCustomizations/"
-fi
-
-curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/NvToolsExt.7z
-7z x NvToolsExt.7z -oNvToolsExt
-mkdir -p "C:/Program Files/NVIDIA Corporation/NvToolsExt"
-cp -r NvToolsExt/* "C:/Program Files/NVIDIA Corporation/NvToolsExt/"
-export NVTOOLSEXT_PATH="C:\\Program Files\\NVIDIA Corporation\\NvToolsExt\\"
-
-if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe"
-then
-    echo "CUDA installation failed"
-    mkdir -p /c/w/build-results
-    7z a "c:\\w\\build-results\\cuda_install_logs.7z" cuda_install_logs
-    exit 1
-fi
-
-cd ..
-rm -rf ./cuda_10.1.243_426.00_win10
-rm -f ./cuda_10.1.243_426.00_win10.exe

.circleci/validate-docker-version.py

@@ -1,44 +1,43 @@
 #!/usr/bin/env python3
+
+import urllib.request
+import re
+
+import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
 import cimodel.data.caffe2_build_definitions as caffe2_build_definitions
-import cimodel.data.simple.util.docker_constants as pytorch_docker_constants
 
-from yaml import load
+RE_VERSION = re.compile(r'allDeployedVersions = "([0-9,]+)"')
 
-try:
+URL_TEMPLATE = (
-    from yaml import CLoader as Loader
+    "https://raw.githubusercontent.com/pytorch/ossci-job-dsl/"
-except ImportError:
+    "master/src/main/groovy/ossci/{}/DockerVersion.groovy"
-    from yaml import Loader
+)
-
-
-def load_config(filename=".circleci/config.yml"):
-    with open(filename, "r") as fh:
-        return load("".join(fh.readlines()), Loader)
-
-
-def load_tags_for_projects(workflow_config):
-    return {
-        v["ecr_gc_job"]["project"]: v["ecr_gc_job"]["tags_to_keep"]
-        for v in workflow_config["workflows"]["ecr_gc"]["jobs"]
-        if isinstance(v, dict) and "ecr_gc_job" in v
-    }
-
-
-def check_version(job, tags, expected_version):
-    valid_versions = tags[job].split(",")
-    if expected_version not in valid_versions:
-        raise RuntimeError(
-            "We configured {} to use Docker version {}; but this "
-            "version is not configured in job ecr_gc_job_for_{}.  Non-deployed versions will be "
-            "garbage collected two weeks after they are created.  DO NOT LAND "
-            "THIS TO MASTER without also updating ossci-job-dsl with this version."
-            "\n\nDeployed versions: {}".format(job, expected_version, job, tags[job])
-        )
 
+def check_version(job, expected_version):
+    url = URL_TEMPLATE.format(job)
+    with urllib.request.urlopen(url) as f:
+        contents = f.read().decode('utf-8')
+        m = RE_VERSION.search(contents)
+        if not m:
+            raise RuntimeError(
+                "Unbelievable! I could not find the variable allDeployedVersions in "
+                "{}; did the organization of ossci-job-dsl change?\n\nFull contents:\n{}"
+                .format(url, contents)
+            )
+        valid_versions = [int(v) for v in m.group(1).split(',')]
+        if expected_version not in valid_versions:
+            raise RuntimeError(
+                "We configured {} to use Docker version {}; but this "
+                "version is not deployed in {}.  Non-deployed versions will be "
+                "garbage collected two weeks after they are created.  DO NOT LAND "
+                "THIS TO MASTER without also updating ossci-job-dsl with this version."
+                "\n\nDeployed versions: {}"
+                .format(job, expected_version, url, m.group(1))
+            )
 
 def validate_docker_version():
-    tags = load_tags_for_projects(load_config())
+    check_version('pytorch', pytorch_build_definitions.DOCKER_IMAGE_VERSION)
-    check_version("pytorch", tags, pytorch_docker_constants.DOCKER_IMAGE_TAG)
+    check_version('caffe2', caffe2_build_definitions.DOCKER_IMAGE_VERSION)
-    check_version("caffe2", tags, caffe2_build_definitions.DOCKER_IMAGE_VERSION)
 
 
 if __name__ == "__main__":

.circleci/verbatim-sources/binary-build-params.yml

@@ -52,15 +52,3 @@ binary_mac_params: &binary_mac_params
   environment:
     BUILD_ENVIRONMENT: << parameters.build_environment >>
 
-binary_windows_params: &binary_windows_params
-  parameters:
-    build_environment:
-      type: string
-      default: ""
-    executor:
-      type: string
-      default: "windows-cpu-with-nvidia-cuda"
-  environment:
-    BUILD_ENVIRONMENT: << parameters.build_environment >>
-    BUILD_FOR_SYSTEM: windows
-    JOB_EXECUTOR: <<parameters.executor>>

.circleci/verbatim-sources/binary-build-tests.yml

@@ -0,0 +1,20 @@
+
+# There is currently no testing for libtorch TODO
+#  binary_linux_libtorch_2.7m_cpu_test:
+#    environment:
+#      BUILD_ENVIRONMENT: "libtorch 2.7m cpu"
+#    resource_class: gpu.medium
+#    <<: *binary_linux_test
+#
+#  binary_linux_libtorch_2.7m_cu90_test:
+#    environment:
+#      BUILD_ENVIRONMENT: "libtorch 2.7m cu90"
+#    resource_class: gpu.medium
+#    <<: *binary_linux_test
+#
+#  binary_linux_libtorch_2.7m_cu100_test:
+#    environment:
+#      BUILD_ENVIRONMENT: "libtorch 2.7m cu100"
+#    resource_class: gpu.medium
+#    <<: *binary_linux_test
+

.circleci/verbatim-sources/binary-job-specs.yml

@@ -2,7 +2,7 @@
     <<: *binary_linux_build_params
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
+    - should_run_job
     - run:
         <<: *binary_checkout
     - run:
@@ -19,8 +19,8 @@
             elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then
               retry apt-get update
               retry apt-get -y install expect moreutils
-              retry conda install -y -c eumetsat expect
+              conda install -y -c eumetsat expect
-              retry conda install -y cmake
+              conda install -y cmake
             fi
     - run:
         name: Update compiler to devtoolset7
@@ -41,28 +41,10 @@
         no_output_timeout: "1h"
         command: |
             source "/pytorch/.circleci/scripts/binary_linux_build.sh"
-    - run:
-        name: Output binary sizes
-        no_output_timeout: "1m"
-        command: |
-            ls -lah /final_pkgs
-    - run:
-        name: save binary size
-        no_output_timeout: "5m"
-        command: |
-            source /env
-            cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
-            pip3 install requests && \
-            SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
-            python3 /pytorch/.circleci/scripts/upload_binary_size_to_scuba.py || exit 0
-
     - persist_to_workspace:
         root: /
         paths: final_pkgs
 
-    - store_artifacts:
-        path: /final_pkgs
-
     # This should really just be another step of the binary_linux_build job above.
     # This isn't possible right now b/c the build job uses the docker executor
     # (otherwise they'd be really really slow) but this one uses the macine
@@ -74,7 +56,7 @@
         image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
+    - should_run_job
     # TODO: We shouldn't attach the workspace multiple times
     - attach_workspace:
         at: /home/circleci/project
@@ -87,7 +69,7 @@
     - run:
         name: Prepare test code
         no_output_timeout: "1h"
-        command: .circleci/scripts/binary_linux_test.sh
+        command: ~/workspace/.circleci/scripts/binary_linux_test.sh
     - run:
         <<: *binary_run_in_docker
 
@@ -97,7 +79,7 @@
         image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
+    - should_run_job
     - setup_linux_system_environment
     - setup_ci_environment
     - attach_workspace:
@@ -109,7 +91,7 @@
     - run:
         name: Upload
         no_output_timeout: "1h"
-        command: .circleci/scripts/binary_linux_upload.sh
+        command: ~/workspace/.circleci/scripts/binary_linux_upload.sh
 
   # Nighlty build smoke tests defaults
   # These are the second-round smoke tests. These make sure that the binaries are
@@ -121,7 +103,10 @@
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    - checkout
+    - attach_workspace:
+        at: ~/workspace
+    - attach_workspace:
+        at: /home/circleci/project
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -145,9 +130,12 @@
   smoke_mac_test:
     <<: *binary_linux_test_upload_params
     macos:
-      xcode: "9.4.1"
+      xcode: "9.0"
     steps:
-      - checkout
+      - attach_workspace:
+          at: ~/workspace
+      - attach_workspace: # TODO - we can `cp` from ~/workspace
+          at: /Users/distiller/project
       - run:
           <<: *binary_checkout
       - run:
@@ -170,10 +158,10 @@
   binary_mac_build:
     <<: *binary_mac_params
     macos:
-      xcode: "9.4.1"
+      xcode: "9.0"
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
+    - should_run_job
     - run:
         <<: *binary_checkout
     - run:
@@ -211,10 +199,10 @@
   binary_mac_upload: &binary_mac_upload
     <<: *binary_mac_params
     macos:
-      xcode: "9.4.1"
+      xcode: "9.0"
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
+    - should_run_job
     - run:
         <<: *binary_checkout
     - run:
@@ -239,6 +227,7 @@
     steps:
     - attach_workspace:
         at: ~/workspace
+    - should_run_job
     - checkout
     - run_brew_for_ios_build
     - run:
@@ -258,14 +247,15 @@
     - persist_to_workspace:
         root: /Users/distiller/workspace/
         paths: ios
-
+  
-  binary_ios_upload:
+  binary_ios_upload: 
     <<: *pytorch_ios_params
     macos:
       xcode: "11.2.1"
     steps:
     - attach_workspace:
         at: ~/workspace
+    - should_run_job
     - checkout
     - run_brew_for_ios_build
     - run:
@@ -275,108 +265,3 @@
           script="/Users/distiller/project/.circleci/scripts/binary_ios_upload.sh"
           cat "$script"
           source "$script"
-
-  binary_windows_build:
-    <<: *binary_windows_params
-    parameters:
-      build_environment:
-        type: string
-        default: ""
-      executor:
-        type: string
-        default: "windows-cpu-with-nvidia-cuda"
-    executor: <<parameters.executor>>
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
-    - run:
-        <<: *binary_checkout
-    - run:
-        <<: *binary_populate_env
-    - run:
-        name: Build
-        no_output_timeout: "1h"
-        command: |
-          set -eux -o pipefail
-          script="/c/w/p/.circleci/scripts/binary_windows_build.sh"
-          cat "$script"
-          source "$script"
-    - persist_to_workspace:
-        root: "C:/w"
-        paths: final_pkgs
-
-  binary_windows_test:
-    <<: *binary_windows_params
-    parameters:
-      build_environment:
-        type: string
-        default: ""
-      executor:
-        type: string
-        default: "windows-cpu-with-nvidia-cuda"
-    executor: <<parameters.executor>>
-    steps:
-    - checkout
-    - attach_workspace:
-        at: c:/users/circleci/project
-    - run:
-        <<: *binary_checkout
-    - run:
-        <<: *binary_populate_env
-    - run:
-        name: Test
-        no_output_timeout: "1h"
-        command: |
-          set -eux -o pipefail
-          script="/c/w/p/.circleci/scripts/binary_windows_test.sh"
-          cat "$script"
-          source "$script"
-
-  binary_windows_upload:
-    <<: *binary_windows_params
-    docker:
-      - image: continuumio/miniconda
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
-    - attach_workspace:
-        at: /root/workspace
-    - run:
-        <<: *binary_checkout
-    - run:
-        <<: *binary_populate_env
-    - run:
-        name: Upload
-        no_output_timeout: "10m"
-        command: |
-          set -eux -o pipefail
-          script="/pytorch/.circleci/scripts/binary_windows_upload.sh"
-          cat "$script"
-          source "$script"
-
-  smoke_windows_test:
-    <<: *binary_windows_params
-    parameters:
-      build_environment:
-        type: string
-        default: ""
-      executor:
-        type: string
-        default: "windows-cpu-with-nvidia-cuda"
-    executor: <<parameters.executor>>
-    steps:
-    - checkout
-    - run:
-        <<: *binary_checkout
-    - run:
-        <<: *binary_populate_env
-    - run:
-        name: Test
-        no_output_timeout: "1h"
-        command: |
-          set -eux -o pipefail
-          export TEST_NIGHTLY_PACKAGE=1
-          script="/c/w/p/.circleci/scripts/binary_windows_test.sh"
-          cat "$script"
-          source "$script"
-

.circleci/verbatim-sources/binary_update_htmls.yml

@@ -10,7 +10,8 @@
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    - checkout
+    - attach_workspace:
+        at: ~/workspace
     - setup_linux_system_environment
     - run:
         <<: *binary_checkout
@@ -27,15 +28,6 @@
     # make sure it has the same upload folder as the job it's attached to. This
     # function is idempotent, so it won't hurt anything; it's just a little
     # unnescessary"
-    - run:
-        name: define PIP_UPLOAD_FOLDER
-        command: |
-          our_upload_folder=nightly/
-          # On tags upload to test instead
-          if [[ -n "${CIRCLE_TAG}" ]]; then
-            our_upload_folder=test/
-          fi
-          echo "export PIP_UPLOAD_FOLDER=${our_upload_folder}" >> ${BASH_ENV}
     - run:
         name: Update s3 htmls
         no_output_timeout: "1h"
@@ -50,3 +42,55 @@
           }
           retry pip install awscli==1.6
           "/home/circleci/project/builder/cron/update_s3_htmls.sh"
+
+  # Update s3 htmls for the nightlies
+  update_s3_htmls_for_nightlies:
+    environment:
+      PIP_UPLOAD_FOLDER: "nightly/"
+    <<: *update_s3_htmls
+
+  # Update s3 htmls for the nightlies for devtoolset7
+  update_s3_htmls_for_nightlies_devtoolset7:
+    environment:
+      PIP_UPLOAD_FOLDER: "nightly/devtoolset7/"
+    <<: *update_s3_htmls
+
+
+  # upload_binary_logs job
+  # The builder hud at pytorch.org/builder shows the sizes of all the binaries
+  # over time. It gets this info from html files stored in S3, which this job
+  # populates every day.
+  upload_binary_sizes: &upload_binary_sizes
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - setup_linux_system_environment
+    - run:
+        <<: *binary_checkout
+    - run:
+        <<: *binary_install_miniconda
+    - run:
+        name: Upload binary sizes
+        no_output_timeout: "1h"
+        command: |
+          set +x
+          echo "declare -x \"AWS_ACCESS_KEY_ID=${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}\"" > /home/circleci/project/env
+          echo "declare -x \"AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}\"" >> /home/circleci/project/env
+          export DATE="$(date -u +%Y_%m_%d)"
+          retry () {
+              $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+          }
+          source /home/circleci/project/env
+          set -eux -o pipefail
+
+          # This is hardcoded to match binary_install_miniconda.sh
+          export PATH="/home/circleci/project/miniconda/bin:$PATH"
+          # Not any awscli will work. Most won't. This one will work
+          retry conda create -qyn aws36 python=3.6
+          source activate aws36
+          pip install awscli==1.16.46
+
+          "/home/circleci/project/builder/cron/upload_binary_sizes.sh"
+

.circleci/verbatim-sources/build-parameters/promote-build-params.yml

@@ -1,14 +0,0 @@
-
-promote_common: &promote_common
-  docker:
-    - image: pytorch/release
-  parameters:
-    package_name:
-      description: "package name to promote"
-      type: string
-      default: ""
-  environment:
-    PACKAGE_NAME: << parameters.package_name >>
-    ANACONDA_API_TOKEN: ${CONDA_PYTORCHBOT_TOKEN}
-    AWS_ACCESS_KEY_ID: ${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}
-    AWS_SECRET_ACCESS_KEY: ${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}

.circleci/verbatim-sources/build-parameters/pytorch-build-params.yml

@@ -1,85 +0,0 @@
-pytorch_params: &pytorch_params
-  parameters:
-    build_environment:
-      type: string
-      default: ""
-    docker_image:
-      type: string
-      default: ""
-    resource_class:
-      type: string
-      default: "large"
-    use_cuda_docker_runtime:
-      type: string
-      default: ""
-    build_only:
-      type: string
-      default: ""
-  environment:
-    BUILD_ENVIRONMENT: << parameters.build_environment >>
-    DOCKER_IMAGE: << parameters.docker_image >>
-    USE_CUDA_DOCKER_RUNTIME: << parameters.use_cuda_docker_runtime >>
-    BUILD_ONLY: << parameters.build_only >>
-  resource_class: << parameters.resource_class >>
-
-pytorch_ios_params: &pytorch_ios_params
-  parameters:
-    build_environment:
-      type: string
-      default: ""
-    ios_arch:
-      type: string
-      default: ""
-    ios_platform:
-      type: string
-      default: ""
-    op_list:
-      type: string
-      default: ""
-  environment:
-    BUILD_ENVIRONMENT: << parameters.build_environment >>
-    IOS_ARCH: << parameters.ios_arch >>
-    IOS_PLATFORM: << parameters.ios_platform >>
-    SELECTED_OP_LIST: << parameters.op_list >>
-
-pytorch_windows_params: &pytorch_windows_params
-  parameters:
-    executor:
-      type: string
-      default: "windows-cpu-with-nvidia-cuda"
-    build_environment:
-      type: string
-      default: ""
-    test_name:
-      type: string
-      default: ""
-    cuda_version:
-      type: string
-      default: "10"
-    python_version:
-      type: string
-      default: "3.6"
-    vc_version:
-      type: string
-      default: "14.11"
-    vc_year:
-      type: string
-      default: "2017"
-    vc_product:
-      type: string
-      default: "BuildTools"
-    use_cuda:
-      type: string
-      default: ""
-  environment:
-    BUILD_ENVIRONMENT: <<parameters.build_environment>>
-    SCCACHE_BUCKET: "ossci-compiler-cache"
-    CUDA_VERSION: <<parameters.cuda_version>>
-    PYTHON_VERSION: <<parameters.python_version>>
-    VC_VERSION: <<parameters.vc_version>>
-    VC_YEAR: <<parameters.vc_year>>
-    VC_PRODUCT: <<parameters.vc_product>>
-    USE_CUDA: <<parameters.use_cuda>>
-    TORCH_CUDA_ARCH_LIST: "7.5"
-    JOB_BASE_NAME: <<parameters.test_name>>
-    JOB_EXECUTOR: <<parameters.executor>>

.circleci/verbatim-sources/caffe2-build-params.yml

@@ -25,3 +25,4 @@ caffe2_params: &caffe2_params
     DOCKER_IMAGE: << parameters.docker_image >>
     BUILD_ONLY: << parameters.build_only >>
   resource_class: << parameters.resource_class >>
+

.circleci/verbatim-sources/caffe2-job-specs.yml

@@ -4,8 +4,9 @@
       image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
+    - should_run_job
     - setup_linux_system_environment
+    - checkout
     - setup_ci_environment
     - run:
         name: Build
@@ -63,7 +64,7 @@
       image: ubuntu-1604:201903-01
     steps:
     # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
+    - should_run_job
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -123,9 +124,10 @@
   caffe2_macos_build:
     <<: *caffe2_params
     macos:
-      xcode: "9.4.1"
+      xcode: "9.0"
     steps:
       # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
+      - should_run_job
       - checkout
       - run_brew_for_macos_build
       - run:
@@ -149,7 +151,7 @@
             # Install Anaconda if we need to
             if [ -n "${CAFFE2_USE_ANACONDA}" ]; then
               rm -rf ${TMPDIR}/anaconda
-              curl --retry 3 -o ${TMPDIR}/conda.sh https://repo.anaconda.com/miniconda/Miniconda${ANACONDA_VERSION}-latest-MacOSX-x86_64.sh
+              curl -o ${TMPDIR}/conda.sh https://repo.continuum.io/miniconda/Miniconda${ANACONDA_VERSION}-latest-MacOSX-x86_64.sh
               chmod +x ${TMPDIR}/conda.sh
               /bin/bash ${TMPDIR}/conda.sh -b -p ${TMPDIR}/anaconda
               rm -f ${TMPDIR}/conda.sh
@@ -160,7 +162,7 @@
             pip -q install numpy
 
             # Install sccache
-            sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+            sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
             sudo chmod +x /usr/local/bin/sccache
             export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
 

.circleci/verbatim-sources/commands.yml

@@ -1,23 +1,18 @@
 commands:
-
+  # NB: This command must be run as the first command in a job. It
-  # Must be run after attaching workspace from previous steps
+  # attaches the workspace at ~/workspace; this workspace is generated
-  load_shared_env:
+  # by the setup job. Note that ~/workspace is not the default working
-    description: "Loads .circleci/shared/env_file into ${BASH_ENV}"
+  # directory (that's ~/project).
-    parameters:
+  should_run_job:
-      # For some weird reason we decide to reattach our workspace to ~/workspace so
+    description: "Test if the job should run or not"
-      # in the vein of making it simple let's assume our share env_file is here
-      root:
-        type: string
-        default: "~/workspace"
     steps:
+      - attach_workspace:
+          name: Attaching workspace
+          at: ~/workspace
       - run:
-          name: "Load .circleci/shared/env_file into ${BASH_ENV}"
+          name: Should run job
-          command: |
+          no_output_timeout: "2m"
-            if [[ -f  "<< parameters.root >>/.circleci/shared/env_file" ]]; then
+          command: ~/workspace/.circleci/scripts/should_run_job.sh
-              cat << parameters.root >>/.circleci/shared/env_file >> ${BASH_ENV}
-            else
-              echo "We didn't have a shared env file, that's weird"
-            fi
 
   # This system setup script is meant to run before the CI-related scripts, e.g.,
   # installing Git client, checking out code, setting up CI env, and
@@ -27,14 +22,14 @@ commands:
       - run:
           name: Set Up System Environment
           no_output_timeout: "1h"
-          command: .circleci/scripts/setup_linux_system_environment.sh
+          command: ~/workspace/.circleci/scripts/setup_linux_system_environment.sh
 
   setup_ci_environment:
     steps:
       - run:
           name: Set Up CI Environment After attach_workspace
           no_output_timeout: "1h"
-          command: .circleci/scripts/setup_ci_environment.sh
+          command: ~/workspace/.circleci/scripts/setup_ci_environment.sh
 
   brew_update:
     description: "Update Homebrew and install base formulae"
@@ -93,41 +88,3 @@ commands:
       - brew_update
       - brew_install:
           formulae: libtool
-
-  optional_merge_target_branch:
-    steps:
-      - run:
-          name: (Optional) Merge target branch
-          no_output_timeout: "10m"
-          command: |
-            if [ -n "$CIRCLE_PULL_REQUEST" ]; then
-              PR_NUM=$(basename $CIRCLE_PULL_REQUEST)
-              CIRCLE_PR_BASE_BRANCH=$(curl -s https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$PR_NUM | jq -r '.base.ref')
-              if [[ "${BUILD_ENVIRONMENT}" == *"xla"* || "${BUILD_ENVIRONMENT}" == *"gcc5"* ]] ; then
-                set -x
-                git config --global user.email "circleci.ossci@gmail.com"
-                git config --global user.name "CircleCI"
-                git config remote.origin.url https://github.com/pytorch/pytorch.git
-                git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master
-                git fetch --tags --progress https://github.com/pytorch/pytorch.git +refs/heads/master:refs/remotes/origin/master --depth=100 --quiet
-                # PRs generated from ghstack has format CIRCLE_PR_BASE_BRANCH=gh/xxx/1234/base
-                if [[ "${CIRCLE_PR_BASE_BRANCH}" == "gh/"* ]]; then
-                  CIRCLE_PR_BASE_BRANCH=master
-                fi
-                export GIT_MERGE_TARGET=`git log -n 1 --pretty=format:"%H" origin/$CIRCLE_PR_BASE_BRANCH`
-                echo "GIT_MERGE_TARGET: " ${GIT_MERGE_TARGET}
-                export GIT_COMMIT=${CIRCLE_SHA1}
-                echo "GIT_COMMIT: " ${GIT_COMMIT}
-                git checkout -f ${GIT_COMMIT}
-                git reset --hard ${GIT_COMMIT}
-                git merge --allow-unrelated-histories --no-edit --no-ff ${GIT_MERGE_TARGET}
-                echo "Merged $CIRCLE_PR_BASE_BRANCH branch before building in environment $BUILD_ENVIRONMENT"
-                set +x
-              else
-                echo "No need to merge with $CIRCLE_PR_BASE_BRANCH, skipping..."
-              fi
-            else
-              echo "This is not a pull request, skipping..."
-            fi
-
-

.circleci/verbatim-sources/docker_build_job.yml

@@ -0,0 +1,21 @@
+  docker_build_job:
+      parameters:
+        image_name:
+          type: string
+          default: ""
+      machine:
+        image: ubuntu-1604:201903-01
+      resource_class: large
+      environment:
+        IMAGE_NAME: << parameters.image_name >>
+      steps:
+        - checkout
+        - run:
+            name: build_docker_image_<< parameters.image_name >>
+            no_output_timeout: "1h"
+            command: |
+              set +x
+              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
+              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
+              set -x
+              cd .circleci/docker && ./build_docker.sh

.circleci/verbatim-sources/header-section.yml

@@ -1,34 +1,21 @@
 # WARNING: DO NOT EDIT THIS FILE DIRECTLY!!!
 # See the README.md in this directory.
 
-# IMPORTANT: To update Docker image version, please follow
+# IMPORTANT: To update Docker image version, please first update
-# the instructions at
+# https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/pytorch/DockerVersion.groovy and
-# https://github.com/pytorch/pytorch/wiki/Docker-image-build-on-CircleCI
+# https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/caffe2/DockerVersion.groovy,
+# and then update DOCKER_IMAGE_VERSION at the top of the following files:
+# * cimodel/data/pytorch_build_definitions.py
+# * cimodel/data/caffe2_build_definitions.py
+# And the inline copies of the variable in
+# * verbatim-sources/job-specs-custom.yml
+#   (grep for DOCKER_IMAGE)
 
 version: 2.1
 
-parameters:
-  run_binary_tests:
-    type: boolean
-    default: false
-
 docker_config_defaults: &docker_config_defaults
   user: jenkins
   aws_auth:
     # This IAM user only allows read-write access to ECR
     aws_access_key_id: ${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_WRITE_V4}
     aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_WRITE_V4}
-
-executors:
-  windows-with-nvidia-gpu:
-    machine:
-      resource_class: windows.gpu.nvidia.medium
-      image: windows-server-2019-nvidia:stable
-      shell: bash.exe
-
-  windows-cpu-with-nvidia-cuda:
-    machine:
-      # we will change to CPU host when it's ready
-      resource_class: windows.xlarge
-      image: windows-server-2019-vs2019:stable
-      shell: bash.exe

.circleci/verbatim-sources/job-specs-custom.yml

@@ -2,12 +2,13 @@
     environment:
       BUILD_ENVIRONMENT: pytorch-python-doc-push
       # TODO: stop hardcoding this
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3:405"
     resource_class: large
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    - checkout
+    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
+    - should_run_job
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -38,26 +39,21 @@
 
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
-          mkdir -p ~/workspace/build_artifacts
-          docker cp $id:/var/lib/jenkins/workspace/pytorch.github.io/docs/master ~/workspace/build_artifacts
-
           # Save the docs build so we can debug any problems
           export DEBUG_COMMIT_DOCKER_IMAGE=${COMMIT_DOCKER_IMAGE}-debug
           docker commit "$id" ${DEBUG_COMMIT_DOCKER_IMAGE}
           time docker push ${DEBUG_COMMIT_DOCKER_IMAGE}
-    - store_artifacts:
-        path: ~/workspace/build_artifacts/master
-        destination: docs
 
   pytorch_cpp_doc_push:
     environment:
       BUILD_ENVIRONMENT: pytorch-cpp-doc-push
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3:405"
     resource_class: large
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    - checkout
+    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
+    - should_run_job
     - setup_linux_system_environment
     - setup_ci_environment
     - run:
@@ -97,8 +93,10 @@
     environment:
       BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-build
     macos:
-      xcode: "9.4.1"
+      xcode: "9.0"
     steps:
+      # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
+      - should_run_job
       - checkout
       - run_brew_for_macos_build
       - run:
@@ -109,7 +107,7 @@
             export IN_CIRCLECI=1
 
             # Install sccache
-            sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+            sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
             sudo chmod +x /usr/local/bin/sccache
             export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
 
@@ -122,20 +120,24 @@
             chmod a+x .jenkins/pytorch/macos-build.sh
             unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
 
+            # copy with -a to preserve relative structure (e.g., symlinks), and be recursive
+            cp -a ~/project ~/workspace
+
       - persist_to_workspace:
-          root: /Users/distiller/workspace/
+          root: ~/workspace
           paths:
             - miniconda3
+            - project
 
   pytorch_macos_10_13_py3_test:
     environment:
       BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-test
     macos:
-      xcode: "9.4.1"
+      xcode: "9.0"
     steps:
-      - checkout
+      # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-      - attach_workspace:
+      # This workspace also carries binaries from the build job
-          at: ~/workspace
+      - should_run_job
       - run_brew_for_macos_build
       - run:
           name: Test
@@ -144,22 +146,74 @@
             set -e
             export IN_CIRCLECI=1
 
+            # copy with -a to preserve relative structure (e.g., symlinks), and be recursive
+            cp -a ~/workspace/project/. ~/project
+
             chmod a+x .jenkins/pytorch/macos-test.sh
             unbuffer .jenkins/pytorch/macos-test.sh 2>&1 | ts
       - store_test_results:
           path: test/test-reports
 
+  pytorch_macos_10_13_cuda9_2_cudnn7_py3_build:
+    environment:
+      BUILD_ENVIRONMENT: pytorch-macos-10.13-cuda9.2-cudnn7-py3-build
+    macos:
+      xcode: "9.0"
+    steps:
+      # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
+      - should_run_job
+      - checkout
+      - run_brew_for_macos_build
+      - run:
+          name: Build
+          no_output_timeout: "1h"
+          command: |
+            set -e
+
+            export IN_CIRCLECI=1
+
+            # Install CUDA 9.2
+            sudo rm -rf ~/cuda_9.2.64_mac_installer.app || true
+            curl https://s3.amazonaws.com/ossci-macos/cuda_9.2.64_mac_installer.zip -o ~/cuda_9.2.64_mac_installer.zip
+            unzip ~/cuda_9.2.64_mac_installer.zip -d ~/
+            sudo ~/cuda_9.2.64_mac_installer.app/Contents/MacOS/CUDAMacOSXInstaller --accept-eula --no-window
+            sudo cp /usr/local/cuda/lib/libcuda.dylib /Developer/NVIDIA/CUDA-9.2/lib/libcuda.dylib
+            sudo rm -rf /usr/local/cuda || true
+
+            # Install cuDNN 7.1 for CUDA 9.2
+            curl https://s3.amazonaws.com/ossci-macos/cudnn-9.2-osx-x64-v7.1.tgz -o ~/cudnn-9.2-osx-x64-v7.1.tgz
+            rm -rf ~/cudnn-9.2-osx-x64-v7.1 && mkdir ~/cudnn-9.2-osx-x64-v7.1
+            tar -xzvf ~/cudnn-9.2-osx-x64-v7.1.tgz -C ~/cudnn-9.2-osx-x64-v7.1
+            sudo cp ~/cudnn-9.2-osx-x64-v7.1/cuda/include/cudnn.h /Developer/NVIDIA/CUDA-9.2/include/
+            sudo cp ~/cudnn-9.2-osx-x64-v7.1/cuda/lib/libcudnn* /Developer/NVIDIA/CUDA-9.2/lib/
+            sudo chmod a+r /Developer/NVIDIA/CUDA-9.2/include/cudnn.h /Developer/NVIDIA/CUDA-9.2/lib/libcudnn*
+
+            # Install sccache
+            sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+            sudo chmod +x /usr/local/bin/sccache
+            export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+            # This IAM user allows write access to S3 bucket for sccache
+            set +x
+            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}
+            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}
+            set -x
+
+            git submodule sync && git submodule update -q --init --recursive
+            chmod a+x .jenkins/pytorch/macos-build.sh
+            unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
+
   pytorch_android_gradle_build:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
       PYTHON_VERSION: "3.6"
     resource_class: large
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    - checkout
+    - should_run_job
     - setup_linux_system_environment
+    - checkout
     - setup_ci_environment
     - run:
         name: pytorch android gradle build
@@ -193,7 +247,7 @@
           export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_arm_v7a" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
-          mkdir -p ~/workspace/build_android_install_arm_v7a
+          mkdir ~/workspace/build_android_install_arm_v7a
           docker cp $id_arm_v7a:/var/lib/jenkins/workspace/build_android/install ~/workspace/build_android_install_arm_v7a
 
           # x86_64
@@ -203,7 +257,7 @@
           export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_x86_64" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
-          mkdir -p ~/workspace/build_android_install_x86_64
+          mkdir ~/workspace/build_android_install_x86_64
           docker cp $id_x86_64:/var/lib/jenkins/workspace/build_android/install ~/workspace/build_android_install_x86_64
 
           # arm-v8a
@@ -213,7 +267,7 @@
           export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_arm_v8a" bash) 2>&1'
           echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
-          mkdir -p ~/workspace/build_android_install_arm_v8a
+          mkdir ~/workspace/build_android_install_arm_v8a
           docker cp $id_arm_v8a:/var/lib/jenkins/workspace/build_android/install ~/workspace/build_android_install_arm_v8a
 
           docker cp ~/workspace/build_android_install_arm_v7a $id_x86_32:/var/lib/jenkins/workspace/build_android_install_arm_v7a
@@ -230,26 +284,6 @@
           output_image=$docker_image_libtorch_android_x86_32-gradle
           docker commit "$id_x86_32" ${output_image}
           time docker push ${output_image}
-    - run:
-        name: save binary size
-        no_output_timeout: "5m"
-        command: |
-          docker_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-x86_32-gradle
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image})
-          echo "docker-id: $id"
-          cat \<< EOL | docker exec -u jenkins -i "$id" bash
-          # ============================== Begin Docker ==============================
-          cd workspace
-          source ./env
-          export ANDROID_BUILD_TYPE="prebuild"
-          export COMMIT_TIME=\$(git log --max-count=1 --format=%ct || echo 0)
-          export CIRCLE_BUILD_NUM="${CIRCLE_BUILD_NUM}"
-          export CIRCLE_SHA1="${CIRCLE_SHA1}"
-          export CIRCLE_BRANCH="${CIRCLE_BRANCH}"
-          export SCRIBE_GRAPHQL_ACCESS_TOKEN="${SCRIBE_GRAPHQL_ACCESS_TOKEN}"
-          python .circleci/scripts/upload_binary_size_to_scuba.py android
-          # ==============================  End Docker  ==============================
-          EOL
     - store_artifacts:
         path: ~/workspace/build_android_artifacts/artifacts.tgz
         destination: artifacts.tgz
@@ -257,13 +291,13 @@
   pytorch_android_publish_snapshot:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-publish-snapshot
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
       PYTHON_VERSION: "3.6"
     resource_class: large
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    - checkout
+    - should_run_job
     - setup_linux_system_environment
     - checkout
     - setup_ci_environment
@@ -293,13 +327,13 @@
   pytorch_android_gradle_build-x86_32:
     environment:
       BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-only-x86_32
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:209062ef-ab58-422a-b295-36c4eed6e906"
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
       PYTHON_VERSION: "3.6"
     resource_class: large
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    - checkout
+    - should_run_job
     - run:
         name: filter out not PR runs
         no_output_timeout: "5m"
@@ -332,26 +366,6 @@
           output_image=${docker_image_libtorch_android_x86_32}-gradle
           docker commit "$id" ${output_image}
           time docker push ${output_image}
-    - run:
-        name: save binary size
-        no_output_timeout: "5m"
-        command: |
-          docker_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-x86_32-gradle
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image})
-          echo "docker-id: $id"
-          cat \<< EOL | docker exec -u jenkins -i "$id" bash
-          # ============================== Begin Docker ==============================
-          cd workspace
-          source ./env
-          export ANDROID_BUILD_TYPE="prebuild-single"
-          export COMMIT_TIME=\$(git log --max-count=1 --format=%ct || echo 0)
-          export CIRCLE_BUILD_NUM="${CIRCLE_BUILD_NUM}"
-          export CIRCLE_SHA1="${CIRCLE_SHA1}"
-          export CIRCLE_BRANCH="${CIRCLE_BRANCH}"
-          export SCRIBE_GRAPHQL_ACCESS_TOKEN="${SCRIBE_GRAPHQL_ACCESS_TOKEN}"
-          python .circleci/scripts/upload_binary_size_to_scuba.py android
-          # ==============================  End Docker  ==============================
-          EOL
     - store_artifacts:
         path: ~/workspace/build_android_x86_32_artifacts/artifacts.tgz
         destination: artifacts.tgz
@@ -361,8 +375,10 @@
     macos:
       xcode: "11.2.1"
     steps:
+      # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
+      - should_run_job
       - checkout
-      - run_brew_for_ios_build
+      - run_brew_for_ios_build    
       - run:
           name: Run Fastlane
           no_output_timeout: "1h"
@@ -394,44 +410,30 @@
             WORKSPACE=/Users/distiller/workspace
             PROJ_ROOT=/Users/distiller/project
             export TCLLIBPATH="/usr/local/lib"
-
             # Install conda
-            curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            curl -o ~/Downloads/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
-            chmod +x ~/conda.sh
+            chmod +x ~/Downloads/conda.sh
-            /bin/bash ~/conda.sh -b -p ~/anaconda
+            /bin/bash ~/Downloads/conda.sh -b -p ~/anaconda
             export PATH="~/anaconda/bin:${PATH}"
             source ~/anaconda/bin/activate
-
             # Install dependencies
-            retry () {
+            conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing requests --yes
-                $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
-            }
-
-            retry conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing requests --yes
-
             # sync submodules
             cd ${PROJ_ROOT}
             git submodule sync
             git submodule update --init --recursive
-
             # export
             export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
-
             # run build script
             chmod a+x ${PROJ_ROOT}/scripts/build_ios.sh
             echo "IOS_ARCH: ${IOS_ARCH}"
             echo "IOS_PLATFORM: ${IOS_PLATFORM}"
-
+            export BUILD_PYTORCH_MOBILE=1
-            #check the custom build flag
-            echo "SELECTED_OP_LIST: ${SELECTED_OP_LIST}"
-            if [ -n "${SELECTED_OP_LIST}" ]; then
-                export SELECTED_OP_LIST="${PROJ_ROOT}/ios/TestApp/custom_build/${SELECTED_OP_LIST}"
-            fi
             export IOS_ARCH=${IOS_ARCH}
             export IOS_PLATFORM=${IOS_PLATFORM}
             unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts
       - run:
-          name: Run Build Test
+          name: Run Build Tests
           no_output_timeout: "30m"
           command: |
             set -e
@@ -443,11 +445,7 @@
               exit 1
             fi
             echo ${IOS_DEV_TEAM_ID}
-            if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then
+            ruby ${PROJ_ROOT}/scripts/xcode_build.rb -i ${PROJ_ROOT}/build_ios/install -x ${PROJ_ROOT}/ios/TestApp/TestApp.xcodeproj -p ${IOS_PLATFORM} -c ${PROFILE} -t ${IOS_DEV_TEAM_ID}
-              ruby ${PROJ_ROOT}/scripts/xcode_build.rb -i ${PROJ_ROOT}/build_ios/install -x ${PROJ_ROOT}/ios/TestApp/TestApp.xcodeproj -p ${IOS_PLATFORM} -c ${PROFILE} -t ${IOS_DEV_TEAM_ID}
-            else
-              ruby ${PROJ_ROOT}/scripts/xcode_build.rb -i ${PROJ_ROOT}/build_ios/install -x ${PROJ_ROOT}/ios/TestApp/TestApp.xcodeproj -p ${IOS_PLATFORM}
-            fi
             if ! [ "$?" -eq "0" ]; then
               echo 'xcodebuild failed!'
               exit 1
@@ -457,14 +455,15 @@
           no_output_timeout: "2h"
           command: |
             set -e
-            if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then
+            if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then 
               echo "not SIMULATOR build, skip it."
               exit 0
             fi
             WORKSPACE=/Users/distiller/workspace
             PROJ_ROOT=/Users/distiller/project
             source ~/anaconda/bin/activate
-            pip install torch torchvision --progress-bar off
+            #install the latest version of PyTorch and TorchVision
+            pip install torch torchvision
             #run unit test
             cd ${PROJ_ROOT}/ios/TestApp/benchmark
             python trace_model.py
@@ -472,106 +471,4 @@
             cd ${PROJ_ROOT}/ios/TestApp
             instruments -s -devices
             fastlane scan
-  pytorch_linux_bazel_build:
+            
-    <<: *pytorch_params
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    - checkout
-    - setup_linux_system_environment
-    - setup_ci_environment
-    - run:
-        name: Bazel Build
-        no_output_timeout: "1h"
-        command: |
-          set -e
-          # Pull Docker image and run build
-          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
-          time docker pull ${DOCKER_IMAGE} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
-
-          echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
-
-          git submodule sync && git submodule update -q --init --recursive
-
-          docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
-
-          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
-
-          # Push intermediate Docker image for next phase to use
-          if [ -z "${BUILD_ONLY}" ]; then
-            # Augment our output image name with bazel to avoid collisions
-            output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
-            export COMMIT_DOCKER_IMAGE=$output_image
-            docker commit "$id" ${COMMIT_DOCKER_IMAGE}
-            time docker push ${COMMIT_DOCKER_IMAGE}
-          fi
-
-  pytorch_linux_bazel_test:
-    <<: *pytorch_params
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    - checkout
-    - setup_linux_system_environment
-    - setup_ci_environment
-    - run:
-        name: Test
-        no_output_timeout: "90m"
-        command: |
-          set -e
-          output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
-          export COMMIT_DOCKER_IMAGE=$output_image
-          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
-
-          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
-
-          if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          else
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          fi
-
-          retrieve_test_reports() {
-            echo "retrieving test reports"
-            docker cp -L $id:/var/lib/jenkins/workspace/bazel-testlogs ./ || echo 'No test reports found!'
-          }
-          trap "retrieve_test_reports" ERR
-
-          if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          else
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          fi
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
-
-          retrieve_test_reports
-          docker stats --all --no-stream
-    - store_test_results:
-        path: bazel-testlogs
-
-  pytorch_doc_test:
-    environment:
-      BUILD_ENVIRONMENT: pytorch-doc-test
-      # TODO: stop hardcoding this
-      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:209062ef-ab58-422a-b295-36c4eed6e906"
-    resource_class: medium
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    - checkout
-    - setup_linux_system_environment
-    - setup_ci_environment
-    - run:
-        name: Doc test
-        no_output_timeout: "30m"
-        command: |
-          set -ex
-          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
-          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
-          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && . ./.jenkins/pytorch/docs-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

.circleci/verbatim-sources/job-specs-setup.yml

@@ -27,3 +27,4 @@
       - persist_to_workspace:
           root: .
           paths: .circleci/scripts
+

.circleci/verbatim-sources/job-specs/binary-build-tests.yml

@@ -1,14 +0,0 @@
-
-# There is currently no testing for libtorch TODO
-#  binary_linux_libtorch_3.6m_cpu_test:
-#    environment:
-#      BUILD_ENVIRONMENT: "libtorch 3.6m cpu"
-#    resource_class: gpu.medium
-#    <<: *binary_linux_test
-#
-#  binary_linux_libtorch_3.6m_cu90_test:
-#    environment:
-#      BUILD_ENVIRONMENT: "libtorch 3.6m cu90"
-#    resource_class: gpu.medium
-#    <<: *binary_linux_test
-#

.circleci/verbatim-sources/job-specs/docker_jobs.yml

@@ -1,135 +0,0 @@
-  docker_build_job:
-      parameters:
-        image_name:
-          type: string
-          default: ""
-      machine:
-        image: ubuntu-1604:201903-01
-      resource_class: large
-      environment:
-        IMAGE_NAME: << parameters.image_name >>
-        # Enable 'docker manifest'
-        DOCKER_CLI_EXPERIMENTAL: "enabled"
-        DOCKER_BUILDKIT: 1
-      steps:
-        - checkout
-        - run:
-            name: Calculate docker tag
-            command: |
-              set -x
-              mkdir .circleci/shared
-              # git keeps a hash of all sub trees
-              echo "export DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker)" >> .circleci/shared/env_file
-        # Saves our calculated docker tag to our workpace for later use
-        - persist_to_workspace:
-            root: .
-            paths:
-              - .circleci/shared/
-        - load_shared_env:
-            root: .
-        - run:
-            name: Check if image should be built
-            command: |
-              set +x
-              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
-              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-              eval $(aws ecr get-login --no-include-email --region us-east-1)
-              set -x
-              PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker")
-              # Check if image already exists, if it does then skip building it
-              if docker manifest inspect "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/${IMAGE_NAME}:${DOCKER_TAG}"; then
-                circleci-agent step halt
-                # circleci-agent step halt doesn't actually halt the step so we need to
-                # explicitly exit the step here ourselves before it causes too much trouble
-                exit 0
-              fi
-              # If no image exists but the hash is the same as the previous hash then we should error out here
-              if [[ ${PREVIOUS_DOCKER_TAG} = ${DOCKER_TAG} ]]; then
-                echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
-                echo "       contact the PyTorch team to restore the original images"
-                exit 1
-              fi
-        - run:
-            name: build_docker_image_<< parameters.image_name >>
-            no_output_timeout: "1h"
-            command: |
-              set +x
-              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
-              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-              set -x
-              cd .circleci/docker && ./build_docker.sh
-  docker_for_ecr_gc_build_job:
-      machine:
-        image: ubuntu-1604:201903-01
-      steps:
-        - checkout
-        - run:
-            name: build_docker_image_for_ecr_gc
-            no_output_timeout: "1h"
-            command: |
-              cd .circleci/ecr_gc_docker
-              docker build . -t 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
-              set +x
-              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
-              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-              eval $(aws ecr get-login --no-include-email --region us-east-1)
-              set -x
-              docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
-  ecr_gc_job:
-      parameters:
-        project:
-          type: string
-          default: "pytorch"
-        tags_to_keep:  # comma separate values
-          type: string
-      environment:
-        PROJECT: << parameters.project >>
-        # TODO: Remove legacy image tags once we feel comfortable with new docker image tags
-        IMAGE_TAG: << parameters.tags_to_keep >>
-      docker:
-        - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
-          aws_auth:
-            aws_access_key_id: ${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
-            aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-
-      steps:
-        - checkout
-        - run:
-            # NOTE: see 'docker_build_job' for how these tags actually get built
-            name: dynamically generate tags to keep
-            no_output_timeout: "1h"
-            command: |
-              GENERATED_IMAGE_TAG=$(\
-                git log --oneline --pretty='%H' .circleci/docker \
-                  | xargs -I '{}' git rev-parse '{}:.circleci/docker' \
-                  | paste -sd "," -)
-              echo "export GENERATED_IMAGE_TAG='${GENERATED_IMAGE_TAG}'" >> ${BASH_ENV}
-        - run:
-            name: garbage collecting for ecr images
-            no_output_timeout: "1h"
-            command: |
-              set +x
-              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
-              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-              set -x
-              /usr/bin/gc.py --filter-prefix ${PROJECT}  --ignore-tags "${IMAGE_TAG},${GENERATED_IMAGE_TAG}"
-
-  docker_hub_index_job:
-      docker:
-        - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
-          aws_auth:
-            aws_access_key_id: ${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
-            aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-
-      steps:
-        - run:
-            name: garbage collecting for ecr images
-            no_output_timeout: "1h"
-            command: |
-              set +x
-              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
-              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-              export DOCKER_HUB_USERNAME=${CIRCLECI_DOCKER_HUB_USERNAME}
-              export DOCKER_HUB_PASSWORD=${CIRCLECI_DOCKER_HUB_PASSWORD}
-              set -x
-              /usr/bin/docker_hub.py

.circleci/verbatim-sources/job-specs/job-specs-promote.yml

@@ -1,18 +0,0 @@
-
-  promote_s3:
-    <<: *promote_common
-    steps:
-      - checkout
-      - run:
-          name: Running promote script
-          command: |
-            scripts/release/promote/wheel_to_s3.sh
-
-  promote_conda:
-    <<: *promote_common
-    steps:
-      - checkout
-      - run:
-          name: Running promote script
-          command: |
-            scripts/release/promote/conda_to_conda.sh

.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml

@@ -1,304 +0,0 @@
-jobs:
-  pytorch_linux_build:
-    <<: *pytorch_params
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
-    - setup_linux_system_environment
-    - checkout
-    - optional_merge_target_branch
-    - setup_ci_environment
-    - run:
-        name: Build
-        no_output_timeout: "1h"
-        command: |
-          set -e
-          # Pull Docker image and run build
-          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
-          time docker pull ${DOCKER_IMAGE} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
-
-          git submodule sync && git submodule update -q --init --recursive
-
-          docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
-
-          if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
-            export PARALLEL_FLAGS="export ATEN_THREADING=TBB USE_TBB=1 "
-          elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
-            export PARALLEL_FLAGS="export ATEN_THREADING=NATIVE "
-          fi
-          echo "Parallel backend flags: "${PARALLEL_FLAGS}
-
-          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo '"$PARALLEL_FLAGS"' && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
-
-          # Push intermediate Docker image for next phase to use
-          if [ -z "${BUILD_ONLY}" ]; then
-            # Note [Special build images]
-            # The xla build uses the same docker image as
-            # pytorch-linux-trusty-py3.6-gcc5.4-build. In the push step, we have to
-            # distinguish between them so the test can pick up the correct image.
-            output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}
-            if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then
-              export COMMIT_DOCKER_IMAGE=$output_image-xla
-            elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then
-              export COMMIT_DOCKER_IMAGE=$output_image-libtorch
-            elif [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
-              export COMMIT_DOCKER_IMAGE=$output_image-paralleltbb
-            elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
-              export COMMIT_DOCKER_IMAGE=$output_image-parallelnative
-            elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-x86_64"* ]]; then
-              export COMMIT_DOCKER_IMAGE=$output_image-android-x86_64
-            elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-arm-v7a"* ]]; then
-              export COMMIT_DOCKER_IMAGE=$output_image-android-arm-v7a
-            elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-arm-v8a"* ]]; then
-              export COMMIT_DOCKER_IMAGE=$output_image-android-arm-v8a
-            elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-x86_32"* ]]; then
-              export COMMIT_DOCKER_IMAGE=$output_image-android-x86_32
-            elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-vulkan-x86_32"* ]]; then
-              export COMMIT_DOCKER_IMAGE=$output_image-android-vulkan-x86_32
-            else
-              export COMMIT_DOCKER_IMAGE=$output_image
-            fi
-            docker commit "$id" ${COMMIT_DOCKER_IMAGE}
-            time docker push ${COMMIT_DOCKER_IMAGE}
-          fi
-
-  pytorch_linux_test:
-    <<: *pytorch_params
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - checkout
-    - setup_linux_system_environment
-    - setup_ci_environment
-    - run:
-        name: Download Docker image
-        no_output_timeout: "90m"
-        command: |
-          set -e
-          # See Note [Special build images]
-          output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}
-          if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then
-            export COMMIT_DOCKER_IMAGE=$output_image-xla
-          elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then
-            export COMMIT_DOCKER_IMAGE=$output_image-libtorch
-          elif [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
-            export COMMIT_DOCKER_IMAGE=$output_image-paralleltbb
-          elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
-            export COMMIT_DOCKER_IMAGE=$output_image-parallelnative
-          else
-            export COMMIT_DOCKER_IMAGE=$output_image
-          fi
-          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
-
-          if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
-            export PARALLEL_FLAGS="export ATEN_THREADING=TBB USE_TBB=1 "
-          elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
-            export PARALLEL_FLAGS="export ATEN_THREADING=NATIVE "
-          fi
-          echo "Parallel backend flags: "${PARALLEL_FLAGS}
-
-          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
-
-          if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          else
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          fi
-
-          # Pass environment variables to the next step
-          # See https://circleci.com/docs/2.0/env-vars/#using-parameters-and-bash-environment
-          echo "export PARALLEL_FLAGS=\"${PARALLEL_FLAGS}\"" >> $BASH_ENV
-          echo "export id=$id" >> $BASH_ENV
-    - run:
-        name: Check for no AVX instruction by default
-        no_output_timeout: "20m"
-        command: |
-          set -e
-          is_vanilla_build() {
-            if [ "${BUILD_ENVIRONMENT}" == "pytorch-linux-bionic-py3.6-clang9-test" ]; then
-              return 0
-            fi
-            if [ "${BUILD_ENVIRONMENT}" == "pytorch-linux-xenial-py3.6-gcc5.4-test" ]; then
-              return 0
-            fi
-            return 1
-          }
-
-          if is_vanilla_build; then
-            echo "apt-get update && apt-get install -y qemu-user" | docker exec -u root -i "$id" bash
-            echo "cd workspace/build; qemu-x86_64 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU" | docker exec -u jenkins -i "$id" bash
-          else
-            echo "Skipping for ${BUILD_ENVIRONMENT}"
-          fi
-    - run:
-        name: Run tests
-        no_output_timeout: "90m"
-        command: |
-          set -e
-
-          if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "${PARALLEL_FLAGS}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          else
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}" && echo "${PARALLEL_FLAGS}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          fi
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
-    - run:
-        name: Report results
-        no_output_timeout: "5m"
-        command: |
-          set -e
-          docker stats --all --no-stream
-          echo "cd workspace; python test/print_test_stats.py test" | docker exec -u jenkins -i "$id" bash
-          echo "Retrieving test reports"
-          docker cp $id:/var/lib/jenkins/workspace/test/test-reports ./ || echo 'No test reports found!'
-        when: always
-    - store_test_results:
-        path: test-reports
-
-  pytorch_windows_build:
-    <<: *pytorch_windows_params
-    parameters:
-      executor:
-        type: string
-        default: "windows-cpu-with-nvidia-cuda"
-      build_environment:
-        type: string
-        default: ""
-      test_name:
-        type: string
-        default: ""
-      cuda_version:
-        type: string
-        default: "10"
-      python_version:
-        type: string
-        default: "3.6"
-      vc_version:
-        type: string
-        default: "14.11"
-      vc_year:
-        type: string
-        default: "2017"
-      vc_product:
-        type: string
-        default: "BuildTools"
-      use_cuda:
-        type: string
-        default: ""
-    executor: <<parameters.executor>>
-    steps:
-      - checkout
-      - run:
-          name: Install VS2017
-          command: |
-            if [[ "${VC_YEAR}" == "2017" ]]; then
-              powershell .circleci/scripts/vs_install.ps1
-            fi
-      - run:
-          name: Install Cuda
-          no_output_timeout: 30m
-          command: |
-            if [[ "${USE_CUDA}" == "1" ]]; then
-              .circleci/scripts/windows_cuda_install.sh
-            fi
-      - run:
-          name: Install Cudnn
-          command : |
-            if [[ "${USE_CUDA}" == "1" ]]; then
-              cd c:/
-              curl --retry 3 -O https://ossci-windows.s3.amazonaws.com/cudnn-10.1-windows10-x64-v7.6.4.38.zip
-              7z x cudnn-10.1-windows10-x64-v7.6.4.38.zip -ocudnn
-              cp -r cudnn/cuda/* "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/"
-            fi
-      - run:
-          name: Build
-          no_output_timeout: "90m"
-          command: |
-            set -e
-            set +x
-            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_WIN_BUILD_V1}
-            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_WIN_BUILD_V1}
-            set -x
-            .jenkins/pytorch/win-build.sh
-      - persist_to_workspace:
-          root: "C:/w"
-          paths: build-results
-      - store_artifacts:
-          path: C:/w/build-results
-
-  pytorch_windows_test:
-    <<: *pytorch_windows_params
-    parameters:
-      executor:
-        type: string
-        default: "windows-cpu-with-nvidia-cuda"
-      build_environment:
-        type: string
-        default: ""
-      test_name:
-        type: string
-        default: ""
-      cuda_version:
-        type: string
-        default: "10"
-      python_version:
-        type: string
-        default: "3.6"
-      vc_version:
-        type: string
-        default: "14.11"
-      vc_year:
-        type: string
-        default: "2017"
-      vc_product:
-        type: string
-        default: "BuildTools"
-      use_cuda:
-        type: string
-        default: ""
-    executor: <<parameters.executor>>
-    steps:
-      - checkout
-      - attach_workspace:
-          at: c:/users/circleci/workspace
-      - run:
-          name: Install VS2017
-          command: |
-            if [[ "${VC_YEAR}" == "2017" ]]; then
-              powershell .circleci/scripts/vs_install.ps1
-            fi
-      - run:
-          name: Install Cuda
-          no_output_timeout: 30m
-          command: |
-            if [[ "${CUDA_VERSION}" != "cpu" && "${JOB_EXECUTOR}" != "windows-with-nvidia-gpu" ]]; then
-              .circleci/scripts/windows_cuda_install.sh
-            fi
-      - run:
-          name: Install Cudnn
-          command : |
-            if [[ "${CUDA_VERSION}" != "cpu" && "${JOB_EXECUTOR}" != "windows-with-nvidia-gpu" ]]; then
-              cd c:/
-              curl --retry 3 -O https://ossci-windows.s3.amazonaws.com/cudnn-10.1-windows10-x64-v7.6.4.38.zip
-              7z x cudnn-10.1-windows10-x64-v7.6.4.38.zip -ocudnn
-              cp -r cudnn/cuda/* "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/"
-            fi
-      - run:
-          name: Test
-          no_output_timeout: "30m"
-          command: |
-            set -e
-            export IN_CIRCLECI=1
-            set +x
-            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_WIN_BUILD_V1}
-            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_WIN_BUILD_V1}
-            set -x
-            .jenkins/pytorch/win-test.sh
-      - store_test_results:
-          path: test/test-reports

.circleci/verbatim-sources/nightly-binary-build-defaults.yml

@@ -26,18 +26,18 @@
 # (smoke tests and upload jobs do not need the pytorch repo).
 binary_checkout: &binary_checkout
   name: Checkout pytorch/builder repo
-  command: .circleci/scripts/binary_checkout.sh
+  command: ~/workspace/.circleci/scripts/binary_checkout.sh
 
 # Parses circleci arguments in a consistent way, essentially routing to the
 # correct pythonXgccXcudaXos build we want
 binary_populate_env: &binary_populate_env
   name: Set up binary env variables
-  command: .circleci/scripts/binary_populate_env.sh
+  command: ~/workspace/.circleci/scripts/binary_populate_env.sh
 
 binary_install_miniconda: &binary_install_miniconda
   name: Install miniconda
   no_output_timeout: "1h"
-  command: .circleci/scripts/binary_install_miniconda.sh
+  command: ~/workspace/.circleci/scripts/binary_install_miniconda.sh
 
 # This section is used in the binary_test and smoke_test jobs. It expects
 # 'binary_populate_env' to have populated /home/circleci/project/env and it
@@ -47,4 +47,4 @@ binary_run_in_docker: &binary_run_in_docker
   name: Run in docker
   # This step only runs on circleci linux machine executors that themselves
   # need to start docker images
-  command: .circleci/scripts/binary_run_in_docker.sh
+  command: ~/workspace/.circleci/scripts/binary_run_in_docker.sh

.circleci/verbatim-sources/pytorch-build-params.yml

@@ -0,0 +1,39 @@
+pytorch_params: &pytorch_params
+  parameters:
+    build_environment:
+      type: string
+      default: ""
+    docker_image:
+      type: string
+      default: ""
+    resource_class:
+      type: string
+      default: "large"
+    use_cuda_docker_runtime:
+      type: string
+      default: ""
+  environment:
+    BUILD_ENVIRONMENT: << parameters.build_environment >>
+    DOCKER_IMAGE: << parameters.docker_image >>
+    USE_CUDA_DOCKER_RUNTIME: << parameters.use_cuda_docker_runtime >>
+  resource_class: << parameters.resource_class >>
+
+pytorch_ios_params: &pytorch_ios_params
+  parameters:
+    build_environment:
+      type: string
+      default: ""
+    ios_arch:
+      type: string
+      default: ""
+    ios_platform:
+      type: string
+      default: ""
+  environment:
+    BUILD_ENVIRONMENT: << parameters.build_environment >>
+    IOS_ARCH: << parameters.ios_arch >>
+    IOS_PLATFORM: << parameters.ios_platform >>
+
+
+
+

.circleci/verbatim-sources/pytorch-job-specs.yml

@@ -0,0 +1,141 @@
+jobs:
+  pytorch_linux_build:
+    <<: *pytorch_params
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
+    - should_run_job
+    - setup_linux_system_environment
+    - checkout
+    - setup_ci_environment
+    - run:
+        name: Build
+        no_output_timeout: "1h"
+        command: |
+          set -e
+          # Pull Docker image and run build
+          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
+          time docker pull ${DOCKER_IMAGE} >/dev/null
+          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+
+          # NB: Temporarily disable the rebase logic in v1.4.0, don't merge this change into master
+          # # TODO We may want to move the rebase logic to a separate step after checkout
+          # # Rebase to master only if in xenial_py3_6_gcc5_4 case
+          # if [[ "${CIRCLE_BRANCH}" != "master" && "${BUILD_ENVIRONMENT}" == *"gcc5"* ]]; then
+          #   echo "Merge master branch into $CIRCLE_BRANCH before build in environment $BUILD_ENVIRONMENT"
+          #   set -x
+          #   git config --global user.email "circleci.ossci@gmail.com"
+          #   git config --global user.name "CircleCI"
+          #   git config remote.origin.url https://github.com/pytorch/pytorch.git
+          #   git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master
+          #   git fetch --tags --progress https://github.com/pytorch/pytorch.git +refs/heads/master:refs/remotes/origin/master --depth=100 --quiet
+          #   export GIT_MERGE_TARGET=`git log -n 1 --pretty=format:"%H" origin/master`
+          #   echo "GIT_MERGE_TARGET: " ${GIT_MERGE_TARGET}
+          #   export GIT_COMMIT=${CIRCLE_SHA1}
+          #   echo "GIT_COMMIT: " ${GIT_COMMIT}
+          #   git checkout -f ${GIT_COMMIT}
+          #   git reset --hard ${GIT_COMMIT}
+          #   git merge --allow-unrelated-histories --no-edit --no-ff ${GIT_MERGE_TARGET}
+          #   set +x
+          # else
+          #   echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
+          # fi
+
+          git submodule sync && git submodule update -q --init --recursive
+
+          docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
+
+          if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
+            export PARALLEL_FLAGS="export ATEN_THREADING=TBB USE_TBB=1 "
+          elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
+            export PARALLEL_FLAGS="export ATEN_THREADING=NATIVE "
+          fi
+          echo "Parallel backend flags: "${PARALLEL_FLAGS}
+
+          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo '"$PARALLEL_FLAGS"' && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+
+          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+
+          # Push intermediate Docker image for next phase to use
+          if [ -z "${BUILD_ONLY}" ]; then
+            # Note [Special build images]
+            # The xla build uses the same docker image as
+            # pytorch-linux-trusty-py3.6-gcc5.4-build. In the push step, we have to
+            # distinguish between them so the test can pick up the correct image.
+            output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+            if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then
+              export COMMIT_DOCKER_IMAGE=$output_image-xla
+            elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then
+              export COMMIT_DOCKER_IMAGE=$output_image-libtorch
+            elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-x86_64"* ]]; then
+              export COMMIT_DOCKER_IMAGE=$output_image-android-x86_64
+            elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-arm-v7a"* ]]; then
+              export COMMIT_DOCKER_IMAGE=$output_image-android-arm-v7a
+            elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-arm-v8a"* ]]; then
+              export COMMIT_DOCKER_IMAGE=$output_image-android-arm-v8a
+            elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-x86_32"* ]]; then
+              export COMMIT_DOCKER_IMAGE=$output_image-android-x86_32
+            else
+              export COMMIT_DOCKER_IMAGE=$output_image
+            fi
+            docker commit "$id" ${COMMIT_DOCKER_IMAGE}
+            time docker push ${COMMIT_DOCKER_IMAGE}
+          fi
+
+  pytorch_linux_test:
+    <<: *pytorch_params
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
+    - should_run_job
+    - setup_linux_system_environment
+    - setup_ci_environment
+    - run:
+        name: Test
+        no_output_timeout: "90m"
+        command: |
+          set -e
+          # See Note [Special build images]
+          output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+          if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then
+            export COMMIT_DOCKER_IMAGE=$output_image-xla
+          elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then
+            export COMMIT_DOCKER_IMAGE=$output_image-libtorch
+          else
+            export COMMIT_DOCKER_IMAGE=$output_image
+          fi
+          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
+
+          if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
+            export PARALLEL_FLAGS="export ATEN_THREADING=TBB USE_TBB=1 "
+          elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
+            export PARALLEL_FLAGS="export ATEN_THREADING=NATIVE "
+          fi
+          echo "Parallel backend flags: "${PARALLEL_FLAGS}
+
+          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
+
+          if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
+            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          else
+            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          fi
+
+          retrieve_test_reports() {
+            echo "retrieving test reports"
+            docker cp $id:/var/lib/jenkins/workspace/test/test-reports ./ || echo 'No test reports found!'
+          }
+          trap "retrieve_test_reports" ERR
+
+          if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
+            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "${PARALLEL_FLAGS}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          else
+            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "${PARALLEL_FLAGS}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          fi
+          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+
+          retrieve_test_reports
+    - store_test_results:
+        path: test-reports

.circleci/verbatim-sources/workflows-binary-build-header.yml

@@ -0,0 +1,4 @@
+
+##############################################################################
+# Daily binary build trigger
+##############################################################################

.circleci/verbatim-sources/workflows-binary-builds-smoke-subset.yml

@@ -0,0 +1,101 @@
+      # Binary builds (subset, to smoke test that they'll work)
+      #
+      # NB: If you modify this file, you need to also modify
+      # the binary_and_smoke_tests_on_pr variable in
+      # pytorch-ci-hud to adjust the list of whitelisted builds
+      # at https://github.com/ezyang/pytorch-ci-hud/blob/master/src/BuildHistoryDisplay.js
+
+      - binary_linux_build:
+          name: binary_linux_manywheel_2_7mu_cpu_devtoolset7_build
+          build_environment: "manywheel 2.7mu cpu devtoolset7"
+          requires:
+            - setup
+          docker_image: "pytorch/manylinux-cuda100"
+      - binary_linux_build:
+          name: binary_linux_manywheel_3_7m_cu100_devtoolset7_build
+          build_environment: "manywheel 3.7m cu100 devtoolset7"
+          requires:
+            - setup
+          docker_image: "pytorch/manylinux-cuda100"
+      - binary_linux_build:
+          name: binary_linux_conda_2_7_cpu_devtoolset7_build
+          build_environment: "conda 2.7 cpu devtoolset7"
+          requires:
+            - setup
+          docker_image: "pytorch/conda-cuda"
+      # This binary build is currently broken, see https://github_com/pytorch/pytorch/issues/16710
+      # - binary_linux_conda_3_6_cu90_devtoolset7_build
+      - binary_linux_build:
+          name: binary_linux_libtorch_2_7m_cpu_devtoolset7_shared-with-deps_build
+          build_environment: "libtorch 2.7m cpu devtoolset7"
+          requires:
+            - setup
+          libtorch_variant: "shared-with-deps"
+          docker_image: "pytorch/manylinux-cuda100"
+      - binary_linux_build:
+          name: binary_linux_libtorch_2_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build
+          build_environment: "libtorch 2.7m cpu gcc5.4_cxx11-abi"
+          requires:
+            - setup
+          libtorch_variant: "shared-with-deps"
+          docker_image: "pytorch/pytorch-binary-docker-image-ubuntu16.04:latest"
+      # TODO we should test a libtorch cuda build, but they take too long
+      # - binary_linux_libtorch_2_7m_cu90_devtoolset7_static-without-deps_build
+      - binary_mac_build:
+          name: binary_macos_wheel_3_6_cpu_build
+          build_environment: "wheel 3.6 cpu"
+          requires:
+            - setup
+      - binary_mac_build:
+          name: binary_macos_conda_2_7_cpu_build
+          build_environment: "conda 2.7 cpu"
+          requires:
+            - setup
+      - binary_mac_build:
+          name: binary_macos_libtorch_2_7_cpu_build
+          build_environment: "libtorch 2.7 cpu"
+          requires:
+            - setup
+
+      - binary_linux_test:
+          name: binary_linux_manywheel_2_7mu_cpu_devtoolset7_test
+          build_environment: "manywheel 2.7mu cpu devtoolset7"
+          requires:
+            - setup
+            - binary_linux_manywheel_2_7mu_cpu_devtoolset7_build
+          docker_image: "pytorch/manylinux-cuda100"
+      - binary_linux_test:
+          name: binary_linux_manywheel_3_7m_cu100_devtoolset7_test
+          build_environment: "manywheel 3.7m cu100 devtoolset7"
+          requires:
+            - setup
+            - binary_linux_manywheel_3_7m_cu100_devtoolset7_build
+          docker_image: "pytorch/manylinux-cuda100"
+          use_cuda_docker_runtime: "1"
+          resource_class: gpu.medium
+      - binary_linux_test:
+          name: binary_linux_conda_2_7_cpu_devtoolset7_test
+          build_environment: "conda 2.7 cpu devtoolset7"
+          requires:
+            - setup
+            - binary_linux_conda_2_7_cpu_devtoolset7_build
+          docker_image: "pytorch/conda-cuda"
+      # This binary build is currently broken, see https://github_com/pytorch/pytorch/issues/16710
+      # - binary_linux_conda_3_6_cu90_devtoolset7_test:
+      - binary_linux_test:
+          name: binary_linux_libtorch_2_7m_cpu_devtoolset7_shared-with-deps_test
+          build_environment: "libtorch 2.7m cpu devtoolset7"
+          requires:
+            - setup
+            - binary_linux_libtorch_2_7m_cpu_devtoolset7_shared-with-deps_build
+          libtorch_variant: "shared-with-deps"
+          docker_image: "pytorch/manylinux-cuda100"
+      - binary_linux_test:
+          name: binary_linux_libtorch_2_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_test
+          build_environment: "libtorch 2.7m cpu gcc5.4_cxx11-abi"
+          requires:
+            - setup
+            - binary_linux_libtorch_2_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build
+          libtorch_variant: "shared-with-deps"
+          docker_image: "pytorch/pytorch-binary-docker-image-ubuntu16.04:latest"
+

.circleci/verbatim-sources/workflows-docker-builder.yml

@@ -0,0 +1,66 @@
+  docker_build:
+    triggers:
+      - schedule:
+          cron: "0 15 * * 0"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - docker_build_job:
+          name: "pytorch-linux-bionic-clang9-thrift-llvmdev"
+          image_name: "pytorch-linux-bionic-clang9-thrift-llvmdev"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7"
+          image_name: "pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
+          image_name: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-cuda8-cudnn7-py2"
+          image_name: "pytorch-linux-xenial-cuda8-cudnn7-py2"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-cuda8-cudnn7-py3"
+          image_name: "pytorch-linux-xenial-cuda8-cudnn7-py3"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-cuda9-cudnn7-py2"
+          image_name: "pytorch-linux-xenial-cuda9-cudnn7-py2"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-cuda9-cudnn7-py3"
+          image_name: "pytorch-linux-xenial-cuda9-cudnn7-py3"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
+          image_name: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py2.7.9"
+          image_name: "pytorch-linux-xenial-py2.7.9"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py2.7"
+          image_name: "pytorch-linux-xenial-py2.7"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
+          image_name: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py3-clang5-asan"
+          image_name: "pytorch-linux-xenial-py3-clang5-asan"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py3.5"
+          image_name: "pytorch-linux-xenial-py3.5"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py3.6-clang7"
+          image_name: "pytorch-linux-xenial-py3.6-clang7"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py3.6-gcc4.8"
+          image_name: "pytorch-linux-xenial-py3.6-gcc4.8"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py3.6-gcc5.4"
+          image_name: "pytorch-linux-xenial-py3.6-gcc5.4"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py3.6-gcc7.2"
+          image_name: "pytorch-linux-xenial-py3.6-gcc7.2"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-py3.6-gcc7"
+          image_name: "pytorch-linux-xenial-py3.6-gcc7"
+      - docker_build_job:
+          name: "pytorch-linux-xenial-pynightly"
+          image_name: "pytorch-linux-xenial-pynightly"

.circleci/verbatim-sources/workflows-nightly-android-binary-builds.yml

@@ -0,0 +1,56 @@
+      - pytorch_linux_build:
+          name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build
+          build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32"
+          requires:
+            - setup
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+          filters:
+            branches:
+              only: nightly
+      - pytorch_linux_build:
+          name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build
+          build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64"
+          requires:
+            - setup
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+          filters:
+            branches:
+              only: nightly
+      - pytorch_linux_build:
+          name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build
+          build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a"
+          requires:
+            - setup
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+          filters:
+            branches:
+              only: nightly
+      - pytorch_linux_build:
+          name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build
+          build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a"
+          requires:
+            - setup
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:405"
+          filters:
+            branches:
+              only: nightly
+
+      - pytorch_android_gradle_build:
+          name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build
+          requires:
+            - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build
+            - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build
+            - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build
+            - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build
+          filters:
+            branches:
+              only: nightly
+
+      - pytorch_android_publish_snapshot:
+          name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_android_publish_snapshot
+          requires:
+            - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build
+          context: org-member
+          filters:
+            branches:
+              only: nightly

.circleci/verbatim-sources/workflows-nightly-ios-binary-builds.yml

@@ -0,0 +1,33 @@
+      # Pytorch iOS binary builds
+      - binary_ios_build:
+          name: pytorch_ios_11_2_1_nightly_x86_64_build
+          build_environment: "libtorch-ios-11.2.1-nightly-x86_64-build"
+          context: org-member
+          ios_platform: "SIMULATOR"
+          ios_arch: "x86_64"
+          requires: 
+            - setup
+          filters:
+            branches:
+              only: nightly
+      - binary_ios_build:
+          name: pytorch_ios_11_2_1_nightly_arm64_build
+          build_environment: "libtorch-ios-11.2.1-nightly-arm64-build"
+          context: org-member
+          ios_arch: "arm64"
+          ios_platform: "OS"
+          requires: 
+            - setup
+          filters:
+            branches:
+              only: nightly
+      - binary_ios_upload:
+          build_environment: "libtorch-ios-11.2.1-nightly-binary-build-upload"
+          context: org-member
+          requires:
+            - setup
+            - pytorch_ios_11_2_1_nightly_x86_64_build
+            - pytorch_ios_11_2_1_nightly_arm64_build
+          filters:
+            branches:
+              only: nightly

.circleci/verbatim-sources/workflows-nightly-uploads-header.yml

@@ -0,0 +1,11 @@
+      #- binary_linux_libtorch_2.7m_cpu_test:
+      #    requires:
+      #      - binary_linux_libtorch_2.7m_cpu_build
+      #- binary_linux_libtorch_2.7m_cu90_test:
+      #    requires:
+      #      - binary_linux_libtorch_2.7m_cu90_build
+      #- binary_linux_libtorch_2.7m_cu100_test:
+      #    requires:
+      #      - binary_linux_libtorch_2.7m_cu100_build
+
+      # Nightly uploads

.circleci/verbatim-sources/workflows-pytorch-android-gradle-build.yml

@@ -0,0 +1,12 @@
+      - pytorch_android_gradle_build-x86_32:
+          name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32
+          requires:
+            - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build
+
+      - pytorch_android_gradle_build:
+          name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build
+          requires:
+            - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build
+            - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build
+            - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build
+            - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build

.circleci/verbatim-sources/workflows-pytorch-ge-config-tests.yml

@@ -0,0 +1,16 @@
+      - pytorch_linux_test:
+          name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_legacy_test
+          requires:
+            - setup
+            - pytorch_linux_xenial_py3_6_gcc5_4_build
+          build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_legacy-test"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:405"
+          resource_class: large
+      - pytorch_linux_test:
+          name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_simple_test
+          requires:
+            - setup
+            - pytorch_linux_xenial_py3_6_gcc5_4_build
+          build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_simple-test"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:405"
+          resource_class: large